2024-09-21T09:18:54.295 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:18:54.323 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:18:54.417 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646 branch: main description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7913646' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi076.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM+SO4siWHc5OcG93/b0yF5QTvR4Kf8fw/IxkAnJl5F3B71xUboyRCk7rwRMXy6RRWqWfwphb5372iT/bQzmv68= smithi191.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHdeAjFv+Yw7ltrcM6fxDb1ZLgJtXPHmbHt/854w/9TMoFVuQMv4ai8luSICbTOCcUXyhWp/jbHiGfGFgti1Qn4= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-21T09:18:54.417 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:18:54.417 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:18:54.418 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:18:54.419 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:18:54.440 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:18:54.440 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:18:54.440 INFO:teuthology.packaging:ref: None 2024-09-21T09:18:54.440 INFO:teuthology.packaging:tag: None 2024-09-21T09:18:54.440 INFO:teuthology.packaging:branch: main 2024-09-21T09:18:54.440 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:18:54.440 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:18:54.743 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:18:54.744 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:18:54.751 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:18:54.751 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:18:54.763 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:18:54.788 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:18:54.795 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:18:54.821 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi076.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:13:36.830743', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM+SO4siWHc5OcG93/b0yF5QTvR4Kf8fw/IxkAnJl5F3B71xUboyRCk7rwRMXy6RRWqWfwphb5372iT/bQzmv68='} 2024-09-21T09:18:54.842 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi191.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:13:36.832117', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHdeAjFv+Yw7ltrcM6fxDb1ZLgJtXPHmbHt/854w/9TMoFVuQMv4ai8luSICbTOCcUXyhWp/jbHiGfGFgti1Qn4='} 2024-09-21T09:18:54.843 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:18:54.851 INFO:teuthology.task.internal:roles: ubuntu@smithi076.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-21T09:18:54.851 INFO:teuthology.task.internal:roles: ubuntu@smithi191.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-21T09:18:54.851 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:18:54.915 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fcf744c0160>, signals=[15]) 2024-09-21T09:18:54.915 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:18:54.921 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:18:54.921 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi076.front.sepia.ceph.com 2024-09-21T09:18:54.922 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:18:55.000 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi191.front.sepia.ceph.com 2024-09-21T09:18:55.001 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:18:55.096 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:18:55.102 DEBUG:teuthology.orchestra.run.smithi076:> uname -m 2024-09-21T09:18:55.120 INFO:teuthology.orchestra.run.smithi076.stdout:x86_64 2024-09-21T09:18:55.120 DEBUG:teuthology.orchestra.run.smithi076:> cat /etc/os-release 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:NAME="CentOS Stream" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:VERSION="9" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:ID="centos" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:VERSION_ID="9" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:ANSI_COLOR="0;31" 2024-09-21T09:18:55.178 INFO:teuthology.orchestra.run.smithi076.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:18:55.179 INFO:teuthology.orchestra.run.smithi076.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:18:55.179 INFO:teuthology.orchestra.run.smithi076.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:18:55.179 INFO:teuthology.orchestra.run.smithi076.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:18:55.179 INFO:teuthology.orchestra.run.smithi076.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:18:55.179 INFO:teuthology.orchestra.run.smithi076.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:18:55.180 INFO:teuthology.lock.ops:Updating smithi076.front.sepia.ceph.com on lock server 2024-09-21T09:18:55.200 DEBUG:teuthology.orchestra.run.smithi191:> uname -m 2024-09-21T09:18:55.219 INFO:teuthology.orchestra.run.smithi191.stdout:x86_64 2024-09-21T09:18:55.219 DEBUG:teuthology.orchestra.run.smithi191:> cat /etc/os-release 2024-09-21T09:18:55.276 INFO:teuthology.orchestra.run.smithi191.stdout:NAME="CentOS Stream" 2024-09-21T09:18:55.276 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION="9" 2024-09-21T09:18:55.276 INFO:teuthology.orchestra.run.smithi191.stdout:ID="centos" 2024-09-21T09:18:55.276 INFO:teuthology.orchestra.run.smithi191.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:VERSION_ID="9" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:ANSI_COLOR="0;31" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:18:55.277 INFO:teuthology.orchestra.run.smithi191.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:18:55.278 INFO:teuthology.lock.ops:Updating smithi191.front.sepia.ceph.com on lock server 2024-09-21T09:18:55.300 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:18:55.309 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:18:55.314 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:18:55.314 DEBUG:teuthology.orchestra.run.smithi076:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:18:55.317 DEBUG:teuthology.orchestra.run.smithi191:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:18:55.333 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:18:55.338 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:18:55.339 DEBUG:teuthology.orchestra.run.smithi076:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:18:55.374 DEBUG:teuthology.orchestra.run.smithi191:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:18:55.482 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:18:55.537 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:18:55.551 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:18:55.551 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:18:55.551 DEBUG:teuthology.orchestra.run.smithi076:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:18:55.552 DEBUG:teuthology.orchestra.run.smithi191:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:18:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:18:55.568 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2024-09-21T09:18:55.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:18:55.569 DEBUG:teuthology.orchestra.run.smithi076:> uname -r 2024-09-21T09:18:55.626 INFO:teuthology.orchestra.run.smithi191.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:18:55.626 INFO:teuthology.task.kernel:Running kernel on smithi191: 5.14.0-503.el9.x86_64 2024-09-21T09:18:55.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum install -y kernel 2024-09-21T09:18:55.627 INFO:teuthology.orchestra.run.smithi076.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:18:55.627 INFO:teuthology.task.kernel:Running kernel on smithi076: 5.14.0-503.el9.x86_64 2024-09-21T09:18:55.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum install -y kernel 2024-09-21T09:18:58.129 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.3 MB 00:00 2024-09-21T09:18:59.446 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - BaseOS 4.2 MB/s | 8.3 MB 00:01 2024-09-21T09:19:00.797 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - AppStream 13 MB/s | 20 MB 00:01 2024-09-21T09:19:01.190 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-21T09:19:05.049 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.5 MB 00:00 2024-09-21T09:19:06.841 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 19 kB 00:00 2024-09-21T09:19:07.946 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 23 MB/s | 23 MB 00:00 2024-09-21T09:19:08.656 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - CRB 1.8 MB/s | 6.5 MB 00:03 2024-09-21T09:19:10.673 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 19 kB 00:00 2024-09-21T09:19:11.561 INFO:teuthology.orchestra.run.smithi076.stdout:Extra Packages for Enterprise Linux 30 MB/s | 23 MB 00:00 2024-09-21T09:19:13.327 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-21T09:19:15.334 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:19:15.334 INFO:teuthology.orchestra.run.smithi191.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:19:15.419 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:19:15.430 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:19:15.431 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:19:15.432 INFO:teuthology.orchestra.run.smithi191.stdout:Install 4 Packages 2024-09-21T09:19:15.432 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:19:15.432 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 79 M 2024-09-21T09:19:15.432 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 126 M 2024-09-21T09:19:15.432 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:19:15.571 INFO:teuthology.orchestra.run.smithi191.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 386 kB/s | 36 kB 00:00 2024-09-21T09:19:16.047 INFO:teuthology.orchestra.run.smithi191.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 28 MB/s | 16 MB 00:00 2024-09-21T09:19:16.788 INFO:teuthology.orchestra.run.smithi191.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 26 MB/s | 35 MB 00:01 2024-09-21T09:19:17.072 INFO:teuthology.orchestra.run.smithi191.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 19 MB/s | 29 MB 00:01 2024-09-21T09:19:17.072 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:17.072 INFO:teuthology.orchestra.run.smithi191.stdout:Total 48 MB/s | 79 MB 00:01 2024-09-21T09:19:17.542 INFO:teuthology.orchestra.run.smithi076.stdout:lab-extras 2.6 kB/s | 1.7 kB 00:00 2024-09-21T09:19:17.760 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:19:18.101 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:19:18.101 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:19:19.060 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:19:19.061 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:19:19.406 INFO:teuthology.orchestra.run.smithi076.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:19:19.406 INFO:teuthology.orchestra.run.smithi076.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:19:19.496 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:19:19.515 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: Package Architecture Version Repository Size 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout:Install 4 Packages 2024-09-21T09:19:19.516 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:19:19.517 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 79 M 2024-09-21T09:19:19.517 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 126 M 2024-09-21T09:19:19.517 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:19:19.626 INFO:teuthology.orchestra.run.smithi076.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 409 kB/s | 36 kB 00:00 2024-09-21T09:19:19.985 INFO:teuthology.orchestra.run.smithi076.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-09-21T09:19:20.485 INFO:teuthology.orchestra.run.smithi076.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 36 MB/s | 35 MB 00:00 2024-09-21T09:19:20.802 INFO:teuthology.orchestra.run.smithi076.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-21T09:19:20.802 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:19:20.802 INFO:teuthology.orchestra.run.smithi076.stdout:Total 61 MB/s | 79 MB 00:01 2024-09-21T09:19:21.478 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:19:21.502 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:19:21.835 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:19:21.835 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:19:22.248 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:19:22.338 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:22.794 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:19:22.794 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:19:25.070 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:25.210 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:19:25.217 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:26.019 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:19:26.110 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:28.884 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:19:29.001 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:30.038 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:30.144 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:19:33.818 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:19:33.851 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:19:34.999 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:19:38.690 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:13.594 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:13.634 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:13.999 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:14.000 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:20:14.000 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:20:14.000 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:14.851 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:20:14.852 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:20:15.216 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:15.318 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:15.319 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:15.319 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:20:15.319 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:20:15.320 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:20:15.320 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-21T09:20:15.320 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi191.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:20:15.320 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:15.378 INFO:teuthology.orchestra.run.smithi191.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:15.378 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:15.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo rpm -qi grub2-tools 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Name : grub2-tools 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Epoch : 1 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Version : 2.06 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Release : 82.el9 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Architecture: x86_64 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Group : Unspecified 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Size : 8274670 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:License : GPLv3+ 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:20:15.416 INFO:teuthology.orchestra.run.smithi191.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:Packager : builder@centos.org 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:Vendor : CentOS 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:Summary : Support tools for GRUB. 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:Description : 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:hardware devices. 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:20:15.417 INFO:teuthology.orchestra.run.smithi191.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:20:15.418 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:20:15.418 INFO:teuthology.task.kernel:Updating grub on smithi191 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:20:15.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:20:16.065 INFO:teuthology.orchestra.run.smithi191.stderr:Generating grub configuration file ... 2024-09-21T09:20:17.067 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:17.114 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:17.408 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:17.408 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:20:17.408 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:20:17.408 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:20:17.415 INFO:teuthology.orchestra.run.smithi191.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:20:17.434 INFO:teuthology.orchestra.run.smithi191.stderr:done 2024-09-21T09:20:17.436 DEBUG:teuthology.orchestra.run.smithi191:> mktemp 2024-09-21T09:20:17.471 INFO:teuthology.orchestra.run.smithi191.stdout:/tmp/tmp.S1Xy5IZiKm 2024-09-21T09:20:17.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.S1Xy5IZiKm 2024-09-21T09:20:17.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0666 /tmp/tmp.S1Xy5IZiKm 2024-09-21T09:20:17.673 DEBUG:teuthology.orchestra.remote:smithi191:/tmp/tmp.S1Xy5IZiKm is 6KB 2024-09-21T09:20:17.722 DEBUG:teuthology.orchestra.run.smithi191:> rm -fr /tmp/tmp.S1Xy5IZiKm 2024-09-21T09:20:17.737 DEBUG:teuthology.orchestra.run.smithi191:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:20:17.801 INFO:teuthology.orchestra.run.smithi191.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:20:17.802 INFO:teuthology.orchestra.run.smithi191.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:20:17.802 INFO:teuthology.orchestra.run.smithi191.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:20:17.802 INFO:teuthology.orchestra.run.smithi191.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:20:17.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:20:17.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo shutdown -r now 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:20:18.245 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:20:18.246 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:20:18.609 DEBUG:teuthology.orchestra.run.smithi076:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:18.693 INFO:teuthology.orchestra.run.smithi076.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:18.693 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:18.693 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:20:18.693 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:20:18.693 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:20:18.693 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:20:18.693 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi076.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:20:18.694 DEBUG:teuthology.orchestra.run.smithi076:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:20:18.752 INFO:teuthology.orchestra.run.smithi076.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:20:18.752 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:20:18.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo rpm -qi grub2-tools 2024-09-21T09:20:18.794 INFO:teuthology.orchestra.run.smithi076.stdout:Name : grub2-tools 2024-09-21T09:20:18.794 INFO:teuthology.orchestra.run.smithi076.stdout:Epoch : 1 2024-09-21T09:20:18.794 INFO:teuthology.orchestra.run.smithi076.stdout:Version : 2.06 2024-09-21T09:20:18.794 INFO:teuthology.orchestra.run.smithi076.stdout:Release : 82.el9 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Architecture: x86_64 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Group : Unspecified 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Size : 8274670 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:License : GPLv3+ 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Packager : builder@centos.org 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Vendor : CentOS 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Summary : Support tools for GRUB. 2024-09-21T09:20:18.795 INFO:teuthology.orchestra.run.smithi076.stdout:Description : 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout:hardware devices. 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:20:18.796 INFO:teuthology.orchestra.run.smithi076.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:20:18.797 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:20:18.797 INFO:teuthology.task.kernel:Updating grub on smithi076 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:20:18.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:20:19.521 INFO:teuthology.orchestra.run.smithi076.stderr:Generating grub configuration file ... 2024-09-21T09:20:20.829 INFO:teuthology.orchestra.run.smithi076.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:20:20.856 INFO:teuthology.orchestra.run.smithi076.stderr:done 2024-09-21T09:20:20.858 DEBUG:teuthology.orchestra.run.smithi076:> mktemp 2024-09-21T09:20:20.874 INFO:teuthology.orchestra.run.smithi076.stdout:/tmp/tmp.o9kADvuHNI 2024-09-21T09:20:20.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.o9kADvuHNI 2024-09-21T09:20:20.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod 0666 /tmp/tmp.o9kADvuHNI 2024-09-21T09:20:21.080 DEBUG:teuthology.orchestra.remote:smithi076:/tmp/tmp.o9kADvuHNI is 6KB 2024-09-21T09:20:21.128 DEBUG:teuthology.orchestra.run.smithi076:> rm -fr /tmp/tmp.o9kADvuHNI 2024-09-21T09:20:21.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:20:21.208 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:20:21.208 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:20:21.208 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:20:21.208 INFO:teuthology.orchestra.run.smithi076.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:20:21.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:20:21.375 DEBUG:teuthology.orchestra.run.smithi076:> sudo shutdown -r now 2024-09-21T09:20:47.957 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-21T09:20:47.957 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-09-21T09:20:47.958 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:20:51.381 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:20:51.381 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:20:51.382 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:06.287 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.191 2024-09-21T09:21:09.871 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.76 2024-09-21T09:21:15.290 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-09-21T09:21:15.291 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:18.350 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.191 2024-09-21T09:21:18.872 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:21:18.873 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:21.935 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.76 2024-09-21T09:21:30.358 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-09-21T09:21:30.359 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:21:34.011 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:21:34.012 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:22:02.194 DEBUG:teuthology.orchestra.run.smithi191:> true 2024-09-21T09:22:02.674 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-09-21T09:22:02.674 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:22:02.674 DEBUG:teuthology.orchestra.run.smithi191:> uname -r 2024-09-21T09:22:02.688 INFO:teuthology.orchestra.run.smithi191.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:22:02.689 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:22:02.689 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:22:02.689 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:22:03.690 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-21T09:22:03.690 DEBUG:teuthology.orchestra.run.smithi191:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:22:03.778 INFO:teuthology.orchestra.run.smithi191.stdout:ttyS1 2024-09-21T09:22:03.805 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:34.014 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:22:49.015 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:22:49.015 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:22:49.459 DEBUG:teuthology.orchestra.run.smithi076:> true 2024-09-21T09:22:49.882 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:22:49.882 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:22:49.882 DEBUG:teuthology.orchestra.run.smithi076:> uname -r 2024-09-21T09:22:49.899 INFO:teuthology.orchestra.run.smithi076.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:22:49.900 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:22:49.900 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:22:49.900 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:22:50.900 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:22:50.901 DEBUG:teuthology.orchestra.run.smithi076:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:22:50.975 INFO:teuthology.orchestra.run.smithi076.stdout:ttyS1 2024-09-21T09:22:50.995 DEBUG:teuthology.parallel:result is None 2024-09-21T09:22:50.995 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:22:51.002 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:22:51.003 DEBUG:teuthology.orchestra.run.smithi076:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:22:51.005 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:22:51.024 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:22:51.030 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:22:51.037 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:22:51.038 DEBUG:teuthology.orchestra.run.smithi076:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:22:51.062 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:22:51.130 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:22:51.137 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:22:51.137 DEBUG:teuthology.orchestra.run.smithi076:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:22:51.140 DEBUG:teuthology.orchestra.run.smithi191:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:22:51.167 INFO:teuthology.orchestra.run.smithi076.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:51.173 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:51.178 INFO:teuthology.orchestra.run.smithi076.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:51.184 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:22:51.186 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:22:51.192 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:22:51.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:22:51.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:22:51.253 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:22:51.261 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:22:51.262 DEBUG:teuthology.orchestra.run.smithi076:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:22:51.290 DEBUG:teuthology.orchestra.run.smithi191:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:22:51.311 DEBUG:teuthology.orchestra.run.smithi076:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:51.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:51.434 DEBUG:teuthology.orchestra.run.smithi076:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:51.516 DEBUG:teuthology.orchestra.run.smithi076:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:51.557 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:22:51.557 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:22:51.622 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:51.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:22:51.713 DEBUG:teuthology.orchestra.run.smithi191:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:51.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:22:51.839 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:22:51.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:22:51.905 DEBUG:teuthology.orchestra.run.smithi076:> sudo service rsyslog restart 2024-09-21T09:22:51.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo service rsyslog restart 2024-09-21T09:22:51.948 INFO:teuthology.orchestra.run.smithi076.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:22:52.010 INFO:teuthology.orchestra.run.smithi191.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:22:52.407 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:22:52.420 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:22:52.421 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:22:52.444 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:22:52.459 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T09:22:52.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo service auditd rotate 2024-09-21T09:22:52.501 INFO:teuthology.orchestra.run.smithi076.stdout:Rotating logs: 2024-09-21T09:22:52.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo service auditd rotate 2024-09-21T09:22:52.573 INFO:teuthology.orchestra.run.smithi191.stdout:Rotating logs: 2024-09-21T09:22:52.575 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:22:52.575 DEBUG:teuthology.orchestra.run.smithi076:> /usr/sbin/getenforce 2024-09-21T09:22:52.595 INFO:teuthology.orchestra.run.smithi076.stdout:Permissive 2024-09-21T09:22:52.595 DEBUG:teuthology.orchestra.run.smithi191:> /usr/sbin/getenforce 2024-09-21T09:22:52.684 INFO:teuthology.orchestra.run.smithi191.stdout:Permissive 2024-09-21T09:22:52.684 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi076.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi191.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:22:52.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:22:52.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:22:52.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:22:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:22:52.736 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:22:52.736 DEBUG:teuthology.orchestra.run.smithi076:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:22:52.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:22:52.821 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:22:52.830 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-21T09:22:52.833 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-21T09:22:52.936 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:22:52.946 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:22:52.947 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi076.front.sepia.ceph.com,smithi191.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:26:37.713 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi076.front.sepia.ceph.com'), Remote(name='ubuntu@smithi191.front.sepia.ceph.com')] 2024-09-21T09:26:37.714 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:26:37.715 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi076.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:26:37.792 DEBUG:teuthology.orchestra.run.smithi076:> true 2024-09-21T09:26:37.867 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi076.front.sepia.ceph.com' 2024-09-21T09:26:37.868 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-09-21T09:26:37.868 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi191.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:26:37.944 DEBUG:teuthology.orchestra.run.smithi191:> true 2024-09-21T09:26:38.027 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi191.front.sepia.ceph.com' 2024-09-21T09:26:38.027 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:26:38.040 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:26:38.040 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:26:38.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:26:38.043 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:26:38.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:26:38.074 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:26:38.093 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:26:38.098 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:26:38.113 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:26:38.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: ntpd: command not found 2024-09-21T09:26:38.133 INFO:teuthology.orchestra.run.smithi076.stdout:506 Cannot talk to daemon 2024-09-21T09:26:38.139 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: ntpd: command not found 2024-09-21T09:26:38.150 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:26:38.150 INFO:teuthology.orchestra.run.smithi191.stdout:506 Cannot talk to daemon 2024-09-21T09:26:38.165 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:26:38.167 INFO:teuthology.orchestra.run.smithi076.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:26:38.179 INFO:teuthology.orchestra.run.smithi191.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:26:38.230 INFO:teuthology.orchestra.run.smithi076.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:26:38.234 INFO:teuthology.orchestra.run.smithi076.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:26:38.234 INFO:teuthology.orchestra.run.smithi076.stdout:=============================================================================== 2024-09-21T09:26:38.234 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.234 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.234 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.234 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.241 INFO:teuthology.orchestra.run.smithi191.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:26:38.244 INFO:teuthology.orchestra.run.smithi191.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:26:38.244 INFO:teuthology.orchestra.run.smithi191.stdout:=============================================================================== 2024-09-21T09:26:38.244 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.244 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.244 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.245 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:26:38.245 INFO:teuthology.run_tasks:Running task pexec... 2024-09-21T09:26:38.258 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-21T09:26:38.258 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi076.front.sepia.ceph.com 2024-09-21T09:26:38.258 DEBUG:teuthology.orchestra.run.smithi076:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:26:38.259 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi191.front.sepia.ceph.com 2024-09-21T09:26:38.259 DEBUG:teuthology.orchestra.run.smithi191:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:26:38.636 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:26:38.639 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:26:38.649 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:26:38.649 INFO:teuthology.orchestra.run.smithi076.stdout: Package Architecture Version Repository Size 2024-09-21T09:26:38.649 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:26:38.649 INFO:teuthology.orchestra.run.smithi076.stdout:Removing: 2024-09-21T09:26:38.649 INFO:teuthology.orchestra.run.smithi076.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout:Removing unused dependencies: 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout:Remove 2 Packages 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout:Freed space: 5.5 M 2024-09-21T09:26:38.650 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:Removing: 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:Removing unused dependencies: 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout:Remove 2 Packages 2024-09-21T09:26:38.651 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:38.652 INFO:teuthology.orchestra.run.smithi191.stdout:Freed space: 5.5 M 2024-09-21T09:26:38.652 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:26:38.654 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:26:38.654 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:26:38.654 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:26:38.655 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:26:38.716 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:26:38.716 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:26:38.721 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:26:38.721 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:26:38.868 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:26:38.878 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:26:38.932 INFO:teuthology.orchestra.run.smithi076.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:26:38.936 INFO:teuthology.orchestra.run.smithi191.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:26:38.998 INFO:teuthology.orchestra.run.smithi076.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:39.004 INFO:teuthology.orchestra.run.smithi191.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:39.478 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:39.478 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:26:39.587 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:26:39.587 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:26:39.705 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:26:39.705 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:39.705 INFO:teuthology.orchestra.run.smithi191.stdout:Removed: 2024-09-21T09:26:39.705 INFO:teuthology.orchestra.run.smithi191.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:39.705 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:39.705 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:26:39.867 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:26:39.867 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:39.867 INFO:teuthology.orchestra.run.smithi076.stdout:Removed: 2024-09-21T09:26:39.867 INFO:teuthology.orchestra.run.smithi076.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:39.867 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:39.867 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:26:40.527 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:51 ago on Sat 21 Sep 2024 09:25:49 AM UTC. 2024-09-21T09:26:40.662 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:40.663 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:26:40.664 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:26:40.664 INFO:teuthology.orchestra.run.smithi191.stdout:Install 7 Packages 2024-09-21T09:26:40.664 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:40.687 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 5.2 M 2024-09-21T09:26:40.687 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 19 M 2024-09-21T09:26:40.687 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:26:40.720 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:51 ago on Sat 21 Sep 2024 09:25:49 AM UTC. 2024-09-21T09:26:40.854 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:26:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:40.856 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:26:40.856 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:26:40.856 INFO:teuthology.orchestra.run.smithi076.stdout:Install 7 Packages 2024-09-21T09:26:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:40.870 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 5.2 M 2024-09-21T09:26:40.870 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 19 M 2024-09-21T09:26:40.870 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:26:40.952 INFO:teuthology.orchestra.run.smithi191.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 395 kB/s | 44 kB 00:00 2024-09-21T09:26:41.002 INFO:teuthology.orchestra.run.smithi191.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 651 kB/s | 106 kB 00:00 2024-09-21T09:26:41.027 INFO:teuthology.orchestra.run.smithi191.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 952 kB/s | 72 kB 00:00 2024-09-21T09:26:41.052 INFO:teuthology.orchestra.run.smithi191.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.6 MB/s | 84 kB 00:00 2024-09-21T09:26:41.169 INFO:teuthology.orchestra.run.smithi191.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 2.7 MB/s | 906 kB 00:00 2024-09-21T09:26:41.228 INFO:teuthology.orchestra.run.smithi191.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 4.1 MB/s | 837 kB 00:00 2024-09-21T09:26:41.297 INFO:teuthology.orchestra.run.smithi076.stdout:(1/7): libnvme-1.9-3.el9.x86_64.rpm 1.1 MB/s | 106 kB 00:00 2024-09-21T09:26:41.320 INFO:teuthology.orchestra.run.smithi191.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 12 MB/s | 3.2 MB 00:00 2024-09-21T09:26:41.320 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:26:41.320 INFO:teuthology.orchestra.run.smithi191.stdout:Total 8.2 MB/s | 5.2 MB 00:00 2024-09-21T09:26:41.322 INFO:teuthology.orchestra.run.smithi076.stdout:(2/7): nvmetcli-0.7-3.el9.noarch.rpm 362 kB/s | 44 kB 00:00 2024-09-21T09:26:41.364 INFO:teuthology.orchestra.run.smithi076.stdout:(3/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 5.4 MB/s | 906 kB 00:00 2024-09-21T09:26:41.389 INFO:teuthology.orchestra.run.smithi076.stdout:(4/7): python3-configshell-1.1.30-1.el9.noarch. 776 kB/s | 72 kB 00:00 2024-09-21T09:26:41.415 INFO:teuthology.orchestra.run.smithi076.stdout:(5/7): python3-kmod-0.9-32.el9.x86_64.rpm 909 kB/s | 84 kB 00:00 2024-09-21T09:26:41.431 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:26:41.446 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:26:41.446 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:26:41.457 INFO:teuthology.orchestra.run.smithi076.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 8.9 MB/s | 837 kB 00:00 2024-09-21T09:26:41.747 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:26:41.748 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:26:42.199 INFO:teuthology.orchestra.run.smithi076.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 3.9 MB/s | 3.2 MB 00:00 2024-09-21T09:26:42.199 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:26:42.199 INFO:teuthology.orchestra.run.smithi076.stdout:Total 3.9 MB/s | 5.2 MB 00:01 2024-09-21T09:26:42.218 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:26:42.308 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:26:42.314 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:26:42.323 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:26:42.323 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:26:42.352 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:26:42.403 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:26:42.611 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:26:42.618 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:26:42.619 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:26:42.650 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:43.078 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:26:43.086 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:43.114 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:43.151 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:26:43.195 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:26:43.223 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:26:43.227 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:43.289 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:43.431 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:26:43.462 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:43.900 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:26:43.933 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:44.040 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:26:44.102 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:26:44.289 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:26:44.746 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:26:45.122 DEBUG:teuthology.parallel:result is None 2024-09-21T09:26:45.128 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:45.128 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:26:45.128 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:26:45.128 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:26:45.128 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:26:45.129 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:26:45.129 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:26:45.516 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:26:45.927 DEBUG:teuthology.parallel:result is None 2024-09-21T09:26:45.927 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:26:45.935 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:26:45.935 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:26:45.935 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:26:45.936 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:26:45.940 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:26:45.941 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:26:45.941 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:26:45.941 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:45.941 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:45.941 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:45.941 INFO:teuthology.packaging:branch: reef 2024-09-21T09:26:45.941 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:45.942 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:26:45.943 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:26:45.943 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:45.943 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:45.943 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:45.944 INFO:teuthology.packaging:branch: reef 2024-09-21T09:26:45.944 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:45.944 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:26:46.194 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/ 2024-09-21T09:26:46.194 INFO:teuthology.task.install.rpm:Package version is 18.2.4-871.g6a1bdf85 2024-09-21T09:26:46.219 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/ 2024-09-21T09:26:46.219 INFO:teuthology.task.install.rpm:Package version is 18.2.4-871.g6a1bdf85 2024-09-21T09:26:46.307 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:26:46.307 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:26:46.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:26:46.324 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:26:46.324 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:26:46.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:26:46.338 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-21T09:26:46.339 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:46.339 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:46.339 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:46.339 INFO:teuthology.packaging:branch: reef 2024-09-21T09:26:46.339 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:46.339 DEBUG:teuthology.orchestra.run.smithi191:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:26:46.350 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-21T09:26:46.351 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:26:46.351 INFO:teuthology.packaging:ref: None 2024-09-21T09:26:46.351 INFO:teuthology.packaging:tag: None 2024-09-21T09:26:46.351 INFO:teuthology.packaging:branch: reef 2024-09-21T09:26:46.351 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:26:46.351 DEBUG:teuthology.orchestra.run.smithi076:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:26:46.407 DEBUG:teuthology.orchestra.run.smithi191:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:26:46.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:26:46.484 DEBUG:teuthology.orchestra.run.smithi191:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:26:46.514 DEBUG:teuthology.orchestra.run.smithi076:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:26:46.538 INFO:teuthology.orchestra.run.smithi076.stdout:check_obsoletes = 1 2024-09-21T09:26:46.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum clean all 2024-09-21T09:26:46.549 INFO:teuthology.orchestra.run.smithi191.stdout:check_obsoletes = 1 2024-09-21T09:26:46.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum clean all 2024-09-21T09:26:46.778 INFO:teuthology.orchestra.run.smithi191.stdout:45 files removed 2024-09-21T09:26:46.783 INFO:teuthology.orchestra.run.smithi076.stdout:45 files removed 2024-09-21T09:26:46.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-radosgw 2024-09-21T09:26:46.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-radosgw 2024-09-21T09:26:47.359 INFO:teuthology.orchestra.run.smithi191.stdout:ceph packages for x86_64 284 kB/s | 77 kB 00:00 2024-09-21T09:26:47.364 INFO:teuthology.orchestra.run.smithi076.stdout:ceph packages for x86_64 285 kB/s | 77 kB 00:00 2024-09-21T09:26:47.643 INFO:teuthology.orchestra.run.smithi191.stdout:ceph noarch packages 48 kB/s | 11 kB 00:00 2024-09-21T09:26:47.657 INFO:teuthology.orchestra.run.smithi076.stdout:ceph noarch packages 47 kB/s | 11 kB 00:00 2024-09-21T09:26:47.919 INFO:teuthology.orchestra.run.smithi191.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-09-21T09:26:47.933 INFO:teuthology.orchestra.run.smithi076.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-09-21T09:26:48.619 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.3 MB 00:00 2024-09-21T09:26:49.641 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - BaseOS 4.9 MB/s | 8.3 MB 00:01 2024-09-21T09:26:51.780 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - AppStream 9.9 MB/s | 20 MB 00:02 2024-09-21T09:26:52.476 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - AppStream 12 MB/s | 20 MB 00:01 2024-09-21T09:26:57.996 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - CRB 2.9 MB/s | 6.5 MB 00:02 2024-09-21T09:26:58.001 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.5 MB 00:01 2024-09-21T09:27:00.302 INFO:teuthology.orchestra.run.smithi076.stdout:CentOS Stream 9 - Extras packages 20 kB/s | 19 kB 00:00 2024-09-21T09:27:00.596 INFO:teuthology.orchestra.run.smithi191.stdout:CentOS Stream 9 - Extras packages 16 kB/s | 19 kB 00:01 2024-09-21T09:27:01.207 INFO:teuthology.orchestra.run.smithi076.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:27:01.517 INFO:teuthology.orchestra.run.smithi191.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:27:06.655 INFO:teuthology.orchestra.run.smithi076.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-09-21T09:27:07.139 INFO:teuthology.orchestra.run.smithi191.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-21T09:27:08.877 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-radosgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 7.7 M 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout:Upgrading: 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout: librados2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.3 M 2024-09-21T09:27:08.879 INFO:teuthology.orchestra.run.smithi076.stdout: librbd1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-base x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 5.1 M 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 18 M 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-selinux x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 25 k 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T09:27:08.880 INFO:teuthology.orchestra.run.smithi076.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: libcephfs2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 711 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: libradosstriper1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 477 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: librgw2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.5 M 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: python3-ceph-argparse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 45 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: python3-ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 129 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cephfs x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 161 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:27:08.881 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rados x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 320 k 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 300 k 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 99 k 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout:Install 28 Packages 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout:Upgrade 2 Packages 2024-09-21T09:27:08.882 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:08.883 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 53 M 2024-09-21T09:27:08.883 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:27:09.403 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:27:09.404 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:09.404 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 7.7 M 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrading: 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: librados2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.3 M 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 5.1 M 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 18 M 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 25 k 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:27:09.405 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 711 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 477 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.5 M 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T09:27:09.406 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 45 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 129 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 161 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 320 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 300 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 99 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout:Install 28 Packages 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout:Upgrade 2 Packages 2024-09-21T09:27:09.407 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:09.408 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 53 M 2024-09-21T09:27:09.408 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:27:09.489 INFO:teuthology.orchestra.run.smithi076.stdout:(1/30): ceph-base-18.2.4-871.g6a1bdf85.el9.x86_ 9.6 MB/s | 5.1 MB 00:00 2024-09-21T09:27:09.539 INFO:teuthology.orchestra.run.smithi076.stdout:(2/30): ceph-selinux-18.2.4-871.g6a1bdf85.el9.x 505 kB/s | 25 kB 00:00 2024-09-21T09:27:09.623 INFO:teuthology.orchestra.run.smithi076.stdout:(3/30): ceph-radosgw-18.2.4-871.g6a1bdf85.el9.x 12 MB/s | 7.7 MB 00:00 2024-09-21T09:27:09.665 INFO:teuthology.orchestra.run.smithi076.stdout:(4/30): libcephfs2-18.2.4-871.g6a1bdf85.el9.x86 5.5 MB/s | 711 kB 00:00 2024-09-21T09:27:09.690 INFO:teuthology.orchestra.run.smithi076.stdout:(5/30): libradosstriper1-18.2.4-871.g6a1bdf85.e 7.0 MB/s | 477 kB 00:00 2024-09-21T09:27:09.732 INFO:teuthology.orchestra.run.smithi076.stdout:(6/30): python3-ceph-argparse-18.2.4-871.g6a1bd 1.1 MB/s | 45 kB 00:00 2024-09-21T09:27:09.791 INFO:teuthology.orchestra.run.smithi076.stdout:(7/30): python3-ceph-common-18.2.4-871.g6a1bdf8 2.2 MB/s | 129 kB 00:00 2024-09-21T09:27:09.858 INFO:teuthology.orchestra.run.smithi076.stdout:(8/30): python3-cephfs-18.2.4-871.g6a1bdf85.el9 2.4 MB/s | 161 kB 00:00 2024-09-21T09:27:09.950 INFO:teuthology.orchestra.run.smithi076.stdout:(9/30): librgw2-18.2.4-871.g6a1bdf85.el9.x86_64 16 MB/s | 4.5 MB 00:00 2024-09-21T09:27:09.975 INFO:teuthology.orchestra.run.smithi076.stdout:(10/30): python3-rados-18.2.4-871.g6a1bdf85.el9 2.7 MB/s | 320 kB 00:00 2024-09-21T09:27:10.000 INFO:teuthology.orchestra.run.smithi076.stdout:(11/30): python3-rbd-18.2.4-871.g6a1bdf85.el9.x 5.9 MB/s | 300 kB 00:00 2024-09-21T09:27:10.026 INFO:teuthology.orchestra.run.smithi076.stdout:(12/30): python3-rgw-18.2.4-871.g6a1bdf85.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-21T09:27:10.218 INFO:teuthology.orchestra.run.smithi076.stdout:(13/30): ceph-common-18.2.4-871.g6a1bdf85.el9.x 15 MB/s | 18 MB 00:01 2024-09-21T09:27:10.240 INFO:teuthology.orchestra.run.smithi191.stdout:(1/30): ceph-base-18.2.4-871.g6a1bdf85.el9.x86_ 9.8 MB/s | 5.1 MB 00:00 2024-09-21T09:27:10.251 INFO:teuthology.orchestra.run.smithi076.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 426 kB/s | 107 kB 00:00 2024-09-21T09:27:10.290 INFO:teuthology.orchestra.run.smithi191.stdout:(2/30): ceph-selinux-18.2.4-871.g6a1bdf85.el9.x 503 kB/s | 25 kB 00:00 2024-09-21T09:27:10.301 INFO:teuthology.orchestra.run.smithi076.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 544 kB/s | 45 kB 00:00 2024-09-21T09:27:10.352 INFO:teuthology.orchestra.run.smithi076.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 492 kB/s | 160 kB 00:00 2024-09-21T09:27:10.366 INFO:teuthology.orchestra.run.smithi191.stdout:(3/30): libcephfs2-18.2.4-871.g6a1bdf85.el9.x86 9.3 MB/s | 711 kB 00:00 2024-09-21T09:27:10.385 INFO:teuthology.orchestra.run.smithi076.stdout:(17/30): librdkafka-1.6.1-102.el9.x86_64.rpm 4.8 MB/s | 662 kB 00:00 2024-09-21T09:27:10.411 INFO:teuthology.orchestra.run.smithi076.stdout:(18/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-09-21T09:27:10.444 INFO:teuthology.orchestra.run.smithi076.stdout:(19/30): python3-prettytable-0.7.2-27.el9.noarc 459 kB/s | 42 kB 00:00 2024-09-21T09:27:10.474 INFO:teuthology.orchestra.run.smithi191.stdout:(4/30): ceph-radosgw-18.2.4-871.g6a1bdf85.el9.x 10 MB/s | 7.7 MB 00:00 2024-09-21T09:27:10.486 INFO:teuthology.orchestra.run.smithi076.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 1.1 MB/s | 111 kB 00:00 2024-09-21T09:27:10.508 INFO:teuthology.orchestra.run.smithi191.stdout:(5/30): libradosstriper1-18.2.4-871.g6a1bdf85.e 3.3 MB/s | 477 kB 00:00 2024-09-21T09:27:10.512 INFO:teuthology.orchestra.run.smithi076.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.1 MB/s | 26 kB 00:00 2024-09-21T09:27:10.537 INFO:teuthology.orchestra.run.smithi076.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:27:10.562 INFO:teuthology.orchestra.run.smithi076.stdout:(23/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.1 MB/s | 308 kB 00:00 2024-09-21T09:27:10.575 INFO:teuthology.orchestra.run.smithi191.stdout:(6/30): python3-ceph-argparse-18.2.4-871.g6a1bd 673 kB/s | 45 kB 00:00 2024-09-21T09:27:10.588 INFO:teuthology.orchestra.run.smithi076.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-21T09:27:10.621 INFO:teuthology.orchestra.run.smithi076.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 5.6 MB/s | 191 kB 00:00 2024-09-21T09:27:10.634 INFO:teuthology.orchestra.run.smithi191.stdout:(7/30): python3-ceph-common-18.2.4-871.g6a1bdf8 2.2 MB/s | 129 kB 00:00 2024-09-21T09:27:10.655 INFO:teuthology.orchestra.run.smithi076.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.9 MB/s | 839 kB 00:00 2024-09-21T09:27:10.705 INFO:teuthology.orchestra.run.smithi076.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 19 MB/s | 1.6 MB 00:00 2024-09-21T09:27:10.709 INFO:teuthology.orchestra.run.smithi191.stdout:(8/30): python3-cephfs-18.2.4-871.g6a1bdf85.el9 2.1 MB/s | 161 kB 00:00 2024-09-21T09:27:10.781 INFO:teuthology.orchestra.run.smithi076.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 14 MB/s | 4.4 MB 00:00 2024-09-21T09:27:10.792 INFO:teuthology.orchestra.run.smithi191.stdout:(9/30): python3-rados-18.2.4-871.g6a1bdf85.el9. 3.8 MB/s | 320 kB 00:00 2024-09-21T09:27:10.851 INFO:teuthology.orchestra.run.smithi191.stdout:(10/30): librgw2-18.2.4-871.g6a1bdf85.el9.x86_6 12 MB/s | 4.5 MB 00:00 2024-09-21T09:27:10.876 INFO:teuthology.orchestra.run.smithi191.stdout:(11/30): python3-rbd-18.2.4-871.g6a1bdf85.el9.x 3.5 MB/s | 300 kB 00:00 2024-09-21T09:27:10.935 INFO:teuthology.orchestra.run.smithi191.stdout:(12/30): python3-rgw-18.2.4-871.g6a1bdf85.el9.x 1.2 MB/s | 99 kB 00:00 2024-09-21T09:27:11.010 INFO:teuthology.orchestra.run.smithi191.stdout:(13/30): boost-program-options-1.75.0-8.el9.x86 801 kB/s | 107 kB 00:00 2024-09-21T09:27:11.039 INFO:teuthology.orchestra.run.smithi076.stdout:(29/30): librbd1-18.2.4-871.g6a1bdf85.el9.x86_6 9.0 MB/s | 3.0 MB 00:00 2024-09-21T09:27:11.060 INFO:teuthology.orchestra.run.smithi191.stdout:(14/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 902 kB/s | 45 kB 00:00 2024-09-21T09:27:11.086 INFO:teuthology.orchestra.run.smithi191.stdout:(15/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.0 MB/s | 160 kB 00:00 2024-09-21T09:27:11.098 INFO:teuthology.orchestra.run.smithi076.stdout:(30/30): librados2-18.2.4-871.g6a1bdf85.el9.x86 7.5 MB/s | 3.3 MB 00:00 2024-09-21T09:27:11.102 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:11.102 INFO:teuthology.orchestra.run.smithi076.stdout:Total 24 MB/s | 53 MB 00:02 2024-09-21T09:27:11.169 INFO:teuthology.orchestra.run.smithi191.stdout:(16/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-09-21T09:27:11.173 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:27:11.195 INFO:teuthology.orchestra.run.smithi191.stdout:(17/30): librdkafka-1.6.1-102.el9.x86_64.rpm 4.8 MB/s | 662 kB 00:00 2024-09-21T09:27:11.208 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:27:11.208 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:27:11.236 INFO:teuthology.orchestra.run.smithi191.stdout:(18/30): python3-prettytable-0.7.2-27.el9.noarc 631 kB/s | 42 kB 00:00 2024-09-21T09:27:11.279 INFO:teuthology.orchestra.run.smithi191.stdout:(19/30): fmt-8.1.1-5.el9.x86_64.rpm 1.3 MB/s | 111 kB 00:00 2024-09-21T09:27:11.320 INFO:teuthology.orchestra.run.smithi191.stdout:(20/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 3.6 MB/s | 308 kB 00:00 2024-09-21T09:27:11.346 INFO:teuthology.orchestra.run.smithi191.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.0 MB/s | 26 kB 00:00 2024-09-21T09:27:11.371 INFO:teuthology.orchestra.run.smithi191.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:27:11.396 INFO:teuthology.orchestra.run.smithi191.stdout:(23/30): libunwind-1.6.2-1.el9.x86_64.rpm 2.6 MB/s | 67 kB 00:00 2024-09-21T09:27:11.455 INFO:teuthology.orchestra.run.smithi191.stdout:(24/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 14 MB/s | 839 kB 00:00 2024-09-21T09:27:11.480 INFO:teuthology.orchestra.run.smithi191.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 7.5 MB/s | 191 kB 00:00 2024-09-21T09:27:11.539 INFO:teuthology.orchestra.run.smithi191.stdout:(26/30): libarrow-9.0.0-11.el9.x86_64.rpm 17 MB/s | 4.4 MB 00:00 2024-09-21T09:27:11.589 INFO:teuthology.orchestra.run.smithi191.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 15 MB/s | 1.6 MB 00:00 2024-09-21T09:27:11.661 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:27:11.661 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:27:11.884 INFO:teuthology.orchestra.run.smithi191.stdout:(28/30): librbd1-18.2.4-871.g6a1bdf85.el9.x86_6 10 MB/s | 3.0 MB 00:00 2024-09-21T09:27:11.931 INFO:teuthology.orchestra.run.smithi191.stdout:(29/30): librados2-18.2.4-871.g6a1bdf85.el9.x86 8.4 MB/s | 3.3 MB 00:00 2024-09-21T09:27:12.165 INFO:teuthology.orchestra.run.smithi191.stdout:(30/30): ceph-common-18.2.4-871.g6a1bdf85.el9.x 7.5 MB/s | 18 MB 00:02 2024-09-21T09:27:12.169 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:12.170 INFO:teuthology.orchestra.run.smithi191.stdout:Total 19 MB/s | 53 MB 00:02 2024-09-21T09:27:12.237 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:27:12.271 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:27:12.271 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:27:12.727 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:27:12.727 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:27:13.039 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:27:13.126 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-21T09:27:13.154 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-21T09:27:13.204 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-21T09:27:13.580 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-21T09:27:13.637 INFO:teuthology.orchestra.run.smithi076.stdout: Upgrading : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:27:13.738 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:27:13.821 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:27:13.871 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:27:13.919 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:27:13.955 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-21T09:27:13.999 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 9/32 2024-09-21T09:27:14.066 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:27:14.079 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:27:14.099 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:27:14.149 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:27:14.174 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-21T09:27:14.204 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-21T09:27:14.261 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-21T09:27:14.261 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-21T09:27:14.299 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-21T09:27:14.340 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-21T09:27:14.380 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-21T09:27:14.407 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-21T09:27:14.443 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-21T09:27:14.489 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-21T09:27:14.578 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-21T09:27:14.702 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:27:14.787 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:27:14.802 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-21T09:27:14.848 INFO:teuthology.orchestra.run.smithi076.stdout: Upgrading : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:27:14.876 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:27:14.921 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:27:14.939 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:27:14.977 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 21/32 2024-09-21T09:27:14.984 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:27:15.013 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-21T09:27:15.020 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-21T09:27:15.056 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 9/32 2024-09-21T09:27:15.122 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:27:15.148 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:27:15.214 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:27:15.319 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-21T09:27:15.356 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-21T09:27:15.396 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-21T09:27:15.427 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-21T09:27:15.454 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-21T09:27:15.458 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:27:15.481 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-21T09:27:15.517 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-21T09:27:15.534 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:27:15.555 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-21T09:27:15.607 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 25/32 2024-09-21T09:27:15.844 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 26/32 2024-09-21T09:27:15.862 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-21T09:27:15.913 INFO:teuthology.orchestra.run.smithi191.stdout: Upgrading : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:27:16.003 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T09:27:16.042 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 21/32 2024-09-21T09:27:16.086 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-21T09:27:16.492 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-21T09:27:16.523 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:27:16.595 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T09:27:16.671 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 25/32 2024-09-21T09:27:16.709 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 26/32 2024-09-21T09:27:17.983 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:27:18.025 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:27:18.570 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:27:18.611 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:27:18.712 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:27:18.712 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:27:18.712 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:27:18.712 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:18.746 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:27:18.798 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:27:18.830 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:27:19.368 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T09:27:19.416 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:27:19.517 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T09:27:19.517 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:27:19.517 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:27:19.517 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:19.551 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /sys 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /proc 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /mnt 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /var/tmp 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /home 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /root 2024-09-21T09:27:28.839 INFO:teuthology.orchestra.run.smithi076.stdout:skipping the directory /tmp 2024-09-21T09:27:28.840 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:29.611 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:27:29.643 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:27:29.644 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:29.644 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:27:29.644 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:29.644 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:29.644 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:29.645 INFO:teuthology.orchestra.run.smithi076.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:27:29.731 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:27:29.731 INFO:teuthology.orchestra.run.smithi076.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /sys 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /proc 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /mnt 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /var/tmp 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /home 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /root 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout:skipping the directory /tmp 2024-09-21T09:27:29.759 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:30.542 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:27:30.566 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T09:27:30.566 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:30.566 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:27:30.567 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:30.567 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:27:30.567 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:30.568 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:27:30.644 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T09:27:30.644 INFO:teuthology.orchestra.run.smithi191.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:27:30.860 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:27:30.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/32 2024-09-21T09:27:30.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/32 2024-09-21T09:27:30.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/32 2024-09-21T09:27:30.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 8/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 9/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 13/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-21T09:27:30.861 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:27:30.862 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-21T09:27:30.863 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/32 2024-09-21T09:27:31.748 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout:Upgraded: 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.749 INFO:teuthology.orchestra.run.smithi076.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.750 INFO:teuthology.orchestra.run.smithi076.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:31.751 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:27:31.833 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 8/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 9/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T09:27:31.834 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 13/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-21T09:27:31.835 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-21T09:27:31.836 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-21T09:27:31.836 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-21T09:27:31.836 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T09:27:31.836 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-21T09:27:31.836 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/32 2024-09-21T09:27:32.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-test 2024-09-21T09:27:32.700 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:26 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:27:32.837 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-test x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 40 M 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout: libcephsqlite x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 166 k 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:27:32.838 INFO:teuthology.orchestra.run.smithi076.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout:Install 4 Packages 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 40 M 2024-09-21T09:27:32.839 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 164 M 2024-09-21T09:27:32.840 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:27:32.843 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T09:27:32.843 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:32.843 INFO:teuthology.orchestra.run.smithi191.stdout:Upgraded: 2024-09-21T09:27:32.843 INFO:teuthology.orchestra.run.smithi191.stdout: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.843 INFO:teuthology.orchestra.run.smithi191.stdout: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:27:32.844 INFO:teuthology.orchestra.run.smithi191.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:32.845 INFO:teuthology.orchestra.run.smithi191.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T09:27:32.846 INFO:teuthology.orchestra.run.smithi191.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:27:32.846 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:32.846 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:27:33.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-test 2024-09-21T09:27:33.300 INFO:teuthology.orchestra.run.smithi076.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.5 MB/s | 304 kB 00:00 2024-09-21T09:27:33.325 INFO:teuthology.orchestra.run.smithi076.stdout:(2/4): libcephsqlite-18.2.4-871.g6a1bdf85.el9.x 726 kB/s | 166 kB 00:00 2024-09-21T09:27:33.350 INFO:teuthology.orchestra.run.smithi076.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-21T09:27:33.784 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:26 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:33.920 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:27:33.920 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:33.920 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:27:33.920 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:33.920 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:27:33.920 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 40 M 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 166 k 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout:Install 4 Packages 2024-09-21T09:27:33.921 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:33.922 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 40 M 2024-09-21T09:27:33.922 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 164 M 2024-09-21T09:27:33.922 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:27:34.226 INFO:teuthology.orchestra.run.smithi191.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.5 MB/s | 304 kB 00:00 2024-09-21T09:27:34.251 INFO:teuthology.orchestra.run.smithi191.stdout:(2/4): libcephsqlite-18.2.4-871.g6a1bdf85.el9.x 729 kB/s | 166 kB 00:00 2024-09-21T09:27:34.277 INFO:teuthology.orchestra.run.smithi191.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-21T09:27:34.409 INFO:teuthology.orchestra.run.smithi076.stdout:(4/4): ceph-test-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 40 MB 00:01 2024-09-21T09:27:34.411 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:34.411 INFO:teuthology.orchestra.run.smithi076.stdout:Total 26 MB/s | 40 MB 00:01 2024-09-21T09:27:34.439 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:27:34.460 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:27:34.461 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:27:34.780 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:27:34.780 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:27:35.261 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:27:35.344 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:27:35.383 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:27:35.394 INFO:teuthology.orchestra.run.smithi191.stdout:(4/4): ceph-test-18.2.4-871.g6a1bdf85.el9.x86_6 29 MB/s | 40 MB 00:01 2024-09-21T09:27:35.396 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:35.396 INFO:teuthology.orchestra.run.smithi191.stdout:Total 27 MB/s | 40 MB 00:01 2024-09-21T09:27:35.410 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:27:35.423 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:27:35.443 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:27:35.443 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:27:35.761 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:27:35.761 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:27:36.251 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:27:36.329 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:27:36.368 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:27:36.396 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:27:39.090 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:27:39.167 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:27:40.087 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T09:27:40.171 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:27:41.277 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:27:41.278 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/4 2024-09-21T09:27:41.278 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/4 2024-09-21T09:27:41.278 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:27:41.560 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:41.561 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:27:41.900 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph 2024-09-21T09:27:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T09:27:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/4 2024-09-21T09:27:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/4 2024-09-21T09:27:42.200 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:27:42.487 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:36 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:42.488 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:27:42.631 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout: ceph x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 6.5 k 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mds x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 2.1 M 2024-09-21T09:27:42.633 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 1.5 M 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-modules-core noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 246 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mon x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.7 M 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-osd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 17 M 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:27:42.634 INFO:teuthology.orchestra.run.smithi076.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:27:42.635 INFO:teuthology.orchestra.run.smithi076.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout:Install 33 Packages 2024-09-21T09:27:42.636 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:42.637 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 29 M 2024-09-21T09:27:42.637 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 104 M 2024-09-21T09:27:42.637 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:27:42.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph 2024-09-21T09:27:43.081 INFO:teuthology.orchestra.run.smithi076.stdout:(1/33): ceph-18.2.4-871.g6a1bdf85.el9.x86_64.rp 56 kB/s | 6.5 kB 00:00 2024-09-21T09:27:43.290 INFO:teuthology.orchestra.run.smithi076.stdout:(2/33): ceph-mds-18.2.4-871.g6a1bdf85.el9.x86_6 6.5 MB/s | 2.1 MB 00:00 2024-09-21T09:27:43.340 INFO:teuthology.orchestra.run.smithi076.stdout:(3/33): ceph-mgr-18.2.4-871.g6a1bdf85.el9.x86_6 3.9 MB/s | 1.5 MB 00:00 2024-09-21T09:27:43.436 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:36 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:43.440 INFO:teuthology.orchestra.run.smithi076.stdout:(4/33): ceph-mgr-modules-core-18.2.4-871.g6a1bd 2.4 MB/s | 246 kB 00:00 2024-09-21T09:27:43.541 INFO:teuthology.orchestra.run.smithi076.stdout:(5/33): ceph-mon-18.2.4-871.g6a1bdf85.el9.x86_6 10 MB/s | 4.7 MB 00:00 2024-09-21T09:27:43.576 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout: ceph x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 6.5 k 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 2.1 M 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 1.5 M 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 246 k 2024-09-21T09:27:43.579 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.7 M 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 17 M 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:27:43.580 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:27:43.581 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:43.582 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:27:43.582 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:43.582 INFO:teuthology.orchestra.run.smithi191.stdout:Install 33 Packages 2024-09-21T09:27:43.582 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:43.582 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 29 M 2024-09-21T09:27:43.582 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 104 M 2024-09-21T09:27:43.583 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:27:43.624 INFO:teuthology.orchestra.run.smithi076.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.3 MB/s | 253 kB 00:00 2024-09-21T09:27:43.666 INFO:teuthology.orchestra.run.smithi076.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-09-21T09:27:43.850 INFO:teuthology.orchestra.run.smithi076.stdout:(8/33): ceph-osd-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-21T09:27:43.875 INFO:teuthology.orchestra.run.smithi076.stdout:(9/33): python3-pycparser-2.20-6.el9.noarch.rpm 648 kB/s | 135 kB 00:00 2024-09-21T09:27:43.925 INFO:teuthology.orchestra.run.smithi076.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 3.2 MB/s | 1.2 MB 00:00 2024-09-21T09:27:43.959 INFO:teuthology.orchestra.run.smithi076.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 2.5 MB/s | 215 kB 00:00 2024-09-21T09:27:44.001 INFO:teuthology.orchestra.run.smithi076.stdout:(12/33): python3-requests-2.25.1-8.el9.noarch.r 829 kB/s | 125 kB 00:00 2024-09-21T09:27:44.005 INFO:teuthology.orchestra.run.smithi191.stdout:(1/33): ceph-18.2.4-871.g6a1bdf85.el9.x86_64.rp 51 kB/s | 6.5 kB 00:00 2024-09-21T09:27:44.076 INFO:teuthology.orchestra.run.smithi076.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 297 kB/s | 35 kB 00:00 2024-09-21T09:27:44.127 INFO:teuthology.orchestra.run.smithi076.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 857 kB/s | 172 kB 00:00 2024-09-21T09:27:44.143 INFO:teuthology.orchestra.run.smithi076.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 359 kB/s | 51 kB 00:00 2024-09-21T09:27:44.160 INFO:teuthology.orchestra.run.smithi076.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 497 kB/s | 42 kB 00:00 2024-09-21T09:27:44.197 INFO:teuthology.orchestra.run.smithi191.stdout:(2/33): ceph-mgr-18.2.4-871.g6a1bdf85.el9.x86_6 4.6 MB/s | 1.5 MB 00:00 2024-09-21T09:27:44.202 INFO:teuthology.orchestra.run.smithi076.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 576 kB/s | 43 kB 00:00 2024-09-21T09:27:44.219 INFO:teuthology.orchestra.run.smithi076.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 729 kB/s | 11 kB 00:00 2024-09-21T09:27:44.239 INFO:teuthology.orchestra.run.smithi191.stdout:(3/33): ceph-mds-18.2.4-871.g6a1bdf85.el9.x86_6 5.8 MB/s | 2.1 MB 00:00 2024-09-21T09:27:44.253 INFO:teuthology.orchestra.run.smithi076.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.5 MB/s | 172 kB 00:00 2024-09-21T09:27:44.270 INFO:teuthology.orchestra.run.smithi076.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 353 kB/s | 18 kB 00:00 2024-09-21T09:27:44.287 INFO:teuthology.orchestra.run.smithi076.stdout:(21/33): python3-jaraco-functools-3.5.0-2.el9.n 1.2 MB/s | 19 kB 00:00 2024-09-21T09:27:44.304 INFO:teuthology.orchestra.run.smithi076.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 696 kB/s | 23 kB 00:00 2024-09-21T09:27:44.320 INFO:teuthology.orchestra.run.smithi076.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 586 kB/s | 20 kB 00:00 2024-09-21T09:27:44.323 INFO:teuthology.orchestra.run.smithi191.stdout:(4/33): ceph-mgr-modules-core-18.2.4-871.g6a1bd 2.9 MB/s | 246 kB 00:00 2024-09-21T09:27:44.337 INFO:teuthology.orchestra.run.smithi076.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.3 MB/s | 46 kB 00:00 2024-09-21T09:27:44.363 INFO:teuthology.orchestra.run.smithi076.stdout:(25/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.9 MB/s | 358 kB 00:00 2024-09-21T09:27:44.388 INFO:teuthology.orchestra.run.smithi076.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.1 MB/s | 79 kB 00:00 2024-09-21T09:27:44.398 INFO:teuthology.orchestra.run.smithi191.stdout:(5/33): ceph-mon-18.2.4-871.g6a1bdf85.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-09-21T09:27:44.405 INFO:teuthology.orchestra.run.smithi076.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 393 kB/s | 16 kB 00:00 2024-09-21T09:27:44.430 INFO:teuthology.orchestra.run.smithi076.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 2.1 MB/s | 90 kB 00:00 2024-09-21T09:27:44.447 INFO:teuthology.orchestra.run.smithi076.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 857 kB/s | 36 kB 00:00 2024-09-21T09:27:44.473 INFO:teuthology.orchestra.run.smithi076.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.0 MB/s | 272 kB 00:00 2024-09-21T09:27:44.498 INFO:teuthology.orchestra.run.smithi076.stdout:(31/33): python3-webob-1.8.8-2.el9.noarch.rpm 3.4 MB/s | 230 kB 00:00 2024-09-21T09:27:44.523 INFO:teuthology.orchestra.run.smithi076.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 398 kB/s | 20 kB 00:00 2024-09-21T09:27:44.549 INFO:teuthology.orchestra.run.smithi076.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.1 MB/s | 427 kB 00:00 2024-09-21T09:27:44.549 INFO:teuthology.orchestra.run.smithi191.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-09-21T09:27:44.553 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:44.553 INFO:teuthology.orchestra.run.smithi076.stdout:Total 15 MB/s | 29 MB 00:01 2024-09-21T09:27:44.670 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:27:44.709 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:27:44.709 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:27:44.757 INFO:teuthology.orchestra.run.smithi191.stdout:(7/33): ceph-osd-18.2.4-871.g6a1bdf85.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-21T09:27:44.782 INFO:teuthology.orchestra.run.smithi191.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 456 kB/s | 106 kB 00:00 2024-09-21T09:27:44.816 INFO:teuthology.orchestra.run.smithi191.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 2.9 MB/s | 1.2 MB 00:00 2024-09-21T09:27:44.841 INFO:teuthology.orchestra.run.smithi191.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 2.1 MB/s | 125 kB 00:00 2024-09-21T09:27:44.866 INFO:teuthology.orchestra.run.smithi191.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-09-21T09:27:44.900 INFO:teuthology.orchestra.run.smithi191.stdout:(12/33): python3-pycparser-2.20-6.el9.noarch.rp 948 kB/s | 135 kB 00:00 2024-09-21T09:27:44.967 INFO:teuthology.orchestra.run.smithi191.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 346 kB/s | 35 kB 00:00 2024-09-21T09:27:44.992 INFO:teuthology.orchestra.run.smithi191.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.1 MB/s | 172 kB 00:00 2024-09-21T09:27:45.018 INFO:teuthology.orchestra.run.smithi191.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 435 kB/s | 51 kB 00:00 2024-09-21T09:27:45.043 INFO:teuthology.orchestra.run.smithi191.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 551 kB/s | 42 kB 00:00 2024-09-21T09:27:45.093 INFO:teuthology.orchestra.run.smithi191.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 432 kB/s | 43 kB 00:00 2024-09-21T09:27:45.119 INFO:teuthology.orchestra.run.smithi191.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 462 kB/s | 11 kB 00:00 2024-09-21T09:27:45.133 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:27:45.133 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:27:45.144 INFO:teuthology.orchestra.run.smithi191.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-09-21T09:27:45.161 INFO:teuthology.orchestra.run.smithi191.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 424 kB/s | 18 kB 00:00 2024-09-21T09:27:45.178 INFO:teuthology.orchestra.run.smithi191.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 1.4 MB/s | 23 kB 00:00 2024-09-21T09:27:45.194 INFO:teuthology.orchestra.run.smithi191.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 583 kB/s | 19 kB 00:00 2024-09-21T09:27:45.211 INFO:teuthology.orchestra.run.smithi191.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 1.2 MB/s | 20 kB 00:00 2024-09-21T09:27:45.237 INFO:teuthology.orchestra.run.smithi191.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.1 MB/s | 46 kB 00:00 2024-09-21T09:27:45.270 INFO:teuthology.orchestra.run.smithi191.stdout:(25/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.6 MB/s | 358 kB 00:00 2024-09-21T09:27:45.296 INFO:teuthology.orchestra.run.smithi191.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 939 kB/s | 79 kB 00:00 2024-09-21T09:27:45.321 INFO:teuthology.orchestra.run.smithi191.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 327 kB/s | 16 kB 00:00 2024-09-21T09:27:45.346 INFO:teuthology.orchestra.run.smithi191.stdout:(28/33): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-09-21T09:27:45.371 INFO:teuthology.orchestra.run.smithi191.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-09-21T09:27:45.397 INFO:teuthology.orchestra.run.smithi191.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-09-21T09:27:45.422 INFO:teuthology.orchestra.run.smithi191.stdout:(31/33): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-21T09:27:45.447 INFO:teuthology.orchestra.run.smithi191.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 398 kB/s | 20 kB 00:00 2024-09-21T09:27:45.472 INFO:teuthology.orchestra.run.smithi191.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-09-21T09:27:45.478 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:45.478 INFO:teuthology.orchestra.run.smithi191.stdout:Total 15 MB/s | 29 MB 00:01 2024-09-21T09:27:45.601 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:27:45.638 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:27:45.638 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:27:46.032 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:27:46.033 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:27:46.363 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:27:46.418 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-21T09:27:46.450 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-21T09:27:46.483 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-21T09:27:46.576 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-21T09:27:46.653 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-21T09:27:46.694 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-21T09:27:46.729 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-21T09:27:46.759 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-21T09:27:46.794 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-21T09:27:46.843 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-21T09:27:46.884 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-21T09:27:46.927 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-21T09:27:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-21T09:27:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-21T09:27:47.073 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-21T09:27:47.221 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-21T09:27:47.239 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:27:47.300 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-21T09:27:47.308 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-21T09:27:47.332 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-21T09:27:47.365 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-21T09:27:47.378 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-21T09:27:47.416 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-21T09:27:47.456 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-21T09:27:47.466 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-21T09:27:47.521 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-21T09:27:47.534 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-21T09:27:47.575 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-21T09:27:47.619 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-21T09:27:47.658 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-21T09:27:47.692 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-21T09:27:47.710 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-21T09:27:47.725 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-21T09:27:47.763 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-21T09:27:47.766 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-21T09:27:47.801 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-21T09:27:47.835 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-21T09:27:47.841 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-21T09:27:47.873 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-21T09:27:47.930 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-21T09:27:47.954 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-21T09:27:47.969 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-21T09:27:48.095 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-21T09:27:48.102 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-21T09:27:48.182 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-21T09:27:48.243 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-21T09:27:48.268 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 28/33 2024-09-21T09:27:48.290 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-21T09:27:48.297 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:27:48.322 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:27:48.323 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:48.323 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:27:48.323 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:48.323 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:48.323 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:48.338 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-21T09:27:48.395 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-21T09:27:48.580 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-21T09:27:48.639 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-21T09:27:48.714 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-21T09:27:48.802 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-21T09:27:48.844 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-21T09:27:48.957 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-21T09:27:49.149 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 28/33 2024-09-21T09:27:49.179 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:27:49.206 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T09:27:49.206 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:49.206 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:27:49.206 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:49.206 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:27:49.206 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:49.790 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:27:49.820 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:27:49.820 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:49.820 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:27:49.820 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:49.820 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:49.820 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:50.266 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:27:50.290 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:27:50.290 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:50.291 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:27:50.291 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:50.291 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:50.291 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:50.622 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:27:50.625 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:27:50.649 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:27:50.650 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:50.650 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:27:50.650 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:50.650 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:50.650 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:50.651 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T09:27:50.652 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:50.652 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:27:50.652 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:50.652 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:27:50.652 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:50.722 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:27:51.106 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:27:51.130 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T09:27:51.130 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:51.130 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:27:51.130 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:51.130 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:27:51.130 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:51.331 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:27:51.355 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T09:27:51.355 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:51.355 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:27:51.356 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:51.356 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:27:51.356 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:51.420 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 6/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-21T09:27:52.320 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-21T09:27:52.321 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-21T09:27:52.322 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-21T09:27:52.323 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-21T09:27:52.323 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-21T09:27:52.323 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-21T09:27:52.323 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-21T09:27:52.323 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-21T09:27:52.323 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-21T09:27:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T09:27:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/33 2024-09-21T09:27:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/33 2024-09-21T09:27:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 6/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-21T09:27:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-21T09:27:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-21T09:27:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-21T09:27:53.311 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-21T09:27:53.311 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:53.311 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:27:53.312 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:27:53.313 INFO:teuthology.orchestra.run.smithi076.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:53.314 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:27:53.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-base 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:27:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:27:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:54.112 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:27:54.321 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:27:54.409 INFO:teuthology.orchestra.run.smithi076.stdout:Package ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:27:54.452 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:54.453 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:27:54.453 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:27:54.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-base 2024-09-21T09:27:54.530 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install cephadm 2024-09-21T09:27:55.098 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:55.106 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:49 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:27:55.186 INFO:teuthology.orchestra.run.smithi191.stdout:Package ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:27:55.228 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:27:55.229 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:27:55.229 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:27:55.237 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:55.237 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 226 k 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:27:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:55.239 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 226 k 2024-09-21T09:27:55.239 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 217 k 2024-09-21T09:27:55.239 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:27:55.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install cephadm 2024-09-21T09:27:55.466 INFO:teuthology.orchestra.run.smithi076.stdout:cephadm-18.2.4-871.g6a1bdf85.el9.noarch.rpm 995 kB/s | 226 kB 00:00 2024-09-21T09:27:55.467 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:55.467 INFO:teuthology.orchestra.run.smithi076.stdout:Total 988 kB/s | 226 kB 00:00 2024-09-21T09:27:55.467 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:27:55.472 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:27:55.472 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:27:55.475 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:27:55.476 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:27:55.548 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:27:55.866 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:55.979 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:55.997 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 226 k 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:55.998 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:27:55.999 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:55.999 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:27:55.999 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:55.999 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 226 k 2024-09-21T09:27:55.999 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 217 k 2024-09-21T09:27:55.999 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:27:56.070 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:56.239 INFO:teuthology.orchestra.run.smithi191.stdout:cephadm-18.2.4-871.g6a1bdf85.el9.noarch.rpm 943 kB/s | 226 kB 00:00 2024-09-21T09:27:56.240 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:56.240 INFO:teuthology.orchestra.run.smithi191.stdout:Total 938 kB/s | 226 kB 00:00 2024-09-21T09:27:56.240 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:27:56.245 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:27:56.245 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:27:56.248 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:27:56.248 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:27:56.337 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:27:56.446 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:56.683 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:56.683 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:56.683 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:27:56.683 INFO:teuthology.orchestra.run.smithi076.stdout: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:27:56.684 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:56.684 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:27:56.802 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:56.893 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:56.911 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:27:57.253 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:57.497 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:51 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:27:57.517 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/1 2024-09-21T09:27:57.517 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:57.517 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:27:57.517 INFO:teuthology.orchestra.run.smithi191.stdout: cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:27:57.517 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:57.517 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:27:57.632 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:57.632 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:57.632 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repo Size 2024-09-21T09:27:57.632 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:57.632 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:27:57.632 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 142 k 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 142 k 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 395 k 2024-09-21T09:27:57.633 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:27:57.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:27:57.838 INFO:teuthology.orchestra.run.smithi076.stdout:ceph-immutable-object-cache-18.2.4-871.g6a1bdf8 695 kB/s | 142 kB 00:00 2024-09-21T09:27:57.839 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:57.839 INFO:teuthology.orchestra.run.smithi076.stdout:Total 690 kB/s | 142 kB 00:00 2024-09-21T09:27:57.839 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:27:57.847 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:27:57.847 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:27:57.905 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:27:57.905 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:27:58.046 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:27:58.106 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:27:58.132 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:27:58.132 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:58.132 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:27:58.132 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:58.338 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:51 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:27:58.473 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:27:58.473 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:58.473 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2024-09-21T09:27:58.473 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:58.473 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:27:58.473 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 142 k 2024-09-21T09:27:58.474 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:58.474 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:27:58.474 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:27:58.474 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:27:58.474 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:58.474 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 142 k 2024-09-21T09:27:58.475 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 395 k 2024-09-21T09:27:58.475 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:27:58.687 INFO:teuthology.orchestra.run.smithi191.stdout:ceph-immutable-object-cache-18.2.4-871.g6a1bdf8 672 kB/s | 142 kB 00:00 2024-09-21T09:27:58.688 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:27:58.688 INFO:teuthology.orchestra.run.smithi191.stdout:Total 666 kB/s | 142 kB 00:00 2024-09-21T09:27:58.688 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:27:58.695 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:27:58.695 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:27:58.753 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:27:58.753 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:27:58.892 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:27:58.955 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:27:58.979 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:27:58.979 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:27:58.979 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:27:58.979 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:59.047 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:27:59.048 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:59.048 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:27:59.048 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:59.048 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:27:59.048 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:27:59.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-mgr 2024-09-21T09:27:59.842 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:53 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:27:59.909 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T09:27:59.909 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:59.910 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:27:59.910 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:27:59.910 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:27:59.910 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:27:59.936 INFO:teuthology.orchestra.run.smithi076.stdout:Package ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:27:59.979 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:27:59.980 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:27:59.980 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:00.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-mgr-dashboard 2024-09-21T09:28:00.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr 2024-09-21T09:28:00.629 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:54 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:00.714 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:53 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:00.760 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-dashboard noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 3.5 M 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-grafana-dashboards noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 24 k 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-prometheus-alerts noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 16 k 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:28:00.761 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout:Install 5 Packages 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 3.8 M 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 87 M 2024-09-21T09:28:00.762 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:00.806 INFO:teuthology.orchestra.run.smithi191.stdout:Package ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:00.847 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:00.848 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:00.848 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:00.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-dashboard 2024-09-21T09:28:01.084 INFO:teuthology.orchestra.run.smithi076.stdout:(1/5): ceph-prometheus-alerts-18.2.4-871.g6a1bd 111 kB/s | 16 kB 00:00 2024-09-21T09:28:01.109 INFO:teuthology.orchestra.run.smithi076.stdout:(2/5): ceph-grafana-dashboards-18.2.4-871.g6a1b 140 kB/s | 24 kB 00:00 2024-09-21T09:28:01.151 INFO:teuthology.orchestra.run.smithi076.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 462 kB/s | 31 kB 00:00 2024-09-21T09:28:01.177 INFO:teuthology.orchestra.run.smithi076.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-09-21T09:28:01.360 INFO:teuthology.orchestra.run.smithi076.stdout:(5/5): ceph-mgr-dashboard-18.2.4-871.g6a1bdf85. 8.3 MB/s | 3.5 MB 00:00 2024-09-21T09:28:01.364 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:01.364 INFO:teuthology.orchestra.run.smithi076.stdout:Total 6.3 MB/s | 3.8 MB 00:00 2024-09-21T09:28:01.364 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:01.372 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:01.372 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:01.453 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:01.453 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:01.495 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:54 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:01.627 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 3.5 M 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 24 k 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 16 k 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:28:01.628 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout:Install 5 Packages 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 3.8 M 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 87 M 2024-09-21T09:28:01.629 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:01.703 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:01.783 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-21T09:28:01.823 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-21T09:28:01.852 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:28:01.957 INFO:teuthology.orchestra.run.smithi191.stdout:(1/5): ceph-grafana-dashboards-18.2.4-871.g6a1b 162 kB/s | 24 kB 00:00 2024-09-21T09:28:01.982 INFO:teuthology.orchestra.run.smithi191.stdout:(2/5): ceph-prometheus-alerts-18.2.4-871.g6a1bd 93 kB/s | 16 kB 00:00 2024-09-21T09:28:02.024 INFO:teuthology.orchestra.run.smithi191.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 461 kB/s | 31 kB 00:00 2024-09-21T09:28:02.057 INFO:teuthology.orchestra.run.smithi191.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.5 MB/s | 188 kB 00:00 2024-09-21T09:28:02.216 INFO:teuthology.orchestra.run.smithi191.stdout:(5/5): ceph-mgr-dashboard-18.2.4-871.g6a1bdf85. 8.6 MB/s | 3.5 MB 00:00 2024-09-21T09:28:02.220 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:02.220 INFO:teuthology.orchestra.run.smithi191.stdout:Total 6.4 MB/s | 3.8 MB 00:00 2024-09-21T09:28:02.220 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:02.227 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:02.227 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:02.308 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:02.308 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:02.569 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:02.664 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-21T09:28:02.706 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-21T09:28:02.742 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:28:02.899 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 4/5 2024-09-21T09:28:02.939 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:28:03.790 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 4/5 2024-09-21T09:28:03.837 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:28:04.098 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:28:04.098 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 1/5 2024-09-21T09:28:04.098 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2/5 2024-09-21T09:28:04.099 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:28:04.099 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-21T09:28:04.371 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-21T09:28:04.371 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:04.371 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:04.371 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:04.371 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:04.371 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:04.372 INFO:teuthology.orchestra.run.smithi076.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:28:04.372 INFO:teuthology.orchestra.run.smithi076.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:28:04.372 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:04.372 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:04.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-21T09:28:04.990 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 5/5 2024-09-21T09:28:04.990 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.n 1/5 2024-09-21T09:28:04.990 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2/5 2024-09-21T09:28:04.990 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.no 3/5 2024-09-21T09:28:04.990 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-21T09:28:05.166 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:00:59 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-grafana-dashboards-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-dashboard-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-prometheus-alerts-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:05.258 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:05.300 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:05.301 INFO:teuthology.orchestra.run.smithi076.stdout:====================================================================================== 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout:====================================================================================== 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 7.4 M 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:28:05.302 INFO:teuthology.orchestra.run.smithi076.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout:====================================================================================== 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout:Install 12 Packages 2024-09-21T09:28:05.303 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:05.304 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 38 M 2024-09-21T09:28:05.304 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 212 M 2024-09-21T09:28:05.304 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:05.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-21T09:28:05.785 INFO:teuthology.orchestra.run.smithi076.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.4 MB/s | 191 kB 00:00 2024-09-21T09:28:05.835 INFO:teuthology.orchestra.run.smithi076.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 4.2 MB/s | 801 kB 00:00 2024-09-21T09:28:05.860 INFO:teuthology.orchestra.run.smithi076.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 435 kB/s | 33 kB 00:00 2024-09-21T09:28:05.902 INFO:teuthology.orchestra.run.smithi076.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 436 kB/s | 18 kB 00:00 2024-09-21T09:28:05.944 INFO:teuthology.orchestra.run.smithi076.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 924 kB/s | 38 kB 00:00 2024-09-21T09:28:06.053 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:00:59 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:06.086 INFO:teuthology.orchestra.run.smithi076.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 12 MB/s | 3.0 MB 00:00 2024-09-21T09:28:06.186 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:06.187 INFO:teuthology.orchestra.run.smithi191.stdout:====================================================================================== 2024-09-21T09:28:06.187 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:28:06.187 INFO:teuthology.orchestra.run.smithi191.stdout:====================================================================================== 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 7.4 M 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:28:06.188 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout:====================================================================================== 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout:Install 12 Packages 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 38 M 2024-09-21T09:28:06.189 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 212 M 2024-09-21T09:28:06.190 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:06.195 INFO:teuthology.orchestra.run.smithi076.stdout:(7/12): ceph-mgr-diskprediction-local-18.2.4-87 14 MB/s | 7.4 MB 00:00 2024-09-21T09:28:06.220 INFO:teuthology.orchestra.run.smithi076.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.8 MB/s | 245 kB 00:00 2024-09-21T09:28:06.271 INFO:teuthology.orchestra.run.smithi076.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 4.9 MB/s | 250 kB 00:00 2024-09-21T09:28:06.371 INFO:teuthology.orchestra.run.smithi076.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 12 MB/s | 4.9 MB 00:00 2024-09-21T09:28:06.679 INFO:teuthology.orchestra.run.smithi076.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 40 MB/s | 16 MB 00:00 2024-09-21T09:28:06.748 INFO:teuthology.orchestra.run.smithi191.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.3 MB/s | 191 kB 00:00 2024-09-21T09:28:06.773 INFO:teuthology.orchestra.run.smithi191.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 4.7 MB/s | 801 kB 00:00 2024-09-21T09:28:06.871 INFO:teuthology.orchestra.run.smithi076.stdout:(12/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 7.5 MB/s | 5.1 MB 00:00 2024-09-21T09:28:06.872 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:06.872 INFO:teuthology.orchestra.run.smithi076.stdout:Total 24 MB/s | 38 MB 00:01 2024-09-21T09:28:07.065 INFO:teuthology.orchestra.run.smithi191.stdout:(3/12): ceph-mgr-diskprediction-local-18.2.4-87 16 MB/s | 7.4 MB 00:00 2024-09-21T09:28:07.090 INFO:teuthology.orchestra.run.smithi191.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 95 kB/s | 33 kB 00:00 2024-09-21T09:28:07.228 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:07.255 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:07.255 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:07.282 INFO:teuthology.orchestra.run.smithi191.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 201 kB/s | 38 kB 00:00 2024-09-21T09:28:07.350 INFO:teuthology.orchestra.run.smithi191.stdout:(6/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 64 kB/s | 18 kB 00:00 2024-09-21T09:28:07.775 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:07.775 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:07.808 INFO:teuthology.orchestra.run.smithi191.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 534 kB/s | 245 kB 00:00 2024-09-21T09:28:07.883 INFO:teuthology.orchestra.run.smithi191.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.7 MB/s | 3.0 MB 00:01 2024-09-21T09:28:08.000 INFO:teuthology.orchestra.run.smithi191.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.1 MB/s | 250 kB 00:00 2024-09-21T09:28:08.242 INFO:teuthology.orchestra.run.smithi191.stdout:(10/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 5.1 MB/s | 4.9 MB 00:00 2024-09-21T09:28:08.536 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:08.618 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-21T09:28:08.661 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:08.763 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:08.796 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-21T09:28:08.989 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-21T09:28:09.028 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-21T09:28:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-21T09:28:09.284 INFO:teuthology.orchestra.run.smithi191.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 13 MB/s | 16 MB 00:01 2024-09-21T09:28:09.509 INFO:teuthology.orchestra.run.smithi191.stdout:(12/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 3.0 MB/s | 5.1 MB 00:01 2024-09-21T09:28:09.510 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:09.510 INFO:teuthology.orchestra.run.smithi191.stdout:Total 12 MB/s | 38 MB 00:03 2024-09-21T09:28:09.770 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-21T09:28:09.862 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:09.883 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-21T09:28:09.886 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:09.887 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:10.384 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:10.385 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:11.143 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:11.156 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:11.219 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-21T09:28:11.265 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:11.425 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-21T09:28:11.608 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-21T09:28:11.649 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-21T09:28:11.725 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-21T09:28:12.241 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-21T09:28:12.269 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:28:12.384 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-21T09:28:12.519 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-21T09:28:13.755 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:14.792 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-21T09:28:14.823 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:28:14.859 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:28:14.859 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 1/12 2024-09-21T09:28:14.859 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:14.859 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:14.859 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-21T09:28:14.859 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-21T09:28:14.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-21T09:28:14.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-21T09:28:14.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-21T09:28:14.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-21T09:28:14.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:14.860 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:28:15.299 INFO:teuthology.orchestra.run.smithi076.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:15.300 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:15.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-mgr-rook 2024-09-21T09:28:16.271 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:10 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:16.404 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-rook noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 49 k 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:28:16.405 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:16.406 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:16.407 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:16.407 INFO:teuthology.orchestra.run.smithi076.stdout:Install 13 Packages 2024-09-21T09:28:16.407 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:16.407 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 2.2 M 2024-09-21T09:28:16.407 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 26 M 2024-09-21T09:28:16.407 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:16.945 INFO:teuthology.orchestra.run.smithi076.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 221 kB/s | 26 kB 00:00 2024-09-21T09:28:16.970 INFO:teuthology.orchestra.run.smithi076.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 136 kB/s | 19 kB 00:00 2024-09-21T09:28:16.995 INFO:teuthology.orchestra.run.smithi076.stdout:(3/13): ceph-mgr-rook-18.2.4-871.g6a1bdf85.el9. 295 kB/s | 49 kB 00:00 2024-09-21T09:28:17.120 INFO:teuthology.orchestra.run.smithi076.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.4 MB/s | 222 kB 00:00 2024-09-21T09:28:17.146 INFO:teuthology.orchestra.run.smithi076.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 904 kB/s | 159 kB 00:00 2024-09-21T09:28:17.179 INFO:teuthology.orchestra.run.smithi076.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 912 kB/s | 54 kB 00:00 2024-09-21T09:28:17.204 INFO:teuthology.orchestra.run.smithi076.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.3 MB/s | 279 kB 00:00 2024-09-21T09:28:17.238 INFO:teuthology.orchestra.run.smithi076.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 349 kB/s | 32 kB 00:00 2024-09-21T09:28:17.272 INFO:teuthology.orchestra.run.smithi076.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 156 kB/s | 14 kB 00:00 2024-09-21T09:28:17.297 INFO:teuthology.orchestra.run.smithi076.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.5 MB/s | 59 kB 00:00 2024-09-21T09:28:17.322 INFO:teuthology.orchestra.run.smithi076.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.5 MB/s | 90 kB 00:00 2024-09-21T09:28:17.356 INFO:teuthology.orchestra.run.smithi076.stdout:(12/13): python3-google-auth-2.34.0-1.el9.noarc 1.5 MB/s | 220 kB 00:00 2024-09-21T09:28:17.390 INFO:teuthology.orchestra.run.smithi076.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 7.2 MB/s | 1.0 MB 00:00 2024-09-21T09:28:17.392 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:17.392 INFO:teuthology.orchestra.run.smithi076.stdout:Total 2.2 MB/s | 2.2 MB 00:00 2024-09-21T09:28:17.456 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:17.472 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:17.472 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:17.562 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:17.562 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:17.660 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 12/12 2024-09-21T09:28:17.660 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf 1/12 2024-09-21T09:28:17.660 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-21T09:28:17.660 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-21T09:28:17.660 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-21T09:28:17.660 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-21T09:28:17.661 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-21T09:28:17.661 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-21T09:28:17.661 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-21T09:28:17.661 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-21T09:28:17.661 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-21T09:28:17.661 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-21T09:28:18.034 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-21T09:28:18.034 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:18.034 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:18.034 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-diskprediction-local-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:18.035 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:18.097 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:18.264 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-21T09:28:18.325 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-21T09:28:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-21T09:28:18.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-rook 2024-09-21T09:28:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-21T09:28:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-21T09:28:18.595 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-21T09:28:18.697 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-21T09:28:18.744 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-21T09:28:19.002 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:12 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:19.135 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 49 k 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:28:19.137 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout:Install 13 Packages 2024-09-21T09:28:19.138 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:19.139 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 2.2 M 2024-09-21T09:28:19.139 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 26 M 2024-09-21T09:28:19.139 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:19.382 INFO:teuthology.orchestra.run.smithi191.stdout:(1/13): ceph-mgr-rook-18.2.4-871.g6a1bdf85.el9. 299 kB/s | 49 kB 00:00 2024-09-21T09:28:19.475 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-21T09:28:19.525 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-21T09:28:19.558 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-21T09:28:19.606 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-21T09:28:19.628 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:28:19.733 INFO:teuthology.orchestra.run.smithi191.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 37 kB/s | 19 kB 00:00 2024-09-21T09:28:19.775 INFO:teuthology.orchestra.run.smithi191.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 46 kB/s | 26 kB 00:00 2024-09-21T09:28:20.167 INFO:teuthology.orchestra.run.smithi191.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 283 kB/s | 222 kB 00:00 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 1/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-21T09:28:20.440 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-21T09:28:20.441 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-21T09:28:20.441 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-21T09:28:20.441 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-21T09:28:20.450 INFO:teuthology.orchestra.run.smithi191.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 221 kB/s | 159 kB 00:00 2024-09-21T09:28:20.492 INFO:teuthology.orchestra.run.smithi191.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 783 kB/s | 32 kB 00:00 2024-09-21T09:28:20.517 INFO:teuthology.orchestra.run.smithi191.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 585 kB/s | 14 kB 00:00 2024-09-21T09:28:20.542 INFO:teuthology.orchestra.run.smithi191.stdout:(8/13): python3-google-auth-2.34.0-1.el9.noarch 8.7 MB/s | 220 kB 00:00 2024-09-21T09:28:20.576 INFO:teuthology.orchestra.run.smithi191.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 348 kB/s | 279 kB 00:00 2024-09-21T09:28:20.618 INFO:teuthology.orchestra.run.smithi191.stdout:(10/13): python3-kubernetes-26.1.0-2.el9.noarch 14 MB/s | 1.0 MB 00:00 2024-09-21T09:28:20.643 INFO:teuthology.orchestra.run.smithi191.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 3.8 MB/s | 90 kB 00:00 2024-09-21T09:28:20.669 INFO:teuthology.orchestra.run.smithi191.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 643 kB/s | 59 kB 00:00 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:28:20.779 INFO:teuthology.orchestra.run.smithi076.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:20.780 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:20.844 INFO:teuthology.orchestra.run.smithi191.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 79 kB/s | 54 kB 00:00 2024-09-21T09:28:20.850 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:20.850 INFO:teuthology.orchestra.run.smithi191.stdout:Total 1.3 MB/s | 2.2 MB 00:01 2024-09-21T09:28:20.917 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:20.933 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:20.933 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:21.022 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:21.022 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:21.043 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-mgr-cephadm 2024-09-21T09:28:21.549 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:21.654 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:15 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:21.717 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-21T09:28:21.779 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-21T09:28:21.791 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:21.792 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:21.792 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:28:21.792 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:21.792 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:21.792 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 139 k 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout:Install 6 Packages 2024-09-21T09:28:21.793 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:21.794 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 7.0 M 2024-09-21T09:28:21.794 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 33 M 2024-09-21T09:28:21.794 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:21.842 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-21T09:28:21.879 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-21T09:28:21.921 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-21T09:28:22.032 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-21T09:28:22.142 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-21T09:28:22.168 INFO:teuthology.orchestra.run.smithi076.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.6 MB/s | 248 kB 00:00 2024-09-21T09:28:22.197 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-21T09:28:22.218 INFO:teuthology.orchestra.run.smithi076.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 11 MB/s | 548 kB 00:00 2024-09-21T09:28:22.252 INFO:teuthology.orchestra.run.smithi076.stdout:(3/6): ceph-mgr-cephadm-18.2.4-871.g6a1bdf85.el 595 kB/s | 139 kB 00:00 2024-09-21T09:28:22.277 INFO:teuthology.orchestra.run.smithi076.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 983 kB/s | 58 kB 00:00 2024-09-21T09:28:22.319 INFO:teuthology.orchestra.run.smithi076.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.1 MB/s | 74 kB 00:00 2024-09-21T09:28:22.394 INFO:teuthology.orchestra.run.smithi076.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 16 MB/s | 6.0 MB 00:00 2024-09-21T09:28:22.397 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:22.397 INFO:teuthology.orchestra.run.smithi076.stdout:Total 12 MB/s | 7.0 MB 00:00 2024-09-21T09:28:22.467 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:22.478 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:22.478 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:22.578 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:22.578 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:22.883 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:22.925 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-21T09:28:22.978 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-21T09:28:23.011 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-21T09:28:23.061 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-21T09:28:23.068 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-21T09:28:23.089 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:28:23.113 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-21T09:28:23.638 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-21T09:28:23.724 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-21T09:28:23.787 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 13/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 1/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-21T09:28:23.823 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-21T09:28:23.824 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-21T09:28:23.825 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:28:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-21T09:28:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:24.146 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-rook-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:28:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:24.147 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:24.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-mgr-cephadm 2024-09-21T09:28:24.682 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:28:24.683 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/6 2024-09-21T09:28:24.683 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-21T09:28:24.683 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-21T09:28:24.683 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-21T09:28:24.683 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-21T09:28:24.961 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:24.962 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:24.965 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:17 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:25.097 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:25.098 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:25.098 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:28:25.098 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:25.098 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:25.099 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 139 k 2024-09-21T09:28:25.099 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:28:25.099 INFO:teuthology.orchestra.run.smithi191.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:28:25.099 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:28:25.099 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:28:25.099 INFO:teuthology.orchestra.run.smithi191.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:28:25.100 INFO:teuthology.orchestra.run.smithi191.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:28:25.100 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:25.100 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:25.100 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:25.100 INFO:teuthology.orchestra.run.smithi191.stdout:Install 6 Packages 2024-09-21T09:28:25.100 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:25.101 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 7.0 M 2024-09-21T09:28:25.101 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 33 M 2024-09-21T09:28:25.102 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:25.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install ceph-fuse 2024-09-21T09:28:25.545 INFO:teuthology.orchestra.run.smithi191.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.8 MB/s | 248 kB 00:00 2024-09-21T09:28:25.596 INFO:teuthology.orchestra.run.smithi191.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 11 MB/s | 548 kB 00:00 2024-09-21T09:28:25.637 INFO:teuthology.orchestra.run.smithi191.stdout:(3/6): ceph-mgr-cephadm-18.2.4-871.g6a1bdf85.el 618 kB/s | 139 kB 00:00 2024-09-21T09:28:25.663 INFO:teuthology.orchestra.run.smithi191.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 865 kB/s | 58 kB 00:00 2024-09-21T09:28:25.704 INFO:teuthology.orchestra.run.smithi191.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.1 MB/s | 74 kB 00:00 2024-09-21T09:28:25.802 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:19 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:25.933 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:25.933 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repository Size 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 853 k 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout:Installing dependencies: 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout:Install 2 Packages 2024-09-21T09:28:25.934 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:25.935 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 932 k 2024-09-21T09:28:25.935 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 2.7 M 2024-09-21T09:28:25.935 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:26.238 INFO:teuthology.orchestra.run.smithi076.stdout:(1/2): ceph-fuse-18.2.4-871.g6a1bdf85.el9.x86_6 3.3 MB/s | 853 kB 00:00 2024-09-21T09:28:26.355 INFO:teuthology.orchestra.run.smithi076.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 214 kB/s | 80 kB 00:00 2024-09-21T09:28:26.355 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:26.356 INFO:teuthology.orchestra.run.smithi076.stdout:Total 2.2 MB/s | 932 kB 00:00 2024-09-21T09:28:26.368 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:26.390 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:26.390 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:26.470 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:26.470 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:26.658 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:26.795 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:28:26.863 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:28:26.913 INFO:teuthology.orchestra.run.smithi191.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 4.0 MB/s | 6.0 MB 00:01 2024-09-21T09:28:26.916 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:26.916 INFO:teuthology.orchestra.run.smithi191.stdout:Total 3.9 MB/s | 7.0 MB 00:01 2024-09-21T09:28:26.986 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:26.997 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:26.998 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:27.098 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:27.098 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:27.402 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:27.547 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:28:27.547 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/2 2024-09-21T09:28:27.560 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-21T09:28:27.623 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-21T09:28:27.820 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:28:27.820 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:27.820 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:27.820 INFO:teuthology.orchestra.run.smithi076.stdout: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:28:27.821 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:27.821 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:28.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install librados-devel 2024-09-21T09:28:28.143 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-21T09:28:28.242 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-21T09:28:28.304 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-21T09:28:28.327 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:28:28.722 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:22 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:28.859 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repo Size 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout: librados-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 126 k 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 126 k 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 456 k 2024-09-21T09:28:28.860 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:29.076 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 6/6 2024-09-21T09:28:29.076 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 1/6 2024-09-21T09:28:29.076 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-21T09:28:29.076 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-21T09:28:29.076 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-21T09:28:29.076 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-21T09:28:29.078 INFO:teuthology.orchestra.run.smithi076.stdout:librados-devel-18.2.4-871.g6a1bdf85.el9.x86_64. 583 kB/s | 126 kB 00:00 2024-09-21T09:28:29.078 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:29.079 INFO:teuthology.orchestra.run.smithi076.stdout:Total 579 kB/s | 126 kB 00:00 2024-09-21T09:28:29.079 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:29.084 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:29.084 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:29.148 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:29.148 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:29.306 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:29.388 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-21T09:28:29.388 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:29.388 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:29.388 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-mgr-cephadm-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T09:28:29.388 INFO:teuthology.orchestra.run.smithi191.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:28:29.389 INFO:teuthology.orchestra.run.smithi191.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:28:29.389 INFO:teuthology.orchestra.run.smithi191.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:28:29.389 INFO:teuthology.orchestra.run.smithi191.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:28:29.389 INFO:teuthology.orchestra.run.smithi191.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:28:29.389 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:29.389 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:29.409 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:29.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install ceph-fuse 2024-09-21T09:28:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:30.112 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:30.112 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:30.112 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:30.112 INFO:teuthology.orchestra.run.smithi076.stdout: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:30.112 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:30.113 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:30.234 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:23 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:30.323 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install libcephfs2 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repository Size 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 853 k 2024-09-21T09:28:30.366 INFO:teuthology.orchestra.run.smithi191.stdout:Installing dependencies: 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout:Install 2 Packages 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:30.367 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 932 k 2024-09-21T09:28:30.368 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 2.7 M 2024-09-21T09:28:30.368 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:30.848 INFO:teuthology.orchestra.run.smithi191.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 396 kB/s | 80 kB 00:00 2024-09-21T09:28:30.907 INFO:teuthology.orchestra.run.smithi191.stdout:(2/2): ceph-fuse-18.2.4-871.g6a1bdf85.el9.x86_6 3.2 MB/s | 853 kB 00:00 2024-09-21T09:28:30.907 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:30.907 INFO:teuthology.orchestra.run.smithi191.stdout:Total 1.7 MB/s | 932 kB 00:00 2024-09-21T09:28:30.919 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:30.919 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:24 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:30.940 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:30.941 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:31.010 INFO:teuthology.orchestra.run.smithi076.stdout:Package libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:31.021 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:31.021 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:31.054 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:31.055 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:31.055 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:31.131 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install libcephfs-devel 2024-09-21T09:28:31.203 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:31.340 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:28:31.405 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:28:31.712 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:25 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:31.845 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repo Size 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout: libcephfs-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 31 k 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:28:31.846 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:31.847 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 31 k 2024-09-21T09:28:31.847 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 139 k 2024-09-21T09:28:31.847 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:31.999 INFO:teuthology.orchestra.run.smithi076.stdout:libcephfs-devel-18.2.4-871.g6a1bdf85.el9.x86_64 204 kB/s | 31 kB 00:00 2024-09-21T09:28:32.000 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:32.000 INFO:teuthology.orchestra.run.smithi076.stdout:Total 202 kB/s | 31 kB 00:00 2024-09-21T09:28:32.000 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:32.004 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:32.004 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:32.026 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:32.027 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:32.035 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T09:28:32.035 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/2 2024-09-21T09:28:32.113 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:32.221 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:32.320 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:28:32.320 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:32.320 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:32.320 INFO:teuthology.orchestra.run.smithi191.stdout: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:28:32.320 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:32.320 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:32.587 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:32.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librados-devel 2024-09-21T09:28:32.825 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:32.825 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:32.825 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:32.825 INFO:teuthology.orchestra.run.smithi076.stdout: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:32.826 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:32.826 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:32.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install librados2 2024-09-21T09:28:33.242 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:26 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:33.379 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:33.379 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:33.379 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2024-09-21T09:28:33.379 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 126 k 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:28:33.380 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:33.381 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 126 k 2024-09-21T09:28:33.381 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 456 k 2024-09-21T09:28:33.382 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:33.561 INFO:teuthology.orchestra.run.smithi191.stdout:librados-devel-18.2.4-871.g6a1bdf85.el9.x86_64. 699 kB/s | 126 kB 00:00 2024-09-21T09:28:33.562 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:33.562 INFO:teuthology.orchestra.run.smithi191.stdout:Total 694 kB/s | 126 kB 00:00 2024-09-21T09:28:33.562 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:33.567 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:33.567 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:33.579 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:27 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:33.631 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:33.631 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:33.668 INFO:teuthology.orchestra.run.smithi076.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:33.710 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:33.711 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:33.711 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:33.778 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:33.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install librbd1 2024-09-21T09:28:33.875 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:34.293 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:34.371 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:28 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:34.471 INFO:teuthology.orchestra.run.smithi076.stdout:Package librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:34.514 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:34.515 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:34.515 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:34.553 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:34.553 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:34.553 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:34.593 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install python3-rados 2024-09-21T09:28:34.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install libcephfs2 2024-09-21T09:28:35.176 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:29 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:35.264 INFO:teuthology.orchestra.run.smithi076.stdout:Package python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:35.307 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:35.308 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:35.308 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:35.359 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:28 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:35.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install python3-rgw 2024-09-21T09:28:35.450 INFO:teuthology.orchestra.run.smithi191.stdout:Package libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:35.494 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:35.495 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:35.495 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:35.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install libcephfs-devel 2024-09-21T09:28:35.957 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:29 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:36.045 INFO:teuthology.orchestra.run.smithi076.stdout:Package python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:36.088 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:36.089 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:36.089 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:36.143 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:29 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:36.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install python3-cephfs 2024-09-21T09:28:36.275 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 31 k 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 31 k 2024-09-21T09:28:36.276 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 139 k 2024-09-21T09:28:36.277 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:36.423 INFO:teuthology.orchestra.run.smithi191.stdout:libcephfs-devel-18.2.4-871.g6a1bdf85.el9.x86_64 213 kB/s | 31 kB 00:00 2024-09-21T09:28:36.423 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:36.424 INFO:teuthology.orchestra.run.smithi191.stdout:Total 211 kB/s | 31 kB 00:00 2024-09-21T09:28:36.424 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:36.427 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:36.427 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:36.452 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:36.452 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:36.544 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:36.637 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:36.742 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:30 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:36.830 INFO:teuthology.orchestra.run.smithi076.stdout:Package python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:36.874 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:36.875 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:36.875 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:36.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install python3-rbd 2024-09-21T09:28:36.998 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:37.243 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:37.243 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:37.243 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:37.243 INFO:teuthology.orchestra.run.smithi191.stdout: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:37.243 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:37.243 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:37.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librados2 2024-09-21T09:28:37.529 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:31 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:37.617 INFO:teuthology.orchestra.run.smithi076.stdout:Package python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:37.660 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:37.661 INFO:teuthology.orchestra.run.smithi076.stdout:Nothing to do. 2024-09-21T09:28:37.661 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:37.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install rbd-fuse 2024-09-21T09:28:38.017 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:31 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:38.107 INFO:teuthology.orchestra.run.smithi191.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:38.150 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:38.151 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:38.151 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:38.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install librbd1 2024-09-21T09:28:38.312 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:32 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:38.443 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:38.444 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:38.444 INFO:teuthology.orchestra.run.smithi076.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:38.444 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:38.444 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:38.444 INFO:teuthology.orchestra.run.smithi076.stdout: rbd-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 87 k 2024-09-21T09:28:38.445 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:38.445 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:38.445 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:38.445 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:28:38.445 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:38.446 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 87 k 2024-09-21T09:28:38.446 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 227 k 2024-09-21T09:28:38.446 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:38.645 INFO:teuthology.orchestra.run.smithi076.stdout:rbd-fuse-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 438 kB/s | 87 kB 00:00 2024-09-21T09:28:38.645 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:38.646 INFO:teuthology.orchestra.run.smithi076.stdout:Total 436 kB/s | 87 kB 00:00 2024-09-21T09:28:38.646 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:38.653 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:38.653 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:38.716 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:38.716 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:38.813 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:31 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:38.868 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:38.901 INFO:teuthology.orchestra.run.smithi191.stdout:Package librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:38.944 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:38.945 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:38.945 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:38.967 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:39.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rados 2024-09-21T09:28:39.319 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:39.586 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:39.586 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:39.586 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:39.586 INFO:teuthology.orchestra.run.smithi076.stdout: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:39.586 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:39.586 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:39.590 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:32 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:39.678 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:39.721 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:39.722 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:39.722 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:39.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rgw 2024-09-21T09:28:39.830 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install rbd-mirror 2024-09-21T09:28:40.389 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:33 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:40.428 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:40.478 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:40.522 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:40.523 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:40.523 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout: Package Arch Version Repo Size 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout: rbd-mirror x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:40.560 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:40.561 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:40.561 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:28:40.561 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:40.561 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 3.0 M 2024-09-21T09:28:40.561 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 12 M 2024-09-21T09:28:40.561 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:40.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-cephfs 2024-09-21T09:28:40.890 INFO:teuthology.orchestra.run.smithi076.stdout:rbd-mirror-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 9.1 MB/s | 3.0 MB 00:00 2024-09-21T09:28:40.891 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:40.891 INFO:teuthology.orchestra.run.smithi076.stdout:Total 9.0 MB/s | 3.0 MB 00:00 2024-09-21T09:28:40.891 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:40.900 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:40.900 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:40.980 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:40.980 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:41.175 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:41.264 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:41.307 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:41.308 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:41.308 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:41.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install python3-rbd 2024-09-21T09:28:41.404 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:41.475 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:41.500 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:41.500 INFO:teuthology.orchestra.run.smithi076.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:28:41.501 INFO:teuthology.orchestra.run.smithi076.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:28:41.501 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:41.501 INFO:teuthology.orchestra.run.smithi076.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:41.501 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:41.969 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:34 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:42.057 INFO:teuthology.orchestra.run.smithi191.stdout:Package python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T09:28:42.100 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:42.101 INFO:teuthology.orchestra.run.smithi191.stdout:Nothing to do. 2024-09-21T09:28:42.101 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:42.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-fuse 2024-09-21T09:28:42.584 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:42.584 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:42.584 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:42.584 INFO:teuthology.orchestra.run.smithi076.stdout: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:42.584 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:42.584 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:42.762 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:35 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:42.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo yum -y install rbd-nbd 2024-09-21T09:28:42.895 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 87 k 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:28:42.896 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:42.897 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 87 k 2024-09-21T09:28:42.897 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 227 k 2024-09-21T09:28:42.897 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:43.092 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-fuse-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 447 kB/s | 87 kB 00:00 2024-09-21T09:28:43.093 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:43.093 INFO:teuthology.orchestra.run.smithi191.stdout:Total 444 kB/s | 87 kB 00:00 2024-09-21T09:28:43.093 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:43.100 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:43.100 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:43.164 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:43.164 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:43.313 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:43.406 INFO:teuthology.orchestra.run.smithi076.stdout:Last metadata expiration check: 0:01:37 ago on Sat 21 Sep 2024 09:27:06 AM UTC. 2024-09-21T09:28:43.413 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:43.541 INFO:teuthology.orchestra.run.smithi076.stdout:Dependencies resolved. 2024-09-21T09:28:43.541 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:43.541 INFO:teuthology.orchestra.run.smithi076.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout:Installing: 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout: rbd-nbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 172 k 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction Summary 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout:================================================================================ 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout:Install 1 Package 2024-09-21T09:28:43.542 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:43.543 INFO:teuthology.orchestra.run.smithi076.stdout:Total download size: 172 k 2024-09-21T09:28:43.543 INFO:teuthology.orchestra.run.smithi076.stdout:Installed size: 494 k 2024-09-21T09:28:43.543 INFO:teuthology.orchestra.run.smithi076.stdout:Downloading Packages: 2024-09-21T09:28:43.753 INFO:teuthology.orchestra.run.smithi076.stdout:rbd-nbd-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 818 kB/s | 172 kB 00:00 2024-09-21T09:28:43.753 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:43.754 INFO:teuthology.orchestra.run.smithi076.stdout:Total 813 kB/s | 172 kB 00:00 2024-09-21T09:28:43.754 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction check 2024-09-21T09:28:43.757 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:43.761 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction check succeeded. 2024-09-21T09:28:43.762 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction test 2024-09-21T09:28:43.825 INFO:teuthology.orchestra.run.smithi076.stdout:Transaction test succeeded. 2024-09-21T09:28:43.826 INFO:teuthology.orchestra.run.smithi076.stdout:Running transaction 2024-09-21T09:28:43.994 INFO:teuthology.orchestra.run.smithi076.stdout: Preparing : 1/1 2024-09-21T09:28:44.000 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:44.000 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:44.000 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:44.000 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:44.000 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:44.000 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:44.095 INFO:teuthology.orchestra.run.smithi076.stdout: Installing : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:44.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-mirror 2024-09-21T09:28:44.518 INFO:teuthology.orchestra.run.smithi076.stdout: Running scriptlet: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:44.782 INFO:teuthology.orchestra.run.smithi076.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:44.783 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:44.783 INFO:teuthology.orchestra.run.smithi076.stdout:Installed: 2024-09-21T09:28:44.783 INFO:teuthology.orchestra.run.smithi076.stdout: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:44.783 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:28:44.783 INFO:teuthology.orchestra.run.smithi076.stdout:Complete! 2024-09-21T09:28:44.849 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:37 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:44.988 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:44.988 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:44.988 INFO:teuthology.orchestra.run.smithi191.stdout: Package Arch Version Repo Size 2024-09-21T09:28:44.988 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 3.0 M 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 12 M 2024-09-21T09:28:44.989 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:45.013 DEBUG:teuthology.parallel:result is None 2024-09-21T09:28:45.487 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-mirror-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 6.0 MB/s | 3.0 MB 00:00 2024-09-21T09:28:45.488 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:45.488 INFO:teuthology.orchestra.run.smithi191.stdout:Total 6.0 MB/s | 3.0 MB 00:00 2024-09-21T09:28:45.488 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:45.497 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:45.497 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:45.577 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:45.577 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:45.997 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:46.097 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:46.123 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:46.123 INFO:teuthology.orchestra.run.smithi191.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:28:46.123 INFO:teuthology.orchestra.run.smithi191.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:28:46.123 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:46.123 INFO:teuthology.orchestra.run.smithi191.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:28:46.123 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:47.135 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:47.136 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:47.136 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:47.136 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:47.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo yum -y install rbd-nbd 2024-09-21T09:28:47.965 INFO:teuthology.orchestra.run.smithi191.stdout:Last metadata expiration check: 0:01:40 ago on Sat 21 Sep 2024 09:27:07 AM UTC. 2024-09-21T09:28:48.097 INFO:teuthology.orchestra.run.smithi191.stdout:Dependencies resolved. 2024-09-21T09:28:48.097 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout: Package Architecture Version Repository Size 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout:Installing: 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 172 k 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction Summary 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout:================================================================================ 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout:Install 1 Package 2024-09-21T09:28:48.098 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:48.099 INFO:teuthology.orchestra.run.smithi191.stdout:Total download size: 172 k 2024-09-21T09:28:48.099 INFO:teuthology.orchestra.run.smithi191.stdout:Installed size: 494 k 2024-09-21T09:28:48.099 INFO:teuthology.orchestra.run.smithi191.stdout:Downloading Packages: 2024-09-21T09:28:48.308 INFO:teuthology.orchestra.run.smithi191.stdout:rbd-nbd-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 824 kB/s | 172 kB 00:00 2024-09-21T09:28:48.308 INFO:teuthology.orchestra.run.smithi191.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:28:48.308 INFO:teuthology.orchestra.run.smithi191.stdout:Total 819 kB/s | 172 kB 00:00 2024-09-21T09:28:48.309 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction check 2024-09-21T09:28:48.316 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction check succeeded. 2024-09-21T09:28:48.316 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction test 2024-09-21T09:28:48.380 INFO:teuthology.orchestra.run.smithi191.stdout:Transaction test succeeded. 2024-09-21T09:28:48.380 INFO:teuthology.orchestra.run.smithi191.stdout:Running transaction 2024-09-21T09:28:48.544 INFO:teuthology.orchestra.run.smithi191.stdout: Preparing : 1/1 2024-09-21T09:28:48.646 INFO:teuthology.orchestra.run.smithi191.stdout: Installing : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:49.087 INFO:teuthology.orchestra.run.smithi191.stdout: Running scriptlet: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:49.357 INFO:teuthology.orchestra.run.smithi191.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T09:28:49.357 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:49.357 INFO:teuthology.orchestra.run.smithi191.stdout:Installed: 2024-09-21T09:28:49.357 INFO:teuthology.orchestra.run.smithi191.stdout: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T09:28:49.357 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:28:49.357 INFO:teuthology.orchestra.run.smithi191.stdout:Complete! 2024-09-21T09:28:49.602 DEBUG:teuthology.parallel:result is None 2024-09-21T09:28:49.602 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:28:49.602 INFO:teuthology.packaging:ref: None 2024-09-21T09:28:49.602 INFO:teuthology.packaging:tag: None 2024-09-21T09:28:49.602 INFO:teuthology.packaging:branch: reef 2024-09-21T09:28:49.602 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:28:49.603 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:28:49.888 DEBUG:teuthology.orchestra.run.smithi076:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:28:49.913 INFO:teuthology.orchestra.run.smithi076.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T09:28:49.914 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-871.g6a1bdf85.el9 2024-09-21T09:28:49.914 INFO:teuthology.task.install:The correct ceph version 18.2.4-871.g6a1bdf85 is installed. 2024-09-21T09:28:49.915 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:28:49.915 INFO:teuthology.packaging:ref: None 2024-09-21T09:28:49.915 INFO:teuthology.packaging:tag: None 2024-09-21T09:28:49.916 INFO:teuthology.packaging:branch: reef 2024-09-21T09:28:49.916 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:28:49.916 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T09:28:50.192 DEBUG:teuthology.orchestra.run.smithi191:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:28:50.217 INFO:teuthology.orchestra.run.smithi191.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T09:28:50.217 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-871.g6a1bdf85.el9 2024-09-21T09:28:50.217 INFO:teuthology.task.install:The correct ceph version 18.2.4-871.g6a1bdf85 is installed. 2024-09-21T09:28:50.219 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:28:50.219 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:28:50.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:28:50.248 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:28:50.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:28:50.285 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:28:50.285 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:28:50.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:28:50.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:28:50.382 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:28:50.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:28:50.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:28:50.475 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:28:50.476 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:28:50.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:28:50.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:28:50.569 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:28:50.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:28:50.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:28:50.661 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:28:50.661 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:28:50.661 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:28:50.686 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:28:50.750 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:28:50.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:28:50.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:28:50.841 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T09:28:50.947 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:28:50.947 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-09-21T09:28:50.948 INFO:tasks.cephadm:Cluster fsid is e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:28:50.948 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T09:28:50.948 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.76', 'mon.c': '[v2:172.21.15.76:3301,v1:172.21.15.76:6790]', 'mon.b': '172.21.15.191'} 2024-09-21T09:28:50.948 INFO:tasks.cephadm:First mon is mon.a on smithi076 2024-09-21T09:28:50.948 INFO:tasks.cephadm:First mgr is y 2024-09-21T09:28:50.948 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T09:28:50.948 DEBUG:teuthology.orchestra.run.smithi076:> sudo hostname $(hostname -s) 2024-09-21T09:28:50.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo hostname $(hostname -s) 2024-09-21T09:28:50.999 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-09-21T09:28:50.999 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:28:51.139 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/56129/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.3.0-5092-g62da54b4', 'node_name': '172.21.2.5+braggi05', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.3.0-5092.g62da54b4'}, 'url': 'https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-09-20 21:56:15.050681', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://3.chacra.ceph.com/repos/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/', 'archs': ['source', 'x86_64'], 'distro': 'centos'}] 2024-09-21T09:28:51.403 INFO:tasks.util.chacra:got chacra host 4.chacra.ceph.com, ref reef, sha1 6a1bdf854bb5803efd58957907442aa01f41a8fc from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-09-21T09:28:51.405 INFO:tasks.cephadm:Discovered cachra url: https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm 2024-09-21T09:28:51.405 INFO:tasks.cephadm:Downloading cephadm from url: https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm 2024-09-21T09:28:51.405 DEBUG:teuthology.orchestra.run.smithi076:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:51.899 INFO:teuthology.orchestra.run.smithi076.stdout:-rw-r--r--. 1 ubuntu ubuntu 217222 Sep 21 09:28 /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:51.900 DEBUG:teuthology.orchestra.run.smithi191:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:52.449 INFO:teuthology.orchestra.run.smithi191.stdout:-rw-r--r--. 1 ubuntu ubuntu 217222 Sep 21 09:28 /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:52.450 DEBUG:teuthology.orchestra.run.smithi076:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:52.467 DEBUG:teuthology.orchestra.run.smithi191:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:28:52.501 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-09-21T09:28:52.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-21T09:28:52.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-21T09:28:54.069 INFO:teuthology.orchestra.run.smithi191.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:28:54.107 INFO:teuthology.orchestra.run.smithi076.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:29:20.858 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:29:20.858 INFO:teuthology.orchestra.run.smithi191.stdout: "ceph_version": "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)", 2024-09-21T09:29:20.858 INFO:teuthology.orchestra.run.smithi191.stdout: "image_id": "0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de", 2024-09-21T09:29:20.858 INFO:teuthology.orchestra.run.smithi191.stdout: "repo_digests": [ 2024-09-21T09:29:20.858 INFO:teuthology.orchestra.run.smithi191.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6" 2024-09-21T09:29:20.859 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:29:20.859 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:29:33.482 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:29:33.482 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph_version": "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)", 2024-09-21T09:29:33.483 INFO:teuthology.orchestra.run.smithi076.stdout: "image_id": "0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de", 2024-09-21T09:29:33.483 INFO:teuthology.orchestra.run.smithi076.stdout: "repo_digests": [ 2024-09-21T09:29:33.483 INFO:teuthology.orchestra.run.smithi076.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6" 2024-09-21T09:29:33.483 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:29:33.483 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:29:33.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo mkdir -p /etc/ceph 2024-09-21T09:29:33.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo mkdir -p /etc/ceph 2024-09-21T09:29:33.593 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod 777 /etc/ceph 2024-09-21T09:29:33.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 777 /etc/ceph 2024-09-21T09:29:33.662 INFO:tasks.cephadm:Writing seed config... 2024-09-21T09:29:33.663 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-21T09:29:33.663 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-21T09:29:33.663 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T09:29:33.664 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T09:29:33.665 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:29:33.665 DEBUG:teuthology.orchestra.run.smithi076:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T09:29:33.680 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e97bd93e-77fb-11ef-baf6-efdc52797490 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T09:29:33.680 DEBUG:teuthology.orchestra.run.smithi076:mon.a> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service 2024-09-21T09:29:33.724 DEBUG:teuthology.orchestra.run.smithi076:mgr.y> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service 2024-09-21T09:29:33.766 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T09:29:33.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.76 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:29:33.920 INFO:teuthology.orchestra.run.smithi076.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:29:33.920 INFO:teuthology.orchestra.run.smithi076.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'e97bd93e-77fb-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.76', '--skip-admin-label'] 2024-09-21T09:29:33.947 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:29:33.947 INFO:teuthology.orchestra.run.smithi076.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-21T09:29:33.948 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying podman|docker is present... 2024-09-21T09:29:33.969 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:29:33.969 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying lvm2 is present... 2024-09-21T09:29:33.969 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying time synchronization is in place... 2024-09-21T09:29:33.976 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:29:33.976 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:29:33.983 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:29:33.983 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout inactive 2024-09-21T09:29:33.989 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout enabled 2024-09-21T09:29:33.995 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout active 2024-09-21T09:29:33.995 INFO:teuthology.orchestra.run.smithi076.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:29:33.995 INFO:teuthology.orchestra.run.smithi076.stdout:Repeating the final host check... 2024-09-21T09:29:34.017 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:29:34.017 INFO:teuthology.orchestra.run.smithi076.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-21T09:29:34.017 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl is present 2024-09-21T09:29:34.017 INFO:teuthology.orchestra.run.smithi076.stdout:lvcreate is present 2024-09-21T09:29:34.023 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:29:34.023 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:29:34.029 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:29:34.029 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout inactive 2024-09-21T09:29:34.036 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout enabled 2024-09-21T09:29:34.042 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout active 2024-09-21T09:29:34.042 INFO:teuthology.orchestra.run.smithi076.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:29:34.042 INFO:teuthology.orchestra.run.smithi076.stdout:Host looks OK 2024-09-21T09:29:34.042 INFO:teuthology.orchestra.run.smithi076.stdout:Cluster fsid: e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:34.042 INFO:teuthology.orchestra.run.smithi076.stdout:Acquiring lock 139939319246416 on /run/cephadm/e97bd93e-77fb-11ef-baf6-efdc52797490.lock 2024-09-21T09:29:34.042 INFO:teuthology.orchestra.run.smithi076.stdout:Lock 139939319246416 acquired on /run/cephadm/e97bd93e-77fb-11ef-baf6-efdc52797490.lock 2024-09-21T09:29:34.043 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying IP 172.21.15.76 port 3300 ... 2024-09-21T09:29:34.043 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying IP 172.21.15.76 port 6789 ... 2024-09-21T09:29:34.043 INFO:teuthology.orchestra.run.smithi076.stdout:Base mon IP(s) is [172.21.15.76:3300, 172.21.15.76:6789], mon addrv is [v2:172.21.15.76:3300,v1:172.21.15.76:6789] 2024-09-21T09:29:34.047 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.76 metric 100 2024-09-21T09:29:34.047 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.76 metric 100 2024-09-21T09:29:34.049 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T09:29:34.049 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-21T09:29:34.050 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-21T09:29:34.052 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T09:29:34.053 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-21T09:29:34.053 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:29:34.053 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-21T09:29:34.053 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:47e1/64 scope link noprefixroute 2024-09-21T09:29:34.053 INFO:teuthology.orchestra.run.smithi076.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:29:34.053 INFO:teuthology.orchestra.run.smithi076.stdout:Mon IP `172.21.15.76` is in CIDR network `172.21.0.0/20` 2024-09-21T09:29:34.054 INFO:teuthology.orchestra.run.smithi076.stdout:Mon IP `172.21.15.76` is in CIDR network `172.21.0.0/20` 2024-09-21T09:29:34.054 INFO:teuthology.orchestra.run.smithi076.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-21T09:29:34.055 INFO:teuthology.orchestra.run.smithi076.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T09:29:34.055 INFO:teuthology.orchestra.run.smithi076.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:29:34.677 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stdout 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de 2024-09-21T09:29:34.678 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-09-21T09:29:34.678 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stderr Getting image source signatures 2024-09-21T09:29:34.678 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stderr Copying blob sha256:b51f6d1e10fe8de9e06c08c0a23ce0010d2bef42d30a6f89f3ac07fd857715ba 2024-09-21T09:29:34.678 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-21T09:29:34.678 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stderr Copying config sha256:0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de 2024-09-21T09:29:34.678 INFO:teuthology.orchestra.run.smithi076.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-21T09:29:35.631 INFO:teuthology.orchestra.run.smithi076.stdout:ceph: stdout ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable) 2024-09-21T09:29:35.631 INFO:teuthology.orchestra.run.smithi076.stdout:Ceph version: ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable) 2024-09-21T09:29:35.631 INFO:teuthology.orchestra.run.smithi076.stdout:Extracting ceph user uid/gid from container image... 2024-09-21T09:29:36.511 INFO:teuthology.orchestra.run.smithi076.stdout:stat: stdout 167 167 2024-09-21T09:29:36.512 INFO:teuthology.orchestra.run.smithi076.stdout:Creating initial keys... 2024-09-21T09:29:38.064 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph-authtool: stdout AQAAku5mojxGMxAAiI+n4pTrCtwKLQjhIZrbWA== 2024-09-21T09:29:47.808 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph-authtool: stdout AQAFku5m8goRDhAAMU2OCWx1ULam1/V2LUus3w== 2024-09-21T09:29:49.713 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph-authtool: stdout AQANku5mNgz2BBAA2mzteZknW653AX6YaVmFjg== 2024-09-21T09:29:49.714 INFO:teuthology.orchestra.run.smithi076.stdout:Creating initial monmap... 2024-09-21T09:29:50.669 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:29:50.669 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:monmaptool for a [v2:172.21.15.76:3300,v1:172.21.15.76:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:setting min_mon_release = pacific 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/monmaptool: set fsid to e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:29:50.670 INFO:teuthology.orchestra.run.smithi076.stdout:Creating mon... 2024-09-21T09:29:51.641 INFO:teuthology.orchestra.run.smithi076.stdout:create mon.a on 2024-09-21T09:29:51.864 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-21T09:29:52.006 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T09:29:52.173 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490.target. 2024-09-21T09:29:52.174 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490.target. 2024-09-21T09:29:52.395 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a 2024-09-21T09:29:52.395 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service: Unit ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service not loaded. 2024-09-21T09:29:52.555 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490.target.wants/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service → /etc/systemd/system/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@.service. 2024-09-21T09:29:53.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:53 smithi076 podman[28809]: 2024-09-21 09:29:53.005043054 +0000 UTC m=+0.329935092 container start 9ae8933eba309db97a854a0c8c2ab0d05b44ed77e6948c3fafaf1ec6760c7fc1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:29:53.181 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:29:53.181 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout disabled 2024-09-21T09:29:53.189 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:29:53.190 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout inactive 2024-09-21T09:29:53.190 INFO:teuthology.orchestra.run.smithi076.stdout:firewalld.service is not enabled 2024-09-21T09:29:53.190 INFO:teuthology.orchestra.run.smithi076.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:29:53.190 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for mon to start... 2024-09-21T09:29:53.190 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for mon... 2024-09-21T09:29:53.312 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:53 smithi076 ceph-mon[28847]: mkfs e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:53.312 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:53 smithi076 ceph-mon[28847]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:29:53.312 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:53 smithi076 bash[28809]: 9ae8933eba309db97a854a0c8c2ab0d05b44ed77e6948c3fafaf1ec6760c7fc1 2024-09-21T09:29:53.312 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:53 smithi076 systemd[1]: Started Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout cluster: 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout id: e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout services: 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.672785s) 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-21T09:29:54.161 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout data: 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout pgs: 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:mon is available 2024-09-21T09:29:54.162 INFO:teuthology.orchestra.run.smithi076.stdout:Assimilating anything we can from ceph.conf... 2024-09-21T09:29:54.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:54 smithi076 ceph-mon[28847]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:29:54.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:54 smithi076 ceph-mon[28847]: monmap e1: 1 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:29:54.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:54 smithi076 ceph-mon[28847]: fsmap 2024-09-21T09:29:54.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:54 smithi076 ceph-mon[28847]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:29:54.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:54 smithi076 ceph-mon[28847]: mgrmap e1: no daemons active 2024-09-21T09:29:54.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:54 smithi076 ceph-mon[28847]: from='client.? 172.21.15.76:0/2037232647' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T09:29:55.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:55 smithi076 ceph-mon[28847]: from='client.? 172.21.15.76:0/608924646' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:29:55.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:55 smithi076 ceph-mon[28847]: from='client.? 172.21.15.76:0/608924646' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout fsid = e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.76:3300,v1:172.21.15.76:6789] 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:29:55.704 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:29:55.705 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:29:55.705 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:29:55.705 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:29:55.705 INFO:teuthology.orchestra.run.smithi076.stdout:Generating new minimal ceph.conf... 2024-09-21T09:29:56.504 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:56 smithi076 ceph-mon[28847]: from='client.? 172.21.15.76:0/4126716674' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:29:56.758 INFO:teuthology.orchestra.run.smithi076.stdout:Restarting the monitor... 2024-09-21T09:29:57.019 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:56 smithi076 systemd[1]: Stopping Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:29:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[28824]: 2024-09-21T09:29:57.017+0000 7f85f55ed640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:29:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[28824]: 2024-09-21T09:29:57.017+0000 7f85f55ed640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T09:29:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 podman[29163]: 2024-09-21 09:29:57.191368769 +0000 UTC m=+0.272544286 container died 9ae8933eba309db97a854a0c8c2ab0d05b44ed77e6948c3fafaf1ec6760c7fc1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:29:57.708 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 podman[29163]: 2024-09-21 09:29:57.515252281 +0000 UTC m=+0.596427801 container cleanup 9ae8933eba309db97a854a0c8c2ab0d05b44ed77e6948c3fafaf1ec6760c7fc1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS) 2024-09-21T09:29:57.708 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 bash[29163]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:29:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 podman[29181]: 2024-09-21 09:29:57.707466258 +0000 UTC m=+0.518957917 container remove 9ae8933eba309db97a854a0c8c2ab0d05b44ed77e6948c3fafaf1ec6760c7fc1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:29:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:29:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 systemd[1]: Stopped Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:57 smithi076 systemd[1]: Starting Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:29:58.402 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 podman[29275]: 2024-09-21 09:29:58.053399954 +0000 UTC m=+0.015276823 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph:reef 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 podman[29275]: 2024-09-21 09:29:58.175140591 +0000 UTC m=+0.137017446 container create 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 podman[29275]: 2024-09-21 09:29:58.368594273 +0000 UTC m=+0.330471134 container init 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8) 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 podman[29275]: 2024-09-21 09:29:58.373396799 +0000 UTC m=+0.335273654 container start 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable), process ceph-mon, pid 7 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: pidfile_write: ignore empty --pid-file 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: load: jerasure load: lrc 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Git sha 0 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Compile date 2024-09-20 20:34:00 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: DB SUMMARY 2024-09-21T09:29:58.403 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: DB Session ID: Y6RDSY2503EN72HDIAQ3 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84540 ; 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:29:58.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:29:58.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:29:58.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:29:58.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.env: 0x5568f79c0bc0 2024-09-21T09:29:58.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:29:58.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.info_log: 0x5568f8992be0 2024-09-21T09:29:58.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.statistics: (nil) 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.use_fsync: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:29:58.406 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.db_log_dir: 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.wal_dir: 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.write_buffer_manager: 0x5568f89a0320 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:29:58.407 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.unordered_write: 0 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.row_cache: None 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.wal_filter: None 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:29:58.408 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.wal_compression: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:29:58.409 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:29:58.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_open_files: -1 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Compression algorithms supported: 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kZSTD supported: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:29:58.411 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.merge_operator: 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_filter: None 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:29:58.412 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5568f89927e0) 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: index_type: 0 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: data_block_index_type: 0 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: index_shortening: 1 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: checksum: 4 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: no_block_cache: 0 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache: 0x5568f8987350 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache_options: 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: capacity : 536870912 2024-09-21T09:29:58.413 INFO:journalctl@ceph.mon.a.smithi076.stdout: num_shard_bits : 4 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: strict_capacity_limit : 0 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache_compressed: (nil) 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: persistent_cache: (nil) 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_size: 4096 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_size_deviation: 10 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_restart_interval: 16 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: index_block_restart_interval: 1 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: metadata_block_size: 4096 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: partition_filters: 0 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: use_delta_encoding: 1 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: filter_policy: bloomfilter 2024-09-21T09:29:58.414 INFO:journalctl@ceph.mon.a.smithi076.stdout: whole_key_filtering: 1 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: verify_compression: 0 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: format_version: 5 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: enable_index_compression: 1 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_align: 0 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: max_auto_readahead_size: 262144 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: prepopulate_block_cache: 0 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression: NoCompression 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:29:58.415 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.num_levels: 7 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:29:58.416 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:29:58.417 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:29:58.418 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:29:58.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:29:58.420 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.ttl: 2592000 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:29:58.421 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dcd6f1be-f33b-4984-a0ba-b429944fcff3 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910998398900, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910998399768, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81090, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79240, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73441, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726910998, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dcd6f1be-f33b-4984-a0ba-b429944fcff3", "db_session_id": "Y6RDSY2503EN72HDIAQ3", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726910998399863, "job": 1, "event": "recovery_finished"} 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-09-21T09:29:58.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5568f89b2e00 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: DB pointer 0x5568f8aa6000 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** DB Stats ** 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:29:58.423 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: L0 2/0 81.05 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Sum 2/0 81.05 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:29:58.424 INFO:journalctl@ceph.mon.a.smithi076.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative compaction: 0.00 GB write, 25.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval compaction: 0.00 GB write, 25.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: Block cache BinnedLRUCache@0x5568f8987350#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: starting mon.a rank 0 at public addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] at bind addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???) e1 preinit fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:29:58.425 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).mds e1 new map 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).mds e1 print_map 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout: e1 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout: legacy client fscid: -1 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout: No filesystems configured 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:29:58.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T09:29:58.534 INFO:teuthology.orchestra.run.smithi076.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: monmap e1: 1 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: fsmap 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 ceph-mon[29313]: mgrmap e1: no daemons active 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 bash[29275]: 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 2024-09-21T09:29:58.656 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:29:58 smithi076 systemd[1]: Started Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:29:59.612 INFO:teuthology.orchestra.run.smithi076.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-21T09:29:59.634 INFO:teuthology.orchestra.run.smithi076.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:29:59.634 INFO:teuthology.orchestra.run.smithi076.stdout:Creating mgr... 2024-09-21T09:29:59.634 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-21T09:29:59.635 INFO:teuthology.orchestra.run.smithi076.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-21T09:29:59.887 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y 2024-09-21T09:29:59.887 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service: Unit ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service not loaded. 2024-09-21T09:30:00.043 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490.target.wants/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service → /etc/systemd/system/ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@.service. 2024-09-21T09:30:00.268 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:00 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/699241205' entity='client.admin' 2024-09-21T09:30:00.522 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 podman[29553]: 2024-09-21 09:30:00.286900813 +0000 UTC m=+0.132681490 container create 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T09:30:00.522 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 podman[29553]: 2024-09-21 09:30:00.520958826 +0000 UTC m=+0.366739513 container init 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:30:00.644 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:30:00.644 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout disabled 2024-09-21T09:30:00.651 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:30:00.651 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout inactive 2024-09-21T09:30:00.651 INFO:teuthology.orchestra.run.smithi076.stdout:firewalld.service is not enabled 2024-09-21T09:30:00.651 INFO:teuthology.orchestra.run.smithi076.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:30:00.660 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:30:00.660 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout disabled 2024-09-21T09:30:00.667 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:30:00.667 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout inactive 2024-09-21T09:30:00.667 INFO:teuthology.orchestra.run.smithi076.stdout:firewalld.service is not enabled 2024-09-21T09:30:00.667 INFO:teuthology.orchestra.run.smithi076.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-21T09:30:00.667 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for mgr to start... 2024-09-21T09:30:00.668 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for mgr... 2024-09-21T09:30:00.792 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 podman[29553]: 2024-09-21 09:30:00.526027818 +0000 UTC m=+0.371808496 container start 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T09:30:00.792 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 bash[29553]: 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 2024-09-21T09:30:00.792 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 systemd[1]: Started Ceph mgr.y for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:30:00.792 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:00.644+0000 7fdb0d7a3040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:30:00.792 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:00.707+0000 7fdb0d7a3040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:01.079 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:00.859+0000 7fdb0d7a3040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:30:01.079 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:00.928+0000 7fdb0d7a3040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:30:01.079 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:01 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:01.077+0000 7fdb0d7a3040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:01.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:01 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3412465356' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:30:01.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:01 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:01.226+0000 7fdb0d7a3040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:30:01.845 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:01 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:01.549+0000 7fdb0d7a3040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:30:01.845 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:01 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:01.611+0000 7fdb0d7a3040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:30:01.845 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:01 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:01.794+0000 7fdb0d7a3040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:30:01.847 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "fsid": "e97bd93e-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:30:01.850 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.851 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:30:01.852 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:01.852 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:30:01.852 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:30:01.852 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:30:01.852 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:30:01.855 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:30:01.855 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:30:01.855 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.855 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:30:01.855 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:30:01.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:29:53.043588+0000", 2024-09-21T09:30:01.857 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:01.858 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:01.858 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:30:01.858 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:01.858 INFO:teuthology.orchestra.run.smithi076.stdout:mgr not available, waiting (1/15)... 2024-09-21T09:30:03.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:02.665+0000 7fdb0d7a3040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:30:03.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:02.729+0000 7fdb0d7a3040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:30:03.742 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:03.468+0000 7fdb0d7a3040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:30:03.742 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:03.591+0000 7fdb0d7a3040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:30:03.742 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:03.740+0000 7fdb0d7a3040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:04.004 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:03.804+0000 7fdb0d7a3040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:30:04.004 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:03.966+0000 7fdb0d7a3040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:30:04.264 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.037+0000 7fdb0d7a3040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:30:04.264 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.099+0000 7fdb0d7a3040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:30:04.264 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.262+0000 7fdb0d7a3040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:30:04.533 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.333+0000 7fdb0d7a3040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:30:04.533 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.403+0000 7fdb0d7a3040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:30:04.533 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.489+0000 7fdb0d7a3040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:04.862 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4154943091' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:30:04.862 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.553+0000 7fdb0d7a3040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:30:04.862 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:04.615+0000 7fdb0d7a3040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:30:05.176 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:05.176 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:05.176 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "fsid": "e97bd93e-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:30:05.176 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:30:05.176 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:30:05.176 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-21T09:30:05.177 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:30:05.178 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:30:05.181 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:30:05.181 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:30:05.181 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.181 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:30:05.181 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:30:05.182 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:30:05.183 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:05.184 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:29:53.043588+0000", 2024-09-21T09:30:05.184 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:05.184 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:05.184 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:30:05.184 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:05.184 INFO:teuthology.orchestra.run.smithi076.stdout:mgr not available, waiting (2/15)... 2024-09-21T09:30:05.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:05.410+0000 7fdb0d7a3040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:30:06.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: Activating manager daemon y 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: mgrmap e2: y(active, starting, since 0.00281268s) 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: Manager daemon y is now available 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' 2024-09-21T09:30:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:05 smithi076 ceph-mon[29313]: from='mgr.14100 172.21.15.76:0/3628468257' entity='mgr.y' 2024-09-21T09:30:07.566 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:07 smithi076 ceph-mon[29313]: mgrmap e3: y(active, since 1.00683s) 2024-09-21T09:30:08.520 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:08 smithi076 ceph-mon[29313]: mgrmap e4: y(active, since 2s) 2024-09-21T09:30:08.520 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:08 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1414335728' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:30:08.524 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:08.524 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:08.524 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "fsid": "e97bd93e-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:30:08.524 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:30:08.524 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:30:08.525 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:30:08.526 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:30:08.529 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:30:08.529 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:30:08.529 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:30:08.530 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:30:08.531 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:29:53.043588+0000", 2024-09-21T09:30:08.532 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:30:08.532 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:30:08.532 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:30:08.532 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:08.532 INFO:teuthology.orchestra.run.smithi076.stdout:mgr is available 2024-09-21T09:30:09.852 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:09 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2157083899' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:30:09.855 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout fsid = e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:30:09.856 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:30:09.857 INFO:teuthology.orchestra.run.smithi076.stdout:Enabling cephadm module... 2024-09-21T09:30:10.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/389476063' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T09:30:10.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setuser ceph since I am not root 2024-09-21T09:30:10.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setgroup ceph since I am not root 2024-09-21T09:30:10.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:10.693+0000 7fbf7ea04040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:30:10.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:10.748+0000 7fbf7ea04040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:11.235 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:10.897+0000 7fbf7ea04040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:30:11.235 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:10 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:10.962+0000 7fbf7ea04040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:30:11.235 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:11.105+0000 7fbf7ea04040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:11.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:11.246+0000 7fbf7ea04040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:30:11.812 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/389476063' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T09:30:11.812 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-mon[29313]: mgrmap e5: y(active, since 5s) 2024-09-21T09:30:11.812 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:11.560+0000 7fbf7ea04040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:30:11.812 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:11.622+0000 7fbf7ea04040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:30:11.813 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:11.810+0000 7fbf7ea04040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:12.512 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for the mgr to restart... 2024-09-21T09:30:12.513 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for mgr epoch 5... 2024-09-21T09:30:12.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:12 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/469825609' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:30:12.808 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:12.687+0000 7fbf7ea04040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:30:12.808 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:12.752+0000 7fbf7ea04040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:30:13.716 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:13.461+0000 7fbf7ea04040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:30:13.716 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:13.574+0000 7fbf7ea04040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:30:13.992 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:13.714+0000 7fbf7ea04040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:13.992 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:13.774+0000 7fbf7ea04040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:30:13.992 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:13.923+0000 7fbf7ea04040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:30:13.992 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:13.990+0000 7fbf7ea04040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:30:14.264 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.048+0000 7fbf7ea04040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:30:14.264 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.199+0000 7fbf7ea04040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:30:14.264 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.262+0000 7fbf7ea04040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:30:14.533 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.327+0000 7fbf7ea04040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:30:14.534 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.409+0000 7fbf7ea04040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:14.534 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.472+0000 7fbf7ea04040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:30:14.534 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:14.531+0000 7fbf7ea04040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:30:15.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:15.291+0000 7fbf7ea04040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: Active manager daemon y restarted 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: Activating manager daemon y 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: mgrmap e6: y(active, starting, since 0.00339357s) 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:30:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: Manager daemon y is now available 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:30:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:15 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:30:16.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:16 smithi076 ceph-mon[29313]: Found migration_current of "None". Setting to last migration. 2024-09-21T09:30:16.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:16 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:16.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:16 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:16.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:16 smithi076 ceph-mon[29313]: mgrmap e7: y(active, since 1.00464s) 2024-09-21T09:30:16.903 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:16.903 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-21T09:30:16.903 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:30:16.903 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:16.903 INFO:teuthology.orchestra.run.smithi076.stdout:mgr epoch 5 is available 2024-09-21T09:30:16.903 INFO:teuthology.orchestra.run.smithi076.stdout:Setting orchestrator backend to cephadm... 2024-09-21T09:30:17.585 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-mon[29313]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-mon[29313]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:17.392+0000 7fbee66e4640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:30:17] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: Traceback (most recent call last): 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._connections.run(self.expiration_interval) 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:17.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._run(expiration_interval) 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: s = self.context.wrap_socket( 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: return self.sslsocket_class._create( 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self.do_handshake() 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._sslobj.do_handshake() 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:17.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:18.461 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:17] ENGINE Bus STARTING 2024-09-21T09:30:18.461 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:17] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:30:18.461 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: mgrmap e8: y(active, since 2s) 2024-09-21T09:30:18.461 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:17] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:17] ENGINE Bus STARTED 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:17] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:18.462 INFO:journalctl@ceph.mon.a.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: self.do_handshake() 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:18.463 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:18 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:19.426 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-21T09:30:19.426 INFO:teuthology.orchestra.run.smithi076.stdout:Generating ssh key... 2024-09-21T09:30:19.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:19 smithi076 ceph-mon[29313]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:20.488 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: Generating public/private rsa key pair. 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: Your identification has been saved in /tmp/tmpjujtwz8l/key 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: Your public key has been saved in /tmp/tmpjujtwz8l/key.pub 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: The key fingerprint is: 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: SHA256:REIoaWTFw+ZR4QeWE8jU0pf6GWCzs7B4ZFO9vAoFCng ceph-e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: The key's randomart image is: 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: +---[RSA 3072]----+ 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: |..oBoBO=.. | 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: |o.E %oX++ | 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | + = B.Oo. | 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | . * =o+ | 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | + = +S+ | 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | . + . + | 2024-09-21T09:30:20.489 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | . . . | 2024-09-21T09:30:20.490 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | . | 2024-09-21T09:30:20.490 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: | | 2024-09-21T09:30:20.490 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: +----[SHA256]-----+ 2024-09-21T09:30:21.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:21 smithi076 ceph-mon[29313]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:21.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:21 smithi076 ceph-mon[29313]: Generating ssh key... 2024-09-21T09:30:21.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:21 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:21.405 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:21 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:22.048 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuKBRgzXjj1mOtqxeP4s+uUWg3Xh7rN6aGxB6aX0QeaHbF95xQqynLj4gV3m2OKMTDwedlo/ZEBSTmO/6HcjN8caEyzB/mjIIW8D6CQErzqn7wPSKwtZjfeHePtNFAphfPXe5lssxCmGHYjo3+Peiy1aN5fxSvkKcYhA2SHQ2XKjiX7rvDdOtJX8nQz+YPbxHDcqIQueKqtHjyZNp1/ZTKIOfxPj03s6Eec1R4Yyqzts7pxKLf5EbQ6wtedYaetvXbF6ghWKJ0o4jB1cbNHF8kFRNehvxsetVN1xpPXqBiqZIHG1xcYAsImSC8EV6s4Dg01tNyA8GMxMVGD3pcEoApwRuTA6ULqplfIpntJKkYi6Pbhj+r3bLDnEBQz0q3PjCW0RKkdtQNMU0m1LdicLbUtbovthxaddfWhUzRgn5/yExTmn77Md2oiVv1Qx7TcY5IgzjNMqBRnADrSvyjF5lAEkSdjCQijmL/yvlvcxScpKqYw6jv3Xr/d2HLOTSydrM= ceph-e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:22.048 INFO:teuthology.orchestra.run.smithi076.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:30:22.048 INFO:teuthology.orchestra.run.smithi076.stdout:Adding key to root@localhost authorized_keys... 2024-09-21T09:30:22.052 INFO:teuthology.orchestra.run.smithi076.stdout:Adding host smithi076... 2024-09-21T09:30:22.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:22 smithi076 ceph-mon[29313]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:23.718 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:23 smithi076 ceph-mon[29313]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi076", "addr": "172.21.15.76", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:23.718 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:23 smithi076 ceph-mon[29313]: Deploying cephadm binary to smithi076 2024-09-21T09:30:25.974 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout Added host 'smithi076' with addr '172.21.15.76' 2024-09-21T09:30:25.975 INFO:teuthology.orchestra.run.smithi076.stdout:Deploying unmanaged mon service... 2024-09-21T09:30:26.568 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:26 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:26.569 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:26 smithi076 ceph-mon[29313]: Added host smithi076 2024-09-21T09:30:26.569 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:26 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:27.603 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-21T09:30:27.603 INFO:teuthology.orchestra.run.smithi076.stdout:Deploying unmanaged mgr service... 2024-09-21T09:30:27.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:27 smithi076 ceph-mon[29313]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:27.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:27 smithi076 ceph-mon[29313]: Saving service mon spec with placement count:5 2024-09-21T09:30:27.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:27 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:28.963 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-21T09:30:29.598 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:29 smithi076 ceph-mon[29313]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:29.598 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:29 smithi076 ceph-mon[29313]: Saving service mgr spec with placement count:2 2024-09-21T09:30:29.598 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:29 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:30.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:30 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:30 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1654055832' entity='client.admin' 2024-09-21T09:30:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:30 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:32.046 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:31 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3557988969' entity='client.admin' 2024-09-21T09:30:32.048 INFO:teuthology.orchestra.run.smithi076.stdout:Enabling the dashboard module... 2024-09-21T09:30:33.587 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:33.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3828060083' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T09:30:33.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-mon[29313]: from='mgr.14118 172.21.15.76:0/3680161793' entity='mgr.y' 2024-09-21T09:30:33.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setuser ceph since I am not root 2024-09-21T09:30:33.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setgroup ceph since I am not root 2024-09-21T09:30:33.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:33.701+0000 7f46fdc8b040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:30:33.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:33.759+0000 7f46fdc8b040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:34.272 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:33.913+0000 7f46fdc8b040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:30:34.272 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:33.980+0000 7f46fdc8b040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:30:34.273 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:34.126+0000 7f46fdc8b040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:34.273 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:34.270+0000 7f46fdc8b040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:30:34.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3828060083' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T09:30:34.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-mon[29313]: mgrmap e9: y(active, since 18s) 2024-09-21T09:30:34.831 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:34.579+0000 7f46fdc8b040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:30:34.831 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:34.639+0000 7f46fdc8b040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:30:34.831 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:34.828+0000 7f46fdc8b040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:30:35.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:35 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3418353654' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:30:36.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:35.741+0000 7f46fdc8b040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:30:36.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:35.809+0000 7f46fdc8b040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for the mgr to restart... 2024-09-21T09:30:36.173 INFO:teuthology.orchestra.run.smithi076.stdout:Waiting for mgr epoch 9... 2024-09-21T09:30:36.793 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:36.533+0000 7f46fdc8b040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:30:36.793 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:36.653+0000 7f46fdc8b040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:30:37.062 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:36.791+0000 7f46fdc8b040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:30:37.062 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:36.851+0000 7f46fdc8b040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:30:37.063 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:36.994+0000 7f46fdc8b040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:30:37.063 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.060+0000 7f46fdc8b040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:30:37.339 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.119+0000 7f46fdc8b040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:30:37.340 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.272+0000 7f46fdc8b040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:30:37.340 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.337+0000 7f46fdc8b040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:30:37.600 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.403+0000 7f46fdc8b040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:30:37.600 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.483+0000 7f46fdc8b040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:30:37.600 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.540+0000 7f46fdc8b040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:30:37.600 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:37.598+0000 7f46fdc8b040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:30:38.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:38.361+0000 7f46fdc8b040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: Active manager daemon y restarted 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: Activating manager daemon y 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: mgrmap e10: y(active, starting, since 0.00312781s) 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: Manager daemon y is now available 2024-09-21T09:30:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:38.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:30:38.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:30:40.101 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:30:40.101 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-21T09:30:40.101 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:30:40.101 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:30:40.102 INFO:teuthology.orchestra.run.smithi076.stdout:mgr epoch 9 is available 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:39.891+0000 7f4669920640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:30:39] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: Traceback (most recent call last): 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._connections.run(self.expiration_interval) 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._run(expiration_interval) 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:40.102 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: s = self.context.wrap_socket( 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: return self.sslsocket_class._create( 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self.do_handshake() 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._sslobj.do_handshake() 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:40.103 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:30:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:30:40.104 INFO:teuthology.orchestra.run.smithi076.stdout:Generating a dashboard self-signed certificate... 2024-09-21T09:30:40.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:40 smithi076 ceph-mon[29313]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:30:40.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:40 smithi076 ceph-mon[29313]: mgrmap e11: y(active, since 1.00594s) 2024-09-21T09:30:40.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:40 smithi076 ceph-mon[29313]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:30:40.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:41.438 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:39] ENGINE Bus STARTING 2024-09-21T09:30:41.438 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:39] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:30:41.438 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:39] ENGINE Error in HTTPServer.serve 2024-09-21T09:30:41.438 INFO:journalctl@ceph.mon.a.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:30:41.438 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:30:41.438 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: self.do_handshake() 2024-09-21T09:30:41.439 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:39] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:30:39] ENGINE Bus STARTED 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: mgrmap e12: y(active, since 2s) 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:41.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:41 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:41.441 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-21T09:30:41.441 INFO:teuthology.orchestra.run.smithi076.stdout:Creating initial admin user... 2024-09-21T09:30:42.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:42 smithi076 ceph-mon[29313]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:42 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:43 smithi076 ceph-mon[29313]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:43.682 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$oCfHETQco.OMFYUCRfOcm.JJQEueiv1SB4cOkviXWZ.EZkJ/iZcly", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726911042, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T09:30:43.682 INFO:teuthology.orchestra.run.smithi076.stdout:Fetching dashboard port number... 2024-09-21T09:30:44.619 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:44 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1675775400' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T09:30:44.913 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stdout 8443 2024-09-21T09:30:44.923 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:30:44.923 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout disabled 2024-09-21T09:30:44.930 INFO:teuthology.orchestra.run.smithi076.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:30:44.930 INFO:teuthology.orchestra.run.smithi076.stdout:systemctl: stdout inactive 2024-09-21T09:30:44.930 INFO:teuthology.orchestra.run.smithi076.stdout:firewalld.service is not enabled 2024-09-21T09:30:44.930 INFO:teuthology.orchestra.run.smithi076.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout:Ceph Dashboard is now available at: 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout: URL: https://smithi076.front.sepia.ceph.com:8443/ 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout: User: admin 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout: Password: p7diafohpt 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout:Saving cluster configuration to /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config directory 2024-09-21T09:30:44.932 INFO:teuthology.orchestra.run.smithi076.stdout:Enabling autotune for osd_memory_target 2024-09-21T09:30:45.555 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:45 smithi076 ceph-mon[29313]: mgrmap e13: y(active, since 6s) 2024-09-21T09:30:47.387 INFO:teuthology.orchestra.run.smithi076.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-21T09:30:47.387 INFO:teuthology.orchestra.run.smithi076.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-21T09:30:47.387 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.387 INFO:teuthology.orchestra.run.smithi076.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout:Or, if you are only running a single cluster on this host: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: ceph telemetry on 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout:For more information see: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:30:47.388 INFO:teuthology.orchestra.run.smithi076.stdout:Bootstrap complete. 2024-09-21T09:30:47.420 INFO:tasks.cephadm:Fetching config... 2024-09-21T09:30:47.421 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:30:47.421 DEBUG:teuthology.orchestra.run.smithi076:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T09:30:47.437 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T09:30:47.437 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:30:47.437 DEBUG:teuthology.orchestra.run.smithi076:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T09:30:47.492 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T09:30:47.492 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:30:47.492 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/keyring of=/dev/stdout 2024-09-21T09:30:47.557 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T09:30:47.557 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:30:47.558 DEBUG:teuthology.orchestra.run.smithi076:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T09:30:47.613 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T09:30:47.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuKBRgzXjj1mOtqxeP4s+uUWg3Xh7rN6aGxB6aX0QeaHbF95xQqynLj4gV3m2OKMTDwedlo/ZEBSTmO/6HcjN8caEyzB/mjIIW8D6CQErzqn7wPSKwtZjfeHePtNFAphfPXe5lssxCmGHYjo3+Peiy1aN5fxSvkKcYhA2SHQ2XKjiX7rvDdOtJX8nQz+YPbxHDcqIQueKqtHjyZNp1/ZTKIOfxPj03s6Eec1R4Yyqzts7pxKLf5EbQ6wtedYaetvXbF6ghWKJ0o4jB1cbNHF8kFRNehvxsetVN1xpPXqBiqZIHG1xcYAsImSC8EV6s4Dg01tNyA8GMxMVGD3pcEoApwRuTA6ULqplfIpntJKkYi6Pbhj+r3bLDnEBQz0q3PjCW0RKkdtQNMU0m1LdicLbUtbovthxaddfWhUzRgn5/yExTmn77Md2oiVv1Qx7TcY5IgzjNMqBRnADrSvyjF5lAEkSdjCQijmL/yvlvcxScpKqYw6jv3Xr/d2HLOTSydrM= ceph-e97bd93e-77fb-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:30:47.695 INFO:teuthology.orchestra.run.smithi076.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuKBRgzXjj1mOtqxeP4s+uUWg3Xh7rN6aGxB6aX0QeaHbF95xQqynLj4gV3m2OKMTDwedlo/ZEBSTmO/6HcjN8caEyzB/mjIIW8D6CQErzqn7wPSKwtZjfeHePtNFAphfPXe5lssxCmGHYjo3+Peiy1aN5fxSvkKcYhA2SHQ2XKjiX7rvDdOtJX8nQz+YPbxHDcqIQueKqtHjyZNp1/ZTKIOfxPj03s6Eec1R4Yyqzts7pxKLf5EbQ6wtedYaetvXbF6ghWKJ0o4jB1cbNHF8kFRNehvxsetVN1xpPXqBiqZIHG1xcYAsImSC8EV6s4Dg01tNyA8GMxMVGD3pcEoApwRuTA6ULqplfIpntJKkYi6Pbhj+r3bLDnEBQz0q3PjCW0RKkdtQNMU0m1LdicLbUtbovthxaddfWhUzRgn5/yExTmn77Md2oiVv1Qx7TcY5IgzjNMqBRnADrSvyjF5lAEkSdjCQijmL/yvlvcxScpKqYw6jv3Xr/d2HLOTSydrM= ceph-e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:47.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuKBRgzXjj1mOtqxeP4s+uUWg3Xh7rN6aGxB6aX0QeaHbF95xQqynLj4gV3m2OKMTDwedlo/ZEBSTmO/6HcjN8caEyzB/mjIIW8D6CQErzqn7wPSKwtZjfeHePtNFAphfPXe5lssxCmGHYjo3+Peiy1aN5fxSvkKcYhA2SHQ2XKjiX7rvDdOtJX8nQz+YPbxHDcqIQueKqtHjyZNp1/ZTKIOfxPj03s6Eec1R4Yyqzts7pxKLf5EbQ6wtedYaetvXbF6ghWKJ0o4jB1cbNHF8kFRNehvxsetVN1xpPXqBiqZIHG1xcYAsImSC8EV6s4Dg01tNyA8GMxMVGD3pcEoApwRuTA6ULqplfIpntJKkYi6Pbhj+r3bLDnEBQz0q3PjCW0RKkdtQNMU0m1LdicLbUtbovthxaddfWhUzRgn5/yExTmn77Md2oiVv1Qx7TcY5IgzjNMqBRnADrSvyjF5lAEkSdjCQijmL/yvlvcxScpKqYw6jv3Xr/d2HLOTSydrM= ceph-e97bd93e-77fb-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:30:47.771 INFO:teuthology.orchestra.run.smithi191.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCuKBRgzXjj1mOtqxeP4s+uUWg3Xh7rN6aGxB6aX0QeaHbF95xQqynLj4gV3m2OKMTDwedlo/ZEBSTmO/6HcjN8caEyzB/mjIIW8D6CQErzqn7wPSKwtZjfeHePtNFAphfPXe5lssxCmGHYjo3+Peiy1aN5fxSvkKcYhA2SHQ2XKjiX7rvDdOtJX8nQz+YPbxHDcqIQueKqtHjyZNp1/ZTKIOfxPj03s6Eec1R4Yyqzts7pxKLf5EbQ6wtedYaetvXbF6ghWKJ0o4jB1cbNHF8kFRNehvxsetVN1xpPXqBiqZIHG1xcYAsImSC8EV6s4Dg01tNyA8GMxMVGD3pcEoApwRuTA6ULqplfIpntJKkYi6Pbhj+r3bLDnEBQz0q3PjCW0RKkdtQNMU0m1LdicLbUtbovthxaddfWhUzRgn5/yExTmn77Md2oiVv1Qx7TcY5IgzjNMqBRnADrSvyjF5lAEkSdjCQijmL/yvlvcxScpKqYw6jv3Xr/d2HLOTSydrM= ceph-e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:30:47.784 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T09:30:47.979 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:48.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:47 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/156200185' entity='client.admin' 2024-09-21T09:30:50.685 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:50.685 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:50.686 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi076", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:30:50.686 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:50.686 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:50.686 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:51.334 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T09:30:51.334 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T09:30:51.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:51 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/349080228' entity='client.admin' 2024-09-21T09:30:51.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:51.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:52.035 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:54.682 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi191 2024-09-21T09:30:54.682 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:30:54.682 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/etc/ceph/ceph.conf 2024-09-21T09:30:54.701 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:30:54.701 DEBUG:teuthology.orchestra.run.smithi191:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:30:54.759 INFO:tasks.cephadm:Adding host smithi191 to orchestrator... 2024-09-21T09:30:54.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch host add smithi191 2024-09-21T09:30:54.947 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:30:55.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:55.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:55.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:55.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:55.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:54 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:57.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:56 smithi076 ceph-mon[29313]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi191", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:30:58.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:57 smithi076 ceph-mon[29313]: Deploying cephadm binary to smithi191 2024-09-21T09:30:59.180 INFO:teuthology.orchestra.run.smithi076.stdout:Added host 'smithi191' with addr '172.21.15.191' 2024-09-21T09:30:59.497 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:59 smithi076 ceph-mon[29313]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:30:59.498 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:59 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:30:59.498 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:30:59 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:30:59.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T09:31:00.000 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:00.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:00 smithi076 ceph-mon[29313]: Added host smithi191 2024-09-21T09:31:00.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:00 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:00.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:00 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:01.815 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:31:01.815 INFO:teuthology.orchestra.run.smithi076.stdout:[{"addr": "172.21.15.76", "hostname": "smithi076", "labels": [], "status": ""}, {"addr": "172.21.15.191", "hostname": "smithi191", "labels": [], "status": ""}] 2024-09-21T09:31:01.815 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:01 smithi076 ceph-mon[29313]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:01.816 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:01 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:02.400 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T09:31:02.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T09:31:02.592 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:03.772 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi191", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:03.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:04 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:31:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:04 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:04 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:05.480 INFO:teuthology.orchestra.run.smithi076.stderr:adjusted tunables profile to default 2024-09-21T09:31:05.781 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:05 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:31:05.781 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:05 smithi076 ceph-mon[29313]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:05.781 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:05 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3568382035' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:31:06.086 INFO:tasks.cephadm:Adding mon.a on smithi076 2024-09-21T09:31:06.086 INFO:tasks.cephadm:Adding mon.c on smithi076 2024-09-21T09:31:06.086 INFO:tasks.cephadm:Adding mon.b on smithi191 2024-09-21T09:31:06.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch apply mon '3;smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b' 2024-09-21T09:31:06.288 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:06.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:06 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3568382035' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:31:06.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:06 smithi076 ceph-mon[29313]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:07.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:07 smithi076 ceph-mon[29313]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:07.893 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled mon update... 2024-09-21T09:31:08.452 DEBUG:teuthology.orchestra.run.smithi076:mon.c> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.c.service 2024-09-21T09:31:08.455 DEBUG:teuthology.orchestra.run.smithi191:mon.b> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.b.service 2024-09-21T09:31:08.457 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:31:08.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:31:09.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: Saving service mon spec with placement smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b;count:3 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: Deploying daemon mon.b on smithi191 2024-09-21T09:31:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:08 smithi076 ceph-mon[29313]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:09.401 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:11.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:11 smithi076 ceph-mon[29313]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:12.065 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:11 smithi191 systemd[1]: Starting Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:31:12.415 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:31:12.416 INFO:teuthology.orchestra.run.smithi191.stdout:{"epoch":1,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","modified":"2024-09-21T09:29:50.086004Z","created":"2024-09-21T09:29:50.086004Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:3300","nonce":0},{"type":"v1","addr":"172.21.15.76:6789","nonce":0}]},"addr":"172.21.15.76:6789/0","public_addr":"172.21.15.76:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T09:31:12.416 INFO:teuthology.orchestra.run.smithi191.stderr:dumped monmap epoch 1 2024-09-21T09:31:12.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 podman[33249]: 2024-09-21 09:31:12.256616398 +0000 UTC m=+0.019948105 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T09:31:12.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 podman[33249]: 2024-09-21 09:31:12.367315466 +0000 UTC m=+0.130647174 container create dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:31:12.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:12 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/2979819812' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 podman[33249]: 2024-09-21 09:31:12.660061706 +0000 UTC m=+0.423393413 container init dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8) 2024-09-21T09:31:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 podman[33249]: 2024-09-21 09:31:12.664750225 +0000 UTC m=+0.428081935 container start dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T09:31:13.006 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable), process ceph-mon, pid 7 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: pidfile_write: ignore empty --pid-file 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: load: jerasure load: lrc 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Git sha 0 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Compile date 2024-09-20 20:34:00 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: DB SUMMARY 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: DB Session ID: EJIENVHFGNXJD8DWPAL4 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-21T09:31:13.007 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.env: 0x560a767bdbc0 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.info_log: 0x560a770d4be0 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:31:13.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.statistics: (nil) 2024-09-21T09:31:13.009 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.use_fsync: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.db_log_dir: 2024-09-21T09:31:13.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.wal_dir: 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.write_buffer_manager: 0x560a770e2320 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:31:13.011 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.unordered_write: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.row_cache: None 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.wal_filter: None 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:31:13.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.wal_compression: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:31:13.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_open_files: -1 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:31:13.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Compression algorithms supported: 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kZSTD supported: 0 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-09-21T09:31:13.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.merge_operator: 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_filter: None 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560a770d47e0) 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_type: 0 2024-09-21T09:31:13.016 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_block_index_type: 0 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_shortening: 1 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: checksum: 4 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: no_block_cache: 0 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache: 0x560a770c9350 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_options: 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: capacity : 536870912 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: num_shard_bits : 4 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: strict_capacity_limit : 0 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_compressed: (nil) 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: persistent_cache: (nil) 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_size: 4096 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_size_deviation: 10 2024-09-21T09:31:13.017 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_restart_interval: 16 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_block_restart_interval: 1 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: metadata_block_size: 4096 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: partition_filters: 0 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: use_delta_encoding: 1 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: filter_policy: bloomfilter 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: whole_key_filtering: 1 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: verify_compression: 0 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: format_version: 5 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: enable_index_compression: 1 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_align: 0 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_auto_readahead_size: 262144 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: prepopulate_block_cache: 0 2024-09-21T09:31:13.018 INFO:journalctl@ceph.mon.b.smithi191.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression: NoCompression 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.num_levels: 7 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:31:13.019 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:31:13.020 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:31:13.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:31:13.022 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:31:13.023 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.ttl: 2592000 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:31:13.024 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dab05661-a9f5-450c-9f5f-83c8d1de037e 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911072692084, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911072692497, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1648, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911072, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dab05661-a9f5-450c-9f5f-83c8d1de037e", "db_session_id": "EJIENVHFGNXJD8DWPAL4", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911072692591, "job": 1, "event": "recovery_finished"} 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560a770f4e00 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: DB pointer 0x560a771e8000 2024-09-21T09:31:13.025 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** DB Stats ** 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:31:13.026 INFO:journalctl@ceph.mon.b.smithi191.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: L0 1/0 1.61 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: Sum 1/0 1.61 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.9 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:13.027 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: Block cache BinnedLRUCache@0x560a770c9350#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: using public_addr v2:172.21.15.191:0/0 -> [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: starting mon.b rank -1 at public addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] at bind addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(???) e0 preinit fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:13.028 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).mds e1 new map 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).mds e1 print_map 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout: e1 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout: legacy client fscid: -1 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout: No filesystems configured 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:31:13.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3568382035' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: Saving service mon spec with placement smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b;count:3 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:13.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: Deploying daemon mon.b on smithi191 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/2979819812' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 ceph-mon[33301]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 bash[33249]: dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d 2024-09-21T09:31:13.031 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:12 smithi191 systemd[1]: Started Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:31:14.395 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:31:14.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:31:14.613 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:31:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: mon.a calling monitor election 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: mon.b calling monitor election 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: monmap e2: 2 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: fsmap 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: mgrmap e13: y(active, since 39s) 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: overall HEALTH_OK 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:17.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:18.116 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:18.116 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: mon.a calling monitor election 2024-09-21T09:31:18.116 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:18.116 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: mon.b calling monitor election 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: monmap e2: 2 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: fsmap 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: mgrmap e13: y(active, since 39s) 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: overall HEALTH_OK 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:18.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:18.118 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:18.118 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:18.118 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:18.989 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:18 smithi076 ceph-mon[29313]: Deploying daemon mon.c on smithi076 2024-09-21T09:31:18.989 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:18 smithi076 ceph-mon[29313]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:18.989 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:18 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:18 smithi191 ceph-mon[33301]: Deploying daemon mon.c on smithi076 2024-09-21T09:31:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:18 smithi191 ceph-mon[33301]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:18 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:19.510 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:31:19.510 INFO:teuthology.orchestra.run.smithi191.stdout:{"epoch":2,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","modified":"2024-09-21T09:31:12.714659Z","created":"2024-09-21T09:29:50.086004Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:3300","nonce":0},{"type":"v1","addr":"172.21.15.76:6789","nonce":0}]},"addr":"172.21.15.76:6789/0","public_addr":"172.21.15.76:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:3300","nonce":0},{"type":"v1","addr":"172.21.15.191:6789","nonce":0}]},"addr":"172.21.15.191:6789/0","public_addr":"172.21.15.191:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T09:31:19.511 INFO:teuthology.orchestra.run.smithi191.stderr:dumped monmap epoch 2 2024-09-21T09:31:19.996 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:19 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/1222917615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:20.060 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:19 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/1222917615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:20.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 systemd[1]: Starting Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:31:20.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[29313]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:20.890 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 podman[41288]: 2024-09-21 09:31:20.539884155 +0000 UTC m=+0.015400437 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T09:31:20.890 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 podman[41288]: 2024-09-21 09:31:20.647578071 +0000 UTC m=+0.123094352 container create 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:31:20.890 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 podman[41288]: 2024-09-21 09:31:20.858608832 +0000 UTC m=+0.334125122 container init 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:31:20.891 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 podman[41288]: 2024-09-21 09:31:20.863486681 +0000 UTC m=+0.339002973 container start 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T09:31:20.891 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:31:20.891 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable), process ceph-mon, pid 7 2024-09-21T09:31:20.891 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pidfile_write: ignore empty --pid-file 2024-09-21T09:31:20.891 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: load: jerasure load: lrc 2024-09-21T09:31:21.082 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:31:21.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:31:21.109 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:20 smithi191 ceph-mon[33301]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.283 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:31:21.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Git sha 0 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Compile date 2024-09-20 20:34:00 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: DB SUMMARY 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: DB Session ID: M0IMAC4QGLRNWTYJLBB6 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:31:21.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.env: 0x5618a338abc0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.info_log: 0x5618a4138be0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.statistics: (nil) 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.use_fsync: 0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:31:21.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.db_log_dir: 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.wal_dir: 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:31:21.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:31:21.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:31:21.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:31:21.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:31:21.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:31:21.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.write_buffer_manager: 0x5618a4146320 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.unordered_write: 0 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:31:21.378 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.row_cache: None 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.wal_filter: None 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.wal_compression: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:31:21.379 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:31:21.380 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_open_files: -1 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Compression algorithms supported: 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kZSTD supported: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:31:21.381 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.merge_operator: 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_filter: None 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5618a41387e0) 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:31:21.382 INFO:journalctl@ceph.mon.c.smithi076.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: index_type: 0 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: data_block_index_type: 0 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: index_shortening: 1 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: checksum: 4 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: no_block_cache: 0 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache: 0x5618a412d1f0 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache_options: 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: capacity : 536870912 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: num_shard_bits : 4 2024-09-21T09:31:21.383 INFO:journalctl@ceph.mon.c.smithi076.stdout: strict_capacity_limit : 0 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache_compressed: (nil) 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: persistent_cache: (nil) 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_size: 4096 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_size_deviation: 10 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_restart_interval: 16 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: index_block_restart_interval: 1 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: metadata_block_size: 4096 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: partition_filters: 0 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: use_delta_encoding: 1 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: filter_policy: bloomfilter 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: whole_key_filtering: 1 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: verify_compression: 0 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:31:21.384 INFO:journalctl@ceph.mon.c.smithi076.stdout: format_version: 5 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout: enable_index_compression: 1 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_align: 0 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout: max_auto_readahead_size: 262144 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout: prepopulate_block_cache: 0 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression: NoCompression 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.num_levels: 7 2024-09-21T09:31:21.385 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:31:21.386 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:31:21.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:31:21.388 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:31:21.389 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.ttl: 2592000 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:31:21.390 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 581c3213-13f4-4179-98bb-d3d9d16183eb 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911080889807, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911080890187, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1773, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911080, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "581c3213-13f4-4179-98bb-d3d9d16183eb", "db_session_id": "M0IMAC4QGLRNWTYJLBB6", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:31:21.391 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911080890278, "job": 1, "event": "recovery_finished"} 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5618a4158e00 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: DB pointer 0x5618a424c000 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** DB Stats ** 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:31:21.392 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:31:21.393 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:21.393 INFO:journalctl@ceph.mon.c.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:31:21.393 INFO:journalctl@ceph.mon.c.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:31:21.393 INFO:journalctl@ceph.mon.c.smithi076.stdout: L0 1/0 1.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:21.393 INFO:journalctl@ceph.mon.c.smithi076.stdout: Sum 1/0 1.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:31:21.394 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative compaction: 0.00 GB write, 0.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval compaction: 0.00 GB write, 0.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: Block cache BinnedLRUCache@0x5618a412d1f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: using public_addrv [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: starting mon.c rank -1 at public addrs [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] at bind addrs [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(???) e0 preinit fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).mds e1 new map 2024-09-21T09:31:21.395 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout: e1 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout: legacy client fscid: -1 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout: No filesystems configured 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:31:21.396 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3568382035' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: Saving service mon spec with placement smithi076:172.21.15.76=a;smithi076:[v2:172.21.15.76:3301,v1:172.21.15.76:6790]=c;smithi191:172.21.15.191=b;count:3 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:21.397 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: Deploying daemon mon.b on smithi191 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/2979819812' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.a calling monitor election 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.b calling monitor election 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: monmap e2: 2 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: fsmap 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mgrmap e13: y(active, since 39s) 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: overall HEALTH_OK 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: Deploying daemon mon.c on smithi076 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.399 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:21.400 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/1222917615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:21.400 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:21.400 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 ceph-mon[41325]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:31:21.400 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 bash[41288]: 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b 2024-09-21T09:31:21.400 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:20 smithi076 systemd[1]: Started Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:31:26.229 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:31:26.230 INFO:teuthology.orchestra.run.smithi191.stdout:{"epoch":3,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","modified":"2024-09-21T09:31:20.913061Z","created":"2024-09-21T09:29:50.086004Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:3300","nonce":0},{"type":"v1","addr":"172.21.15.76:6789","nonce":0}]},"addr":"172.21.15.76:6789/0","public_addr":"172.21.15.76:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:3300","nonce":0},{"type":"v1","addr":"172.21.15.191:6789","nonce":0}]},"addr":"172.21.15.191:6789/0","public_addr":"172.21.15.191:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:3301","nonce":0},{"type":"v1","addr":"172.21.15.76:6790","nonce":0}]},"addr":"172.21.15.76:6790/0","public_addr":"172.21.15.76:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-21T09:31:26.230 INFO:teuthology.orchestra.run.smithi191.stderr:dumped monmap epoch 3 2024-09-21T09:31:26.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:26.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: mon.a calling monitor election 2024-09-21T09:31:26.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:26.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: mon.b calling monitor election 2024-09-21T09:31:26.230 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: mon.c calling monitor election 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: monmap e3: 3 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: fsmap 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: mgrmap e13: y(active, since 47s) 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: overall HEALTH_OK 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.231 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.232 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.232 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: mon.a calling monitor election 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: mon.b calling monitor election 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: mon.c calling monitor election 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: monmap e3: 3 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: fsmap 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: mgrmap e13: y(active, since 47s) 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: overall HEALTH_OK 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: mon.a calling monitor election 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: mon.b calling monitor election 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: mon.c calling monitor election 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:26.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: monmap e3: 3 mons at {a=[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0],b=[v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0],c=[v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: fsmap 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: mgrmap e13: y(active, since 47s) 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: overall HEALTH_OK 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:26.920 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T09:31:26.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T09:31:27.121 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:27.207 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:26 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:27.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:26 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/513771534' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:27.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:26 smithi191 ceph-mon[33301]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:27.208 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:26 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/513771534' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[29313]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/513771534' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:31:27.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[41325]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:27.244 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:26 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:31:28.316 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.316 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.316 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.316 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.316 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:28.316 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:28.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:28 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:28.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:28 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.214 INFO:teuthology.orchestra.run.smithi076.stdout:# minimal ceph.conf for e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:29.214 INFO:teuthology.orchestra.run.smithi076.stdout:[global] 2024-09-21T09:31:29.214 INFO:teuthology.orchestra.run.smithi076.stdout: fsid = e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:31:29.214 INFO:teuthology.orchestra.run.smithi076.stdout: mon_host = [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] 2024-09-21T09:31:29.214 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: Reconfiguring daemon mon.a on smithi076 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:29.215 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:29.216 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: Reconfiguring daemon mon.a on smithi076 2024-09-21T09:31:29.217 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:29.217 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:31:29.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: Reconfiguring daemon mon.a on smithi076 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1456218943' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1456218943' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:30.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:30.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:30.501 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T09:31:30.501 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:31:30.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:31:30.527 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:31:30.527 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:30.593 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:31:30.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:31:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:30 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1456218943' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:30.625 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:31:30.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:31:30.694 INFO:tasks.cephadm:Adding mgr.y on smithi076 2024-09-21T09:31:30.694 INFO:tasks.cephadm:Adding mgr.x on smithi191 2024-09-21T09:31:30.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch apply mgr '2;smithi076=y;smithi191=x' 2024-09-21T09:31:30.926 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:31:31.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:31 smithi191 ceph-mon[33301]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:31:31.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:31 smithi191 ceph-mon[33301]: Reconfiguring daemon mon.c on smithi076 2024-09-21T09:31:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:31 smithi191 ceph-mon[33301]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:31 smithi076 ceph-mon[29313]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:31:31.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:31 smithi076 ceph-mon[29313]: Reconfiguring daemon mon.c on smithi076 2024-09-21T09:31:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:31 smithi076 ceph-mon[29313]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:31 smithi076 ceph-mon[41325]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:31:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:31 smithi076 ceph-mon[41325]: Reconfiguring daemon mon.c on smithi076 2024-09-21T09:31:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:31 smithi076 ceph-mon[41325]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: Reconfiguring daemon mon.b on smithi191 2024-09-21T09:31:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:32 smithi191 ceph-mon[33301]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: Reconfiguring daemon mon.b on smithi191 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[29313]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: Reconfiguring daemon mon.b on smithi191 2024-09-21T09:31:33.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:32 smithi076 ceph-mon[41325]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:33.263 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled mgr update... 2024-09-21T09:31:34.268 DEBUG:teuthology.orchestra.run.smithi191:mgr.x> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service 2024-09-21T09:31:34.270 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T09:31:34.270 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T09:31:34.270 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:31:34.270 DEBUG:teuthology.orchestra.run.smithi076:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:31:34.286 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:31:34.286 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_1 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi076=y;smithi191=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: Saving service mgr spec with placement smithi076=y;smithi191=x;count:2 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.289 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:34.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:34.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:31:34.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:34.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:34 smithi191 ceph-mon[33301]: Deploying daemon mgr.x on smithi191 2024-09-21T09:31:34.341 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:31:34.341 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:34.341 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 962 Links: 1 2024-09-21T09:31:34.341 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:34.341 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:34.342 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-21 09:31:20.364130835 +0000 2024-09-21T09:31:34.342 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-21 09:26:43.997580711 +0000 2024-09-21T09:31:34.342 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-21 09:26:43.997580711 +0000 2024-09-21T09:31:34.342 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-21 09:26:43.997580711 +0000 2024-09-21T09:31:34.342 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:31:34.407 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-21T09:31:34.407 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-21T09:31:34.407 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000274359 s, 1.9 MB/s 2024-09-21T09:31:34.408 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:31:34.465 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_2 2024-09-21T09:31:34.519 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:31:34.519 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 956 Links: 1 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-21 09:31:20.365130805 +0000 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-21 09:26:43.996580816 +0000 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-21 09:26:43.996580816 +0000 2024-09-21T09:31:34.520 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-21 09:26:43.996580816 +0000 2024-09-21T09:31:34.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:31:34.580 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi076=y;smithi191=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: Saving service mgr spec with placement smithi076=y;smithi191=x;count:2 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:34.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[29313]: Deploying daemon mgr.x on smithi191 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi076=y;smithi191=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: Saving service mgr spec with placement smithi076=y;smithi191=x;count:2 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:34.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:31:34.583 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:34.583 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:34.583 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:34 smithi076 ceph-mon[41325]: Deploying daemon mgr.x on smithi191 2024-09-21T09:31:34.583 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-21T09:31:34.583 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-21T09:31:34.583 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000220137 s, 2.3 MB/s 2024-09-21T09:31:34.586 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:31:34.642 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_3 2024-09-21T09:31:34.698 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:31:34.698 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-21 09:31:20.365130805 +0000 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-21 09:26:44.001580294 +0000 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-21 09:26:44.001580294 +0000 2024-09-21T09:31:34.699 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-21 09:26:44.001580294 +0000 2024-09-21T09:31:34.699 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:31:34.761 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-21T09:31:34.762 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-21T09:31:34.762 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000230475 s, 2.2 MB/s 2024-09-21T09:31:34.763 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:31:34.820 DEBUG:teuthology.orchestra.run.smithi076:> stat /dev/vg_nvme/lv_4 2024-09-21T09:31:34.875 INFO:teuthology.orchestra.run.smithi076.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:31:34.875 INFO:teuthology.orchestra.run.smithi076.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout:Device: 5h/5d Inode: 978 Links: 1 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout:Access: 2024-09-21 09:31:20.365130805 +0000 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout:Modify: 2024-09-21 09:26:44.000580398 +0000 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout:Change: 2024-09-21 09:26:44.000580398 +0000 2024-09-21T09:31:34.876 INFO:teuthology.orchestra.run.smithi076.stdout: Birth: 2024-09-21 09:26:44.000580398 +0000 2024-09-21T09:31:34.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:31:34.938 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records in 2024-09-21T09:31:34.938 INFO:teuthology.orchestra.run.smithi076.stderr:1+0 records out 2024-09-21T09:31:34.939 INFO:teuthology.orchestra.run.smithi076.stderr:512 bytes copied, 0.000172356 s, 3.0 MB/s 2024-09-21T09:31:34.940 DEBUG:teuthology.orchestra.run.smithi076:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:31:34.995 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:31:34.996 DEBUG:teuthology.orchestra.run.smithi191:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:31:35.012 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:31:35.012 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_1 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 983 Links: 1 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-09-21 09:31:11.865114528 +0000 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-09-21 09:26:43.196864378 +0000 2024-09-21T09:31:35.067 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-09-21 09:26:43.196864378 +0000 2024-09-21T09:31:35.068 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-09-21 09:26:43.196864378 +0000 2024-09-21T09:31:35.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:31:35.131 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-09-21T09:31:35.131 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-09-21T09:31:35.131 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000273309 s, 1.9 MB/s 2024-09-21T09:31:35.132 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:31:35.189 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_2 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-09-21 09:31:11.866114511 +0000 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-09-21 09:26:43.193864253 +0000 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-09-21 09:26:43.193864253 +0000 2024-09-21T09:31:35.244 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-09-21 09:26:43.193864253 +0000 2024-09-21T09:31:35.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:31:35.307 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-09-21T09:31:35.309 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-09-21T09:31:35.309 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000204917 s, 2.5 MB/s 2024-09-21T09:31:35.311 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:31:35.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:35 smithi191 ceph-mon[33301]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:35.372 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_3 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-09-21 09:31:11.866114511 +0000 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-09-21 09:26:43.193864253 +0000 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-09-21 09:26:43.193864253 +0000 2024-09-21T09:31:35.429 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-09-21 09:26:43.193864253 +0000 2024-09-21T09:31:35.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:31:35.494 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-09-21T09:31:35.494 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-09-21T09:31:35.494 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.000243196 s, 2.1 MB/s 2024-09-21T09:31:35.496 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:31:35.554 DEBUG:teuthology.orchestra.run.smithi191:> stat /dev/vg_nvme/lv_4 2024-09-21T09:31:35.609 INFO:teuthology.orchestra.run.smithi191.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:31:35.609 INFO:teuthology.orchestra.run.smithi191.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout:Access: 2024-09-21 09:31:35.605692110 +0000 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout:Modify: 2024-09-21 09:26:43.192864212 +0000 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout:Change: 2024-09-21 09:26:43.192864212 +0000 2024-09-21T09:31:35.610 INFO:teuthology.orchestra.run.smithi191.stdout: Birth: 2024-09-21 09:26:43.192864212 +0000 2024-09-21T09:31:35.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:31:35.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:35 smithi076 ceph-mon[29313]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:35.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:35 smithi076 ceph-mon[41325]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:35.676 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records in 2024-09-21T09:31:35.676 INFO:teuthology.orchestra.run.smithi191.stderr:1+0 records out 2024-09-21T09:31:35.676 INFO:teuthology.orchestra.run.smithi191.stderr:512 bytes copied, 0.00017212 s, 3.0 MB/s 2024-09-21T09:31:35.677 DEBUG:teuthology.orchestra.run.smithi191:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:31:35.735 INFO:tasks.cephadm:Deploying osd.0 on smithi076 with /dev/vg_nvme/lv_4... 2024-09-21T09:31:35.735 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:31:35.930 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:36.061 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:35 smithi191 systemd[1]: Starting Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:31:36.322 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 podman[35340]: 2024-09-21 09:31:35.961762158 +0000 UTC m=+0.015518274 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T09:31:36.322 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 podman[35340]: 2024-09-21 09:31:36.071275304 +0000 UTC m=+0.125031396 container create a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, RELEASE=reef-6a1bdf8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T09:31:36.322 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 podman[35340]: 2024-09-21 09:31:36.290580505 +0000 UTC m=+0.344336600 container init a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:31:36.322 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 podman[35340]: 2024-09-21 09:31:36.295637345 +0000 UTC m=+0.349393448 container start a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:31:36.612 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 ceph-mgr[35379]: -- 172.21.15.191:0/2568524791 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c979e54d00 con 0x55c979e29400 2024-09-21T09:31:36.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 bash[35340]: a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 2024-09-21T09:31:36.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 systemd[1]: Started Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:31:36.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:36.415+0000 7f8f10e43040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:31:36.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:36.481+0000 7f8f10e43040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:31:37.018 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:36.637+0000 7f8f10e43040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:31:37.018 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:36.705+0000 7f8f10e43040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:31:37.018 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:36.875+0000 7f8f10e43040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:31:37.018 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:37.015+0000 7f8f10e43040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:31:37.596 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:37.340+0000 7f8f10e43040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:31:37.596 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:37.401+0000 7f8f10e43040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:31:37.597 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-mon[33301]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:37.597 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.597 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.597 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.597 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.597 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[41325]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:37.817 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[29313]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:37.818 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.818 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.818 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.818 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:37.818 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:37 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:37.862 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:37.596+0000 7f8f10e43040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:31:38.517 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:31:38.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi076:vg_nvme/lv_4 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:38.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:38.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:38.517+0000 7f8f10e43040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:38.584+0000 7f8f10e43040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:38.834 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:38.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:31:38.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:31:38.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:38.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:39.291 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:39.562 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.305+0000 7f8f10e43040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:31:39.562 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.420+0000 7f8f10e43040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:31:39.562 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.559+0000 7f8f10e43040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:31:39.838 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-mon[33301]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:31:39.838 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-mon[33301]: Reconfiguring daemon mgr.y on smithi076 2024-09-21T09:31:39.838 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-mon[33301]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:39.838 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.622+0000 7f8f10e43040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:31:39.838 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.768+0000 7f8f10e43040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:31:39.838 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.835+0000 7f8f10e43040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:31:39.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:39 smithi076 ceph-mon[29313]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:31:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:39 smithi076 ceph-mon[29313]: Reconfiguring daemon mgr.y on smithi076 2024-09-21T09:31:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:39 smithi076 ceph-mon[29313]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:39 smithi076 ceph-mon[41325]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:31:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:39 smithi076 ceph-mon[41325]: Reconfiguring daemon mgr.y on smithi076 2024-09-21T09:31:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:39 smithi076 ceph-mon[41325]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:40.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:39.895+0000 7f8f10e43040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:31:40.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:40.052+0000 7f8f10e43040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:31:40.384 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:40.117+0000 7f8f10e43040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:31:40.384 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:40.184+0000 7f8f10e43040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:31:40.384 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:40.265+0000 7f8f10e43040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:31:40.384 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:40.323+0000 7f8f10e43040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:31:40.384 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:40.381+0000 7f8f10e43040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:31:41.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:41.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:41.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:41.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[29313]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:40 smithi076 ceph-mon[41325]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:41.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:31:41.150+0000 7f8f10e43040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:31:41.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:31:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:40 smithi191 ceph-mon[33301]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: Standby manager daemon x started 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: Standby manager daemon x started 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:31:42.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:41 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: Standby manager daemon x started 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4204401120' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:31:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:41 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:43 smithi076 ceph-mon[29313]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-21T09:31:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:43 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:31:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:43 smithi076 ceph-mon[29313]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:43 smithi076 ceph-mon[41325]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-21T09:31:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:43 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:31:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:43 smithi076 ceph-mon[41325]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:43 smithi191 ceph-mon[33301]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-21T09:31:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:43 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:31:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:43 smithi191 ceph-mon[33301]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3429800196' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e79ed056-c2aa-44c5-915a-5680ca7c081e"}]: dispatch 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3429800196' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e79ed056-c2aa-44c5-915a-5680ca7c081e"}]': finished 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[29313]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3429800196' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e79ed056-c2aa-44c5-915a-5680ca7c081e"}]: dispatch 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3429800196' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e79ed056-c2aa-44c5-915a-5680ca7c081e"}]': finished 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[41325]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:31:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:44 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:31:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:44 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3429800196' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e79ed056-c2aa-44c5-915a-5680ca7c081e"}]: dispatch 2024-09-21T09:31:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:44 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3429800196' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e79ed056-c2aa-44c5-915a-5680ca7c081e"}]': finished 2024-09-21T09:31:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:44 smithi191 ceph-mon[33301]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:31:44.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:44 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:31:45.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:45 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1718938773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:31:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:45 smithi191 ceph-mon[33301]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:45.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:45 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1718938773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:31:45.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:45 smithi076 ceph-mon[29313]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:45 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1718938773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:31:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:45 smithi076 ceph-mon[41325]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:47.761 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:47 smithi076 ceph-mon[29313]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:47.762 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:47 smithi076 ceph-mon[41325]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:47 smithi191 ceph-mon[33301]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:49.773 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:49 smithi076 ceph-mon[41325]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:49.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:49 smithi076 ceph-mon[29313]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:49 smithi191 ceph-mon[33301]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:50.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:50 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:31:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:50 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:50.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:50 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:31:50.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:50 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:50.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:31:50.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:31:51.700 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:51 smithi076 ceph-mon[29313]: Deploying daemon osd.0 on smithi076 2024-09-21T09:31:51.700 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:51 smithi076 ceph-mon[29313]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:51.700 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:51 smithi076 ceph-mon[41325]: Deploying daemon osd.0 on smithi076 2024-09-21T09:31:51.700 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:51 smithi076 ceph-mon[41325]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:51 smithi191 ceph-mon[33301]: Deploying daemon osd.0 on smithi076 2024-09-21T09:31:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:51 smithi191 ceph-mon[33301]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:53.590 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:53 smithi076 ceph-mon[29313]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:53.590 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:53 smithi076 ceph-mon[41325]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:53 smithi191 ceph-mon[33301]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:55 smithi191 ceph-mon[33301]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:55.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[29313]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:55.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[41325]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:31:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:57.593 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:57 smithi076 ceph-mon[29313]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:57.593 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:57 smithi076 ceph-mon[41325]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:57.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:57 smithi191 ceph-mon[33301]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:57.910 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 0 on host 'smithi076' 2024-09-21T09:31:58.545 DEBUG:teuthology.orchestra.run.smithi076:osd.0> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T09:31:58.548 INFO:tasks.cephadm:Deploying osd.1 on smithi076 with /dev/vg_nvme/lv_3... 2024-09-21T09:31:58.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:31:58.787 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[29313]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[29313]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.177 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[41325]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:31:59.178 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[41325]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:59.178 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.178 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:58 smithi191 ceph-mon[33301]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:31:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:58 smithi191 ceph-mon[33301]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:31:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:31:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:00.256 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[29313]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:32:00.256 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[29313]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:32:00.256 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:00.256 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[29313]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:00.256 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[41325]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:32:00.256 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[41325]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:32:00.257 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:00.257 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:31:59 smithi076 ceph-mon[41325]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:59 smithi191 ceph-mon[33301]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:32:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:59 smithi191 ceph-mon[33301]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:32:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:59 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:31:59 smithi191 ceph-mon[33301]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:00.603 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[45597]: 2024-09-21T09:32:00.353+0000 7f2263c2d640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:32:00.603 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[45597]: 2024-09-21T09:32:00.356+0000 7f225ba4e640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' 2024-09-21T09:32:01.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[29313]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' 2024-09-21T09:32:01.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:00 smithi076 ceph-mon[41325]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:32:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:32:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:32:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: from='osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549]' entity='osd.0' 2024-09-21T09:32:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:00 smithi191 ceph-mon[33301]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:32:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[29313]: osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549] boot 2024-09-21T09:32:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[29313]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:32:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[41325]: osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549] boot 2024-09-21T09:32:02.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[41325]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:32:02.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:01 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:01 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:01 smithi191 ceph-mon[33301]: osd.0 [v2:172.21.15.76:6802/2043782549,v1:172.21.15.76:6803/2043782549] boot 2024-09-21T09:32:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:01 smithi191 ceph-mon[33301]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:32:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:01 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:32:02.433 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:32:02.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi076:vg_nvme/lv_3 2024-09-21T09:32:02.973 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:03.097 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:02 smithi076 ceph-mon[29313]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:03.097 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:02 smithi076 ceph-mon[41325]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:02 smithi191 ceph-mon[33301]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:03 smithi076 ceph-mon[29313]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:32:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:03 smithi076 ceph-mon[41325]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:32:04.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:03 smithi191 ceph-mon[33301]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:32:05.067 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:05.067 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.067 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:05.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:04 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:04 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi076 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi076 to 16064M 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi076 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi076 to 16064M 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:05 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi076 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi076 to 16064M 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:06.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:06.265 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:06.265 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:05 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:07.163 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:06 smithi076 ceph-mon[29313]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:07.163 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:06 smithi076 ceph-mon[41325]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:06 smithi191 ceph-mon[33301]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:07 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3235004302' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]: dispatch 2024-09-21T09:32:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:07 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]: dispatch 2024-09-21T09:32:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:07 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]': finished 2024-09-21T09:32:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:07 smithi191 ceph-mon[33301]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:32:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:07 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:08.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3235004302' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]: dispatch 2024-09-21T09:32:08.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]: dispatch 2024-09-21T09:32:08.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]': finished 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[41325]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3235004302' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]: dispatch 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]: dispatch 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1de58204-a1cd-4ff3-96f7-e03b83aa0041"}]': finished 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[29313]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:32:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:07 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:08 smithi191 ceph-mon[33301]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:08 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4234972615' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:09.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:08 smithi076 ceph-mon[41325]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:08 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4234972615' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:08 smithi076 ceph-mon[29313]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:08 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4234972615' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:11 smithi076 ceph-mon[29313]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:11 smithi076 ceph-mon[41325]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:11 smithi191 ceph-mon[33301]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:13.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:13 smithi076 ceph-mon[29313]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:13.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:13 smithi076 ceph-mon[41325]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:13 smithi191 ceph-mon[33301]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:14.577 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:32:14.577 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:14.578 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:32:14.578 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:32:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:15.590 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:15 smithi076 ceph-mon[29313]: Deploying daemon osd.1 on smithi076 2024-09-21T09:32:15.590 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:15 smithi076 ceph-mon[29313]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:15.590 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:15 smithi076 ceph-mon[41325]: Deploying daemon osd.1 on smithi076 2024-09-21T09:32:15.591 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:15 smithi076 ceph-mon[41325]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:15 smithi191 ceph-mon[33301]: Deploying daemon osd.1 on smithi076 2024-09-21T09:32:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:15 smithi191 ceph-mon[33301]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.566 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:17 smithi076 ceph-mon[29313]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.566 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:17 smithi076 ceph-mon[41325]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:17 smithi191 ceph-mon[33301]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:19.664 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[29313]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:19.664 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:19.664 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:19.665 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:19.665 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[41325]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:19.665 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:19.665 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:19.665 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:19 smithi191 ceph-mon[33301]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:21.783 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:21 smithi076 ceph-mon[29313]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:21.783 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:21 smithi076 ceph-mon[41325]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:21 smithi191 ceph-mon[33301]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:22.479 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 1 on host 'smithi076' 2024-09-21T09:32:23.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:23.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:23.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:23.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:23.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:23.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:23.119 DEBUG:teuthology.orchestra.run.smithi076:osd.1> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T09:32:23.121 INFO:tasks.cephadm:Deploying osd.2 on smithi076 with /dev/vg_nvme/lv_2... 2024-09-21T09:32:23.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:32:23.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[41325]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[41325]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[41325]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[29313]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[29313]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:23 smithi076 ceph-mon[29313]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:23.966 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:23 smithi191 ceph-mon[33301]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:32:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:23 smithi191 ceph-mon[33301]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:32:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:23 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:23 smithi191 ceph-mon[33301]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:24.346 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[51605]: 2024-09-21T09:32:24.195+0000 7ff899c22640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:32:24.346 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[51605]: 2024-09-21T09:32:24.198+0000 7ff891a43640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:32:24.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[29313]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:24.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[29313]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:32:24.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[29313]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[29313]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[41325]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[41325]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[41325]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' 2024-09-21T09:32:24.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:24 smithi076 ceph-mon[41325]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:24 smithi191 ceph-mon[33301]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:24 smithi191 ceph-mon[33301]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:32:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:24 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:24 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:24 smithi191 ceph-mon[33301]: from='osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860]' entity='osd.1' 2024-09-21T09:32:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:24 smithi191 ceph-mon[33301]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:32:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:25 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:32:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:25 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:32:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:25 smithi191 ceph-mon[33301]: osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860] boot 2024-09-21T09:32:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:25 smithi191 ceph-mon[33301]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:32:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[29313]: osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860] boot 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[29313]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[41325]: osd.1 [v2:172.21.15.76:6810/637600860,v1:172.21.15.76:6811/637600860] boot 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[41325]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:32:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:32:27.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi076 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi076 to 8032M 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:27 smithi191 ceph-mon[33301]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi076 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi076 to 8032M 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[41325]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi076 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi076 to 8032M 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:27.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:27 smithi076 ceph-mon[29313]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:32:28.002 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:32:28.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi076:vg_nvme/lv_2 2024-09-21T09:32:28.215 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:29 smithi076 ceph-mon[29313]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:29.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:29 smithi076 ceph-mon[41325]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:29 smithi191 ceph-mon[33301]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:30.659 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:30.659 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:30.660 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:30.660 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:30.660 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:30.660 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:31.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:31 smithi076 ceph-mon[29313]: from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:31.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:31 smithi076 ceph-mon[29313]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:31.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:31 smithi076 ceph-mon[41325]: from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:31.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:31 smithi076 ceph-mon[41325]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:31 smithi191 ceph-mon[33301]: from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:31 smithi191 ceph-mon[33301]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:32.653 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1040585202' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5de7797-a6a1-4301-8efc-5fe9a50da8cb"}]: dispatch 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1040585202' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5de7797-a6a1-4301-8efc-5fe9a50da8cb"}]': finished 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[29313]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1040585202' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5de7797-a6a1-4301-8efc-5fe9a50da8cb"}]: dispatch 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1040585202' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5de7797-a6a1-4301-8efc-5fe9a50da8cb"}]': finished 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[41325]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:32:32.654 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:32 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1040585202' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f5de7797-a6a1-4301-8efc-5fe9a50da8cb"}]: dispatch 2024-09-21T09:32:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:32 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1040585202' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f5de7797-a6a1-4301-8efc-5fe9a50da8cb"}]': finished 2024-09-21T09:32:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:32 smithi191 ceph-mon[33301]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:32:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:33.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:33 smithi076 ceph-mon[29313]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:33.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:33 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1740889213' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:33.846 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:33 smithi076 ceph-mon[41325]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:33.846 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:33 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1740889213' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:33 smithi191 ceph-mon[33301]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:33 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1740889213' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:35 smithi191 ceph-mon[33301]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:35.870 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:35 smithi076 ceph-mon[29313]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:35.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:35 smithi076 ceph-mon[41325]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:37.553 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:37 smithi076 ceph-mon[29313]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:37.553 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:37 smithi076 ceph-mon[41325]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:37.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:37 smithi191 ceph-mon[33301]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:39.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:39 smithi076 ceph-mon[29313]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:32:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:39 smithi076 ceph-mon[41325]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:32:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:39 smithi191 ceph-mon[33301]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:32:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:40.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:40 smithi076 ceph-mon[29313]: Deploying daemon osd.2 on smithi076 2024-09-21T09:32:40.583 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:40 smithi076 ceph-mon[41325]: Deploying daemon osd.2 on smithi076 2024-09-21T09:32:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:40 smithi191 ceph-mon[33301]: Deploying daemon osd.2 on smithi076 2024-09-21T09:32:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:41 smithi191 ceph-mon[33301]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:41 smithi076 ceph-mon[29313]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:41 smithi076 ceph-mon[41325]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:43.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:43 smithi191 ceph-mon[33301]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:43 smithi076 ceph-mon[29313]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:43 smithi076 ceph-mon[41325]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:44.675 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:44 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:44.675 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:44 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:44.675 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:44 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:44.675 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:44 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:44.675 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:44 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:44.675 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:44 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:44 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:44 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:44 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:45.845 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:45 smithi076 ceph-mon[29313]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:45.845 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:45 smithi076 ceph-mon[41325]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:45 smithi191 ceph-mon[33301]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:46.982 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 2 on host 'smithi076' 2024-09-21T09:32:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[29313]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[29313]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:32:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[41325]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[41325]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:32:47.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:47.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:47 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:47.680 DEBUG:teuthology.orchestra.run.smithi076:osd.2> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T09:32:47.682 INFO:tasks.cephadm:Deploying osd.3 on smithi076 with /dev/vg_nvme/lv_1... 2024-09-21T09:32:47.683 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:32:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:47 smithi191 ceph-mon[33301]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:47 smithi191 ceph-mon[33301]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:32:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:47 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:47 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:47.927 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.730 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[57979]: 2024-09-21T09:32:48.481+0000 7fb803464640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:32:48.730 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[57979]: 2024-09-21T09:32:48.484+0000 7fb7ff28d640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:32:48.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:32:48.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:32:48.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:48.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:48.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:48.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:48.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:32:48.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:48 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:32:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:32:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:49.581 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:49 smithi076 ceph-mon[41325]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:49.581 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:49 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:49.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:49 smithi076 ceph-mon[29313]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:49.581 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:49 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:49 smithi191 ceph-mon[33301]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:49 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:50.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:32:50.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:32:50.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[29313]: osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205] boot 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[29313]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[41325]: osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205] boot 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[41325]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:32:50.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:50 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:50 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:32:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:50 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:32:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:50 smithi191 ceph-mon[33301]: osd.2 [v2:172.21.15.76:6818/2738257205,v1:172.21.15.76:6819/2738257205] boot 2024-09-21T09:32:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:50 smithi191 ceph-mon[33301]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:32:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:50 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:32:51.544 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:32:51.562 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi076:vg_nvme/lv_1 2024-09-21T09:32:51.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[29313]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[29313]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[41325]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[41325]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:32:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:51 smithi191 ceph-mon[33301]: pgmap v74: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-21T09:32:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:51 smithi191 ceph-mon[33301]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:32:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:32:52.230 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:32:52.345 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59596]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:52.345 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59596]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:52.345 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59596]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:52.345 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59596]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:52.622 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59652]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:52.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59652]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:52.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59652]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:52.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[59652]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:52.936 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:52.936 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[41325]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:32:52.936 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[41325]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:52.936 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:52.936 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61954]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:52.936 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61954]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:52.936 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61954]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:52.936 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61954]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:52.936 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:52.937 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[29313]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:32:52.937 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[29313]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:52.937 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:52 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:52 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:32:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:52 smithi191 ceph-mon[33301]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:32:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:52 smithi191 ceph-mon[33301]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:52 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:53.187 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61967]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:53.187 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61967]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:53.187 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61967]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:53.187 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:32:52 smithi076 sudo[61967]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:53.446 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61976]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:53.447 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61976]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:53.447 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61976]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:53.447 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61976]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:53.710 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61992]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:53.710 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61992]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:53.710 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61992]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:53.710 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61992]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:53.711 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61985]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:53.711 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61985]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:53.711 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61985]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:53.711 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61985]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:53.961 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61995]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:53.961 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61995]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:53.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61995]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:53.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 sudo[61995]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:53.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 ceph-mon[29313]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:32:53.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:53 smithi076 ceph-mon[29313]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:32:54.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:53 smithi191 ceph-mon[33301]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:32:54.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:53 smithi191 ceph-mon[33301]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:32:54.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35975]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:54.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35975]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:54.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35975]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:54.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35975]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:54.263 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:53 smithi076 ceph-mon[41325]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:32:54.263 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:53 smithi076 ceph-mon[41325]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:32:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35978]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35978]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35978]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:54 smithi191 sudo[35978]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:54.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:54 smithi076 sudo[62098]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:32:54.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:54 smithi076 sudo[62098]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:54.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:54 smithi076 sudo[62098]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:54.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:54 smithi076 sudo[62098]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:55.284 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:55.284 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi076 2024-09-21T09:32:55.284 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.285 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:55.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:55.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.287 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.287 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi076 to 5354M 2024-09-21T09:32:55.287 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.287 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.287 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.287 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.289 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:55.289 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:55.289 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.289 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 sudo[62195]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:32:55.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 sudo[62195]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:32:55.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 sudo[62195]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:32:55.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 sudo[62195]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:32:55.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:55.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi076 2024-09-21T09:32:55.291 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:55.291 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.291 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.291 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.291 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.291 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.292 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi076 to 5354M 2024-09-21T09:32:55.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:55.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:55.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:55.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi076 2024-09-21T09:32:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:55.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:55.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:55.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:32:55.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi076 to 5354M 2024-09-21T09:32:55.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:55.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:32:55.367 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:32:55.368 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:32:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[41325]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[41325]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[41325]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[29313]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[29313]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:56.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[29313]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:56.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:56 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:56 smithi191 ceph-mon[33301]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:32:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:56 smithi191 ceph-mon[33301]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi076:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:32:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:56 smithi191 ceph-mon[33301]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:32:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:56 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:32:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:57 smithi191 ceph-mon[33301]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:57.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:57 smithi076 ceph-mon[29313]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:57.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:57 smithi076 ceph-mon[41325]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:58 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2466550364' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "357ff6e9-4d95-42ac-8a3f-a9f3c4a48835"}]: dispatch 2024-09-21T09:32:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:58 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2466550364' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "357ff6e9-4d95-42ac-8a3f-a9f3c4a48835"}]': finished 2024-09-21T09:32:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:58 smithi191 ceph-mon[33301]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:32:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:32:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:58 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3857250680' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:58.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2466550364' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "357ff6e9-4d95-42ac-8a3f-a9f3c4a48835"}]: dispatch 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2466550364' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "357ff6e9-4d95-42ac-8a3f-a9f3c4a48835"}]': finished 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[29313]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3857250680' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2466550364' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "357ff6e9-4d95-42ac-8a3f-a9f3c4a48835"}]: dispatch 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2466550364' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "357ff6e9-4d95-42ac-8a3f-a9f3c4a48835"}]': finished 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[41325]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:32:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:58 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3857250680' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:32:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:32:59 smithi191 ceph-mon[33301]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:59.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:32:59 smithi076 ceph-mon[29313]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:32:59.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:32:59 smithi076 ceph-mon[41325]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:01.577 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:01 smithi076 ceph-mon[29313]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:01.577 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:01 smithi076 ceph-mon[41325]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:01 smithi191 ceph-mon[33301]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:03.662 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:03 smithi076 ceph-mon[29313]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:03.662 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:33:03.662 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:03.662 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:03 smithi076 ceph-mon[41325]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:03.662 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:33:03.663 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:03 smithi191 ceph-mon[33301]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:33:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:04.516 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:04 smithi076 ceph-mon[29313]: Deploying daemon osd.3 on smithi076 2024-09-21T09:33:04.516 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:04 smithi076 ceph-mon[41325]: Deploying daemon osd.3 on smithi076 2024-09-21T09:33:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:04 smithi191 ceph-mon[33301]: Deploying daemon osd.3 on smithi076 2024-09-21T09:33:05.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:05 smithi076 ceph-mon[29313]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:05.846 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:05 smithi076 ceph-mon[41325]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:05 smithi191 ceph-mon[33301]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:07.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:07 smithi076 ceph-mon[29313]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:07.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:07 smithi076 ceph-mon[41325]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:07 smithi191 ceph-mon[33301]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:08.714 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:08 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:08.714 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:08 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:08.714 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:08 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:08.714 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:08.715 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:08.715 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:08 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:08 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:08 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:09.778 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:09 smithi076 ceph-mon[29313]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:09.779 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:09 smithi076 ceph-mon[41325]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:09 smithi191 ceph-mon[33301]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:11.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:11 smithi076 ceph-mon[41325]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:11.732 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:11 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:11.732 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:11 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:11.732 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:11 smithi076 ceph-mon[29313]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:11.732 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:11 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:11.732 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:11 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:11.760 INFO:teuthology.orchestra.run.smithi076.stdout:Created osd(s) 3 on host 'smithi076' 2024-09-21T09:33:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:11 smithi191 ceph-mon[33301]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:11 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:11 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:12.419 DEBUG:teuthology.orchestra.run.smithi076:osd.3> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T09:33:12.421 INFO:tasks.cephadm:Deploying osd.4 on smithi191 with /dev/vg_nvme/lv_4... 2024-09-21T09:33:12.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:33:12.623 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:12.741 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[29313]: from='osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:12.741 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[29313]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:12.741 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:12.741 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:12.741 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[41325]: from='osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:12.742 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[41325]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:12.742 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:12.742 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:12 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:12.744 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:12 smithi191 ceph-mon[33301]: from='osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:12.744 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:12 smithi191 ceph-mon[33301]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:33:12.744 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:12.744 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:12 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:13.617 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[29313]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:13.617 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[29313]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:33:13.617 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[29313]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:33:13.617 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:13.617 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[29313]: from='osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:33:13.617 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[29313]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:33:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[41325]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[41325]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:33:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[41325]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:33:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[41325]: from='osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:33:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-mon[41325]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:33:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:13 smithi191 ceph-mon[33301]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:13 smithi191 ceph-mon[33301]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:33:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:13 smithi191 ceph-mon[33301]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:33:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:13 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:13 smithi191 ceph-mon[33301]: from='osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:33:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:13 smithi191 ceph-mon[33301]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]: dispatch 2024-09-21T09:33:14.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[64169]: 2024-09-21T09:33:13.937+0000 7f91f7de3640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:33:14.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:33:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[64169]: 2024-09-21T09:33:13.942+0000 7f91efc04640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:33:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:14 smithi191 ceph-mon[33301]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:33:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:14 smithi191 ceph-mon[33301]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:33:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:14 smithi191 ceph-mon[33301]: from='osd.3 ' entity='osd.3' 2024-09-21T09:33:14.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[41325]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[41325]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[41325]: from='osd.3 ' entity='osd.3' 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[29313]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]}]': finished 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[29313]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:33:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:14.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:14.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:14 smithi076 ceph-mon[29313]: from='osd.3 ' entity='osd.3' 2024-09-21T09:33:15.235 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:33:15.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi191:vg_nvme/lv_4 2024-09-21T09:33:15.449 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866] boot 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:15.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:15.476 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866] boot 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:15.478 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:15 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: osd.3 [v2:172.21.15.76:6826/666372866,v1:172.21.15.76:6827/666372866] boot 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:15.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:15 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:16.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:16 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi076 2024-09-21T09:33:16.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:16 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi076 to 4016M 2024-09-21T09:33:16.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:16 smithi191 ceph-mon[33301]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:33:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:16 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi076 2024-09-21T09:33:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:16 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi076 to 4016M 2024-09-21T09:33:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:16 smithi076 ceph-mon[29313]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:33:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:16 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi076 2024-09-21T09:33:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:16 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi076 to 4016M 2024-09-21T09:33:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:16 smithi076 ceph-mon[41325]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:33:17.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:17 smithi191 ceph-mon[33301]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:17.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:17.802 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:17.803 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:17 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[29313]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[41325]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:17 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:18.530 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:18 smithi191 ceph-mon[33301]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:18 smithi076 ceph-mon[29313]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:18 smithi076 ceph-mon[41325]: from='client.24184 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:19 smithi191 ceph-mon[33301]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:19 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/1857045115' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]: dispatch 2024-09-21T09:33:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:19 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]: dispatch 2024-09-21T09:33:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:19 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]': finished 2024-09-21T09:33:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:19 smithi191 ceph-mon[33301]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:33:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[29313]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/1857045115' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]: dispatch 2024-09-21T09:33:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]: dispatch 2024-09-21T09:33:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]': finished 2024-09-21T09:33:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[29313]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:33:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[41325]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/1857045115' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]: dispatch 2024-09-21T09:33:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]: dispatch 2024-09-21T09:33:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c25d7b87-1abd-4472-af56-e090a83c5835"}]': finished 2024-09-21T09:33:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[41325]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:33:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:20 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/844794563' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:20.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/844794563' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:20.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:20 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/844794563' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:21 smithi191 ceph-mon[33301]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:21.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:21 smithi076 ceph-mon[41325]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:21 smithi076 ceph-mon[29313]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:23 smithi191 ceph-mon[33301]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:23 smithi076 ceph-mon[29313]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:23 smithi076 ceph-mon[41325]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:25.769 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:25 smithi191 ceph-mon[33301]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:25.769 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:33:25.769 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:25.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:25 smithi076 ceph-mon[29313]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:33:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:25 smithi076 ceph-mon[41325]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:33:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:26.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:26 smithi191 ceph-mon[33301]: Deploying daemon osd.4 on smithi191 2024-09-21T09:33:26.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:26 smithi076 ceph-mon[29313]: Deploying daemon osd.4 on smithi191 2024-09-21T09:33:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:26 smithi076 ceph-mon[41325]: Deploying daemon osd.4 on smithi191 2024-09-21T09:33:27.607 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:27 smithi191 ceph-mon[33301]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:27.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:27 smithi076 ceph-mon[29313]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:27 smithi076 ceph-mon[41325]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:29.782 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:29 smithi191 ceph-mon[33301]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:29 smithi076 ceph-mon[29313]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:29 smithi076 ceph-mon[41325]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:30.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:30.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:30.838 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:31.669 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:31 smithi191 ceph-mon[33301]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:31 smithi076 ceph-mon[29313]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:31 smithi076 ceph-mon[41325]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:33.677 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 4 on host 'smithi191' 2024-09-21T09:33:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:33 smithi191 ceph-mon[33301]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[41325]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:33 smithi076 ceph-mon[29313]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:33:34.318 DEBUG:teuthology.orchestra.run.smithi191:osd.4> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T09:33:34.320 INFO:tasks.cephadm:Deploying osd.5 on smithi191 with /dev/vg_nvme/lv_3... 2024-09-21T09:33:34.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:34.864 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T09:33:34.596+0000 7f554dc08640 -1 osd.4 0 waiting for initial osdmap 2024-09-21T09:33:34.864 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 09:33:34 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T09:33:34.600+0000 7f5549a31640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:33:34.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:33:34.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:33:34.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:34.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:34 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:35.132 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:35 smithi191 ceph-mon[33301]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:35 smithi191 ceph-mon[33301]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:33:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:35 smithi191 ceph-mon[33301]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:33:35.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:35 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:35.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:35 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:35.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[29313]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[29313]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[29313]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[41325]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[41325]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[41325]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:35 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:36.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:36 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:33:36.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:36 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:33:36.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:36 smithi191 ceph-mon[33301]: osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433] boot 2024-09-21T09:33:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:36 smithi191 ceph-mon[33301]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:33:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:36 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:36 smithi191 ceph-mon[33301]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:33:36.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[29313]: osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433] boot 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[29313]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[29313]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[41325]: osd.4 [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433] boot 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[41325]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:33:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:36 smithi076 ceph-mon[41325]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:33:37.627 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:37 smithi191 ceph-mon[33301]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:37.627 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:37 smithi191 ceph-mon[33301]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:33:37.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:37 smithi076 ceph-mon[29313]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:37 smithi076 ceph-mon[29313]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:33:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:37 smithi076 ceph-mon[41325]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:37 smithi076 ceph-mon[41325]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:33:39.543 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi191 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi191 to 17088M 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:39.544 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:39 smithi191 ceph-mon[33301]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:33:39.546 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:33:39.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi191:vg_nvme/lv_3 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi191 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi191 to 17088M 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[29313]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:33:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi191 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi191 to 17088M 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:39 smithi076 ceph-mon[41325]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T09:33:39.756 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:40.857 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:40 smithi191 ceph-mon[33301]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T09:33:40.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:40 smithi076 ceph-mon[29313]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T09:33:40.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:40 smithi076 ceph-mon[41325]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T09:33:41.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:41 smithi191 ceph-mon[33301]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:41.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:41 smithi076 ceph-mon[29313]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:41 smithi076 ceph-mon[41325]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:42.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:42 smithi191 ceph-mon[33301]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:42.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:42 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:42.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:42 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:42.717 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:42 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[29313]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[41325]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:33:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:42 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:43.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:43 smithi191 ceph-mon[33301]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-09-21T09:33:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:43 smithi076 ceph-mon[29313]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-09-21T09:33:43.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:43 smithi076 ceph-mon[41325]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-09-21T09:33:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:44 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/1625560738' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]: dispatch 2024-09-21T09:33:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:44 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]: dispatch 2024-09-21T09:33:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:44 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]': finished 2024-09-21T09:33:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:44 smithi191 ceph-mon[33301]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:33:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:44 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:44 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/3670549781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/1625560738' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]': finished 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[29313]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/3670549781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/1625560738' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]: dispatch 2024-09-21T09:33:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "68bcb809-42fe-4390-ae82-4ab0f2f9c54e"}]': finished 2024-09-21T09:33:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[41325]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:33:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:44 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/3670549781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:33:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:45 smithi191 ceph-mon[33301]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:33:45.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:45 smithi076 ceph-mon[29313]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:33:45.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:45 smithi076 ceph-mon[41325]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:33:47.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:47 smithi191 ceph-mon[33301]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:33:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:47 smithi076 ceph-mon[29313]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:33:47.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:47 smithi076 ceph-mon[41325]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:33:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:49 smithi191 ceph-mon[33301]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:33:49.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:49 smithi076 ceph-mon[29313]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:33:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:49 smithi076 ceph-mon[41325]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:33:50.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:50 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:33:50.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:50 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:50.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:33:50.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:50 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:50.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:50 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:33:50.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:50 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:51.582 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:51 smithi191 ceph-mon[33301]: Deploying daemon osd.5 on smithi191 2024-09-21T09:33:51.582 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:51 smithi191 ceph-mon[33301]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:33:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:51 smithi076 ceph-mon[29313]: Deploying daemon osd.5 on smithi191 2024-09-21T09:33:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:51 smithi076 ceph-mon[29313]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:33:51.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:51 smithi076 ceph-mon[41325]: Deploying daemon osd.5 on smithi191 2024-09-21T09:33:51.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:51 smithi076 ceph-mon[41325]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:33:53.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:53 smithi191 ceph-mon[33301]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:53 smithi076 ceph-mon[41325]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:53 smithi076 ceph-mon[29313]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:55 smithi191 ceph-mon[33301]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:55 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[41325]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[29313]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:55 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:57 smithi191 ceph-mon[33301]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:57.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:57 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:57 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:57 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:57.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:57 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:57.824 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:57 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[41325]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:57.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[29313]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:33:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:57 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:58.159 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 5 on host 'smithi191' 2024-09-21T09:33:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:58 smithi191 ceph-mon[33301]: from='osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:33:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:58 smithi191 ceph-mon[33301]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:33:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:58.712 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:58.738 DEBUG:teuthology.orchestra.run.smithi191:osd.5> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T09:33:58.740 INFO:tasks.cephadm:Deploying osd.6 on smithi191 with /dev/vg_nvme/lv_2... 2024-09-21T09:33:58.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[29313]: from='osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[29313]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[41325]: from='osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[41325]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:33:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:58.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:33:59.448 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:33:59.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-mon[33301]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:59.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-mon[33301]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:33:59.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-mon[33301]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:33:59.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:59.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-mon[33301]: from='osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:59.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-mon[33301]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:59.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[29313]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:59.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[29313]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[29313]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[29313]: from='osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[29313]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[41325]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[41325]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[41325]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[41325]: from='osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:33:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:33:59 smithi076 ceph-mon[41325]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:00.112 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T09:33:59.934+0000 7faf2f6da640 -1 osd.5 0 waiting for initial osdmap 2024-09-21T09:34:00.113 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 09:33:59 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T09:33:59.938+0000 7faf274fb640 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:34:00.749 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:00 smithi191 ceph-mon[33301]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:00.749 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:00 smithi191 ceph-mon[33301]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T09:34:00.749 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:00 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.749 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:00 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.749 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:00 smithi191 ceph-mon[33301]: from='osd.5 ' entity='osd.5' 2024-09-21T09:34:00.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[41325]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[41325]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[41325]: from='osd.5 ' entity='osd.5' 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[29313]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[29313]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:00 smithi076 ceph-mon[29313]: from='osd.5 ' entity='osd.5' 2024-09-21T09:34:01.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:34:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:34:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:34:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840] boot 2024-09-21T09:34:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:34:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:01 smithi191 ceph-mon[33301]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840] boot 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[29313]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:34:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:34:01.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:34:01.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-21T09:34:01.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: osd.5 [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840] boot 2024-09-21T09:34:01.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:34:01.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:34:01.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:01 smithi076 ceph-mon[41325]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T09:34:03.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi191 2024-09-21T09:34:03.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi191 to 8544M 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:03.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[41325]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi191 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi191 to 8544M 2024-09-21T09:34:03.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.626 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:03.626 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:03.626 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.626 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:03.626 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:03 smithi076 ceph-mon[29313]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T09:34:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi191 2024-09-21T09:34:03.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi191 to 8544M 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:03.801 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:03 smithi191 ceph-mon[33301]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T09:34:03.803 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:34:03.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi191:vg_nvme/lv_2 2024-09-21T09:34:04.018 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:04.448 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:04 smithi191 ceph-mon[33301]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:34:04.449 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:04 smithi191 ceph-mon[33301]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T09:34:04.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:04 smithi076 ceph-mon[29313]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:34:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:04 smithi076 ceph-mon[29313]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T09:34:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:04 smithi076 ceph-mon[41325]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:34:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:04 smithi076 ceph-mon[41325]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T09:34:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:05 smithi191 ceph-mon[33301]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:05 smithi191 ceph-mon[33301]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T09:34:05.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:05 smithi076 ceph-mon[29313]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:05.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:05 smithi076 ceph-mon[29313]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T09:34:05.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:05 smithi076 ceph-mon[41325]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:05.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:05 smithi076 ceph-mon[41325]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T09:34:06.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:06 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:06.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:06 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:06.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:06 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:06.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:06 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:06 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:06 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:06 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:06 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:06 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:07.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:07 smithi191 ceph-mon[33301]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:07.784 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:07 smithi191 ceph-mon[33301]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:07.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:07 smithi076 ceph-mon[29313]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:07.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:07 smithi076 ceph-mon[29313]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:07 smithi076 ceph-mon[41325]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:07 smithi076 ceph-mon[41325]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:08 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/3888431889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]: dispatch 2024-09-21T09:34:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:08 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]: dispatch 2024-09-21T09:34:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:08 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]': finished 2024-09-21T09:34:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:08 smithi191 ceph-mon[33301]: osdmap e43: 7 total, 6 up, 7 in 2024-09-21T09:34:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:08 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/3888431889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]: dispatch 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]: dispatch 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]': finished 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[29313]: osdmap e43: 7 total, 6 up, 7 in 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/3888431889' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]: dispatch 2024-09-21T09:34:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]: dispatch 2024-09-21T09:34:08.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0592317d-31bb-4c86-81c9-5f134a98cba0"}]': finished 2024-09-21T09:34:08.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[41325]: osdmap e43: 7 total, 6 up, 7 in 2024-09-21T09:34:08.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:08 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:09 smithi191 ceph-mon[33301]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-21T09:34:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:09 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/3355865827' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:09.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:09 smithi191 ceph-mon[33301]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:34:09.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:09 smithi191 ceph-mon[33301]: Cluster is now healthy 2024-09-21T09:34:09.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[29313]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/3355865827' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[29313]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[29313]: Cluster is now healthy 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[41325]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/3355865827' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[41325]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:34:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:09 smithi076 ceph-mon[41325]: Cluster is now healthy 2024-09-21T09:34:11.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:11 smithi191 ceph-mon[33301]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:34:11.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:11 smithi076 ceph-mon[29313]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:34:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:11 smithi076 ceph-mon[41325]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:34:13.855 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:13 smithi191 ceph-mon[33301]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:13.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:13 smithi076 ceph-mon[29313]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:13 smithi076 ceph-mon[41325]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:14.833 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:34:14.833 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:14.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:34:14.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:34:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:15.570 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:15 smithi191 ceph-mon[33301]: Deploying daemon osd.6 on smithi191 2024-09-21T09:34:15.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:15 smithi191 ceph-mon[33301]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:15 smithi076 ceph-mon[29313]: Deploying daemon osd.6 on smithi191 2024-09-21T09:34:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:15 smithi076 ceph-mon[29313]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:15 smithi076 ceph-mon[41325]: Deploying daemon osd.6 on smithi191 2024-09-21T09:34:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:15 smithi076 ceph-mon[41325]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:17.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:17 smithi191 ceph-mon[33301]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:17.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:17 smithi076 ceph-mon[29313]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:17 smithi076 ceph-mon[41325]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:19.752 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:19 smithi191 ceph-mon[33301]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:19.752 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:19.752 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:19.752 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:19 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:19.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[29313]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[41325]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:19 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:21.550 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:21 smithi191 ceph-mon[33301]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:21.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:21 smithi076 ceph-mon[29313]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:21 smithi076 ceph-mon[41325]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:22.294 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 6 on host 'smithi191' 2024-09-21T09:34:22.602 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:22 smithi191 ceph-mon[33301]: from='osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:22.602 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:22 smithi191 ceph-mon[33301]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:22.602 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:22.602 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:22 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[29313]: from='osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[29313]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[41325]: from='osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[41325]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:22 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:22.949 DEBUG:teuthology.orchestra.run.smithi191:osd.6> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T09:34:22.951 INFO:tasks.cephadm:Deploying osd.7 on smithi191 with /dev/vg_nvme/lv_1... 2024-09-21T09:34:22.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:34:23.190 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:23.612 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T09:34:23.484+0000 7f846d112640 -1 osd.6 0 waiting for initial osdmap 2024-09-21T09:34:23.613 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T09:34:23.492+0000 7f8468f3b640 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: from='osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:23 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: from='osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: from='osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:23.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:23 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:24.713 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:24 smithi191 ceph-mon[33301]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:24.713 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:24 smithi191 ceph-mon[33301]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T09:34:24.713 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:24 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:24.713 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:24 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:24.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[29313]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[29313]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[41325]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[41325]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:24 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:25.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:34:25.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:34:25.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:25.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952] boot 2024-09-21T09:34:25.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: osdmap e46: 7 total, 7 up, 7 in 2024-09-21T09:34:25.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:25.758 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:25 smithi191 ceph-mon[33301]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952] boot 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: osdmap e46: 7 total, 7 up, 7 in 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[29313]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:34:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-21T09:34:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: osd.6 [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952] boot 2024-09-21T09:34:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: osdmap e46: 7 total, 7 up, 7 in 2024-09-21T09:34:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:34:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:25 smithi076 ceph-mon[41325]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T09:34:26.586 INFO:teuthology.orchestra.run.smithi191.stdout: 2024-09-21T09:34:26.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi191:vg_nvme/lv_1 2024-09-21T09:34:27.166 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:27.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:27 smithi191 ceph-mon[33301]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:27.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:27 smithi191 ceph-mon[33301]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T09:34:27.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:27 smithi076 ceph-mon[29313]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:27.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:27 smithi076 ceph-mon[29313]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T09:34:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:27 smithi076 ceph-mon[41325]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:27 smithi076 ceph-mon[41325]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T09:34:28.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:28 smithi191 ceph-mon[33301]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T09:34:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:28 smithi076 ceph-mon[29313]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T09:34:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:28 smithi076 ceph-mon[41325]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T09:34:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-21T09:34:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:29 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: pgmap v154: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:29.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:29 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:30 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi191 2024-09-21T09:34:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:30 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi191 to 5696M 2024-09-21T09:34:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:30 smithi191 ceph-mon[33301]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:30 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:30.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi191 2024-09-21T09:34:30.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi191 to 5696M 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[29313]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi191 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi191 to 5696M 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[41325]: from='client.24265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi191:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:34:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:30 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:31 smithi191 ceph-mon[33301]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-09-21T09:34:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:31 smithi191 ceph-mon[33301]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-21T09:34:31.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:31 smithi191 ceph-mon[33301]: Cluster is now healthy 2024-09-21T09:34:31.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:31 smithi076 ceph-mon[29313]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-09-21T09:34:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:31 smithi076 ceph-mon[29313]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-21T09:34:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:31 smithi076 ceph-mon[29313]: Cluster is now healthy 2024-09-21T09:34:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:31 smithi076 ceph-mon[41325]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-09-21T09:34:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:31 smithi076 ceph-mon[41325]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-21T09:34:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:31 smithi076 ceph-mon[41325]: Cluster is now healthy 2024-09-21T09:34:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:32 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/127674287' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]: dispatch 2024-09-21T09:34:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:32 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]: dispatch 2024-09-21T09:34:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:32 smithi191 ceph-mon[33301]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]': finished 2024-09-21T09:34:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:32 smithi191 ceph-mon[33301]: osdmap e51: 8 total, 7 up, 8 in 2024-09-21T09:34:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:32 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:32 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/3398578938' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:32.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/127674287' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[29313]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]': finished 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[29313]: osdmap e51: 8 total, 7 up, 8 in 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/3398578938' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/127674287' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[41325]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08ee88fa-c070-487e-96c8-d85878cc1249"}]': finished 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[41325]: osdmap e51: 8 total, 7 up, 8 in 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:32 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/3398578938' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:34:33.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:33 smithi076 ceph-mon[29313]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-21T09:34:33.773 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:33 smithi076 ceph-mon[41325]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-21T09:34:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:33 smithi191 ceph-mon[33301]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-21T09:34:35.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:35 smithi191 ceph-mon[33301]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:34:35.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:35 smithi076 ceph-mon[29313]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:34:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:35 smithi076 ceph-mon[41325]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:34:37.671 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:37 smithi191 ceph-mon[33301]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:37 smithi076 ceph-mon[29313]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:37 smithi076 ceph-mon[41325]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:38.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:34:38.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:38 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:38.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:34:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:38 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:34:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:38 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:39.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:39 smithi191 ceph-mon[33301]: Deploying daemon osd.7 on smithi191 2024-09-21T09:34:39.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:39 smithi191 ceph-mon[33301]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:39.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:39 smithi076 ceph-mon[29313]: Deploying daemon osd.7 on smithi191 2024-09-21T09:34:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:39 smithi076 ceph-mon[29313]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:39 smithi076 ceph-mon[41325]: Deploying daemon osd.7 on smithi191 2024-09-21T09:34:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:39 smithi076 ceph-mon[41325]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:34:41.800 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:41 smithi191 ceph-mon[33301]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:41.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:41 smithi076 ceph-mon[29313]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:41 smithi076 ceph-mon[41325]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:43.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:43 smithi191 ceph-mon[33301]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:43.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:43 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:43.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:43 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:43.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:43 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[29313]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[41325]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:43 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:45 smithi191 ceph-mon[33301]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:45.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:45 smithi076 ceph-mon[29313]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:45 smithi076 ceph-mon[41325]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:46.568 INFO:teuthology.orchestra.run.smithi191.stdout:Created osd(s) 7 on host 'smithi191' 2024-09-21T09:34:47.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:47.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:47.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.255 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:47.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:47.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:47.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:47.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.256 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:46 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.276 DEBUG:teuthology.orchestra.run.smithi191:osd.7> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T09:34:47.278 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T09:34:47.278 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T09:34:47.306 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.306 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.306 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:47.306 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:47.306 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.306 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:47.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:34:47.308 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:47.308 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:47.308 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.308 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:46 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:47.483 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:34:48.295 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[29313]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[29313]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[29313]: from='osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[29313]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[41325]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[41325]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[41325]: from='osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:48.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:47 smithi076 ceph-mon[41325]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:47 smithi191 ceph-mon[33301]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:34:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:47 smithi191 ceph-mon[33301]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T09:34:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:47 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:47 smithi191 ceph-mon[33301]: from='osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:47 smithi191 ceph-mon[33301]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]: dispatch 2024-09-21T09:34:48.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T09:34:47.999+0000 7fd95e10e640 -1 osd.7 0 waiting for initial osdmap 2024-09-21T09:34:48.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T09:34:48.005+0000 7fd959f37640 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071] boot 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:34:49.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:48 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.342 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:34:49.342 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071] boot 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi191", "root=default"]}]': finished 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:49.343 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-21T09:34:49.344 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: osd.7 [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071] boot 2024-09-21T09:34:49.344 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:34:49.344 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:48 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:34:50.216 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1726911288,"num_in_osds":8,"osd_in_since":1726911271,"num_remapped_pgs":1} 2024-09-21T09:34:50.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:34:50.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[29313]: purged_snaps scrub starts 2024-09-21T09:34:50.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[29313]: purged_snaps scrub ok 2024-09-21T09:34:50.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1145769871' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:34:50.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[29313]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:34:50.280 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: purged_snaps scrub starts 2024-09-21T09:34:50.281 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: purged_snaps scrub ok 2024-09-21T09:34:50.281 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1145769871' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:34:50.281 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:49 smithi076 ceph-mon[41325]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:34:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:49 smithi191 ceph-mon[33301]: purged_snaps scrub starts 2024-09-21T09:34:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:49 smithi191 ceph-mon[33301]: purged_snaps scrub ok 2024-09-21T09:34:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:49 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1145769871' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:34:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:49 smithi191 ceph-mon[33301]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:34:50.451 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: Detected new or changed devices on smithi191 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi191 to 4272M 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[29313]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: Detected new or changed devices on smithi191 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi191 to 4272M 2024-09-21T09:34:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:51.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:51.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:34:51.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:51 smithi076 ceph-mon[41325]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:34:51.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: Detected new or changed devices on smithi191 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi191 to 4272M 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:34:51.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:51 smithi191 ceph-mon[33301]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:34:52.374 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:34:52.374 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":57,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:34:51.455303+0000","last_up_change":"2024-09-21T09:34:48.446984+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6803","nonce":2043782549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6805","nonce":2043782549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6809","nonce":2043782549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6807","nonce":2043782549}]},"public_addr":"172.21.15.76:6803/2043782549","cluster_addr":"172.21.15.76:6805/2043782549","heartbeat_back_addr":"172.21.15.76:6809/2043782549","heartbeat_front_addr":"172.21.15.76:6807/2043782549","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6811","nonce":637600860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6813","nonce":637600860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6817","nonce":637600860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6815","nonce":637600860}]},"public_addr":"172.21.15.76:6811/637600860","cluster_addr":"172.21.15.76:6813/637600860","heartbeat_back_addr":"172.21.15.76:6817/637600860","heartbeat_front_addr":"172.21.15.76:6815/637600860","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6819","nonce":2738257205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6821","nonce":2738257205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6825","nonce":2738257205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6823","nonce":2738257205}]},"public_addr":"172.21.15.76:6819/2738257205","cluster_addr":"172.21.15.76:6821/2738257205","heartbeat_back_addr":"172.21.15.76:6825/2738257205","heartbeat_front_addr":"172.21.15.76:6823/2738257205","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6827","nonce":666372866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6829","nonce":666372866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6833","nonce":666372866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6831","nonce":666372866}]},"public_addr":"172.21.15.76:6827/666372866","cluster_addr":"172.21.15.76:6829/666372866","heartbeat_back_addr":"172.21.15.76:6833/666372866","heartbeat_front_addr":"172.21.15.76:6831/666372866","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:34:52.751 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:52 smithi076 ceph-mon[29313]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:34:52.752 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:52 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1577361819' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:34:52.752 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:52 smithi076 ceph-mon[41325]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:34:52.752 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:52 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1577361819' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:34:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:52 smithi191 ceph-mon[33301]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:34:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:52 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1577361819' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:34:53.083 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T09:32:50.664150+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-09-21T09:34:53.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T09:34:53.318 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:34:53.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:53 smithi076 ceph-mon[29313]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:34:53.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:53 smithi076 ceph-mon[29313]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:34:53.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:53 smithi076 ceph-mon[41325]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:34:53.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:53 smithi076 ceph-mon[41325]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:34:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:53 smithi191 ceph-mon[33301]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:34:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:53 smithi191 ceph-mon[33301]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:34:55.267 INFO:teuthology.orchestra.run.smithi076.stdout:pg_num: 1 2024-09-21T09:34:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:55 smithi191 ceph-mon[33301]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-09-21T09:34:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:55 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/399813657' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:34:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:55 smithi076 ceph-mon[29313]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-09-21T09:34:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:55 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/399813657' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:34:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:55 smithi076 ceph-mon[41325]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-09-21T09:34:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:55 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/399813657' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:34:56.010 INFO:tasks.cephadm:Adding prometheus.a on smithi191 2024-09-21T09:34:56.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch apply prometheus '1;smithi191=a' 2024-09-21T09:34:56.221 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:57.579 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:57 smithi191 ceph-mon[33301]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:34:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:57 smithi076 ceph-mon[29313]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:34:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:57 smithi076 ceph-mon[41325]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:34:57.997 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled prometheus update... 2024-09-21T09:34:58.639 DEBUG:teuthology.orchestra.run.smithi191:prometheus.a> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service 2024-09-21T09:34:58.641 INFO:tasks.cephadm:Adding node-exporter.a on smithi076 2024-09-21T09:34:58.642 INFO:tasks.cephadm:Adding node-exporter.b on smithi191 2024-09-21T09:34:58.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch apply node-exporter '2;smithi076=a;smithi191=b' 2024-09-21T09:34:58.885 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:34:59.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:59.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: Saving service prometheus spec with placement smithi191=a;count:1 2024-09-21T09:34:59.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:59.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:59.340 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:59.340 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:59.340 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:59.340 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: Deploying daemon prometheus.a on smithi191 2024-09-21T09:34:59.340 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:34:58 smithi191 ceph-mon[33301]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: Saving service prometheus spec with placement smithi191=a;count:1 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: Deploying daemon prometheus.a on smithi191 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[29313]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:34:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: Saving service prometheus spec with placement smithi191=a;count:1 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: Deploying daemon prometheus.a on smithi191 2024-09-21T09:34:59.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:34:58 smithi076 ceph-mon[41325]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:35:01.501 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled node-exporter update... 2024-09-21T09:35:01.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:01 smithi191 ceph-mon[33301]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:35:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:01 smithi076 ceph-mon[29313]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:35:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:01 smithi076 ceph-mon[41325]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-09-21T09:35:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:02 smithi191 ceph-mon[33301]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi076=a;smithi191=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:02 smithi191 ceph-mon[33301]: Saving service node-exporter spec with placement smithi076=a;smithi191=b;count:2 2024-09-21T09:35:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:02 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:02 smithi076 ceph-mon[29313]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi076=a;smithi191=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:02 smithi076 ceph-mon[29313]: Saving service node-exporter spec with placement smithi076=a;smithi191=b;count:2 2024-09-21T09:35:02.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:02 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:02 smithi076 ceph-mon[41325]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi076=a;smithi191=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:02 smithi076 ceph-mon[41325]: Saving service node-exporter spec with placement smithi076=a;smithi191=b;count:2 2024-09-21T09:35:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:02 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:03.015 DEBUG:teuthology.orchestra.run.smithi076:node-exporter.a> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service 2024-09-21T09:35:03.018 DEBUG:teuthology.orchestra.run.smithi191:node-exporter.b> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service 2024-09-21T09:35:03.020 INFO:tasks.cephadm:Adding alertmanager.a on smithi076 2024-09-21T09:35:03.020 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch apply alertmanager '1;smithi076=a' 2024-09-21T09:35:03.253 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:03 smithi191 ceph-mon[33301]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:35:03.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:03 smithi076 ceph-mon[29313]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:35:03.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:03 smithi076 ceph-mon[41325]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:35:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:05 smithi191 ceph-mon[33301]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:35:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:05 smithi076 ceph-mon[29313]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:35:05.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:05 smithi076 ceph-mon[41325]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-21T09:35:07.675 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:07 smithi191 ceph-mon[33301]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:07 smithi076 ceph-mon[29313]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:07 smithi076 ceph-mon[41325]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:08.215 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled alertmanager update... 2024-09-21T09:35:09.092 DEBUG:teuthology.orchestra.run.smithi076:alertmanager.a> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@alertmanager.a.service 2024-09-21T09:35:09.095 INFO:tasks.cephadm:Adding grafana.a on smithi191 2024-09-21T09:35:09.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph orch apply grafana '1;smithi191=a' 2024-09-21T09:35:09.347 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:09 smithi191 ceph-mon[33301]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:09.347 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:09 smithi191 ceph-mon[33301]: Saving service alertmanager spec with placement smithi076=a;count:1 2024-09-21T09:35:09.347 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:09 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:09.347 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:09 smithi191 ceph-mon[33301]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[29313]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[29313]: Saving service alertmanager spec with placement smithi076=a;count:1 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[29313]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[41325]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi076=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[41325]: Saving service alertmanager spec with placement smithi076=a;count:1 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:09 smithi076 ceph-mon[41325]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:10.092 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:11.509 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:11 smithi191 ceph-mon[33301]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:11 smithi076 ceph-mon[29313]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:11 smithi076 ceph-mon[41325]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:12.227 INFO:teuthology.orchestra.run.smithi191.stdout:Scheduled grafana update... 2024-09-21T09:35:13.187 DEBUG:teuthology.orchestra.run.smithi191:grafana.a> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@grafana.a.service 2024-09-21T09:35:13.190 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T09:35:13.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:35:13.393 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:13.513 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:13.513 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: Saving service grafana spec with placement smithi191=a;count:1 2024-09-21T09:35:13.513 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.513 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: Saving service grafana spec with placement smithi191=a;count:1 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.515 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:35:13.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi191=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:35:13.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: Saving service grafana spec with placement smithi191=a;count:1 2024-09-21T09:35:13.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:35:14.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: ignoring --setuser ceph since I am not root 2024-09-21T09:35:14.113 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: ignoring --setgroup ceph since I am not root 2024-09-21T09:35:14.113 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-mgr[35379]: -- 172.21.15.191:0/1161268743 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fc85c04d00 con 0x55fc85bd8800 2024-09-21T09:35:14.113 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:13.938+0000 7ff1572fd040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:35:14.113 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:13 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:13.994+0000 7ff1572fd040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:14.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setuser ceph since I am not root 2024-09-21T09:35:14.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setgroup ceph since I am not root 2024-09-21T09:35:14.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:13.939+0000 7f3724221040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:35:14.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:13.996+0000 7f3724221040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:14.423 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:14.144+0000 7f3724221040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:35:14.423 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:14.212+0000 7f3724221040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:35:14.423 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:14.359+0000 7f3724221040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:14.423 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:14.142+0000 7ff1572fd040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:35:14.424 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:14.206+0000 7ff1572fd040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:35:14.424 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:14.350+0000 7ff1572fd040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:14.796 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:14.797 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-mon[33301]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:35:14.797 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-mon[33301]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:35:14.797 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:14.492+0000 7ff1572fd040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:35:14.818 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:14.502+0000 7f3724221040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:35:14.819 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:14.819 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-mon[29313]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:35:14.819 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-mon[29313]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:35:14.819 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' 2024-09-21T09:35:14.819 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-mon[41325]: from='mgr.14150 172.21.15.76:0/2406372032' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:35:14.819 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-mon[41325]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-21T09:35:15.074 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:14.816+0000 7f3724221040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:35:15.074 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:14 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:14.879+0000 7f3724221040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:35:15.074 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:15.070+0000 7f3724221040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:35:15.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:14.794+0000 7ff1572fd040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:35:15.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:14 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:14.854+0000 7ff1572fd040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:35:15.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:15 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:15.035+0000 7ff1572fd040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:35:15.422 INFO:teuthology.orchestra.run.smithi076.stdout:[client.0] 2024-09-21T09:35:15.423 INFO:teuthology.orchestra.run.smithi076.stdout: key = AQBTk+5mxf66GBAAt1vkBnj9c9ck7rc4c+rQwQ== 2024-09-21T09:35:15.798 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/511828797' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:15.798 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-mon[41325]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:15.798 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-mon[41325]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:15.799 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/511828797' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:15.799 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-mon[29313]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:15.799 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-mon[29313]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:15 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/511828797' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:15 smithi191 ceph-mon[33301]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:15 smithi191 ceph-mon[33301]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:16.062 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:15 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:15.991+0000 7f3724221040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:35:16.144 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T09:35:16.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T09:35:16.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T09:35:16.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:35:16.205 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:15 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:15.895+0000 7ff1572fd040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:35:16.205 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:15 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:15.961+0000 7ff1572fd040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:35:16.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:16.059+0000 7f3724221040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:35:16.393 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:35:16.814 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:16 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:16.698+0000 7ff1572fd040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:35:16.814 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:16 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:16.811+0000 7ff1572fd040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:35:17.041 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:16.781+0000 7f3724221040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:35:17.041 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:16.897+0000 7f3724221040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:35:17.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:16 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:16.953+0000 7ff1572fd040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:17.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.014+0000 7ff1572fd040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:35:17.316 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.038+0000 7f3724221040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:35:17.316 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.100+0000 7f3724221040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:35:17.316 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.245+0000 7f3724221040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:35:17.316 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.313+0000 7f3724221040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:35:17.445 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.162+0000 7ff1572fd040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:35:17.445 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.229+0000 7ff1572fd040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:35:17.445 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.289+0000 7ff1572fd040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:35:17.445 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.443+0000 7ff1572fd040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:35:17.595 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.372+0000 7f3724221040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:35:17.595 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.527+0000 7f3724221040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:35:17.595 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.592+0000 7f3724221040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:35:17.715 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.507+0000 7ff1572fd040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:35:17.716 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.572+0000 7ff1572fd040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:35:17.716 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.655+0000 7ff1572fd040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:17.716 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.713+0000 7ff1572fd040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:35:17.857 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.658+0000 7f3724221040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:35:17.857 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.739+0000 7f3724221040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:35:17.858 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.797+0000 7f3724221040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:35:17.858 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:17.855+0000 7f3724221040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:35:17.992 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:17.772+0000 7ff1572fd040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:35:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-mon[33301]: Standby manager daemon x restarted 2024-09-21T09:35:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-mon[33301]: Standby manager daemon x started 2024-09-21T09:35:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:35:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:35:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:35:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:18.567+0000 7ff1572fd040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: [21/Sep/2024:09:35:18] ENGINE Bus STARTING 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: CherryPy Checker: 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: The Application mounted at '' has an empty config. 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: [21/Sep/2024:09:35:18] ENGINE Serving on http://:::9283 2024-09-21T09:35:18.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:35:18 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[35355]: [21/Sep/2024:09:35:18] ENGINE Bus STARTED 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[29313]: Standby manager daemon x restarted 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[29313]: Standby manager daemon x started 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:35:18.884 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[41325]: Standby manager daemon x restarted 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[41325]: Standby manager daemon x started 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/377751621' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:18.634+0000 7f3724221040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:18] ENGINE Bus STARTING 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: CherryPy Checker: 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: The Application mounted at '' has an empty config. 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:18] ENGINE Serving on http://:::9283 2024-09-21T09:35:18.885 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:18] ENGINE Bus STARTED 2024-09-21T09:35:18.886 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:35:18] "GET /metrics HTTP/1.1" 200 23673 "" "Prometheus/2.43.0" 2024-09-21T09:35:19.719 INFO:teuthology.orchestra.run.smithi191.stdout:[client.1] 2024-09-21T09:35:19.719 INFO:teuthology.orchestra.run.smithi191.stdout: key = AQBXk+5mpbqDKhAAXurpioEZp3zbfNKaaflYvw== 2024-09-21T09:35:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-21T09:35:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: Active manager daemon y restarted 2024-09-21T09:35:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: Activating manager daemon y 2024-09-21T09:35:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:35:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: mgrmap e18: y(active, starting, since 0.0070305s), standbys: x 2024-09-21T09:35:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:35:19.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: Manager daemon y is now available 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:19.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:19 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: Active manager daemon y restarted 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: Activating manager daemon y 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: mgrmap e18: y(active, starting, since 0.0070305s), standbys: x 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:35:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: Manager daemon y is now available 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: Active manager daemon y restarted 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: Activating manager daemon y 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: mgrmap e18: y(active, starting, since 0.0070305s), standbys: x 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:35:19.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:35:19.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: Manager daemon y is now available 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:19.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:19 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:35:20.353 DEBUG:teuthology.orchestra.run.smithi191:> set -ex 2024-09-21T09:35:20.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-21T09:35:20.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-21T09:35:20.429 INFO:tasks.cephadm:Creating RBD pool 2024-09-21T09:35:20.429 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create rbd 8 2024-09-21T09:35:20.452 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:20.177+0000 7f3681f9b640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:35:20] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: Traceback (most recent call last): 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._connections.run(self.expiration_interval) 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._run(expiration_interval) 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: s = self.context.wrap_socket( 2024-09-21T09:35:20.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: return self.sslsocket_class._create( 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self.do_handshake() 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: self._sslobj.do_handshake() 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:20.454 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:35:20.650 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:20.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: mgrmap e19: y(active, since 1.0119s), standbys: x 2024-09-21T09:35:20.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:20.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: from='client.? 172.21.15.191:0/3118221914' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: mgrmap e19: y(active, since 1.0119s), standbys: x 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.191:0/3118221914' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:20.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:20 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: mgrmap e19: y(active, since 1.0119s), standbys: x 2024-09-21T09:35:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: from='client.? 172.21.15.191:0/3118221914' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:35:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:35:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:20 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: [21/Sep/2024:09:35:20] ENGINE Bus STARTING 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: [21/Sep/2024:09:35:20] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: [21/Sep/2024:09:35:20] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: self.do_handshake() 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: [21/Sep/2024:09:35:20] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: [21/Sep/2024:09:35:20] ENGINE Bus STARTED 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[41325]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: [21/Sep/2024:09:35:20] ENGINE Bus STARTING 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: [21/Sep/2024:09:35:20] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: [21/Sep/2024:09:35:20] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:21.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:21.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout: self.do_handshake() 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: [21/Sep/2024:09:35:20] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: [21/Sep/2024:09:35:20] ENGINE Bus STARTED 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:21.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:21 smithi076 ceph-mon[29313]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-21T09:35:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: [21/Sep/2024:09:35:20] ENGINE Bus STARTING 2024-09-21T09:35:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: [21/Sep/2024:09:35:20] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:35:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: [21/Sep/2024:09:35:20] ENGINE Error in HTTPServer.serve 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: Traceback (most recent call last): 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._run(expiration_interval) 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: s = self.context.wrap_socket( 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: return self.sslsocket_class._create( 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: self.do_handshake() 2024-09-21T09:35:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._sslobj.do_handshake() 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: [21/Sep/2024:09:35:20] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: [21/Sep/2024:09:35:20] ENGINE Bus STARTED 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:21 smithi191 ceph-mon[33301]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:22.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:22.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:22.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:22 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:35:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:22 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:35:23.743 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'rbd' created 2024-09-21T09:35:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi191 to 4016M 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1289212220' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:23 smithi191 ceph-mon[33301]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi191 to 4016M 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1289212220' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[29313]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi191 to 4016M 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1289212220' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:35:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:23 smithi076 ceph-mon[41325]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-21T09:35:24.964 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- rbd pool init rbd 2024-09-21T09:35:25.026 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:25.026 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:25.026 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[29313]: Deploying daemon alertmanager.a on smithi076 2024-09-21T09:35:25.026 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1289212220' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:35:25.027 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[29313]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:35:25.027 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:25.027 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:25.027 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[41325]: Deploying daemon alertmanager.a on smithi076 2024-09-21T09:35:25.027 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1289212220' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:35:25.027 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:24 smithi076 ceph-mon[41325]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:35:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:24 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:24 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:35:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:24 smithi191 ceph-mon[33301]: Deploying daemon alertmanager.a on smithi076 2024-09-21T09:35:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:24 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1289212220' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:35:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:24 smithi191 ceph-mon[33301]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:35:25.195 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:25 smithi191 ceph-mon[33301]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:25 smithi191 ceph-mon[33301]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:35:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:25 smithi191 ceph-mon[33301]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:35:26.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:25 smithi076 ceph-mon[29313]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:25 smithi076 ceph-mon[29313]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:35:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:25 smithi076 ceph-mon[29313]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:35:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:25 smithi076 ceph-mon[41325]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:25 smithi076 ceph-mon[41325]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:35:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:25 smithi076 ceph-mon[41325]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:35:27.051 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:26 smithi076 ceph-mon[29313]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:35:27.052 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:26 smithi076 ceph-mon[41325]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:35:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:26 smithi191 ceph-mon[33301]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:35:27.840 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:27 smithi076 ceph-mon[29313]: pgmap v10: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:35:27.840 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:27 smithi076 ceph-mon[41325]: pgmap v10: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:35:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:27 smithi191 ceph-mon[33301]: pgmap v10: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:35:29.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:28 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3662902806' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:35:29.015 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:35:28] "GET /metrics HTTP/1.1" 200 23673 "" "Prometheus/2.43.0" 2024-09-21T09:35:29.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:28 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3662902806' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:35:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:28 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3662902806' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:35:30.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:29 smithi076 ceph-mon[29313]: pgmap v11: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:35:30.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:29 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3662902806' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:35:30.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:29 smithi076 ceph-mon[29313]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:35:30.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:29 smithi076 ceph-mon[41325]: pgmap v11: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:35:30.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:29 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3662902806' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:35:30.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:29 smithi076 ceph-mon[41325]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:35:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:29 smithi191 ceph-mon[33301]: pgmap v11: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-21T09:35:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:29 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3662902806' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:35:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:29 smithi191 ceph-mon[33301]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:35:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:30 smithi191 ceph-mon[33301]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:35:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:30 smithi076 ceph-mon[29313]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:35:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:30 smithi076 ceph-mon[41325]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[29313]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[29313]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[29313]: Cluster is now healthy 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[29313]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[41325]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[41325]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[41325]: Cluster is now healthy 2024-09-21T09:35:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:31 smithi076 ceph-mon[41325]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:35:32.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T09:35:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:31 smithi191 ceph-mon[33301]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:31 smithi191 ceph-mon[33301]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:35:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:31 smithi191 ceph-mon[33301]: Cluster is now healthy 2024-09-21T09:35:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:31 smithi191 ceph-mon[33301]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:35:32.372 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 podman[71458]: 2024-09-21 09:35:32.318703267 +0000 UTC m=+0.149136138 volume create de7bf0a0431629da9221da526db75026d604acda3a638e3254bc9748f261eb5a 2024-09-21T09:35:32.689 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 podman[71458]: 2024-09-21 09:35:32.419084385 +0000 UTC m=+0.249517299 container create 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:32.689 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 podman[71458]: 2024-09-21 09:35:32.688455612 +0000 UTC m=+0.518888484 container init 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:32.865 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 podman[71458]: 2024-09-21 09:35:32.693103009 +0000 UTC m=+0.523535880 container start 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.705Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.705Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.706Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.76 port=9094 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.707Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.729Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.729Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.732Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:32.732Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 bash[71458]: 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 2024-09-21T09:35:32.976 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:32 smithi076 systemd[1]: Started Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:35:33.270 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:32] ENGINE Bus STOPPING 2024-09-21T09:35:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:35:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:33] ENGINE Bus STOPPED 2024-09-21T09:35:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:33] ENGINE Bus STARTING 2024-09-21T09:35:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:33] ENGINE Serving on http://:::9283 2024-09-21T09:35:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:35:33] ENGINE Bus STARTED 2024-09-21T09:35:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:33 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:34.116 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:33 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:34.807 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:35:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:34 smithi191 ceph-mon[33301]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T09:35:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:34 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:34 smithi191 ceph-mon[33301]: Deploying daemon grafana.a on smithi191 2024-09-21T09:35:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:34 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2610472484' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:35:35.122 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:34.707Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000297488s 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[29313]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[29313]: Deploying daemon grafana.a on smithi191 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2610472484' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[41325]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[41325]: Deploying daemon grafana.a on smithi191 2024-09-21T09:35:35.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:34 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2610472484' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:35:35.562 INFO:teuthology.orchestra.run.smithi076.stdout:[{"version":23,"timestamp":"2024-09-21T09:35:32.968229+0000","name":"","changes":[{"name":"mgr/mgr/dashboard/GRAFANA_API_SSL_VERIFY","new_value":"false"}]}] 2024-09-21T09:35:35.562 INFO:tasks.ceph_manager:config epoch is 23 2024-09-21T09:35:35.562 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:35:35.562 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:35:35.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:35:35.770 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:35 smithi191 ceph-mon[33301]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:35 smithi076 ceph-mon[29313]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:35 smithi076 ceph-mon[41325]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:36.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:36 smithi076 ceph-mon[41325]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 259 B/s wr, 0 op/s 2024-09-21T09:35:36.945 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:36 smithi076 ceph-mon[29313]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 259 B/s wr, 0 op/s 2024-09-21T09:35:37.347 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:36 smithi191 ceph-mon[33301]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 259 B/s wr, 0 op/s 2024-09-21T09:35:37.669 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:35:37.976 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:37 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/990177849' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:35:37.976 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:37 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/990177849' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:35:38.298 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":21,"flags":0,"active_gid":24326,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6800","nonce":23865699},{"type":"v1","addr":"172.21.15.76:6801","nonce":23865699}]},"active_addr":"172.21.15.76:6801/23865699","active_change":"2024-09-21T09:35:18.636663+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24331,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":3117602468}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":1280485976}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":1616224256}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":3392089630}]}]} 2024-09-21T09:35:38.302 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:35:38.302 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:35:38.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:35:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:37 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/990177849' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:35:38.500 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:39.367 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:35:38] "GET /metrics HTTP/1.1" 200 28482 "" "Prometheus/2.43.0" 2024-09-21T09:35:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:39 smithi076 ceph-mon[29313]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:35:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:39 smithi076 ceph-mon[29313]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:35:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:39 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:39 smithi076 ceph-mon[41325]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:35:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:39 smithi076 ceph-mon[41325]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:35:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:39 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:39 smithi191 ceph-mon[33301]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:35:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:39 smithi191 ceph-mon[33301]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:35:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:39 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:40.460 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:35:40.460 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":66,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:35:38.454824+0000","last_up_change":"2024-09-21T09:34:48.446984+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6803","nonce":2043782549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6805","nonce":2043782549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6809","nonce":2043782549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6807","nonce":2043782549}]},"public_addr":"172.21.15.76:6803/2043782549","cluster_addr":"172.21.15.76:6805/2043782549","heartbeat_back_addr":"172.21.15.76:6809/2043782549","heartbeat_front_addr":"172.21.15.76:6807/2043782549","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6811","nonce":637600860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6813","nonce":637600860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6817","nonce":637600860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6815","nonce":637600860}]},"public_addr":"172.21.15.76:6811/637600860","cluster_addr":"172.21.15.76:6813/637600860","heartbeat_back_addr":"172.21.15.76:6817/637600860","heartbeat_front_addr":"172.21.15.76:6815/637600860","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6819","nonce":2738257205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6821","nonce":2738257205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6825","nonce":2738257205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6823","nonce":2738257205}]},"public_addr":"172.21.15.76:6819/2738257205","cluster_addr":"172.21.15.76:6821/2738257205","heartbeat_back_addr":"172.21.15.76:6825/2738257205","heartbeat_front_addr":"172.21.15.76:6823/2738257205","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6827","nonce":666372866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6829","nonce":666372866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6833","nonce":666372866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6831","nonce":666372866}]},"public_addr":"172.21.15.76:6827/666372866","cluster_addr":"172.21.15.76:6829/666372866","heartbeat_back_addr":"172.21.15.76:6833/666372866","heartbeat_front_addr":"172.21.15.76:6831/666372866","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:35:40.833 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:40 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/789069700' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:40.834 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:40 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/789069700' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:40 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/789069700' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:41.265 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:35:41.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:35:41.459 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:41.582 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:41 smithi076 ceph-mon[29313]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 207 B/s wr, 0 op/s 2024-09-21T09:35:41.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:41 smithi076 ceph-mon[41325]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 207 B/s wr, 0 op/s 2024-09-21T09:35:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:41 smithi191 ceph-mon[33301]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 207 B/s wr, 0 op/s 2024-09-21T09:35:43.056 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:35:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:35:42.709Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002879401s 2024-09-21T09:35:43.444 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:35:43.444 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":66,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:35:38.454824+0000","last_up_change":"2024-09-21T09:34:48.446984+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6803","nonce":2043782549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6805","nonce":2043782549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6809","nonce":2043782549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6807","nonce":2043782549}]},"public_addr":"172.21.15.76:6803/2043782549","cluster_addr":"172.21.15.76:6805/2043782549","heartbeat_back_addr":"172.21.15.76:6809/2043782549","heartbeat_front_addr":"172.21.15.76:6807/2043782549","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6811","nonce":637600860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6813","nonce":637600860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6817","nonce":637600860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6815","nonce":637600860}]},"public_addr":"172.21.15.76:6811/637600860","cluster_addr":"172.21.15.76:6813/637600860","heartbeat_back_addr":"172.21.15.76:6817/637600860","heartbeat_front_addr":"172.21.15.76:6815/637600860","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6819","nonce":2738257205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6821","nonce":2738257205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6825","nonce":2738257205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6823","nonce":2738257205}]},"public_addr":"172.21.15.76:6819/2738257205","cluster_addr":"172.21.15.76:6821/2738257205","heartbeat_back_addr":"172.21.15.76:6825/2738257205","heartbeat_front_addr":"172.21.15.76:6823/2738257205","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6827","nonce":666372866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6829","nonce":666372866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6833","nonce":666372866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6831","nonce":666372866}]},"public_addr":"172.21.15.76:6827/666372866","cluster_addr":"172.21.15.76:6829/666372866","heartbeat_back_addr":"172.21.15.76:6833/666372866","heartbeat_front_addr":"172.21.15.76:6831/666372866","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:35:43.612 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:43 smithi191 systemd[1]: Starting Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:35:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:43 smithi191 ceph-mon[33301]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:35:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:43 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/579895410' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:44.112 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:43 smithi191 podman[64659]: 2024-09-21 09:35:43.575313614 +0000 UTC m=+0.015180548 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-21T09:35:44.112 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:43 smithi191 podman[64659]: 2024-09-21 09:35:43.701566038 +0000 UTC m=+0.141432972 container create d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, version=8.5, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., maintainer=Paul Cuzner , architecture=x86_64, io.openshift.expose-services=, vcs-type=git, distribution-scope=public, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, name=ubi8) 2024-09-21T09:35:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:43 smithi076 ceph-mon[29313]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:35:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:43 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/579895410' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:43 smithi076 ceph-mon[41325]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:35:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:43 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/579895410' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:35:44.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:35:44.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:35:44.182 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:35:44.182 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:35:44.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:35:44.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:35:44.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:35:44.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:35:44.489 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 podman[64659]: 2024-09-21 09:35:44.237762325 +0000 UTC m=+0.677629264 container init d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, release=239.1651231664, architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-type=git, name=ubi8, io.openshift.expose-services=) 2024-09-21T09:35:44.489 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 podman[64659]: 2024-09-21 09:35:44.243505013 +0000 UTC m=+0.683371951 container start d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.32.0, distribution-scope=public, release=239.1651231664, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, architecture=x86_64) 2024-09-21T09:35:44.489 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 bash[64659]: d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e 2024-09-21T09:35:44.489 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 systemd[1]: Started Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:35:44.489 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404308768Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:35:44.489 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.40451137Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:35:44.490 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404531453Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:35:44.490 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404538896Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404545205Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.40455108Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404556945Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404562743Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404568508Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404574195Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404579674Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404584988Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.404590575Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=settings t=2024-09-21T09:35:44.40459666Z level=info msg="App mode production" 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=sqlstore t=2024-09-21T09:35:44.404642142Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=sqlstore t=2024-09-21T09:35:44.404657467Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:35:44.491 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.405594204Z level=info msg="Starting DB migrations" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.405684626Z level=info msg="Executing migration" id="create migration_log table" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.406302324Z level=info msg="Executing migration" id="create user table" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.407034126Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.407816238Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.408636935Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.409411669Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.410186772Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.410995863Z level=info msg="Executing migration" id="create user table v2" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.411795612Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.412773875Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.413782893Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.414374452Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.414785407Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.415602581Z level=info msg="Executing migration" id="Update user table charset" 2024-09-21T09:35:44.492 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.415943717Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.416380265Z level=info msg="Executing migration" id="Add missing user data" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.416769751Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.417203123Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.417660353Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.418096918Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.419238276Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.420182115Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.421924293Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.422368766Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.423223283Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.424248689Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.424598108Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.425469478Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-21T09:35:44.493 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.426441178Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.427165795Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.428182945Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.428963606Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.429805046Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.43075191Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.431583714Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.432609037Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.435498232Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.438092282Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.440669178Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.4434984Z level=info msg="Executing migration" id="create star table" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.446282494Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.446768603Z level=info msg="Executing migration" id="create org table v1" 2024-09-21T09:35:44.494 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.449526043Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.452191875Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.452600476Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.455465929Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.458264395Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.460942465Z level=info msg="Executing migration" id="Update org table charset" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.463779503Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.466725206Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.469516194Z level=info msg="Executing migration" id="create dashboard table" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.472360123Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.472874107Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.473844716Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.47451979Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.475474852Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.47612995Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-21T09:35:44.495 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.477021033Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.477847862Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.478700426Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.479584297Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.480454896Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.481224382Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.482121602Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.482974458Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.483717753Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.484591319Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.485455249Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.486284455Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.48709185Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.488033348Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.488796683Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.489566871Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-21T09:35:44.496 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.490380163Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-21T09:35:44.681 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.681 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.681 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.684 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.688 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.692 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.692 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.695 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:35:44.740 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.491109953Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-21T09:35:44.740 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.491931233Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.492835016Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.493585456Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.494467239Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.495302073Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.496122122Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.497009997Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.499780898Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.502605017Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.505462288Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.508196899Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.511065565Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-21T09:35:44.741 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.513599511Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-21T09:35:44.742 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.516455532Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-21T09:35:44.742 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.519307355Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-21T09:35:44.742 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.521968671Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-21T09:35:44.742 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.524479714Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-21T09:35:44.742 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.527427291Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.530168833Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.53288455Z level=info msg="Executing migration" id="create data_source table" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.53577385Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.538488331Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.538907524Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.541711166Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.542262401Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.544614802Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.547340553Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.549840736Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.552494231Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-21T09:35:44.743 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.555101997Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.557697783Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.560305896Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.563012438Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.565805782Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.568522549Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.571123971Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.573849135Z level=info msg="Executing migration" id="Add uid column" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.576472715Z level=info msg="Executing migration" id="Update uid value" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.578908386Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.581586643Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.584360427Z level=info msg="Executing migration" id="create api_key table" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.586999598Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-21T09:35:44.744 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.589772292Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.590605469Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.59126218Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.592169753Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.592907234Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.593876395Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.595146369Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.595543432Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.595976123Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.596879327Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.597634553Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.597978811Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-21T09:35:44.745 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.598803329Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.599126322Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.599689534Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.600535836Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.600861172Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.601659617Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.60229978Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.60272429Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.603577285Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.60400678Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.604874549Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.605680274Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.606492766Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-21T09:35:44.746 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.606801067Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.607708206Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.608465479Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.609057694Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.609941729Z level=info msg="Executing migration" id="create quota table v1" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.610492621Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.611041642Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-21T09:35:44.747 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.611316136Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-21T09:35:44.748 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.61172137Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-21T09:35:44.748 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.612158368Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-21T09:35:44.748 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.612745225Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-21T09:35:44.748 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.613041622Z level=info msg="Executing migration" id="create session table" 2024-09-21T09:35:44.748 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.613461743Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.613798986Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.614076565Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.614479828Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.614871344Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.615137906Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.615397001Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.615700447Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.615980564Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.616359957Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.616660552Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.617227392Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.617560485Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-21T09:35:44.749 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.618170116Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.618797454Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.619097041Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.619525784Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.620032199Z level=info msg="Executing migration" id="create alert table v1" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.62085036Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.621648289Z level=info msg="Executing migration" id="add index alert state" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.622122882Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.622560457Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.622962695Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.623427643Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.623843572Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-21T09:35:44.750 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.625254087Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.625678642Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.626112614Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.62642905Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.626798771Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.627184218Z level=info msg="Executing migration" id="Add column is_default" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.627876224Z level=info msg="Executing migration" id="Add column frequency" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.628521869Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.629183991Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.629871735Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.630301118Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.630610739Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.630917674Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.633406617Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-21T09:35:44.751 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.636156Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.638886118Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.641549752Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.64201367Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.644729585Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.647171318Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.649715308Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.650186559Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.652949884Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.655568319Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.658207135Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.658665045Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.661443436Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-21T09:35:44.752 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.664077992Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.666844566Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.667309675Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.670081739Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.672634358Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.673598075Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.674203822Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.674936845Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.675665324Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.676128238Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.676975433Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.677387512Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.678200446Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-21T09:35:44.753 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.679032773Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.680581553Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.680975883Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.681894128Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.682525395Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.683488512Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.684002783Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.684886943Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.685633021Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.686619764Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.687208455Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.688055734Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.688885729Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-21T09:35:44.754 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.689682526Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.690010034Z level=info msg="Executing migration" id="Move region to single row" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.690833906Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.691290162Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.691722624Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.692567156Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.693018544Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.693876638Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.694329801Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.694790697Z level=info msg="Executing migration" id="create test_data table" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.69559541Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.696016424Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-21T09:35:44.755 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.696866838Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.69773331Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.700558785Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.700895388Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.703555279Z level=info msg="Executing migration" id="create team table" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.703971722Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.706465829Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.709243316Z level=info msg="Executing migration" id="create team member table" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.709678579Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.712367061Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.71502943Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.717672485Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.718457534Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.721303988Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-21T09:35:44.756 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.724085942Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.726843346Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.727353026Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.730196107Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.733033784Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.735791163Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.736275587Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.739089708Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.741655968Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-21T09:35:44.757 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.744366596Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.747109666Z level=info msg="Executing migration" id="create tag table" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.749728104Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.752383825Z level=info msg="Executing migration" id="create login attempt table" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.755029676Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.757842291Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.760595879Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-21T09:35:44.991 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.76324986Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.766039756Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.76867247Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.771395911Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.774172702Z level=info msg="Executing migration" id="create user auth table" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.776848925Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.77954495Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.780286502Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.781081957Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.781898135Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-21T09:35:44.992 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.782713478Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-21T09:35:44.993 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.78357994Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-21T09:35:44.993 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.784452774Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-21T09:35:44.993 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.785233053Z level=info msg="Executing migration" id="create server_lock table" 2024-09-21T09:35:44.993 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.786051291Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-21T09:35:44.993 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.786899273Z level=info msg="Executing migration" id="create user auth token table" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.787757359Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.788542286Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.789580715Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.790175254Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.790989797Z level=info msg="Executing migration" id="create cache_data table" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.791823562Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.792636586Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.793636368Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.79428881Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.795234441Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.796014979Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.796897366Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-21T09:35:44.994 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.797886231Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.79860426Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.799452893Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.800477285Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.801157445Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.80215609Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.802856832Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.803679053Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.80456039Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.805458962Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.806440637Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.807145077Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-21T09:35:44.995 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.808110379Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.808909274Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.811800053Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.814556979Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.817248594Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.819866829Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.822672852Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.825468279Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.827993555Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.83136565Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.834691605Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.835141778Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.835582695Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-21T09:35:44.996 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.836450237Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.836905127Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.837339833Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.83781381Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.838238466Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.838501774Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.839342746Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.840185822Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.841013411Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.841439833Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.841884731Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.842679321Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.8434873Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-21T09:35:44.997 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.843928513Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-21T09:35:44.998 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.844745296Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.845559134Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.845864347Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.846335431Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.848921091Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.84940392Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.849725664Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.852540737Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.855153972Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.857956329Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.8605156Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-21T09:35:44.999 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.863345861Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.865987714Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.868688346Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.87120562Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.87393408Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.876748396Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.879441582Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.882161932Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.884759369Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.887467299Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.890218258Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.892865389Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.895592496Z level=info msg="Executing migration" id="create alert_image table" 2024-09-21T09:35:45.000 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.898448908Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.9011299Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.901838827Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.90224306Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.902662612Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.902742013Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.903448096Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.904280345Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.905274426Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.905434247Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.905570657Z level=info msg="alerts found to migrate" alerts=0 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.905989873Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.906823081Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.907771687Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-21T09:35:45.001 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.908430104Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.909210238Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.910058349Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.91104468Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.911853667Z level=info msg="Executing migration" id="create data_keys table" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.912597886Z level=info msg="Executing migration" id="create secrets table" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.913594091Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.917979135Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.918937035Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.919794464Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.924034259Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.928130141Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.928983049Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-21T09:35:45.002 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.929882272Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.930675072Z level=info msg="Executing migration" id="create permission table" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.931534818Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.932377934Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.933226348Z level=info msg="Executing migration" id="create role table" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.934088887Z level=info msg="Executing migration" id="add column display_name" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.935059405Z level=info msg="Executing migration" id="add column group_name" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.936028292Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.936902179Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.937665015Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.938524257Z level=info msg="Executing migration" id="create team role table" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.93952401Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.940228587Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.941144818Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.943920618Z level=info msg="Executing migration" id="create user role table" 2024-09-21T09:35:45.003 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.946747327Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.949503066Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.952179347Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.955007108Z level=info msg="Executing migration" id="create builtin role table" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.957840981Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.960312316Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.963327674Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.96572565Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.966259822Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.969168594Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.969839575Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.970298206Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.970688338Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.97111151Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.972067894Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-21T09:35:45.004 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.975072384Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.975635867Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.975931824Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.976312578Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.976783439Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.977146032Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.97742789Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.977795828Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.978196613Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.978636016Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.979585119Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.979883224Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.980291948Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.9807262Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-21T09:35:45.005 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.981155703Z level=info msg="Executing migration" id="add correlation config column" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.982179146Z level=info msg="Executing migration" id="create entity_events table" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.982619042Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.985404656Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.985590885Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.985866129Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.985941328Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.986194392Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.986791553Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.989120642Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.989606003Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:35:45.006 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.992054526Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:35:45.007 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.99486572Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:44.997513143Z level=info msg="Executing migration" id="Drop public config table" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.000202831Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.002872348Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.005621891Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.008268382Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.010958101Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.014075049Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-21T09:35:45.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.016775376Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.019438294Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.022231308Z level=info msg="Executing migration" id="add share column" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.024876774Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.027555092Z level=info msg="Executing migration" id="create file table" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.030291682Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.030786872Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.033528081Z level=info msg="Executing migration" id="create file_meta table" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.036274519Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.03908801Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.04195616Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-21T09:35:45.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.044589749Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.047246721Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.049831083Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.052623598Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.055344264Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.058191573Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.061012261Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.063843372Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.066771036Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.069504471Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-21T09:35:45.365 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.072082996Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.074870186Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.084591668Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.08749036Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.090158921Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.09356052Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.096273208Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.098940292Z level=info msg="Executing migration" id="create folder table" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.101600847Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.104140182Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.107103291Z level=info msg="Executing migration" id="Update folder title length" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.109749697Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=migrator t=2024-09-21T09:35:45.112527885Z level=info msg="migrations completed" performed=485 skipped=0 duration=706.880351ms 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=sqlstore t=2024-09-21T09:35:45.112904011Z level=info msg="Created default organization" 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=plugin.loader t=2024-09-21T09:35:45.17204732Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:35:45.366 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=plugin.loader t=2024-09-21T09:35:45.198608635Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:35:45.367 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=plugin.loader t=2024-09-21T09:35:45.198628644Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:35:45.367 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=secrets t=2024-09-21T09:35:45.198661323Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:35:45.367 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=query_data t=2024-09-21T09:35:45.200307722Z level=info msg="Query Service initialization" 2024-09-21T09:35:45.367 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=live.push_http t=2024-09-21T09:35:45.202556826Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[29313]: Deploying daemon node-exporter.a on smithi076 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[29313]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:35:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[41325]: Deploying daemon node-exporter.a on smithi076 2024-09-21T09:35:45.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:45 smithi076 ceph-mon[41325]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:35:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-mon[33301]: Deploying daemon node-exporter.a on smithi076 2024-09-21T09:35:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-mon[33301]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=infra.usagestats.collector t=2024-09-21T09:35:45.91274116Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=provisioning.datasources t=2024-09-21T09:35:45.968153256Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=provisioning.datasources t=2024-09-21T09:35:46.000017298Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=provisioning.alerting t=2024-09-21T09:35:46.021386058Z level=info msg="starting to provision alerting" 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=provisioning.alerting t=2024-09-21T09:35:46.021435651Z level=info msg="finished to provision alerting" 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=grafanaStorageLogger t=2024-09-21T09:35:46.021560168Z level=info msg="storage starting" 2024-09-21T09:35:46.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=ngalert.state.manager t=2024-09-21T09:35:46.021613134Z level=info msg="Warming state cache for startup" 2024-09-21T09:35:46.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=http.server t=2024-09-21T09:35:46.023546257Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:35:46.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=ngalert.state.manager t=2024-09-21T09:35:46.031866033Z level=info msg="State cache has been initialized" states=0 duration=10.249511ms 2024-09-21T09:35:46.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:35:46.031946412Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:35:46.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=ticker t=2024-09-21T09:35:46.03203994Z level=info msg=starting first_tick=2024-09-21T09:35:50Z 2024-09-21T09:35:46.364 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:35:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[64673]: logger=sqlstore.transactions t=2024-09-21T09:35:46.032050275Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:35:48.033 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:47 smithi076 ceph-mon[29313]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:48.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:47 smithi076 ceph-mon[41325]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:47 smithi191 ceph-mon[33301]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:48.977 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:48 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:48.977 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:48 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:48.978 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:35:48] "GET /metrics HTTP/1.1" 200 28509 "" "Prometheus/2.43.0" 2024-09-21T09:35:48.978 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:48 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:48.978 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:48 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:48 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:35:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:48 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:35:50.014 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:49 smithi076 ceph-mon[29313]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:50.014 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:49 smithi076 ceph-mon[41325]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:49 smithi191 ceph-mon[33301]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:51.834 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:51 smithi076 ceph-mon[29313]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:51.834 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:51 smithi076 ceph-mon[41325]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:51 smithi191 ceph-mon[33301]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:53.990 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:53 smithi076 ceph-mon[29313]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:53.990 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:53 smithi076 ceph-mon[41325]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:53 smithi191 ceph-mon[33301]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:54.623 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:54 smithi076 bash[73635]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:35:55.115 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:54 smithi076 bash[73635]: Getting image source signatures 2024-09-21T09:35:55.115 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:54 smithi076 bash[73635]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:35:55.115 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:54 smithi076 bash[73635]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:35:55.115 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:54 smithi076 bash[73635]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:35:55.987 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:55 smithi076 ceph-mon[29313]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:55.987 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:55 smithi076 ceph-mon[41325]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:55 smithi191 ceph-mon[33301]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:57.123 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:56 smithi076 bash[73635]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:35:57.622 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:57 smithi076 bash[73635]: Writing manifest to image destination 2024-09-21T09:35:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:57 smithi191 ceph-mon[33301]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:58.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:57 smithi076 ceph-mon[29313]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:58.114 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:57 smithi076 ceph-mon[41325]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:58.114 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:57 smithi076 podman[73635]: 2024-09-21 09:35:57.670829965 +0000 UTC m=+3.569079199 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:35:58.114 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:35:58 smithi076 podman[73635]: 2024-09-21 09:35:58.113245783 +0000 UTC m=+4.011495070 container create 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:35:58.823 INFO:teuthology.orchestra.run.smithi076.stdout:34359738418 2024-09-21T09:35:58.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:35:59.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:35:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:35:58] "GET /metrics HTTP/1.1" 200 28509 "" "Prometheus/2.43.0" 2024-09-21T09:35:59.538 INFO:teuthology.orchestra.run.smithi076.stdout:197568495635 2024-09-21T09:35:59.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:35:59.775 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:35:59 smithi076 ceph-mon[29313]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:35:59.775 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:35:59 smithi076 ceph-mon[41325]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:35:59 smithi191 ceph-mon[33301]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 podman[73635]: 2024-09-21 09:36:00.414218975 +0000 UTC m=+6.312468206 container init 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 podman[73635]: 2024-09-21 09:36:00.421129006 +0000 UTC m=+6.319378225 container start 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.425Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.425Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.426Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.426Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.426Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:36:00.751 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:36:00.752 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:36:00.753 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:36:00.754 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[73950]: ts=2024-09-21T09:36:00.427Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:36:00.755 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 bash[73635]: 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 2024-09-21T09:36:00.756 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:36:00 smithi076 systemd[1]: Started Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:01.578 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:01 smithi191 bash[64942]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:36:01.664 INFO:teuthology.orchestra.run.smithi076.stdout:55834574893 2024-09-21T09:36:01.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:36:01.847 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:01 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.848 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:01 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.848 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:01 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.848 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:01 smithi191 ceph-mon[33301]: Deploying daemon node-exporter.b on smithi191 2024-09-21T09:36:01.848 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:01 smithi191 ceph-mon[33301]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[29313]: Deploying daemon node-exporter.b on smithi191 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[29313]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[41325]: Deploying daemon node-exporter.b on smithi191 2024-09-21T09:36:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:01 smithi076 ceph-mon[41325]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:02.066 INFO:teuthology.orchestra.run.smithi076.stdout:163208757274 2024-09-21T09:36:02.067 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:36:02.112 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:01 smithi191 bash[64942]: Getting image source signatures 2024-09-21T09:36:02.112 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:01 smithi191 bash[64942]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:36:02.112 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:01 smithi191 bash[64942]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:36:02.112 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:01 smithi191 bash[64942]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:36:03.612 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 bash[64942]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:36:03.612 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 bash[64942]: Writing manifest to image destination 2024-09-21T09:36:03.709 INFO:teuthology.orchestra.run.smithi076.stdout:231928233999 2024-09-21T09:36:03.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:36:03.898 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-mon[33301]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:03.898 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:03.898 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 podman[64942]: 2024-09-21 09:36:03.554992348 +0000 UTC m=+2.151750489 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:36:03.899 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 podman[64942]: 2024-09-21 09:36:03.653745217 +0000 UTC m=+2.250503397 container create 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:36:03.899 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 podman[64942]: 2024-09-21 09:36:03.897166609 +0000 UTC m=+2.493924750 container init 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:36:04.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:03 smithi076 ceph-mon[29313]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:04.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:03 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:04.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:03 smithi076 ceph-mon[41325]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:04.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:03 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:04.168 INFO:teuthology.orchestra.run.smithi076.stdout:107374182436 2024-09-21T09:36:04.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 podman[64942]: 2024-09-21 09:36:03.902521812 +0000 UTC m=+2.499279952 container start 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.906Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.906Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.906Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.906Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:36:04.363 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:36:04.364 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:36:04.365 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:36:04.366 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.907Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.908Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[65019]: ts=2024-09-21T09:36:03.908Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 bash[64942]: 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 2024-09-21T09:36:04.367 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:36:03 smithi191 systemd[1]: Started Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:04.627 INFO:teuthology.orchestra.run.smithi076.stdout:128849018911 2024-09-21T09:36:04.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:36:04.981 INFO:teuthology.orchestra.run.smithi076.stdout:77309411369 2024-09-21T09:36:04.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:36:05.026 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:05.026 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:05.110 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:05.110 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:05.220 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[29313]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.221 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:05.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:05 smithi076 ceph-mon[41325]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:05.350 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:05 smithi191 ceph-mon[33301]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:07.731 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:07 smithi191 ceph-mon[33301]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:07 smithi076 ceph-mon[29313]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:07 smithi076 ceph-mon[41325]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:08 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:08 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:08 smithi191 ceph-mon[33301]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:08 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:36:08] "GET /metrics HTTP/1.1" 200 28504 "" "Prometheus/2.43.0" 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[41325]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[29313]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:08 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:10.710 INFO:teuthology.orchestra.run.smithi076.stdout:163208757277 2024-09-21T09:36:11.314 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:11.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:11 smithi076 ceph-mon[29313]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:11.794 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:11 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1750526066' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:36:11.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:11 smithi076 ceph-mon[41325]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:11.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:11 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1750526066' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:36:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:11 smithi191 ceph-mon[33301]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:11 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1750526066' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:36:12.327 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757274 got 163208757277 for osd.5 2024-09-21T09:36:12.328 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:12.981 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:12 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3240799829' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:36:12.982 INFO:teuthology.orchestra.run.smithi076.stdout:55834574896 2024-09-21T09:36:12.982 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:12 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3240799829' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:36:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:12 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3240799829' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:36:13.578 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574896 for osd.1 2024-09-21T09:36:13.578 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:13.820 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:13 smithi076 ceph-mon[29313]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:13.820 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:13 smithi076 ceph-mon[41325]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:13.829 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:13.958 INFO:teuthology.orchestra.run.smithi076.stdout:34359738421 2024-09-21T09:36:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:13 smithi191 ceph-mon[33301]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:14.587 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738418 got 34359738421 for osd.0 2024-09-21T09:36:14.587 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:14.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:14 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1033388929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:36:14.828 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:14 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1033388929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:36:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:14 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1033388929' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:36:15.361 INFO:teuthology.orchestra.run.smithi076.stdout:107374182439 2024-09-21T09:36:16.010 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:15 smithi076 ceph-mon[29313]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:16.010 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:15 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3368577063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:36:16.010 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:15 smithi076 ceph-mon[41325]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:16.010 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:15 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3368577063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:36:16.030 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182436 got 107374182439 for osd.3 2024-09-21T09:36:16.031 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:15 smithi191 ceph-mon[33301]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:15 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3368577063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:36:16.793 INFO:teuthology.orchestra.run.smithi076.stdout:197568495640 2024-09-21T09:36:17.509 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495640 for osd.6 2024-09-21T09:36:17.509 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:17.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:17 smithi076 ceph-mon[29313]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:17 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4272408772' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:36:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:17 smithi076 ceph-mon[41325]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:17 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4272408772' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:36:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:17 smithi191 ceph-mon[33301]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:17 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4272408772' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:36:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:18 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:19.113 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:36:18] "GET /metrics HTTP/1.1" 200 28510 "" "Prometheus/2.43.0" 2024-09-21T09:36:19.113 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:18 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:19.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:18 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:19.841 INFO:teuthology.orchestra.run.smithi076.stdout:231928234003 2024-09-21T09:36:19.920 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:19 smithi076 ceph-mon[29313]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:19.920 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:19 smithi076 ceph-mon[41325]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:19 smithi191 ceph-mon[33301]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:20.719 INFO:teuthology.orchestra.run.smithi076.stdout:77309411372 2024-09-21T09:36:20.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:20 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4283837413' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:36:20.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:20 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2008549814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:36:20.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4283837413' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:36:20.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:20 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2008549814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:36:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:20 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4283837413' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:36:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:20 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2008549814' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:36:21.391 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234003 for osd.7 2024-09-21T09:36:21.391 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:21.769 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411369 got 77309411372 for osd.2 2024-09-21T09:36:21.770 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:22.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:21 smithi076 ceph-mon[29313]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:22.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:21 smithi076 ceph-mon[41325]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:21 smithi191 ceph-mon[33301]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:22.451 INFO:teuthology.orchestra.run.smithi076.stdout:128849018915 2024-09-21T09:36:23.003 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:22 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2422839863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:36:23.003 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:22 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2422839863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:36:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:22 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2422839863' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:36:23.625 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018911 got 128849018915 for osd.4 2024-09-21T09:36:23.625 DEBUG:teuthology.parallel:result is None 2024-09-21T09:36:23.626 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:36:23.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:36:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:23 smithi076 ceph-mon[29313]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:23 smithi076 ceph-mon[41325]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:23 smithi191 ceph-mon[33301]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:24.305 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:25.431 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.431 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[29313]: Reconfiguring daemon alertmanager.a on smithi076 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:25.432 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.433 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:36:25.433 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:25 smithi076 ceph-mon[41325]: Reconfiguring daemon alertmanager.a on smithi076 2024-09-21T09:36:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:36:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:25 smithi191 ceph-mon[33301]: Reconfiguring daemon alertmanager.a on smithi076 2024-09-21T09:36:26.411 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:36:26.412 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:36:27.971 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:27 smithi076 ceph-mon[29313]: from='client.14601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:27.971 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:27 smithi076 ceph-mon[29313]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:27.971 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:27 smithi076 ceph-mon[41325]: from='client.14601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:27.971 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:27 smithi076 ceph-mon[41325]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:27 smithi191 ceph-mon[33301]: from='client.14601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:27 smithi191 ceph-mon[33301]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:29.021 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:36:28] "GET /metrics HTTP/1.1" 200 28510 "" "Prometheus/2.43.0" 2024-09-21T09:36:29.022 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:28 smithi076 systemd[1]: Stopping Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:29.430 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-21T09:36:24.657604+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":231340,"kb_used_data":6988,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729876,"statfs":{"total":767960285184,"available":767723393024,"internally_reserved":0,"allocated":7155712,"data_stored":4034970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":229625400},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001702"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.539676+0000","last_change":"2024-09-21T09:35:30.789192+0000","last_active":"2024-09-21T09:35:38.539676+0000","last_peered":"2024-09-21T09:35:38.539676+0000","last_clean":"2024-09-21T09:35:38.539676+0000","last_became_active":"2024-09-21T09:35:24.754596+0000","last_became_peered":"2024-09-21T09:35:24.754596+0000","last_unstale":"2024-09-21T09:35:38.539676+0000","last_undegraded":"2024-09-21T09:35:38.539676+0000","last_fullsized":"2024-09-21T09:35:38.539676+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:39.865349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.457647+0000","last_change":"2024-09-21T09:35:30.789220+0000","last_active":"2024-09-21T09:35:38.457647+0000","last_peered":"2024-09-21T09:35:38.457647+0000","last_clean":"2024-09-21T09:35:38.457647+0000","last_became_active":"2024-09-21T09:35:24.767340+0000","last_became_peered":"2024-09-21T09:35:24.767340+0000","last_unstale":"2024-09-21T09:35:38.457647+0000","last_undegraded":"2024-09-21T09:35:38.457647+0000","last_fullsized":"2024-09-21T09:35:38.457647+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:48.181309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036756400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.767563+0000","last_change":"2024-09-21T09:35:31.069545+0000","last_active":"2024-09-21T09:35:38.767563+0000","last_peered":"2024-09-21T09:35:38.767563+0000","last_clean":"2024-09-21T09:35:38.767563+0000","last_became_active":"2024-09-21T09:35:24.766462+0000","last_became_peered":"2024-09-21T09:35:24.766462+0000","last_unstale":"2024-09-21T09:35:38.767563+0000","last_undegraded":"2024-09-21T09:35:38.767563+0000","last_fullsized":"2024-09-21T09:35:38.767563+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:48.059267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.457481+0000","last_change":"2024-09-21T09:35:30.789143+0000","last_active":"2024-09-21T09:35:38.457481+0000","last_peered":"2024-09-21T09:35:38.457481+0000","last_clean":"2024-09-21T09:35:38.457481+0000","last_became_active":"2024-09-21T09:35:24.766244+0000","last_became_peered":"2024-09-21T09:35:24.766244+0000","last_unstale":"2024-09-21T09:35:38.457481+0000","last_undegraded":"2024-09-21T09:35:38.457481+0000","last_fullsized":"2024-09-21T09:35:38.457481+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:30.932253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.766914+0000","last_change":"2024-09-21T09:35:30.788301+0000","last_active":"2024-09-21T09:35:38.766914+0000","last_peered":"2024-09-21T09:35:38.766914+0000","last_clean":"2024-09-21T09:35:38.766914+0000","last_became_active":"2024-09-21T09:35:24.753321+0000","last_became_peered":"2024-09-21T09:35:24.753321+0000","last_unstale":"2024-09-21T09:35:38.766914+0000","last_undegraded":"2024-09-21T09:35:38.766914+0000","last_fullsized":"2024-09-21T09:35:38.766914+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:52.357133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.692557+0000","last_change":"2024-09-21T09:35:30.783775+0000","last_active":"2024-09-21T09:35:38.692557+0000","last_peered":"2024-09-21T09:35:38.692557+0000","last_clean":"2024-09-21T09:35:38.692557+0000","last_became_active":"2024-09-21T09:35:24.766096+0000","last_became_peered":"2024-09-21T09:35:24.766096+0000","last_unstale":"2024-09-21T09:35:38.692557+0000","last_undegraded":"2024-09-21T09:35:38.692557+0000","last_fullsized":"2024-09-21T09:35:38.692557+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.155489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.767229+0000","last_change":"2024-09-21T09:35:31.069584+0000","last_active":"2024-09-21T09:35:38.767229+0000","last_peered":"2024-09-21T09:35:38.767229+0000","last_clean":"2024-09-21T09:35:38.767229+0000","last_became_active":"2024-09-21T09:35:24.766184+0000","last_became_peered":"2024-09-21T09:35:24.766184+0000","last_unstale":"2024-09-21T09:35:38.767229+0000","last_undegraded":"2024-09-21T09:35:38.767229+0000","last_fullsized":"2024-09-21T09:35:38.767229+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:16.728713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.766925+0000","last_change":"2024-09-21T09:35:30.787751+0000","last_active":"2024-09-21T09:35:38.766925+0000","last_peered":"2024-09-21T09:35:38.766925+0000","last_clean":"2024-09-21T09:35:38.766925+0000","last_became_active":"2024-09-21T09:35:24.753144+0000","last_became_peered":"2024-09-21T09:35:24.753144+0000","last_unstale":"2024-09-21T09:35:38.766925+0000","last_undegraded":"2024-09-21T09:35:38.766925+0000","last_fullsized":"2024-09-21T09:35:38.766925+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:50.550632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":262,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:35:31.069272+0000","last_change":"2024-09-21T09:34:52.468106+0000","last_active":"2024-09-21T09:35:31.069272+0000","last_peered":"2024-09-21T09:35:31.069272+0000","last_clean":"2024-09-21T09:35:31.069272+0000","last_became_active":"2024-09-21T09:34:52.467638+0000","last_became_peered":"2024-09-21T09:34:52.467638+0000","last_unstale":"2024-09-21T09:35:31.069272+0000","last_undegraded":"2024-09-21T09:35:31.069272+0000","last_fullsized":"2024-09-21T09:35:31.069272+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:35.011128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1261568,"data_stored":873130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.73299999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.623},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.66400000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.65200000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.749}]},{"osd":4,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.76700000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.71499999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.58599999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.65500000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.57499999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.435}]},{"osd":7,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.51700000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":283112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54800000000000004,"15min":0.55400000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.497,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.65600000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54300000000000004,"15min":0.55100000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.52000000000000002,"15min":0.52600000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.66000000000000003,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.626},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54800000000000004,"15min":0.55400000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51000000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.624,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54700000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.53700000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.36699999999999999,"15min":0.35299999999999998},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.374,"15min":0.36499999999999999},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.438}]},{"osd":6,"last update":"Sat Sep 21 09:35:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.495},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.45800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.35799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53500000000000003,"15min":0.53000000000000003},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.76700000000000002,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.53300000000000003,"15min":0.53100000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.38400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.53400000000000003,"15min":0.52900000000000003},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68799999999999994,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.499},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.39300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71899999999999997,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:36:05 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.38,"15min":0.372},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.376,"15min":0.36199999999999999},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:35:28 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Sep 21 09:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.46999999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.39200000000000002,"15min":0.39800000000000002},"min":{"1min":0.24199999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.48899999999999999,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.40500000000000003,"15min":0.40899999999999997},"min":{"1min":0.27000000000000002,"5min":0.245,"15min":0.245},"max":{"1min":0.56000000000000005,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.438}]},{"osd":1,"last update":"Sat Sep 21 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.41499999999999998,"15min":0.42099999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.54900000000000004,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.40200000000000002,"15min":0.40600000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53500000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.58699999999999997,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.40000000000000002,"15min":0.40200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.53100000000000003,"5min":0.626,"15min":0.626},"last":0.47599999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86299999999999999,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.39300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.93600000000000005,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.67600000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.58899999999999997,"15min":0.59499999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83599999999999997,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60999999999999999,"15min":0.61299999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.84999999999999998,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.71099999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:35:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1,"5min":1,"15min":1},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.749}]},{"osd":7,"last update":"Sat Sep 21 09:35:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.64300000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.34100000000000003,"15min":0.33900000000000002},"min":{"1min":0.22900000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.503,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.374},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.34999999999999998,"15min":0.34799999999999998},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.49399999999999999,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.24299999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.373,"15min":0.375},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54400000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.309},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.36199999999999999,"15min":0.36099999999999999},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.45000000000000001,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.42499999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:36:24 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.443,"15min":0.45100000000000001},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60599999999999998,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.43099999999999999,"15min":0.44700000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.55300000000000005,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52100000000000002,"15min":0.51800000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68400000000000005,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57999999999999996,"15min":0.58499999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69999999999999996,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:35:24 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":3.7519999999999998,"5min":3.7519999999999998,"15min":3.7519999999999998},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.4119999999999999,"5min":3.4119999999999999,"15min":3.4119999999999999},"last":0.626}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31772,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713380,"statfs":{"total":95995035648,"available":95962501120,"internally_reserved":0,"allocated":671744,"data_stored":282839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.33100000000000002,"15min":0.307},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.314},"min":{"1min":0.26800000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.54900000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.40999999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:35:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.376,"15min":0.36799999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52400000000000002,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.39000000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":0.45500000000000002,"5min":0.62,"15min":0.62},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:36:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.497,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.438},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.432,"15min":0.44400000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.48099999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.5},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59799999999999998,"15min":0.60799999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79300000000000004,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.60299999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:35:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.65700000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.64700000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32352,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712800,"statfs":{"total":95995035648,"available":95961907200,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.34100000000000003,"15min":0.32000000000000001},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.53400000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.312},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.33200000000000002},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.245}]},{"osd":2,"last update":"Sat Sep 21 09:35:59 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60599999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.55900000000000005,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:36:22 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.432,"15min":0.434},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.58399999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.42499999999999999,"15min":0.433},"min":{"1min":0.313,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.76200000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.45400000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:36:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.45800000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57599999999999996,"15min":0.56999999999999995},"min":{"1min":0.50700000000000001,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:35:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.63700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:36:29.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:36:29.799 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:30.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:29 smithi076 ceph-mon[29313]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:30.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:29 smithi076 ceph-mon[41325]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:30.084 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[71487]: ts=2024-09-21T09:36:29.766Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:36:30.084 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:29 smithi076 podman[77018]: 2024-09-21 09:36:29.796992786 +0000 UTC m=+0.186453102 container died 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:29 smithi191 ceph-mon[33301]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:30.350 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:30 smithi076 podman[77018]: 2024-09-21 09:36:30.181020668 +0000 UTC m=+0.570480974 container cleanup 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:30.350 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:30 smithi076 bash[77018]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:36:30.622 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:30 smithi076 podman[77064]: 2024-09-21 09:36:30.440670896 +0000 UTC m=+0.634733083 container remove 38499a9aab2a720ba501e9ceced0acc31f40efd4d7b126d61087e25d9874a9b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:30.622 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:30 smithi076 podman[77064]: 2024-09-21 09:36:30.541469855 +0000 UTC m=+0.735532042 volume remove de7bf0a0431629da9221da526db75026d604acda3a638e3254bc9748f261eb5a 2024-09-21T09:36:31.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:31 smithi076 podman[77095]: 2024-09-21 09:36:31.086234546 +0000 UTC m=+0.876035644 container died 6a9fd0934e52c22e4736aa652f63794296e48f69160eaa8ff0de9a3d1232ddda (image=quay.ceph.io/ceph-ci/ceph:reef, name=peaceful_borg, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T09:36:31.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:31 smithi076 ceph-mon[29313]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:31.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:31 smithi076 ceph-mon[41325]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:31.998 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:31 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:36:31.999 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:31 smithi076 systemd[1]: Stopped Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:31.999 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:31 smithi076 systemd[1]: Starting Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:31 smithi191 ceph-mon[33301]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:32.374 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:36:32.374 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:36:32.623 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 podman[77285]: 2024-09-21 09:36:32.351033694 +0000 UTC m=+0.109010435 volume create d62ce2f7b62824ce0414f2030995d3cdd679c124a116ffa1633a4c8097377420 2024-09-21T09:36:32.623 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 podman[77285]: 2024-09-21 09:36:32.258177196 +0000 UTC m=+0.016153943 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:36:32.623 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 podman[77285]: 2024-09-21 09:36:32.460063602 +0000 UTC m=+0.218040343 container create d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:32.889 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 podman[77285]: 2024-09-21 09:36:32.771559329 +0000 UTC m=+0.529536080 container init d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:32.889 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 podman[77285]: 2024-09-21 09:36:32.776808451 +0000 UTC m=+0.534785192 container start d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:32.889 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.789Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:36:32.889 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.789Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:36:32.889 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.790Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.76 port=9094 2024-09-21T09:36:32.890 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.791Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:36:32.890 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.813Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:36:32.890 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.814Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:36:32.890 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.816Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:36:32.890 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:32.816Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:36:32.890 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 bash[77285]: d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 2024-09-21T09:36:33.226 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:32 smithi076 systemd[1]: Started Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:33.521 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":46,"stamp":"2024-09-21T09:36:30.658608+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":231340,"kb_used_data":6988,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729876,"statfs":{"total":767960285184,"available":767723393024,"internally_reserved":0,"allocated":7155712,"data_stored":4034970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":229625400},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001855"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.539676+0000","last_change":"2024-09-21T09:35:30.789192+0000","last_active":"2024-09-21T09:35:38.539676+0000","last_peered":"2024-09-21T09:35:38.539676+0000","last_clean":"2024-09-21T09:35:38.539676+0000","last_became_active":"2024-09-21T09:35:24.754596+0000","last_became_peered":"2024-09-21T09:35:24.754596+0000","last_unstale":"2024-09-21T09:35:38.539676+0000","last_undegraded":"2024-09-21T09:35:38.539676+0000","last_fullsized":"2024-09-21T09:35:38.539676+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:39.865349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.457647+0000","last_change":"2024-09-21T09:35:30.789220+0000","last_active":"2024-09-21T09:35:38.457647+0000","last_peered":"2024-09-21T09:35:38.457647+0000","last_clean":"2024-09-21T09:35:38.457647+0000","last_became_active":"2024-09-21T09:35:24.767340+0000","last_became_peered":"2024-09-21T09:35:24.767340+0000","last_unstale":"2024-09-21T09:35:38.457647+0000","last_undegraded":"2024-09-21T09:35:38.457647+0000","last_fullsized":"2024-09-21T09:35:38.457647+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:48.181309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036756400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.767563+0000","last_change":"2024-09-21T09:35:31.069545+0000","last_active":"2024-09-21T09:35:38.767563+0000","last_peered":"2024-09-21T09:35:38.767563+0000","last_clean":"2024-09-21T09:35:38.767563+0000","last_became_active":"2024-09-21T09:35:24.766462+0000","last_became_peered":"2024-09-21T09:35:24.766462+0000","last_unstale":"2024-09-21T09:35:38.767563+0000","last_undegraded":"2024-09-21T09:35:38.767563+0000","last_fullsized":"2024-09-21T09:35:38.767563+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:48.059267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.457481+0000","last_change":"2024-09-21T09:35:30.789143+0000","last_active":"2024-09-21T09:35:38.457481+0000","last_peered":"2024-09-21T09:35:38.457481+0000","last_clean":"2024-09-21T09:35:38.457481+0000","last_became_active":"2024-09-21T09:35:24.766244+0000","last_became_peered":"2024-09-21T09:35:24.766244+0000","last_unstale":"2024-09-21T09:35:38.457481+0000","last_undegraded":"2024-09-21T09:35:38.457481+0000","last_fullsized":"2024-09-21T09:35:38.457481+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:30.932253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.766914+0000","last_change":"2024-09-21T09:35:30.788301+0000","last_active":"2024-09-21T09:35:38.766914+0000","last_peered":"2024-09-21T09:35:38.766914+0000","last_clean":"2024-09-21T09:35:38.766914+0000","last_became_active":"2024-09-21T09:35:24.753321+0000","last_became_peered":"2024-09-21T09:35:24.753321+0000","last_unstale":"2024-09-21T09:35:38.766914+0000","last_undegraded":"2024-09-21T09:35:38.766914+0000","last_fullsized":"2024-09-21T09:35:38.766914+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:52.357133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.692557+0000","last_change":"2024-09-21T09:35:30.783775+0000","last_active":"2024-09-21T09:35:38.692557+0000","last_peered":"2024-09-21T09:35:38.692557+0000","last_clean":"2024-09-21T09:35:38.692557+0000","last_became_active":"2024-09-21T09:35:24.766096+0000","last_became_peered":"2024-09-21T09:35:24.766096+0000","last_unstale":"2024-09-21T09:35:38.692557+0000","last_undegraded":"2024-09-21T09:35:38.692557+0000","last_fullsized":"2024-09-21T09:35:38.692557+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.155489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.767229+0000","last_change":"2024-09-21T09:35:31.069584+0000","last_active":"2024-09-21T09:35:38.767229+0000","last_peered":"2024-09-21T09:35:38.767229+0000","last_clean":"2024-09-21T09:35:38.767229+0000","last_became_active":"2024-09-21T09:35:24.766184+0000","last_became_peered":"2024-09-21T09:35:24.766184+0000","last_unstale":"2024-09-21T09:35:38.767229+0000","last_undegraded":"2024-09-21T09:35:38.767229+0000","last_fullsized":"2024-09-21T09:35:38.767229+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:16.728713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:35:38.766925+0000","last_change":"2024-09-21T09:35:30.787751+0000","last_active":"2024-09-21T09:35:38.766925+0000","last_peered":"2024-09-21T09:35:38.766925+0000","last_clean":"2024-09-21T09:35:38.766925+0000","last_became_active":"2024-09-21T09:35:24.753144+0000","last_became_peered":"2024-09-21T09:35:24.753144+0000","last_unstale":"2024-09-21T09:35:38.766925+0000","last_undegraded":"2024-09-21T09:35:38.766925+0000","last_fullsized":"2024-09-21T09:35:38.766925+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:50.550632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":262,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-21T09:35:31.069272+0000","last_change":"2024-09-21T09:34:52.468106+0000","last_active":"2024-09-21T09:35:31.069272+0000","last_peered":"2024-09-21T09:35:31.069272+0000","last_clean":"2024-09-21T09:35:31.069272+0000","last_became_active":"2024-09-21T09:34:52.467638+0000","last_became_peered":"2024-09-21T09:34:52.467638+0000","last_unstale":"2024-09-21T09:35:31.069272+0000","last_undegraded":"2024-09-21T09:35:31.069272+0000","last_fullsized":"2024-09-21T09:35:31.069272+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:35.011128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234006,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1261568,"data_stored":873130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.626}]},{"osd":1,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.52200000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495643,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.54600000000000004,"15min":0.53600000000000003},"min":{"1min":0.47399999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.40899999999999997,"5min":0.376,"15min":0.376},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.52700000000000002,"15min":0.52100000000000002},"min":{"1min":0.42799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.52700000000000002,"15min":0.51900000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.64300000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.47899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57199999999999995,"15min":0.57099999999999995},"min":{"1min":0.44800000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.72899999999999998,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.52600000000000002,"15min":0.51900000000000002},"min":{"1min":0.39000000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.55300000000000005,"15min":0.54900000000000004},"min":{"1min":0.38100000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.68899999999999995}]},{"osd":4,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.442,"15min":0.437},"min":{"1min":0.36599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63200000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.41999999999999998,"15min":0.41299999999999998},"min":{"1min":0.308,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64400000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.45900000000000002,"15min":0.45400000000000001},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.66800000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.45500000000000002,"15min":0.44600000000000001},"min":{"1min":0.379,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.63600000000000001,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.40400000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757281,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":283112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54800000000000004,"15min":0.55400000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.505}]},{"osd":1,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.54300000000000004,"15min":0.55100000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.52000000000000002,"15min":0.52600000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.66000000000000003,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54800000000000004,"15min":0.55400000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.65200000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.624,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.624},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54700000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Sep 21 09:36:06 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.36699999999999999,"15min":0.35299999999999998},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.374,"15min":0.36499999999999999},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:36:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.41799999999999998,"15min":0.41399999999999998},"min":{"1min":0.27200000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.45300000000000001,"15min":0.44400000000000001},"min":{"1min":0.27800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.434}]},{"osd":7,"last update":"Sat Sep 21 09:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.47399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018917,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.51300000000000001,"15min":0.505},"min":{"1min":0.35799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.53500000000000003,"15min":0.53000000000000003},"min":{"1min":0.36299999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.76700000000000002,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.63300000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.53300000000000003,"15min":0.53100000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.38400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72999999999999998,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64700000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.53400000000000003,"15min":0.52900000000000003},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68799999999999994,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.51500000000000001,"15min":0.50900000000000001},"min":{"1min":0.39300000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71899999999999997,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:36:05 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.38,"15min":0.372},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.376,"15min":0.36199999999999999},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 21 09:35:28 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.51200000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182442,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:26 2024","interfaces":[{"interface":"back","average":{"1min":0.312,"5min":0.374,"15min":0.39200000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.435,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.217},{"interface":"front","average":{"1min":0.34000000000000002,"5min":0.39000000000000001,"15min":0.40400000000000003},"min":{"1min":0.252,"5min":0.245,"15min":0.245},"max":{"1min":0.46600000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:36:26 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.40100000000000002,"15min":0.41599999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.51100000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.313},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.38800000000000001,"15min":0.40100000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.52100000000000002,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:36:26 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.432,"15min":0.44800000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.58999999999999997,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.39100000000000001,"15min":0.39900000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.47599999999999998,"5min":0.626,"15min":0.626},"last":0.27500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86299999999999999,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.39300000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.93600000000000005,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.58899999999999997,"15min":0.59499999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83599999999999997,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60999999999999999,"15min":0.61299999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.84999999999999998,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:36:26 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82399999999999995,"5min":1,"15min":1},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.57599999999999996,"15min":0.57799999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.81499999999999995,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:35:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.5},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.56499999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.34100000000000003,"15min":0.33900000000000002},"min":{"1min":0.22900000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.503,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.313},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.34999999999999998,"15min":0.34799999999999998},"min":{"1min":0.22900000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.49399999999999999,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.224}]},{"osd":1,"last update":"Sat Sep 21 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.373,"15min":0.375},"min":{"1min":0.23300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54400000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.36199999999999999,"15min":0.36099999999999999},"min":{"1min":0.26500000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.45000000000000001,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35999999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:36:24 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.443,"15min":0.45100000000000001},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60599999999999998,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.43099999999999999,"15min":0.44700000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.55300000000000005,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.52100000000000002,"15min":0.51800000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.68400000000000005,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.628},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57999999999999996,"15min":0.58499999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69999999999999996,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.56299999999999994}]},{"osd":6,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.438,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.72599999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":3.7519999999999998,"5min":3.7519999999999998,"15min":3.7519999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.4119999999999999,"5min":3.4119999999999999,"15min":3.4119999999999999},"last":0.58599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574900,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31772,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713380,"statfs":{"total":95995035648,"available":95962501120,"internally_reserved":0,"allocated":671744,"data_stored":282839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.33100000000000002,"15min":0.307},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.314},"min":{"1min":0.26800000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.54900000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:35:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.376,"15min":0.36799999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52400000000000002,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.39000000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":0.45500000000000002,"5min":0.62,"15min":0.62},"last":0.376}]},{"osd":3,"last update":"Sat Sep 21 09:36:20 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.497,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.432,"15min":0.44400000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.48099999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:35:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 21 09:36:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59799999999999998,"15min":0.60799999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.79300000000000004,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:36:27 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.40200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.60799999999999998,"15min":0.61699999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76100000000000001,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:35:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.627},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.48799999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738425,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32352,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712800,"statfs":{"total":95995035648,"available":95961907200,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:35:34 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.34100000000000003,"15min":0.32000000000000001},"min":{"1min":0.23899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.53400000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.33200000000000002},"min":{"1min":0.26800000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:35:59 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.60599999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.55900000000000005,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.31}]},{"osd":3,"last update":"Sat Sep 21 09:36:22 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.432,"15min":0.434},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.58399999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.42499999999999999,"15min":0.433},"min":{"1min":0.313,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.47899999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.76200000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.57799999999999996,"15min":0.57599999999999996},"min":{"1min":0.45400000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:36:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.45800000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57599999999999996,"15min":0.56999999999999995},"min":{"1min":0.50700000000000001,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.437}]},{"osd":6,"last update":"Sat Sep 21 09:36:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.59999999999999998,"15min":0.60799999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.72099999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.443},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.61499999999999999,"15min":0.623},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.68400000000000005,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.39500000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:35:51 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.501},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.40799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:36:33.523 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:36:33.523 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:36:33.523 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:36:33.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:36:33.721 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[41325]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[41325]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[29313]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:33.835 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[29313]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:33.836 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:33.836 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:33.836 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:33 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:33.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:33 smithi191 ceph-mon[33301]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:36:33.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:33 smithi191 ceph-mon[33301]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:33.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:33.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:33.974 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:33 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:35.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:34 smithi191 ceph-mon[33301]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:36:35.021 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:34 smithi191 ceph-mon[33301]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:36:35.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:34 smithi076 ceph-mon[29313]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:36:35.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:34 smithi076 ceph-mon[29313]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:36:35.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:34 smithi076 ceph-mon[41325]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:36:35.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:34 smithi076 ceph-mon[41325]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:36:35.110 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:34.791Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000722865s 2024-09-21T09:36:35.668 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:36:35.668 INFO:teuthology.orchestra.run.smithi076.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:36:35.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:35 smithi191 ceph-mon[33301]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:35.952 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:35 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/75760809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:36:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:35 smithi076 ceph-mon[29313]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:35 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/75760809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:36:36.073 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:35 smithi076 ceph-mon[41325]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:36.073 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:35 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/75760809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:36:36.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 systemd[1]: Stopping Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:36.440 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:36:36.440 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T09:36:36.440 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T09:36:36.449 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi076.front.sepia.ceph.com 2024-09-21T09:36:36.449 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- bash -c 'ceph fs volume create foo' 2024-09-21T09:36:36.650 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:36.656 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.373Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.374Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.374Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[62550]: ts=2024-09-21T09:36:36.374Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:36:36.657 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 podman[65898]: 2024-09-21 09:36:36.383687203 +0000 UTC m=+0.126336205 container died 06b044499e69fcdfe9df27551c7e989dabbfec5df19781c3e71e52c9175439f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:36.940 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 podman[65898]: 2024-09-21 09:36:36.746366728 +0000 UTC m=+0.489015720 container cleanup 06b044499e69fcdfe9df27551c7e989dabbfec5df19781c3e71e52c9175439f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:36.940 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 bash[65898]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:36:37.362 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:36 smithi191 podman[65917]: 2024-09-21 09:36:36.939020749 +0000 UTC m=+0.557800227 container remove 06b044499e69fcdfe9df27551c7e989dabbfec5df19781c3e71e52c9175439f5 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:37.362 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:36:37.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 systemd[1]: Stopped Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:37.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.166s CPU time. 2024-09-21T09:36:37.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 systemd[1]: Starting Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:36:37.630 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 podman[66014]: 2024-09-21 09:36:37.275787766 +0000 UTC m=+0.016445058 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:36:37.675 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 podman[66014]: 2024-09-21 09:36:37.390547353 +0000 UTC m=+0.131204697 container create 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:37.675 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 podman[66014]: 2024-09-21 09:36:37.600565433 +0000 UTC m=+0.341222712 container init 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:37.675 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 podman[66014]: 2024-09-21 09:36:37.605404575 +0000 UTC m=+0.346061845 container start 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:36:37.985 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-mon[29313]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:37.985 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-mon[41325]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:37.985 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STOPPING 2024-09-21T09:36:37.985 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:36:37.985 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STOPPED 2024-09-21T09:36:37.985 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STARTING 2024-09-21T09:36:37.986 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Serving on http://:::9283 2024-09-21T09:36:37.986 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STARTED 2024-09-21T09:36:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-mon[33301]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:38.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.628Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:36:38.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.628Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:36:38.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.628Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi191 (none))" 2024-09-21T09:36:38.114 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.628Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:36:38.114 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.629Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:36:38.114 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.630Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:36:38.114 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.631Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:36:38.114 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.632Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:36:38.114 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.632Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.634Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.634Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.794µs 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.634Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.635Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.636Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.636Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.016µs wal_replay_duration=1.702919ms wbl_replay_duration=133ns total_replay_duration=1.73933ms 2024-09-21T09:36:38.115 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.637Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.637Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.637Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.657Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.976212ms db_storage=701ns remote_storage=941ns web_handler=375ns query_engine=586ns scrape=374.776µs scrape_sd=75.574µs notify=11.736µs notify_sd=17.322µs rules=19.182055ms tracing=5.849µs 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.657Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:36:37.657Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 bash[66014]: 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc 2024-09-21T09:36:38.116 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:36:37 smithi191 systemd[1]: Started Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:36:38.250 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STOPPING 2024-09-21T09:36:38.251 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:36:38.251 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STOPPED 2024-09-21T09:36:38.251 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:37] ENGINE Bus STARTING 2024-09-21T09:36:38.251 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Serving on http://:::9283 2024-09-21T09:36:38.251 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Bus STARTED 2024-09-21T09:36:38.251 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Bus STOPPING 2024-09-21T09:36:38.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:36:38.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Bus STOPPED 2024-09-21T09:36:38.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Bus STARTING 2024-09-21T09:36:38.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Serving on http://:::9283 2024-09-21T09:36:38.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:36:38] ENGINE Bus STARTED 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi076.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi076.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi076.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi076.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:38.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:38.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:38.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:38 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi076.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi076.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi191.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi191.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:38 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:39 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:40.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-21T09:36:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-21T09:36:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:36:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[29290]: 2024-09-21T09:36:39.878+0000 7f2e7135b640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:40.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T09:36:41.023 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:36:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: fsmap foo:0 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: Saving service mds.foo spec with placement count:2 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:40 smithi191 ceph-mon[33301]: pgmap v54: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:41.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: fsmap foo:0 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: Saving service mds.foo spec with placement count:2 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[29313]: pgmap v54: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-21T09:36:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: fsmap foo:0 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: Saving service mds.foo spec with placement count:2 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:40 smithi076 ceph-mon[41325]: pgmap v54: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:41 smithi191 ceph-mon[33301]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:36:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:41 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:41 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:42.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:41 smithi076 ceph-mon[29313]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:36:42.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:41 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:42.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:41 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:42.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:41 smithi076 ceph-mon[41325]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:36:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:41 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:41 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.123 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:36:42.795Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003878794s 2024-09-21T09:36:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: Deploying daemon mds.foo.smithi076.jymebx on smithi076 2024-09-21T09:36:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[29313]: pgmap v57: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:43.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:43.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:43.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: Deploying daemon mds.foo.smithi076.jymebx on smithi076 2024-09-21T09:36:43.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:42 smithi076 ceph-mon[41325]: pgmap v57: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:36:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: Deploying daemon mds.foo.smithi076.jymebx on smithi076 2024-09-21T09:36:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:42 smithi191 ceph-mon[33301]: pgmap v57: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:45.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T09:36:45.386 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:45 smithi191 ceph-mon[33301]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:46.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:45 smithi076 ceph-mon[41325]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:45 smithi076 ceph-mon[29313]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:47.995 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:47.995 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:47.995 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:47.995 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:47.995 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:47.996 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:47.996 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:47.996 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:47 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:48.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:48.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:48.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:48.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:48.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:36:48.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:47 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:48.399 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:36:48.413 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:36:48.413 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:36:48.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:36:48.614 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:48.905 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: Deploying daemon mds.foo.smithi191.dhrpzx on smithi191 2024-09-21T09:36:48.905 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:boot 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: daemon mds.foo.smithi076.jymebx assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: Cluster is now healthy 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: fsmap foo:0 1 up:standby 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:creating} 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: daemon mds.foo.smithi076.jymebx is now active in filesystem foo as rank 0 2024-09-21T09:36:48.906 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:48 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:36:48] "GET /metrics HTTP/1.1" 200 33416 "" "Prometheus/2.43.0" 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: Deploying daemon mds.foo.smithi191.dhrpzx on smithi191 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:boot 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: daemon mds.foo.smithi076.jymebx assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:36:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: Cluster is now healthy 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: fsmap foo:0 1 up:standby 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: fsmap foo:1 {0=foo.smithi076.jymebx=up:creating} 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: daemon mds.foo.smithi076.jymebx is now active in filesystem foo as rank 0 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: Deploying daemon mds.foo.smithi191.dhrpzx on smithi191 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:boot 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: daemon mds.foo.smithi076.jymebx assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: Cluster is now healthy 2024-09-21T09:36:49.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: fsmap foo:0 1 up:standby 2024-09-21T09:36:49.017 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:creating} 2024-09-21T09:36:49.017 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: daemon mds.foo.smithi076.jymebx is now active in filesystem foo as rank 0 2024-09-21T09:36:49.017 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:48 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:36:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:49 smithi191 ceph-mon[33301]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:49 smithi191 ceph-mon[33301]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:active 2024-09-21T09:36:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:49 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 2024-09-21T09:36:49.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:49 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:49 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:49 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[29313]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:49.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[29313]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:active 2024-09-21T09:36:49.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[29313]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[41325]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[41325]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:active 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:49 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.662 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:36:50.738 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: mds.? [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] up:boot 2024-09-21T09:36:50.738 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:36:50.738 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:50.738 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: mds.? [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] up:boot 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:50.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1501309288' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:36:50.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: mds.? [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] up:boot 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:50 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1501309288' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:36:51.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:51.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:50 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1501309288' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:36:51.721 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":21,"flags":0,"active_gid":24326,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6800","nonce":23865699},{"type":"v1","addr":"172.21.15.76:6801","nonce":23865699}]},"active_addr":"172.21.15.76:6801/23865699","active_change":"2024-09-21T09:35:18.636663+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24331,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":3117602468}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":1280485976}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":1616224256}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":3392089630}]}]} 2024-09-21T09:36:51.724 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:36:51.724 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:36:51.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:36:51.859 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:51 smithi191 ceph-mon[33301]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T09:36:52.059 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:52.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:51 smithi076 ceph-mon[29313]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T09:36:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:51 smithi076 ceph-mon[41325]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-21T09:36:52.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:52 smithi076 ceph-mon[29313]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:active 2024-09-21T09:36:52.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:52 smithi076 ceph-mon[29313]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:52.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:52 smithi076 ceph-mon[41325]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:active 2024-09-21T09:36:52.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:52 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:53.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:52 smithi191 ceph-mon[33301]: mds.? [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] up:active 2024-09-21T09:36:53.090 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:52 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:53.822 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:36:53.823 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":71,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:36:41.895694+0000","last_up_change":"2024-09-21T09:34:48.446984+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6803","nonce":2043782549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6805","nonce":2043782549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6809","nonce":2043782549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6807","nonce":2043782549}]},"public_addr":"172.21.15.76:6803/2043782549","cluster_addr":"172.21.15.76:6805/2043782549","heartbeat_back_addr":"172.21.15.76:6809/2043782549","heartbeat_front_addr":"172.21.15.76:6807/2043782549","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6811","nonce":637600860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6813","nonce":637600860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6817","nonce":637600860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6815","nonce":637600860}]},"public_addr":"172.21.15.76:6811/637600860","cluster_addr":"172.21.15.76:6813/637600860","heartbeat_back_addr":"172.21.15.76:6817/637600860","heartbeat_front_addr":"172.21.15.76:6815/637600860","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6819","nonce":2738257205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6821","nonce":2738257205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6825","nonce":2738257205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6823","nonce":2738257205}]},"public_addr":"172.21.15.76:6819/2738257205","cluster_addr":"172.21.15.76:6821/2738257205","heartbeat_back_addr":"172.21.15.76:6825/2738257205","heartbeat_front_addr":"172.21.15.76:6823/2738257205","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6827","nonce":666372866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6829","nonce":666372866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6833","nonce":666372866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6831","nonce":666372866}]},"public_addr":"172.21.15.76:6827/666372866","cluster_addr":"172.21.15.76:6829/666372866","heartbeat_back_addr":"172.21.15.76:6833/666372866","heartbeat_front_addr":"172.21.15.76:6831/666372866","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[29313]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[41325]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:53 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:53 smithi191 ceph-mon[33301]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:36:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:53 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:53 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:53 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:54.621 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:36:54.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:36:54.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:54 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1832472537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:54.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:54 smithi076 ceph-mon[41325]: mds.? [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] up:standby 2024-09-21T09:36:54.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:54 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:54.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:54 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1832472537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:54.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:54 smithi076 ceph-mon[29313]: mds.? [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] up:standby 2024-09-21T09:36:54.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:54 smithi076 ceph-mon[29313]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:54 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1832472537' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:54 smithi191 ceph-mon[33301]: mds.? [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] up:standby 2024-09-21T09:36:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:54 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:36:55.138 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:55.918 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:55 smithi076 ceph-mon[29313]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:36:55.918 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:55 smithi076 ceph-mon[41325]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:36:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:55 smithi191 ceph-mon[33301]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:36:57.651 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:36:57.651 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":71,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:36:41.895694+0000","last_up_change":"2024-09-21T09:34:48.446984+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6803","nonce":2043782549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6805","nonce":2043782549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6809","nonce":2043782549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":2043782549},{"type":"v1","addr":"172.21.15.76:6807","nonce":2043782549}]},"public_addr":"172.21.15.76:6803/2043782549","cluster_addr":"172.21.15.76:6805/2043782549","heartbeat_back_addr":"172.21.15.76:6809/2043782549","heartbeat_front_addr":"172.21.15.76:6807/2043782549","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6811","nonce":637600860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6813","nonce":637600860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6817","nonce":637600860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":637600860},{"type":"v1","addr":"172.21.15.76:6815","nonce":637600860}]},"public_addr":"172.21.15.76:6811/637600860","cluster_addr":"172.21.15.76:6813/637600860","heartbeat_back_addr":"172.21.15.76:6817/637600860","heartbeat_front_addr":"172.21.15.76:6815/637600860","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6819","nonce":2738257205}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6821","nonce":2738257205}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6825","nonce":2738257205}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":2738257205},{"type":"v1","addr":"172.21.15.76:6823","nonce":2738257205}]},"public_addr":"172.21.15.76:6819/2738257205","cluster_addr":"172.21.15.76:6821/2738257205","heartbeat_back_addr":"172.21.15.76:6825/2738257205","heartbeat_front_addr":"172.21.15.76:6823/2738257205","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6827","nonce":666372866}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6829","nonce":666372866}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6833","nonce":666372866}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":666372866},{"type":"v1","addr":"172.21.15.76:6831","nonce":666372866}]},"public_addr":"172.21.15.76:6827/666372866","cluster_addr":"172.21.15.76:6829/666372866","heartbeat_back_addr":"172.21.15.76:6833/666372866","heartbeat_front_addr":"172.21.15.76:6831/666372866","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:36:58.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/54527443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:58.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:58.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:57 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/54527443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:36:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:36:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:36:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:36:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:36:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:57 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/54527443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:36:59.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:36:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:36:58] "GET /metrics HTTP/1.1" 200 33416 "" "Prometheus/2.43.0" 2024-09-21T09:36:59.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:36:59.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:36:59.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:36:59.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:36:59.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:36:59.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:36:59.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:36:59.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:36:59.681 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.684 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.688 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.688 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.688 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.689 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.705 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:36:59.714 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:00.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:36:59 smithi076 ceph-mon[29313]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:37:00.037 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:36:59 smithi076 ceph-mon[41325]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:37:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:36:59 smithi191 ceph-mon[33301]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:37:01.825 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:01 smithi076 ceph-mon[29313]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:37:01.825 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:01 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:01.825 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:01 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:01.825 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:01 smithi076 ceph-mon[41325]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:37:01.825 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:01 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:01.825 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:01 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:01 smithi191 ceph-mon[33301]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-21T09:37:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:01 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:01 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:04.011 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:03 smithi076 ceph-mon[29313]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:37:04.011 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:03 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:04.011 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:03 smithi076 ceph-mon[41325]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:37:04.011 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:03 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:03 smithi191 ceph-mon[33301]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:37:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:03 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:05.955 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:05 smithi076 ceph-mon[29313]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:37:05.955 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:05 smithi076 ceph-mon[41325]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:37:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:05 smithi191 ceph-mon[33301]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:37:08.014 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:07 smithi076 ceph-mon[29313]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:08.015 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:07 smithi076 ceph-mon[41325]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:07 smithi191 ceph-mon[33301]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:09.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:37:08] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.43.0" 2024-09-21T09:37:10.007 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:09 smithi076 ceph-mon[29313]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:10.007 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:09 smithi076 ceph-mon[41325]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:09 smithi191 ceph-mon[33301]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:12.007 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:11 smithi076 ceph-mon[29313]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:12.007 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:11 smithi076 ceph-mon[41325]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:12.029 INFO:teuthology.orchestra.run.smithi076.stdout:107374182451 2024-09-21T09:37:12.030 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:37:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:11 smithi191 ceph-mon[33301]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:13.122 INFO:teuthology.orchestra.run.smithi076.stdout:77309411383 2024-09-21T09:37:13.123 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:37:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:13 smithi191 ceph-mon[33301]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:14.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:13 smithi076 ceph-mon[29313]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:13 smithi076 ceph-mon[41325]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:14.377 INFO:teuthology.orchestra.run.smithi076.stdout:34359738433 2024-09-21T09:37:14.378 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:37:15.859 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:15 smithi076 ceph-mon[29313]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:15.860 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:15 smithi076 ceph-mon[41325]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:15 smithi191 ceph-mon[33301]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:17.183 INFO:teuthology.orchestra.run.smithi076.stdout:197568495652 2024-09-21T09:37:17.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:37:17.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[41325]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[29313]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:17.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:17.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:17 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:18.053 INFO:teuthology.orchestra.run.smithi076.stdout:128849018926 2024-09-21T09:37:18.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:37:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:17 smithi191 ceph-mon[33301]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:17 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:17 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:17 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:17 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:17 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:18.346 INFO:teuthology.orchestra.run.smithi076.stdout:163208757291 2024-09-21T09:37:18.346 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:37:18.719 INFO:teuthology.orchestra.run.smithi076.stdout:55834574910 2024-09-21T09:37:18.719 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:37:19.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-09-21T09:37:19.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:37:18] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-09-21T09:37:19.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:18 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:19.100 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:19.107 INFO:teuthology.orchestra.run.smithi076.stdout:231928234015 2024-09-21T09:37:19.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:37:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-09-21T09:37:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-09-21T09:37:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T09:37:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:18 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:19.115 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:19.116 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:19 smithi191 ceph-mon[33301]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:19 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-09-21T09:37:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:19 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-21T09:37:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:19 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:37:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:19 smithi191 ceph-mon[33301]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:37:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[29313]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[29313]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[41325]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T09:37:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:19 smithi076 ceph-mon[41325]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:37:20.423 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:20.698 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:20.815 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:20 smithi076 ceph-mon[29313]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:37:20.815 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:20 smithi076 ceph-mon[41325]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:37:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:20 smithi191 ceph-mon[33301]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:37:22.081 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:21 smithi076 ceph-mon[29313]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:22.081 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:21 smithi076 ceph-mon[41325]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:21 smithi191 ceph-mon[33301]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:23.591 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:23.918 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:23 smithi076 ceph-mon[29313]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:23.919 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:23 smithi076 ceph-mon[41325]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:23 smithi191 ceph-mon[33301]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:24.169 INFO:teuthology.orchestra.run.smithi076.stdout:34359738436 2024-09-21T09:37:24.883 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738436 for osd.0 2024-09-21T09:37:24.883 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:25.057 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:24 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3105530734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:37:25.057 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:24 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3105530734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:37:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:24 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3105530734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:37:25.491 INFO:teuthology.orchestra.run.smithi076.stdout:231928234018 2024-09-21T09:37:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:25 smithi191 ceph-mon[33301]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:25 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/911047848' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:37:26.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:25 smithi076 ceph-mon[41325]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:26.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:25 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/911047848' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:37:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:25 smithi076 ceph-mon[29313]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:25 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/911047848' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:37:26.539 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:26.695 INFO:teuthology.orchestra.run.smithi076.stdout:128849018929 2024-09-21T09:37:27.039 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:26 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1784610844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:37:27.039 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:26 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1784610844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:37:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:26 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1784610844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:37:27.820 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018926 got 128849018929 for osd.4 2024-09-21T09:37:27.820 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:28.086 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:27 smithi076 ceph-mon[41325]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:28.086 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:27 smithi076 ceph-mon[29313]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:27 smithi191 ceph-mon[33301]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:28.156 INFO:teuthology.orchestra.run.smithi076.stdout:77309411387 2024-09-21T09:37:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:28 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/551475436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:37:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:28 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/551475436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:37:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:28 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/551475436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:37:29.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:37:28] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2024-09-21T09:37:29.309 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234015 got 231928234018 for osd.7 2024-09-21T09:37:29.309 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:29.736 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411387 for osd.2 2024-09-21T09:37:29.736 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:29.974 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:29 smithi076 ceph-mon[29313]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:29.974 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:29 smithi076 ceph-mon[41325]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:29 smithi191 ceph-mon[33301]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:30.189 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:31 smithi191 ceph-mon[33301]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:32.121 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:31 smithi076 ceph-mon[29313]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:32.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:31 smithi076 ceph-mon[41325]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:32.225 INFO:teuthology.orchestra.run.smithi076.stdout:55834574913 2024-09-21T09:37:33.020 INFO:teuthology.orchestra.run.smithi076.stdout:107374182455 2024-09-21T09:37:33.021 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:32 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1035853747' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:37:33.021 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:32 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1035853747' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:37:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:32 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1035853747' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:37:33.120 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574910 got 55834574913 for osd.1 2024-09-21T09:37:33.120 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:33.937 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:33 smithi076 ceph-mon[29313]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:33.937 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:33 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2255470933' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:37:33.937 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:33 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:33.938 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:33 smithi076 ceph-mon[41325]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:33.938 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:33 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2255470933' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:37:33.938 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:33 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:33 smithi191 ceph-mon[33301]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:33 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2255470933' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:37:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:33 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:34.554 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182451 got 107374182455 for osd.3 2024-09-21T09:37:34.555 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:35.797 INFO:teuthology.orchestra.run.smithi076.stdout:197568495657 2024-09-21T09:37:36.104 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:35 smithi076 ceph-mon[29313]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:36.104 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:35 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1712466554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:37:36.105 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:35 smithi076 ceph-mon[41325]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:36.105 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:35 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1712466554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:37:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:35 smithi191 ceph-mon[33301]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:35 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1712466554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:37:36.493 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495652 got 197568495657 for osd.6 2024-09-21T09:37:36.494 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:36.862 INFO:teuthology.orchestra.run.smithi076.stdout:163208757295 2024-09-21T09:37:37.566 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757291 got 163208757295 for osd.5 2024-09-21T09:37:37.566 DEBUG:teuthology.parallel:result is None 2024-09-21T09:37:37.566 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:37:37.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:37:37.763 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:37 smithi191 ceph-mon[33301]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:37 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2029398061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:37:38.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:37 smithi076 ceph-mon[29313]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:38.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:37 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2029398061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:37:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:37 smithi076 ceph-mon[41325]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:37 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2029398061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:37:39.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:37:38] "GET /metrics HTTP/1.1" 200 33711 "" "Prometheus/2.43.0" 2024-09-21T09:37:39.502 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:37:39.503 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:37:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:39 smithi191 ceph-mon[33301]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:40.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:39 smithi076 ceph-mon[41325]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:40.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:39 smithi076 ceph-mon[29313]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:40.299 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-09-21T09:37:38.672184+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":232120,"kb_used_data":7756,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729096,"statfs":{"total":767960285184,"available":767722594304,"internally_reserved":0,"allocated":7942144,"data_stored":4446144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":229625400},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002502"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763655+0000","last_change":"2024-09-21T09:36:40.896108+0000","last_active":"2024-09-21T09:37:18.763655+0000","last_peered":"2024-09-21T09:37:18.763655+0000","last_clean":"2024-09-21T09:37:18.763655+0000","last_became_active":"2024-09-21T09:36:40.895551+0000","last_became_peered":"2024-09-21T09:36:40.895551+0000","last_unstale":"2024-09-21T09:37:18.763655+0000","last_undegraded":"2024-09-21T09:37:18.763655+0000","last_fullsized":"2024-09-21T09:37:18.763655+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:24.331410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905478+0000","last_change":"2024-09-21T09:36:39.881562+0000","last_active":"2024-09-21T09:37:18.905478+0000","last_peered":"2024-09-21T09:37:18.905478+0000","last_clean":"2024-09-21T09:37:18.905478+0000","last_became_active":"2024-09-21T09:36:39.881282+0000","last_became_peered":"2024-09-21T09:36:39.881282+0000","last_unstale":"2024-09-21T09:37:18.905478+0000","last_undegraded":"2024-09-21T09:37:18.905478+0000","last_fullsized":"2024-09-21T09:37:18.905478+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:58.476757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770127+0000","last_change":"2024-09-21T09:36:40.085128+0000","last_active":"2024-09-21T09:37:18.770127+0000","last_peered":"2024-09-21T09:37:18.770127+0000","last_clean":"2024-09-21T09:37:18.770127+0000","last_became_active":"2024-09-21T09:36:40.084892+0000","last_became_peered":"2024-09-21T09:36:40.084892+0000","last_unstale":"2024-09-21T09:37:18.770127+0000","last_undegraded":"2024-09-21T09:37:18.770127+0000","last_fullsized":"2024-09-21T09:37:18.770127+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:42.183643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768432+0000","last_change":"2024-09-21T09:36:39.880523+0000","last_active":"2024-09-21T09:37:18.768432+0000","last_peered":"2024-09-21T09:37:18.768432+0000","last_clean":"2024-09-21T09:37:18.768432+0000","last_became_active":"2024-09-21T09:36:39.880257+0000","last_became_peered":"2024-09-21T09:36:39.880257+0000","last_unstale":"2024-09-21T09:37:18.768432+0000","last_undegraded":"2024-09-21T09:37:18.768432+0000","last_fullsized":"2024-09-21T09:37:18.768432+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:07.507882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768730+0000","last_change":"2024-09-21T09:36:40.084600+0000","last_active":"2024-09-21T09:37:18.768730+0000","last_peered":"2024-09-21T09:37:18.768730+0000","last_clean":"2024-09-21T09:37:18.768730+0000","last_became_active":"2024-09-21T09:36:40.084332+0000","last_became_peered":"2024-09-21T09:36:40.084332+0000","last_unstale":"2024-09-21T09:37:18.768730+0000","last_undegraded":"2024-09-21T09:37:18.768730+0000","last_fullsized":"2024-09-21T09:37:18.768730+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:53.585344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905336+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:37:18.905336+0000","last_peered":"2024-09-21T09:37:18.905336+0000","last_clean":"2024-09-21T09:37:18.905336+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:37:18.905336+0000","last_undegraded":"2024-09-21T09:37:18.905336+0000","last_fullsized":"2024-09-21T09:37:18.905336+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764135+0000","last_change":"2024-09-21T09:36:40.895387+0000","last_active":"2024-09-21T09:37:18.764135+0000","last_peered":"2024-09-21T09:37:18.764135+0000","last_clean":"2024-09-21T09:37:18.764135+0000","last_became_active":"2024-09-21T09:36:40.895107+0000","last_became_peered":"2024-09-21T09:36:40.895107+0000","last_unstale":"2024-09-21T09:37:18.764135+0000","last_undegraded":"2024-09-21T09:37:18.764135+0000","last_fullsized":"2024-09-21T09:37:18.764135+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:31:56.935414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768368+0000","last_change":"2024-09-21T09:36:39.873811+0000","last_active":"2024-09-21T09:37:18.768368+0000","last_peered":"2024-09-21T09:37:18.768368+0000","last_clean":"2024-09-21T09:37:18.768368+0000","last_became_active":"2024-09-21T09:36:39.873558+0000","last_became_peered":"2024-09-21T09:36:39.873558+0000","last_unstale":"2024-09-21T09:37:18.768368+0000","last_undegraded":"2024-09-21T09:37:18.768368+0000","last_fullsized":"2024-09-21T09:37:18.768368+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:41.470084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905737+0000","last_change":"2024-09-21T09:36:39.879194+0000","last_active":"2024-09-21T09:37:18.905737+0000","last_peered":"2024-09-21T09:37:18.905737+0000","last_clean":"2024-09-21T09:37:18.905737+0000","last_became_active":"2024-09-21T09:36:39.878521+0000","last_became_peered":"2024-09-21T09:36:39.878521+0000","last_unstale":"2024-09-21T09:37:18.905737+0000","last_undegraded":"2024-09-21T09:37:18.905737+0000","last_fullsized":"2024-09-21T09:37:18.905737+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:13.986085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.769927+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:37:18.769927+0000","last_peered":"2024-09-21T09:37:18.769927+0000","last_clean":"2024-09-21T09:37:18.769927+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:37:18.769927+0000","last_undegraded":"2024-09-21T09:37:18.769927+0000","last_fullsized":"2024-09-21T09:37:18.769927+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905690+0000","last_change":"2024-09-21T09:36:39.880390+0000","last_active":"2024-09-21T09:37:18.905690+0000","last_peered":"2024-09-21T09:37:18.905690+0000","last_clean":"2024-09-21T09:37:18.905690+0000","last_became_active":"2024-09-21T09:36:39.880142+0000","last_became_peered":"2024-09-21T09:36:39.880142+0000","last_unstale":"2024-09-21T09:37:18.905690+0000","last_undegraded":"2024-09-21T09:37:18.905690+0000","last_fullsized":"2024-09-21T09:37:18.905690+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:03.490045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765668+0000","last_change":"2024-09-21T09:36:40.176250+0000","last_active":"2024-09-21T09:37:18.765668+0000","last_peered":"2024-09-21T09:37:18.765668+0000","last_clean":"2024-09-21T09:37:18.765668+0000","last_became_active":"2024-09-21T09:36:40.175949+0000","last_became_peered":"2024-09-21T09:36:40.175949+0000","last_unstale":"2024-09-21T09:37:18.765668+0000","last_undegraded":"2024-09-21T09:37:18.765668+0000","last_fullsized":"2024-09-21T09:37:18.765668+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:47.315693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905663+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:37:18.905663+0000","last_peered":"2024-09-21T09:37:18.905663+0000","last_clean":"2024-09-21T09:37:18.905663+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:37:18.905663+0000","last_undegraded":"2024-09-21T09:37:18.905663+0000","last_fullsized":"2024-09-21T09:37:18.905663+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767917+0000","last_change":"2024-09-21T09:36:40.176231+0000","last_active":"2024-09-21T09:37:18.767917+0000","last_peered":"2024-09-21T09:37:18.767917+0000","last_clean":"2024-09-21T09:37:18.767917+0000","last_became_active":"2024-09-21T09:36:40.175855+0000","last_became_peered":"2024-09-21T09:36:40.175855+0000","last_unstale":"2024-09-21T09:37:18.767917+0000","last_undegraded":"2024-09-21T09:37:18.767917+0000","last_fullsized":"2024-09-21T09:37:18.767917+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:05:31.249444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770141+0000","last_change":"2024-09-21T09:36:39.878498+0000","last_active":"2024-09-21T09:37:18.770141+0000","last_peered":"2024-09-21T09:37:18.770141+0000","last_clean":"2024-09-21T09:37:18.770141+0000","last_became_active":"2024-09-21T09:36:39.878237+0000","last_became_peered":"2024-09-21T09:36:39.878237+0000","last_unstale":"2024-09-21T09:37:18.770141+0000","last_undegraded":"2024-09-21T09:37:18.770141+0000","last_fullsized":"2024-09-21T09:37:18.770141+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:16.444276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.891302+0000","last_change":"2024-09-21T09:37:19.891302+0000","last_active":"2024-09-21T09:37:19.891302+0000","last_peered":"2024-09-21T09:37:19.891302+0000","last_clean":"2024-09-21T09:37:19.891302+0000","last_became_active":"2024-09-21T09:37:19.890936+0000","last_became_peered":"2024-09-21T09:37:19.890936+0000","last_unstale":"2024-09-21T09:37:19.891302+0000","last_undegraded":"2024-09-21T09:37:19.891302+0000","last_fullsized":"2024-09-21T09:37:19.891302+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:08.589525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763558+0000","last_change":"2024-09-21T09:36:39.890215+0000","last_active":"2024-09-21T09:37:18.763558+0000","last_peered":"2024-09-21T09:37:18.763558+0000","last_clean":"2024-09-21T09:37:18.763558+0000","last_became_active":"2024-09-21T09:36:39.889954+0000","last_became_peered":"2024-09-21T09:36:39.889954+0000","last_unstale":"2024-09-21T09:37:18.763558+0000","last_undegraded":"2024-09-21T09:37:18.763558+0000","last_fullsized":"2024-09-21T09:37:18.763558+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:39.812907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765911+0000","last_change":"2024-09-21T09:36:39.895269+0000","last_active":"2024-09-21T09:37:18.765911+0000","last_peered":"2024-09-21T09:37:18.765911+0000","last_clean":"2024-09-21T09:37:18.765911+0000","last_became_active":"2024-09-21T09:36:39.894941+0000","last_became_peered":"2024-09-21T09:36:39.894941+0000","last_unstale":"2024-09-21T09:37:18.765911+0000","last_undegraded":"2024-09-21T09:37:18.765911+0000","last_fullsized":"2024-09-21T09:37:18.765911+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:38.574808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768053+0000","last_change":"2024-09-21T09:36:39.876805+0000","last_active":"2024-09-21T09:37:18.768053+0000","last_peered":"2024-09-21T09:37:18.768053+0000","last_clean":"2024-09-21T09:37:18.768053+0000","last_became_active":"2024-09-21T09:36:39.876308+0000","last_became_peered":"2024-09-21T09:36:39.876308+0000","last_unstale":"2024-09-21T09:37:18.768053+0000","last_undegraded":"2024-09-21T09:37:18.768053+0000","last_fullsized":"2024-09-21T09:37:18.768053+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:00.708481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766077+0000","last_change":"2024-09-21T09:36:40.177666+0000","last_active":"2024-09-21T09:37:18.766077+0000","last_peered":"2024-09-21T09:37:18.766077+0000","last_clean":"2024-09-21T09:37:18.766077+0000","last_became_active":"2024-09-21T09:36:40.177355+0000","last_became_peered":"2024-09-21T09:36:40.177355+0000","last_unstale":"2024-09-21T09:37:18.766077+0000","last_undegraded":"2024-09-21T09:37:18.766077+0000","last_fullsized":"2024-09-21T09:37:18.766077+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:17.332240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765792+0000","last_change":"2024-09-21T09:36:39.884045+0000","last_active":"2024-09-21T09:37:18.765792+0000","last_peered":"2024-09-21T09:37:18.765792+0000","last_clean":"2024-09-21T09:37:18.765792+0000","last_became_active":"2024-09-21T09:36:39.883609+0000","last_became_peered":"2024-09-21T09:36:39.883609+0000","last_unstale":"2024-09-21T09:37:18.765792+0000","last_undegraded":"2024-09-21T09:37:18.765792+0000","last_fullsized":"2024-09-21T09:37:18.765792+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:54:17.035411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766095+0000","last_change":"2024-09-21T09:36:40.178208+0000","last_active":"2024-09-21T09:37:18.766095+0000","last_peered":"2024-09-21T09:37:18.766095+0000","last_clean":"2024-09-21T09:37:18.766095+0000","last_became_active":"2024-09-21T09:36:40.177971+0000","last_became_peered":"2024-09-21T09:36:40.177971+0000","last_unstale":"2024-09-21T09:37:18.766095+0000","last_undegraded":"2024-09-21T09:37:18.766095+0000","last_fullsized":"2024-09-21T09:37:18.766095+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:21.635698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763211+0000","last_change":"2024-09-21T09:36:39.884071+0000","last_active":"2024-09-21T09:37:18.763211+0000","last_peered":"2024-09-21T09:37:18.763211+0000","last_clean":"2024-09-21T09:37:18.763211+0000","last_became_active":"2024-09-21T09:36:39.883795+0000","last_became_peered":"2024-09-21T09:36:39.883795+0000","last_unstale":"2024-09-21T09:37:18.763211+0000","last_undegraded":"2024-09-21T09:37:18.763211+0000","last_fullsized":"2024-09-21T09:37:18.763211+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:21:54.070798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768074+0000","last_change":"2024-09-21T09:36:40.980176+0000","last_active":"2024-09-21T09:37:18.768074+0000","last_peered":"2024-09-21T09:37:18.768074+0000","last_clean":"2024-09-21T09:37:18.768074+0000","last_became_active":"2024-09-21T09:36:40.979895+0000","last_became_peered":"2024-09-21T09:36:40.979895+0000","last_unstale":"2024-09-21T09:37:18.768074+0000","last_undegraded":"2024-09-21T09:37:18.768074+0000","last_fullsized":"2024-09-21T09:37:18.768074+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:44:57.080348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905301+0000","last_change":"2024-09-21T09:34:52.468106+0000","last_active":"2024-09-21T09:37:18.905301+0000","last_peered":"2024-09-21T09:37:18.905301+0000","last_clean":"2024-09-21T09:37:18.905301+0000","last_became_active":"2024-09-21T09:34:52.467638+0000","last_became_peered":"2024-09-21T09:34:52.467638+0000","last_unstale":"2024-09-21T09:37:18.905301+0000","last_undegraded":"2024-09-21T09:37:18.905301+0000","last_fullsized":"2024-09-21T09:37:18.905301+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:35.011128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.890448+0000","last_change":"2024-09-21T09:35:30.787751+0000","last_active":"2024-09-21T09:37:19.890448+0000","last_peered":"2024-09-21T09:37:19.890448+0000","last_clean":"2024-09-21T09:37:19.890448+0000","last_became_active":"2024-09-21T09:35:24.753144+0000","last_became_peered":"2024-09-21T09:35:24.753144+0000","last_unstale":"2024-09-21T09:37:19.890448+0000","last_undegraded":"2024-09-21T09:37:19.890448+0000","last_fullsized":"2024-09-21T09:37:19.890448+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:50.550632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763515+0000","last_change":"2024-09-21T09:36:39.883277+0000","last_active":"2024-09-21T09:37:18.763515+0000","last_peered":"2024-09-21T09:37:18.763515+0000","last_clean":"2024-09-21T09:37:18.763515+0000","last_became_active":"2024-09-21T09:36:39.882971+0000","last_became_peered":"2024-09-21T09:36:39.882971+0000","last_unstale":"2024-09-21T09:37:18.763515+0000","last_undegraded":"2024-09-21T09:37:18.763515+0000","last_fullsized":"2024-09-21T09:37:18.763515+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:16.821287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768073+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:37:18.768073+0000","last_peered":"2024-09-21T09:37:18.768073+0000","last_clean":"2024-09-21T09:37:18.768073+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:37:18.768073+0000","last_undegraded":"2024-09-21T09:37:18.768073+0000","last_fullsized":"2024-09-21T09:37:18.768073+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.229581+0000","last_change":"2024-09-21T09:35:31.069584+0000","last_active":"2024-09-21T09:37:20.229581+0000","last_peered":"2024-09-21T09:37:20.229581+0000","last_clean":"2024-09-21T09:37:20.229581+0000","last_became_active":"2024-09-21T09:35:24.766184+0000","last_became_peered":"2024-09-21T09:35:24.766184+0000","last_unstale":"2024-09-21T09:37:20.229581+0000","last_undegraded":"2024-09-21T09:37:20.229581+0000","last_fullsized":"2024-09-21T09:37:20.229581+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:16.728713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768209+0000","last_change":"2024-09-21T09:36:39.884020+0000","last_active":"2024-09-21T09:37:18.768209+0000","last_peered":"2024-09-21T09:37:18.768209+0000","last_clean":"2024-09-21T09:37:18.768209+0000","last_became_active":"2024-09-21T09:36:39.882217+0000","last_became_peered":"2024-09-21T09:36:39.882217+0000","last_unstale":"2024-09-21T09:37:18.768209+0000","last_undegraded":"2024-09-21T09:37:18.768209+0000","last_fullsized":"2024-09-21T09:37:18.768209+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:06.639754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768663+0000","last_change":"2024-09-21T09:36:40.176330+0000","last_active":"2024-09-21T09:37:18.768663+0000","last_peered":"2024-09-21T09:37:18.768663+0000","last_clean":"2024-09-21T09:37:18.768663+0000","last_became_active":"2024-09-21T09:36:40.176046+0000","last_became_peered":"2024-09-21T09:36:40.176046+0000","last_unstale":"2024-09-21T09:37:18.768663+0000","last_undegraded":"2024-09-21T09:37:18.768663+0000","last_fullsized":"2024-09-21T09:37:18.768663+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:55.382440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.766818+0000","last_change":"2024-09-21T09:35:30.783775+0000","last_active":"2024-09-21T09:37:19.766818+0000","last_peered":"2024-09-21T09:37:19.766818+0000","last_clean":"2024-09-21T09:37:19.766818+0000","last_became_active":"2024-09-21T09:35:24.766096+0000","last_became_peered":"2024-09-21T09:35:24.766096+0000","last_unstale":"2024-09-21T09:37:19.766818+0000","last_undegraded":"2024-09-21T09:37:19.766818+0000","last_fullsized":"2024-09-21T09:37:19.766818+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.155489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767588+0000","last_change":"2024-09-21T09:36:39.878248+0000","last_active":"2024-09-21T09:37:18.767588+0000","last_peered":"2024-09-21T09:37:18.767588+0000","last_clean":"2024-09-21T09:37:18.767588+0000","last_became_active":"2024-09-21T09:36:39.877981+0000","last_became_peered":"2024-09-21T09:36:39.877981+0000","last_unstale":"2024-09-21T09:37:18.767588+0000","last_undegraded":"2024-09-21T09:37:18.767588+0000","last_fullsized":"2024-09-21T09:37:18.767588+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:03.622897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768089+0000","last_change":"2024-09-21T09:36:40.969925+0000","last_active":"2024-09-21T09:37:18.768089+0000","last_peered":"2024-09-21T09:37:18.768089+0000","last_clean":"2024-09-21T09:37:18.768089+0000","last_became_active":"2024-09-21T09:36:40.969439+0000","last_became_peered":"2024-09-21T09:36:40.969439+0000","last_unstale":"2024-09-21T09:37:18.768089+0000","last_undegraded":"2024-09-21T09:37:18.768089+0000","last_fullsized":"2024-09-21T09:37:18.768089+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:28.961156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.890488+0000","last_change":"2024-09-21T09:35:30.788301+0000","last_active":"2024-09-21T09:37:19.890488+0000","last_peered":"2024-09-21T09:37:19.890488+0000","last_clean":"2024-09-21T09:37:19.890488+0000","last_became_active":"2024-09-21T09:35:24.753321+0000","last_became_peered":"2024-09-21T09:35:24.753321+0000","last_unstale":"2024-09-21T09:37:19.890488+0000","last_undegraded":"2024-09-21T09:37:19.890488+0000","last_fullsized":"2024-09-21T09:37:19.890488+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:52.357133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766345+0000","last_change":"2024-09-21T09:36:39.879835+0000","last_active":"2024-09-21T09:37:18.766345+0000","last_peered":"2024-09-21T09:37:18.766345+0000","last_clean":"2024-09-21T09:37:18.766345+0000","last_became_active":"2024-09-21T09:36:39.879284+0000","last_became_peered":"2024-09-21T09:36:39.879284+0000","last_unstale":"2024-09-21T09:37:18.766345+0000","last_undegraded":"2024-09-21T09:37:18.766345+0000","last_fullsized":"2024-09-21T09:37:18.766345+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:43.176096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767834+0000","last_change":"2024-09-21T09:36:40.896338+0000","last_active":"2024-09-21T09:37:18.767834+0000","last_peered":"2024-09-21T09:37:18.767834+0000","last_clean":"2024-09-21T09:37:18.767834+0000","last_became_active":"2024-09-21T09:36:40.895996+0000","last_became_peered":"2024-09-21T09:36:40.895996+0000","last_unstale":"2024-09-21T09:37:18.767834+0000","last_undegraded":"2024-09-21T09:37:18.767834+0000","last_fullsized":"2024-09-21T09:37:18.767834+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:33.010247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770245+0000","last_change":"2024-09-21T09:36:39.879239+0000","last_active":"2024-09-21T09:37:18.770245+0000","last_peered":"2024-09-21T09:37:18.770245+0000","last_clean":"2024-09-21T09:37:18.770245+0000","last_became_active":"2024-09-21T09:36:39.878988+0000","last_became_peered":"2024-09-21T09:36:39.878988+0000","last_unstale":"2024-09-21T09:37:18.770245+0000","last_undegraded":"2024-09-21T09:37:18.770245+0000","last_fullsized":"2024-09-21T09:37:18.770245+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:58.839553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.582448+0000","last_change":"2024-09-21T09:35:30.789143+0000","last_active":"2024-09-21T09:37:20.582448+0000","last_peered":"2024-09-21T09:37:20.582448+0000","last_clean":"2024-09-21T09:37:20.582448+0000","last_became_active":"2024-09-21T09:35:24.766244+0000","last_became_peered":"2024-09-21T09:35:24.766244+0000","last_unstale":"2024-09-21T09:37:20.582448+0000","last_undegraded":"2024-09-21T09:37:20.582448+0000","last_fullsized":"2024-09-21T09:37:20.582448+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:30.932253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767587+0000","last_change":"2024-09-21T09:36:40.980366+0000","last_active":"2024-09-21T09:37:18.767587+0000","last_peered":"2024-09-21T09:37:18.767587+0000","last_clean":"2024-09-21T09:37:18.767587+0000","last_became_active":"2024-09-21T09:36:40.979656+0000","last_became_peered":"2024-09-21T09:36:40.979656+0000","last_unstale":"2024-09-21T09:37:18.767587+0000","last_undegraded":"2024-09-21T09:37:18.767587+0000","last_fullsized":"2024-09-21T09:37:18.767587+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:22.779091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767871+0000","last_change":"2024-09-21T09:36:39.885841+0000","last_active":"2024-09-21T09:37:18.767871+0000","last_peered":"2024-09-21T09:37:18.767871+0000","last_clean":"2024-09-21T09:37:18.767871+0000","last_became_active":"2024-09-21T09:36:39.885556+0000","last_became_peered":"2024-09-21T09:36:39.885556+0000","last_unstale":"2024-09-21T09:37:18.767871+0000","last_undegraded":"2024-09-21T09:37:18.767871+0000","last_fullsized":"2024-09-21T09:37:18.767871+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:14:14.747580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.229814+0000","last_change":"2024-09-21T09:35:31.069545+0000","last_active":"2024-09-21T09:37:20.229814+0000","last_peered":"2024-09-21T09:37:20.229814+0000","last_clean":"2024-09-21T09:37:20.229814+0000","last_became_active":"2024-09-21T09:35:24.766462+0000","last_became_peered":"2024-09-21T09:35:24.766462+0000","last_unstale":"2024-09-21T09:37:20.229814+0000","last_undegraded":"2024-09-21T09:37:20.229814+0000","last_fullsized":"2024-09-21T09:37:20.229814+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:48.059267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.769166+0000","last_change":"2024-09-21T09:36:40.084429+0000","last_active":"2024-09-21T09:37:18.769166+0000","last_peered":"2024-09-21T09:37:18.769166+0000","last_clean":"2024-09-21T09:37:18.769166+0000","last_became_active":"2024-09-21T09:36:40.084124+0000","last_became_peered":"2024-09-21T09:36:40.084124+0000","last_unstale":"2024-09-21T09:37:18.769166+0000","last_undegraded":"2024-09-21T09:37:18.769166+0000","last_fullsized":"2024-09-21T09:37:18.769166+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:13:42.274683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764216+0000","last_change":"2024-09-21T09:36:39.886972+0000","last_active":"2024-09-21T09:37:18.764216+0000","last_peered":"2024-09-21T09:37:18.764216+0000","last_clean":"2024-09-21T09:37:18.764216+0000","last_became_active":"2024-09-21T09:36:39.885739+0000","last_became_peered":"2024-09-21T09:36:39.885739+0000","last_unstale":"2024-09-21T09:37:18.764216+0000","last_undegraded":"2024-09-21T09:37:18.764216+0000","last_fullsized":"2024-09-21T09:37:18.764216+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:03.951390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.582862+0000","last_change":"2024-09-21T09:35:30.789220+0000","last_active":"2024-09-21T09:37:20.582862+0000","last_peered":"2024-09-21T09:37:20.582862+0000","last_clean":"2024-09-21T09:37:20.582862+0000","last_became_active":"2024-09-21T09:35:24.767340+0000","last_became_peered":"2024-09-21T09:35:24.767340+0000","last_unstale":"2024-09-21T09:37:20.582862+0000","last_undegraded":"2024-09-21T09:37:20.582862+0000","last_fullsized":"2024-09-21T09:37:20.582862+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:48.181309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036756400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763118+0000","last_change":"2024-09-21T09:36:40.933241+0000","last_active":"2024-09-21T09:37:18.763118+0000","last_peered":"2024-09-21T09:37:18.763118+0000","last_clean":"2024-09-21T09:37:18.763118+0000","last_became_active":"2024-09-21T09:36:40.932905+0000","last_became_peered":"2024-09-21T09:36:40.932905+0000","last_unstale":"2024-09-21T09:37:18.763118+0000","last_undegraded":"2024-09-21T09:37:18.763118+0000","last_fullsized":"2024-09-21T09:37:18.763118+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:07.761909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768806+0000","last_change":"2024-09-21T09:36:39.882383+0000","last_active":"2024-09-21T09:37:18.768806+0000","last_peered":"2024-09-21T09:37:18.768806+0000","last_clean":"2024-09-21T09:37:18.768806+0000","last_became_active":"2024-09-21T09:36:39.881756+0000","last_became_peered":"2024-09-21T09:36:39.881756+0000","last_unstale":"2024-09-21T09:37:18.768806+0000","last_undegraded":"2024-09-21T09:37:18.768806+0000","last_fullsized":"2024-09-21T09:37:18.768806+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:27.240205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.767540+0000","last_change":"2024-09-21T09:35:30.789192+0000","last_active":"2024-09-21T09:37:19.767540+0000","last_peered":"2024-09-21T09:37:19.767540+0000","last_clean":"2024-09-21T09:37:19.767540+0000","last_became_active":"2024-09-21T09:35:24.754596+0000","last_became_peered":"2024-09-21T09:35:24.754596+0000","last_unstale":"2024-09-21T09:37:19.767540+0000","last_undegraded":"2024-09-21T09:37:19.767540+0000","last_fullsized":"2024-09-21T09:37:19.767540+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:39.865349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765772+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:37:18.765772+0000","last_peered":"2024-09-21T09:37:18.765772+0000","last_clean":"2024-09-21T09:37:18.765772+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:37:18.765772+0000","last_undegraded":"2024-09-21T09:37:18.765772+0000","last_fullsized":"2024-09-21T09:37:18.765772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766300+0000","last_change":"2024-09-21T09:36:40.177744+0000","last_active":"2024-09-21T09:37:18.766300+0000","last_peered":"2024-09-21T09:37:18.766300+0000","last_clean":"2024-09-21T09:37:18.766300+0000","last_became_active":"2024-09-21T09:36:40.177500+0000","last_became_peered":"2024-09-21T09:36:40.177500+0000","last_unstale":"2024-09-21T09:37:18.766300+0000","last_undegraded":"2024-09-21T09:37:18.766300+0000","last_fullsized":"2024-09-21T09:37:18.766300+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:35.517184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766015+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:37:18.766015+0000","last_peered":"2024-09-21T09:37:18.766015+0000","last_clean":"2024-09-21T09:37:18.766015+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:37:18.766015+0000","last_undegraded":"2024-09-21T09:37:18.766015+0000","last_fullsized":"2024-09-21T09:37:18.766015+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767608+0000","last_change":"2024-09-21T09:36:40.980042+0000","last_active":"2024-09-21T09:37:18.767608+0000","last_peered":"2024-09-21T09:37:18.767608+0000","last_clean":"2024-09-21T09:37:18.767608+0000","last_became_active":"2024-09-21T09:36:40.979616+0000","last_became_peered":"2024-09-21T09:36:40.979616+0000","last_unstale":"2024-09-21T09:37:18.767608+0000","last_undegraded":"2024-09-21T09:37:18.767608+0000","last_fullsized":"2024-09-21T09:37:18.767608+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:00.254599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905554+0000","last_change":"2024-09-21T09:36:39.880134+0000","last_active":"2024-09-21T09:37:18.905554+0000","last_peered":"2024-09-21T09:37:18.905554+0000","last_clean":"2024-09-21T09:37:18.905554+0000","last_became_active":"2024-09-21T09:36:39.879853+0000","last_became_peered":"2024-09-21T09:36:39.879853+0000","last_unstale":"2024-09-21T09:37:18.905554+0000","last_undegraded":"2024-09-21T09:37:18.905554+0000","last_fullsized":"2024-09-21T09:37:18.905554+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:42.799198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770194+0000","last_change":"2024-09-21T09:36:39.879888+0000","last_active":"2024-09-21T09:37:18.770194+0000","last_peered":"2024-09-21T09:37:18.770194+0000","last_clean":"2024-09-21T09:37:18.770194+0000","last_became_active":"2024-09-21T09:36:39.879622+0000","last_became_peered":"2024-09-21T09:36:39.879622+0000","last_unstale":"2024-09-21T09:37:18.770194+0000","last_undegraded":"2024-09-21T09:37:18.770194+0000","last_fullsized":"2024-09-21T09:37:18.770194+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:02:23.355344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763310+0000","last_change":"2024-09-21T09:36:40.979593+0000","last_active":"2024-09-21T09:37:18.763310+0000","last_peered":"2024-09-21T09:37:18.763310+0000","last_clean":"2024-09-21T09:37:18.763310+0000","last_became_active":"2024-09-21T09:36:40.979205+0000","last_became_peered":"2024-09-21T09:36:40.979205+0000","last_unstale":"2024-09-21T09:37:18.763310+0000","last_undegraded":"2024-09-21T09:37:18.763310+0000","last_fullsized":"2024-09-21T09:37:18.763310+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:25.107383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768866+0000","last_change":"2024-09-21T09:36:39.880706+0000","last_active":"2024-09-21T09:37:18.768866+0000","last_peered":"2024-09-21T09:37:18.768866+0000","last_clean":"2024-09-21T09:37:18.768866+0000","last_became_active":"2024-09-21T09:36:39.880094+0000","last_became_peered":"2024-09-21T09:36:39.880094+0000","last_unstale":"2024-09-21T09:37:18.768866+0000","last_undegraded":"2024-09-21T09:37:18.768866+0000","last_fullsized":"2024-09-21T09:37:18.768866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:46.918541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763070+0000","last_change":"2024-09-21T09:36:40.979477+0000","last_active":"2024-09-21T09:37:18.763070+0000","last_peered":"2024-09-21T09:37:18.763070+0000","last_clean":"2024-09-21T09:37:18.763070+0000","last_became_active":"2024-09-21T09:36:40.979165+0000","last_became_peered":"2024-09-21T09:36:40.979165+0000","last_unstale":"2024-09-21T09:37:18.763070+0000","last_undegraded":"2024-09-21T09:37:18.763070+0000","last_fullsized":"2024-09-21T09:37:18.763070+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:14.737992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.891466+0000","last_change":"2024-09-21T09:37:19.891466+0000","last_active":"2024-09-21T09:37:19.891466+0000","last_peered":"2024-09-21T09:37:19.891466+0000","last_clean":"2024-09-21T09:37:19.891466+0000","last_became_active":"2024-09-21T09:37:19.891039+0000","last_became_peered":"2024-09-21T09:37:19.891039+0000","last_unstale":"2024-09-21T09:37:19.891466+0000","last_undegraded":"2024-09-21T09:37:19.891466+0000","last_fullsized":"2024-09-21T09:37:19.891466+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:13.728638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763142+0000","last_change":"2024-09-21T09:36:39.886829+0000","last_active":"2024-09-21T09:37:18.763142+0000","last_peered":"2024-09-21T09:37:18.763142+0000","last_clean":"2024-09-21T09:37:18.763142+0000","last_became_active":"2024-09-21T09:36:39.886574+0000","last_became_peered":"2024-09-21T09:36:39.886574+0000","last_unstale":"2024-09-21T09:37:18.763142+0000","last_undegraded":"2024-09-21T09:37:18.763142+0000","last_fullsized":"2024-09-21T09:37:18.763142+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:22.122621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768586+0000","last_change":"2024-09-21T09:36:40.084441+0000","last_active":"2024-09-21T09:37:18.768586+0000","last_peered":"2024-09-21T09:37:18.768586+0000","last_clean":"2024-09-21T09:37:18.768586+0000","last_became_active":"2024-09-21T09:36:40.084159+0000","last_became_peered":"2024-09-21T09:36:40.084159+0000","last_unstale":"2024-09-21T09:37:18.768586+0000","last_undegraded":"2024-09-21T09:37:18.768586+0000","last_fullsized":"2024-09-21T09:37:18.768586+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:06.332538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767537+0000","last_change":"2024-09-21T09:36:39.883492+0000","last_active":"2024-09-21T09:37:18.767537+0000","last_peered":"2024-09-21T09:37:18.767537+0000","last_clean":"2024-09-21T09:37:18.767537+0000","last_became_active":"2024-09-21T09:36:39.883094+0000","last_became_peered":"2024-09-21T09:36:39.883094+0000","last_unstale":"2024-09-21T09:37:18.767537+0000","last_undegraded":"2024-09-21T09:37:18.767537+0000","last_fullsized":"2024-09-21T09:37:18.767537+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:48:13.399373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764136+0000","last_change":"2024-09-21T09:36:40.979475+0000","last_active":"2024-09-21T09:37:18.764136+0000","last_peered":"2024-09-21T09:37:18.764136+0000","last_clean":"2024-09-21T09:37:18.764136+0000","last_became_active":"2024-09-21T09:36:40.979165+0000","last_became_peered":"2024-09-21T09:36:40.979165+0000","last_unstale":"2024-09-21T09:37:18.764136+0000","last_undegraded":"2024-09-21T09:37:18.764136+0000","last_fullsized":"2024-09-21T09:37:18.764136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:44.963935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765822+0000","last_change":"2024-09-21T09:36:39.882983+0000","last_active":"2024-09-21T09:37:18.765822+0000","last_peered":"2024-09-21T09:37:18.765822+0000","last_clean":"2024-09-21T09:37:18.765822+0000","last_became_active":"2024-09-21T09:36:39.882708+0000","last_became_peered":"2024-09-21T09:36:39.882708+0000","last_unstale":"2024-09-21T09:37:18.765822+0000","last_undegraded":"2024-09-21T09:37:18.765822+0000","last_fullsized":"2024-09-21T09:37:18.765822+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:56.046813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.762901+0000","last_change":"2024-09-21T09:36:40.176237+0000","last_active":"2024-09-21T09:37:18.762901+0000","last_peered":"2024-09-21T09:37:18.762901+0000","last_clean":"2024-09-21T09:37:18.762901+0000","last_became_active":"2024-09-21T09:36:40.175943+0000","last_became_peered":"2024-09-21T09:36:40.175943+0000","last_unstale":"2024-09-21T09:37:18.762901+0000","last_undegraded":"2024-09-21T09:37:18.762901+0000","last_fullsized":"2024-09-21T09:37:18.762901+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:15.513812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.769165+0000","last_change":"2024-09-21T09:36:39.886552+0000","last_active":"2024-09-21T09:37:18.769165+0000","last_peered":"2024-09-21T09:37:18.769165+0000","last_clean":"2024-09-21T09:37:18.769165+0000","last_became_active":"2024-09-21T09:36:39.886299+0000","last_became_peered":"2024-09-21T09:36:39.886299+0000","last_unstale":"2024-09-21T09:37:18.769165+0000","last_undegraded":"2024-09-21T09:37:18.769165+0000","last_fullsized":"2024-09-21T09:37:18.769165+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:57.230097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766255+0000","last_change":"2024-09-21T09:36:40.177700+0000","last_active":"2024-09-21T09:37:18.766255+0000","last_peered":"2024-09-21T09:37:18.766255+0000","last_clean":"2024-09-21T09:37:18.766255+0000","last_became_active":"2024-09-21T09:36:40.177425+0000","last_became_peered":"2024-09-21T09:36:40.177425+0000","last_unstale":"2024-09-21T09:37:18.766255+0000","last_undegraded":"2024-09-21T09:37:18.766255+0000","last_fullsized":"2024-09-21T09:37:18.766255+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:36:14.013782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770089+0000","last_change":"2024-09-21T09:36:39.881146+0000","last_active":"2024-09-21T09:37:18.770089+0000","last_peered":"2024-09-21T09:37:18.770089+0000","last_clean":"2024-09-21T09:37:18.770089+0000","last_became_active":"2024-09-21T09:36:39.880857+0000","last_became_peered":"2024-09-21T09:36:39.880857+0000","last_unstale":"2024-09-21T09:37:18.770089+0000","last_undegraded":"2024-09-21T09:37:18.770089+0000","last_fullsized":"2024-09-21T09:37:18.770089+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:44.106709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765656+0000","last_change":"2024-09-21T09:36:39.897283+0000","last_active":"2024-09-21T09:37:18.765656+0000","last_peered":"2024-09-21T09:37:18.765656+0000","last_clean":"2024-09-21T09:37:18.765656+0000","last_became_active":"2024-09-21T09:36:39.897012+0000","last_became_peered":"2024-09-21T09:36:39.897012+0000","last_unstale":"2024-09-21T09:37:18.765656+0000","last_undegraded":"2024-09-21T09:37:18.765656+0000","last_fullsized":"2024-09-21T09:37:18.765656+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:55.761167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770232+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:37:18.770232+0000","last_peered":"2024-09-21T09:37:18.770232+0000","last_clean":"2024-09-21T09:37:18.770232+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:37:18.770232+0000","last_undegraded":"2024-09-21T09:37:18.770232+0000","last_fullsized":"2024-09-21T09:37:18.770232+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763378+0000","last_change":"2024-09-21T09:36:40.933183+0000","last_active":"2024-09-21T09:37:18.763378+0000","last_peered":"2024-09-21T09:37:18.763378+0000","last_clean":"2024-09-21T09:37:18.763378+0000","last_became_active":"2024-09-21T09:36:40.932869+0000","last_became_peered":"2024-09-21T09:36:40.932869+0000","last_unstale":"2024-09-21T09:37:18.763378+0000","last_undegraded":"2024-09-21T09:37:18.763378+0000","last_fullsized":"2024-09-21T09:37:18.763378+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:27.920189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764072+0000","last_change":"2024-09-21T09:36:39.882084+0000","last_active":"2024-09-21T09:37:18.764072+0000","last_peered":"2024-09-21T09:37:18.764072+0000","last_clean":"2024-09-21T09:37:18.764072+0000","last_became_active":"2024-09-21T09:36:39.881520+0000","last_became_peered":"2024-09-21T09:36:39.881520+0000","last_unstale":"2024-09-21T09:37:18.764072+0000","last_undegraded":"2024-09-21T09:37:18.764072+0000","last_fullsized":"2024-09-21T09:37:18.764072+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:46:47.489475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.774504+0000","last_change":"2024-09-21T09:37:19.774504+0000","last_active":"2024-09-21T09:37:19.774504+0000","last_peered":"2024-09-21T09:37:19.774504+0000","last_clean":"2024-09-21T09:37:19.774504+0000","last_became_active":"2024-09-21T09:37:19.774089+0000","last_became_peered":"2024-09-21T09:37:19.774089+0000","last_unstale":"2024-09-21T09:37:19.774504+0000","last_undegraded":"2024-09-21T09:37:19.774504+0000","last_fullsized":"2024-09-21T09:37:19.774504+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:26:09.377745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768785+0000","last_change":"2024-09-21T09:36:39.880733+0000","last_active":"2024-09-21T09:37:18.768785+0000","last_peered":"2024-09-21T09:37:18.768785+0000","last_clean":"2024-09-21T09:37:18.768785+0000","last_became_active":"2024-09-21T09:36:39.880476+0000","last_became_peered":"2024-09-21T09:36:39.880476+0000","last_unstale":"2024-09-21T09:37:18.768785+0000","last_undegraded":"2024-09-21T09:37:18.768785+0000","last_fullsized":"2024-09-21T09:37:18.768785+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:15:59.915859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28364,"kb_used_data":1344,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716788,"statfs":{"total":95995035648,"available":95965990912,"internally_reserved":0,"allocated":1376256,"data_stored":925421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.38,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.70599999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.62},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.51200000000000001,"15min":0.51000000000000001},"min":{"1min":0.37,"5min":0.215,"15min":0.215},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.71299999999999997,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51100000000000001,"15min":0.50800000000000001},"min":{"1min":0.38600000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.44700000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.69999999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53400000000000003,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66100000000000003,"5min":0.873,"15min":0.873},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55600000000000005,"15min":0.56000000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66200000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.749,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.308,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.57099999999999995,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.5},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.64800000000000002,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.505,"5min":0.48699999999999999,"15min":0.48399999999999999},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.54300000000000004}]},{"osd":6,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.623,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.39500000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71499999999999997,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.64700000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28356,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716796,"statfs":{"total":95995035648,"available":95965999104,"internally_reserved":0,"allocated":1368064,"data_stored":925294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54500000000000004,"15min":0.53500000000000003},"min":{"1min":0.33300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.72299999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54700000000000004,"15min":0.54600000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69699999999999995,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73999999999999999,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53100000000000003,"15min":0.52100000000000002},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.745,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.57399999999999995}]},{"osd":2,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54500000000000004,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68200000000000005,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56699999999999995,"15min":0.56899999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.67100000000000004,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55600000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.52900000000000003,"15min":0.52000000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67300000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55700000000000005,"15min":0.55000000000000004},"min":{"1min":0.42399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.53300000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.44,"15min":0.436},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61799999999999999,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.42699999999999999,"15min":0.41499999999999998},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70199999999999996,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.46100000000000002,"15min":0.44800000000000001},"min":{"1min":0.35599999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.48199999999999998,"15min":0.47799999999999998},"min":{"1min":0.34000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70299999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.628},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69099999999999995,"5min":1.002,"15min":1.002},"last":0.64000000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":334275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54400000000000004,"15min":0.55300000000000005},"min":{"1min":0.39000000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.622},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.59599999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53700000000000003,"15min":0.54900000000000004},"min":{"1min":0.38800000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.50700000000000001,"15min":0.50900000000000001},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.67100000000000004}]},{"osd":2,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52000000000000002,"15min":0.52600000000000002},"min":{"1min":0.40699999999999997,"5min":0.372,"15min":0.372},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.70399999999999996}]},{"osd":3,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.65600000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.376,"15min":0.35599999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.51100000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.36899999999999999},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53500000000000003,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.42799999999999999,"15min":0.41699999999999998},"min":{"1min":0.28899999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65900000000000003,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45800000000000002,"15min":0.44600000000000001},"min":{"1min":0.318,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63900000000000001,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.64100000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.53900000000000003,"15min":0.55300000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.627,"5min":0.749,"15min":0.749},"last":0.498}]}]},{"osd":4,"up_from":30,"seq":128849018931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27784,"kb_used_data":764,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717368,"statfs":{"total":95995035648,"available":95966584832,"internally_reserved":0,"allocated":782336,"data_stored":335314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51500000000000001,"15min":0.50600000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70599999999999996,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.64600000000000002,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.5}]},{"osd":1,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.52200000000000002,"15min":0.52700000000000002},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61299999999999999,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.54100000000000004,"15min":0.53400000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.75900000000000001,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.55600000000000005}]},{"osd":2,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72299999999999998,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.51700000000000002,"15min":0.51000000000000001},"min":{"1min":0.374,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.54200000000000004,"15min":0.53900000000000003},"min":{"1min":0.40600000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64900000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68899999999999995,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.39600000000000002,"15min":0.378},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.439,"5min":0.39200000000000002,"15min":0.36699999999999999},"min":{"1min":0.29999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53600000000000003,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.33400000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:36:31 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.42399999999999999,"15min":0.41799999999999998},"min":{"1min":0.33600000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.59299999999999997,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.42899999999999999,"15min":0.41799999999999998},"min":{"1min":0.33800000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 21 09:36:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.47999999999999998,"15min":0.47099999999999997},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.45800000000000002,"15min":0.45500000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.41999999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.376,"15min":0.39300000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.093,"5min":2.093,"15min":2.093},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.38700000000000001,"15min":0.40300000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.39700000000000002,"15min":0.41499999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.49,"5min":1.49,"15min":1.49},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.377,"15min":0.39700000000000002},"min":{"1min":0.24299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.378}]},{"osd":2,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41499999999999998,"15min":0.443},"min":{"1min":0.27700000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40000000000000002,"15min":0.40200000000000002},"min":{"1min":0.26400000000000001,"5min":0.254,"15min":0.254},"max":{"1min":2.1299999999999999,"5min":2.1299999999999999,"15min":2.1299999999999999},"last":0.441}]},{"osd":4,"last update":"Sat Sep 21 09:36:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86699999999999999,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.38300000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77300000000000002,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.59599999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.8160000000000001,"5min":1.8160000000000001,"15min":1.8160000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.436,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.624,"5min":0.58499999999999996,"15min":0.58099999999999996},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.7789999999999999,"5min":1.7789999999999999,"15min":1.7789999999999999},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:36:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76700000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81499999999999995,"5min":0.875,"15min":0.875},"last":0.77700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":765952,"data_stored":334220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.45900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.245},{"interface":"front","average":{"1min":0.29899999999999999,"5min":0.34000000000000002,"15min":0.34499999999999997},"min":{"1min":0.224,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.38600000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36299999999999999,"15min":0.372},"min":{"1min":0.24099999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.47299999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.307},{"interface":"front","average":{"1min":0.318,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.42499999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.42599999999999999,"15min":0.44600000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.495,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.41199999999999998,"15min":0.44},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.48499999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:36:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.60899999999999999,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:37:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52200000000000002,"15min":0.52000000000000002},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.67800000000000005,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57799999999999996,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76300000000000001,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.441}]},{"osd":6,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56200000000000006,"15min":0.56100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.374,"5min":0.371,"15min":0.371},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:36:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.68100000000000005,"15min":0.70199999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.747,"5min":3.7519999999999998,"15min":3.7519999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.64800000000000002,"15min":0.66500000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.76600000000000001,"5min":3.4119999999999999,"15min":3.4119999999999999},"last":0.434}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31856,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713296,"statfs":{"total":95995035648,"available":95962415104,"internally_reserved":0,"allocated":757760,"data_stored":333350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:34 2024","interfaces":[{"interface":"back","average":{"1min":0.32800000000000001,"5min":0.33800000000000002,"15min":0.309},"min":{"1min":0.24199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.47699999999999998,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.32600000000000001,"5min":0.34799999999999998,"15min":0.316},"min":{"1min":0.254,"5min":0.218,"15min":0.218},"max":{"1min":0.443,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.432}]},{"osd":2,"last update":"Sat Sep 21 09:36:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.376,"15min":0.36799999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.53000000000000003,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.38800000000000001,"15min":0.38300000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:37:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40300000000000002,"15min":0.41299999999999998},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.42399999999999999,"15min":0.441},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.58499999999999996,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:36:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.80500000000000005,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.73399999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:37:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56799999999999995,"15min":0.58199999999999996},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79900000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.59899999999999998,"15min":0.60799999999999998},"min":{"1min":0.42099999999999999,"5min":0.371,"15min":0.371},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.748}]},{"osd":6,"last update":"Sat Sep 21 09:37:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.40300000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59799999999999998,"15min":0.61399999999999999},"min":{"1min":0.433,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75800000000000001,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 21 09:36:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59199999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.88100000000000001,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84199999999999997,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.55700000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32444,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712708,"statfs":{"total":95995035648,"available":95961812992,"internally_reserved":0,"allocated":1359872,"data_stored":924576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:36:34 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.34899999999999998,"15min":0.32300000000000001},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36599999999999999,"15min":0.33500000000000002},"min":{"1min":0.245,"5min":0.217,"15min":0.217},"max":{"1min":0.52600000000000002,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.376}]},{"osd":2,"last update":"Sat Sep 21 09:37:01 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38800000000000001,"15min":0.39000000000000001},"min":{"1min":0.23899999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.504,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.38300000000000001,"15min":0.376},"min":{"1min":0.26700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.52700000000000002,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.307}]},{"osd":3,"last update":"Sat Sep 21 09:37:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52300000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.41399999999999998,"15min":0.42899999999999999},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:36:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.41999999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.73999999999999999,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.70199999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.437,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83599999999999997,"5min":1.04,"15min":1.04},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.59199999999999997,"15min":0.60599999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77900000000000003,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.623},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61299999999999999,"15min":0.623},"min":{"1min":0.438,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76800000000000002,"5min":0.874,"15min":0.874},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.875,"5min":1.075,"15min":1.075},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:37:40.304 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:37:40.502 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:40.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:40 smithi076 ceph-mon[29313]: from='client.14757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:40 smithi076 ceph-mon[41325]: from='client.14757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:40 smithi191 ceph-mon[33301]: from='client.14757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:42.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:41 smithi076 ceph-mon[29313]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:42.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:41 smithi076 ceph-mon[41325]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:41 smithi191 ceph-mon[33301]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:42.465 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:37:42.466 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:37:43.258 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":88,"stamp":"2024-09-21T09:37:40.672593+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":232120,"kb_used_data":7756,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729096,"statfs":{"total":767960285184,"available":767722594304,"internally_reserved":0,"allocated":7942144,"data_stored":4446144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12744,"internal_metadata":229625400},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002559"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763655+0000","last_change":"2024-09-21T09:36:40.896108+0000","last_active":"2024-09-21T09:37:18.763655+0000","last_peered":"2024-09-21T09:37:18.763655+0000","last_clean":"2024-09-21T09:37:18.763655+0000","last_became_active":"2024-09-21T09:36:40.895551+0000","last_became_peered":"2024-09-21T09:36:40.895551+0000","last_unstale":"2024-09-21T09:37:18.763655+0000","last_undegraded":"2024-09-21T09:37:18.763655+0000","last_fullsized":"2024-09-21T09:37:18.763655+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:24.331410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905478+0000","last_change":"2024-09-21T09:36:39.881562+0000","last_active":"2024-09-21T09:37:18.905478+0000","last_peered":"2024-09-21T09:37:18.905478+0000","last_clean":"2024-09-21T09:37:18.905478+0000","last_became_active":"2024-09-21T09:36:39.881282+0000","last_became_peered":"2024-09-21T09:36:39.881282+0000","last_unstale":"2024-09-21T09:37:18.905478+0000","last_undegraded":"2024-09-21T09:37:18.905478+0000","last_fullsized":"2024-09-21T09:37:18.905478+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:58.476757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770127+0000","last_change":"2024-09-21T09:36:40.085128+0000","last_active":"2024-09-21T09:37:18.770127+0000","last_peered":"2024-09-21T09:37:18.770127+0000","last_clean":"2024-09-21T09:37:18.770127+0000","last_became_active":"2024-09-21T09:36:40.084892+0000","last_became_peered":"2024-09-21T09:36:40.084892+0000","last_unstale":"2024-09-21T09:37:18.770127+0000","last_undegraded":"2024-09-21T09:37:18.770127+0000","last_fullsized":"2024-09-21T09:37:18.770127+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:42.183643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768432+0000","last_change":"2024-09-21T09:36:39.880523+0000","last_active":"2024-09-21T09:37:18.768432+0000","last_peered":"2024-09-21T09:37:18.768432+0000","last_clean":"2024-09-21T09:37:18.768432+0000","last_became_active":"2024-09-21T09:36:39.880257+0000","last_became_peered":"2024-09-21T09:36:39.880257+0000","last_unstale":"2024-09-21T09:37:18.768432+0000","last_undegraded":"2024-09-21T09:37:18.768432+0000","last_fullsized":"2024-09-21T09:37:18.768432+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:07.507882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768730+0000","last_change":"2024-09-21T09:36:40.084600+0000","last_active":"2024-09-21T09:37:18.768730+0000","last_peered":"2024-09-21T09:37:18.768730+0000","last_clean":"2024-09-21T09:37:18.768730+0000","last_became_active":"2024-09-21T09:36:40.084332+0000","last_became_peered":"2024-09-21T09:36:40.084332+0000","last_unstale":"2024-09-21T09:37:18.768730+0000","last_undegraded":"2024-09-21T09:37:18.768730+0000","last_fullsized":"2024-09-21T09:37:18.768730+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:53.585344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905336+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:37:18.905336+0000","last_peered":"2024-09-21T09:37:18.905336+0000","last_clean":"2024-09-21T09:37:18.905336+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:37:18.905336+0000","last_undegraded":"2024-09-21T09:37:18.905336+0000","last_fullsized":"2024-09-21T09:37:18.905336+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764135+0000","last_change":"2024-09-21T09:36:40.895387+0000","last_active":"2024-09-21T09:37:18.764135+0000","last_peered":"2024-09-21T09:37:18.764135+0000","last_clean":"2024-09-21T09:37:18.764135+0000","last_became_active":"2024-09-21T09:36:40.895107+0000","last_became_peered":"2024-09-21T09:36:40.895107+0000","last_unstale":"2024-09-21T09:37:18.764135+0000","last_undegraded":"2024-09-21T09:37:18.764135+0000","last_fullsized":"2024-09-21T09:37:18.764135+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:31:56.935414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768368+0000","last_change":"2024-09-21T09:36:39.873811+0000","last_active":"2024-09-21T09:37:18.768368+0000","last_peered":"2024-09-21T09:37:18.768368+0000","last_clean":"2024-09-21T09:37:18.768368+0000","last_became_active":"2024-09-21T09:36:39.873558+0000","last_became_peered":"2024-09-21T09:36:39.873558+0000","last_unstale":"2024-09-21T09:37:18.768368+0000","last_undegraded":"2024-09-21T09:37:18.768368+0000","last_fullsized":"2024-09-21T09:37:18.768368+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:41.470084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905737+0000","last_change":"2024-09-21T09:36:39.879194+0000","last_active":"2024-09-21T09:37:18.905737+0000","last_peered":"2024-09-21T09:37:18.905737+0000","last_clean":"2024-09-21T09:37:18.905737+0000","last_became_active":"2024-09-21T09:36:39.878521+0000","last_became_peered":"2024-09-21T09:36:39.878521+0000","last_unstale":"2024-09-21T09:37:18.905737+0000","last_undegraded":"2024-09-21T09:37:18.905737+0000","last_fullsized":"2024-09-21T09:37:18.905737+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:13.986085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.769927+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:37:18.769927+0000","last_peered":"2024-09-21T09:37:18.769927+0000","last_clean":"2024-09-21T09:37:18.769927+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:37:18.769927+0000","last_undegraded":"2024-09-21T09:37:18.769927+0000","last_fullsized":"2024-09-21T09:37:18.769927+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905690+0000","last_change":"2024-09-21T09:36:39.880390+0000","last_active":"2024-09-21T09:37:18.905690+0000","last_peered":"2024-09-21T09:37:18.905690+0000","last_clean":"2024-09-21T09:37:18.905690+0000","last_became_active":"2024-09-21T09:36:39.880142+0000","last_became_peered":"2024-09-21T09:36:39.880142+0000","last_unstale":"2024-09-21T09:37:18.905690+0000","last_undegraded":"2024-09-21T09:37:18.905690+0000","last_fullsized":"2024-09-21T09:37:18.905690+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:03.490045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765668+0000","last_change":"2024-09-21T09:36:40.176250+0000","last_active":"2024-09-21T09:37:18.765668+0000","last_peered":"2024-09-21T09:37:18.765668+0000","last_clean":"2024-09-21T09:37:18.765668+0000","last_became_active":"2024-09-21T09:36:40.175949+0000","last_became_peered":"2024-09-21T09:36:40.175949+0000","last_unstale":"2024-09-21T09:37:18.765668+0000","last_undegraded":"2024-09-21T09:37:18.765668+0000","last_fullsized":"2024-09-21T09:37:18.765668+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:47.315693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905663+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:37:18.905663+0000","last_peered":"2024-09-21T09:37:18.905663+0000","last_clean":"2024-09-21T09:37:18.905663+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:37:18.905663+0000","last_undegraded":"2024-09-21T09:37:18.905663+0000","last_fullsized":"2024-09-21T09:37:18.905663+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767917+0000","last_change":"2024-09-21T09:36:40.176231+0000","last_active":"2024-09-21T09:37:18.767917+0000","last_peered":"2024-09-21T09:37:18.767917+0000","last_clean":"2024-09-21T09:37:18.767917+0000","last_became_active":"2024-09-21T09:36:40.175855+0000","last_became_peered":"2024-09-21T09:36:40.175855+0000","last_unstale":"2024-09-21T09:37:18.767917+0000","last_undegraded":"2024-09-21T09:37:18.767917+0000","last_fullsized":"2024-09-21T09:37:18.767917+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:05:31.249444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770141+0000","last_change":"2024-09-21T09:36:39.878498+0000","last_active":"2024-09-21T09:37:18.770141+0000","last_peered":"2024-09-21T09:37:18.770141+0000","last_clean":"2024-09-21T09:37:18.770141+0000","last_became_active":"2024-09-21T09:36:39.878237+0000","last_became_peered":"2024-09-21T09:36:39.878237+0000","last_unstale":"2024-09-21T09:37:18.770141+0000","last_undegraded":"2024-09-21T09:37:18.770141+0000","last_fullsized":"2024-09-21T09:37:18.770141+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:16.444276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.891302+0000","last_change":"2024-09-21T09:37:19.891302+0000","last_active":"2024-09-21T09:37:19.891302+0000","last_peered":"2024-09-21T09:37:19.891302+0000","last_clean":"2024-09-21T09:37:19.891302+0000","last_became_active":"2024-09-21T09:37:19.890936+0000","last_became_peered":"2024-09-21T09:37:19.890936+0000","last_unstale":"2024-09-21T09:37:19.891302+0000","last_undegraded":"2024-09-21T09:37:19.891302+0000","last_fullsized":"2024-09-21T09:37:19.891302+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:08.589525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763558+0000","last_change":"2024-09-21T09:36:39.890215+0000","last_active":"2024-09-21T09:37:18.763558+0000","last_peered":"2024-09-21T09:37:18.763558+0000","last_clean":"2024-09-21T09:37:18.763558+0000","last_became_active":"2024-09-21T09:36:39.889954+0000","last_became_peered":"2024-09-21T09:36:39.889954+0000","last_unstale":"2024-09-21T09:37:18.763558+0000","last_undegraded":"2024-09-21T09:37:18.763558+0000","last_fullsized":"2024-09-21T09:37:18.763558+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:39.812907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765911+0000","last_change":"2024-09-21T09:36:39.895269+0000","last_active":"2024-09-21T09:37:18.765911+0000","last_peered":"2024-09-21T09:37:18.765911+0000","last_clean":"2024-09-21T09:37:18.765911+0000","last_became_active":"2024-09-21T09:36:39.894941+0000","last_became_peered":"2024-09-21T09:36:39.894941+0000","last_unstale":"2024-09-21T09:37:18.765911+0000","last_undegraded":"2024-09-21T09:37:18.765911+0000","last_fullsized":"2024-09-21T09:37:18.765911+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:38.574808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768053+0000","last_change":"2024-09-21T09:36:39.876805+0000","last_active":"2024-09-21T09:37:18.768053+0000","last_peered":"2024-09-21T09:37:18.768053+0000","last_clean":"2024-09-21T09:37:18.768053+0000","last_became_active":"2024-09-21T09:36:39.876308+0000","last_became_peered":"2024-09-21T09:36:39.876308+0000","last_unstale":"2024-09-21T09:37:18.768053+0000","last_undegraded":"2024-09-21T09:37:18.768053+0000","last_fullsized":"2024-09-21T09:37:18.768053+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:00.708481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766077+0000","last_change":"2024-09-21T09:36:40.177666+0000","last_active":"2024-09-21T09:37:18.766077+0000","last_peered":"2024-09-21T09:37:18.766077+0000","last_clean":"2024-09-21T09:37:18.766077+0000","last_became_active":"2024-09-21T09:36:40.177355+0000","last_became_peered":"2024-09-21T09:36:40.177355+0000","last_unstale":"2024-09-21T09:37:18.766077+0000","last_undegraded":"2024-09-21T09:37:18.766077+0000","last_fullsized":"2024-09-21T09:37:18.766077+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:17.332240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765792+0000","last_change":"2024-09-21T09:36:39.884045+0000","last_active":"2024-09-21T09:37:18.765792+0000","last_peered":"2024-09-21T09:37:18.765792+0000","last_clean":"2024-09-21T09:37:18.765792+0000","last_became_active":"2024-09-21T09:36:39.883609+0000","last_became_peered":"2024-09-21T09:36:39.883609+0000","last_unstale":"2024-09-21T09:37:18.765792+0000","last_undegraded":"2024-09-21T09:37:18.765792+0000","last_fullsized":"2024-09-21T09:37:18.765792+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:54:17.035411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766095+0000","last_change":"2024-09-21T09:36:40.178208+0000","last_active":"2024-09-21T09:37:18.766095+0000","last_peered":"2024-09-21T09:37:18.766095+0000","last_clean":"2024-09-21T09:37:18.766095+0000","last_became_active":"2024-09-21T09:36:40.177971+0000","last_became_peered":"2024-09-21T09:36:40.177971+0000","last_unstale":"2024-09-21T09:37:18.766095+0000","last_undegraded":"2024-09-21T09:37:18.766095+0000","last_fullsized":"2024-09-21T09:37:18.766095+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:21.635698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763211+0000","last_change":"2024-09-21T09:36:39.884071+0000","last_active":"2024-09-21T09:37:18.763211+0000","last_peered":"2024-09-21T09:37:18.763211+0000","last_clean":"2024-09-21T09:37:18.763211+0000","last_became_active":"2024-09-21T09:36:39.883795+0000","last_became_peered":"2024-09-21T09:36:39.883795+0000","last_unstale":"2024-09-21T09:37:18.763211+0000","last_undegraded":"2024-09-21T09:37:18.763211+0000","last_fullsized":"2024-09-21T09:37:18.763211+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:21:54.070798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768074+0000","last_change":"2024-09-21T09:36:40.980176+0000","last_active":"2024-09-21T09:37:18.768074+0000","last_peered":"2024-09-21T09:37:18.768074+0000","last_clean":"2024-09-21T09:37:18.768074+0000","last_became_active":"2024-09-21T09:36:40.979895+0000","last_became_peered":"2024-09-21T09:36:40.979895+0000","last_unstale":"2024-09-21T09:37:18.768074+0000","last_undegraded":"2024-09-21T09:37:18.768074+0000","last_fullsized":"2024-09-21T09:37:18.768074+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:44:57.080348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":270,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905301+0000","last_change":"2024-09-21T09:34:52.468106+0000","last_active":"2024-09-21T09:37:18.905301+0000","last_peered":"2024-09-21T09:37:18.905301+0000","last_clean":"2024-09-21T09:37:18.905301+0000","last_became_active":"2024-09-21T09:34:52.467638+0000","last_became_peered":"2024-09-21T09:34:52.467638+0000","last_unstale":"2024-09-21T09:37:18.905301+0000","last_undegraded":"2024-09-21T09:37:18.905301+0000","last_fullsized":"2024-09-21T09:37:18.905301+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:35.011128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.890448+0000","last_change":"2024-09-21T09:35:30.787751+0000","last_active":"2024-09-21T09:37:19.890448+0000","last_peered":"2024-09-21T09:37:19.890448+0000","last_clean":"2024-09-21T09:37:19.890448+0000","last_became_active":"2024-09-21T09:35:24.753144+0000","last_became_peered":"2024-09-21T09:35:24.753144+0000","last_unstale":"2024-09-21T09:37:19.890448+0000","last_undegraded":"2024-09-21T09:37:19.890448+0000","last_fullsized":"2024-09-21T09:37:19.890448+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:59:50.550632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763515+0000","last_change":"2024-09-21T09:36:39.883277+0000","last_active":"2024-09-21T09:37:18.763515+0000","last_peered":"2024-09-21T09:37:18.763515+0000","last_clean":"2024-09-21T09:37:18.763515+0000","last_became_active":"2024-09-21T09:36:39.882971+0000","last_became_peered":"2024-09-21T09:36:39.882971+0000","last_unstale":"2024-09-21T09:37:18.763515+0000","last_undegraded":"2024-09-21T09:37:18.763515+0000","last_fullsized":"2024-09-21T09:37:18.763515+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:16.821287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768073+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:37:18.768073+0000","last_peered":"2024-09-21T09:37:18.768073+0000","last_clean":"2024-09-21T09:37:18.768073+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:37:18.768073+0000","last_undegraded":"2024-09-21T09:37:18.768073+0000","last_fullsized":"2024-09-21T09:37:18.768073+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.229581+0000","last_change":"2024-09-21T09:35:31.069584+0000","last_active":"2024-09-21T09:37:20.229581+0000","last_peered":"2024-09-21T09:37:20.229581+0000","last_clean":"2024-09-21T09:37:20.229581+0000","last_became_active":"2024-09-21T09:35:24.766184+0000","last_became_peered":"2024-09-21T09:35:24.766184+0000","last_unstale":"2024-09-21T09:37:20.229581+0000","last_undegraded":"2024-09-21T09:37:20.229581+0000","last_fullsized":"2024-09-21T09:37:20.229581+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:16.728713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768209+0000","last_change":"2024-09-21T09:36:39.884020+0000","last_active":"2024-09-21T09:37:18.768209+0000","last_peered":"2024-09-21T09:37:18.768209+0000","last_clean":"2024-09-21T09:37:18.768209+0000","last_became_active":"2024-09-21T09:36:39.882217+0000","last_became_peered":"2024-09-21T09:36:39.882217+0000","last_unstale":"2024-09-21T09:37:18.768209+0000","last_undegraded":"2024-09-21T09:37:18.768209+0000","last_fullsized":"2024-09-21T09:37:18.768209+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:06.639754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768663+0000","last_change":"2024-09-21T09:36:40.176330+0000","last_active":"2024-09-21T09:37:18.768663+0000","last_peered":"2024-09-21T09:37:18.768663+0000","last_clean":"2024-09-21T09:37:18.768663+0000","last_became_active":"2024-09-21T09:36:40.176046+0000","last_became_peered":"2024-09-21T09:36:40.176046+0000","last_unstale":"2024-09-21T09:37:18.768663+0000","last_undegraded":"2024-09-21T09:37:18.768663+0000","last_fullsized":"2024-09-21T09:37:18.768663+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:55.382440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.766818+0000","last_change":"2024-09-21T09:35:30.783775+0000","last_active":"2024-09-21T09:37:19.766818+0000","last_peered":"2024-09-21T09:37:19.766818+0000","last_clean":"2024-09-21T09:37:19.766818+0000","last_became_active":"2024-09-21T09:35:24.766096+0000","last_became_peered":"2024-09-21T09:35:24.766096+0000","last_unstale":"2024-09-21T09:37:19.766818+0000","last_undegraded":"2024-09-21T09:37:19.766818+0000","last_fullsized":"2024-09-21T09:37:19.766818+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:34.155489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767588+0000","last_change":"2024-09-21T09:36:39.878248+0000","last_active":"2024-09-21T09:37:18.767588+0000","last_peered":"2024-09-21T09:37:18.767588+0000","last_clean":"2024-09-21T09:37:18.767588+0000","last_became_active":"2024-09-21T09:36:39.877981+0000","last_became_peered":"2024-09-21T09:36:39.877981+0000","last_unstale":"2024-09-21T09:37:18.767588+0000","last_undegraded":"2024-09-21T09:37:18.767588+0000","last_fullsized":"2024-09-21T09:37:18.767588+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:27:03.622897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768089+0000","last_change":"2024-09-21T09:36:40.969925+0000","last_active":"2024-09-21T09:37:18.768089+0000","last_peered":"2024-09-21T09:37:18.768089+0000","last_clean":"2024-09-21T09:37:18.768089+0000","last_became_active":"2024-09-21T09:36:40.969439+0000","last_became_peered":"2024-09-21T09:36:40.969439+0000","last_unstale":"2024-09-21T09:37:18.768089+0000","last_undegraded":"2024-09-21T09:37:18.768089+0000","last_fullsized":"2024-09-21T09:37:18.768089+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:58:28.961156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.890488+0000","last_change":"2024-09-21T09:35:30.788301+0000","last_active":"2024-09-21T09:37:19.890488+0000","last_peered":"2024-09-21T09:37:19.890488+0000","last_clean":"2024-09-21T09:37:19.890488+0000","last_became_active":"2024-09-21T09:35:24.753321+0000","last_became_peered":"2024-09-21T09:35:24.753321+0000","last_unstale":"2024-09-21T09:37:19.890488+0000","last_undegraded":"2024-09-21T09:37:19.890488+0000","last_fullsized":"2024-09-21T09:37:19.890488+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:52.357133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766345+0000","last_change":"2024-09-21T09:36:39.879835+0000","last_active":"2024-09-21T09:37:18.766345+0000","last_peered":"2024-09-21T09:37:18.766345+0000","last_clean":"2024-09-21T09:37:18.766345+0000","last_became_active":"2024-09-21T09:36:39.879284+0000","last_became_peered":"2024-09-21T09:36:39.879284+0000","last_unstale":"2024-09-21T09:37:18.766345+0000","last_undegraded":"2024-09-21T09:37:18.766345+0000","last_fullsized":"2024-09-21T09:37:18.766345+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:43.176096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767834+0000","last_change":"2024-09-21T09:36:40.896338+0000","last_active":"2024-09-21T09:37:18.767834+0000","last_peered":"2024-09-21T09:37:18.767834+0000","last_clean":"2024-09-21T09:37:18.767834+0000","last_became_active":"2024-09-21T09:36:40.895996+0000","last_became_peered":"2024-09-21T09:36:40.895996+0000","last_unstale":"2024-09-21T09:37:18.767834+0000","last_undegraded":"2024-09-21T09:37:18.767834+0000","last_fullsized":"2024-09-21T09:37:18.767834+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:33.010247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770245+0000","last_change":"2024-09-21T09:36:39.879239+0000","last_active":"2024-09-21T09:37:18.770245+0000","last_peered":"2024-09-21T09:37:18.770245+0000","last_clean":"2024-09-21T09:37:18.770245+0000","last_became_active":"2024-09-21T09:36:39.878988+0000","last_became_peered":"2024-09-21T09:36:39.878988+0000","last_unstale":"2024-09-21T09:37:18.770245+0000","last_undegraded":"2024-09-21T09:37:18.770245+0000","last_fullsized":"2024-09-21T09:37:18.770245+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:58.839553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.582448+0000","last_change":"2024-09-21T09:35:30.789143+0000","last_active":"2024-09-21T09:37:20.582448+0000","last_peered":"2024-09-21T09:37:20.582448+0000","last_clean":"2024-09-21T09:37:20.582448+0000","last_became_active":"2024-09-21T09:35:24.766244+0000","last_became_peered":"2024-09-21T09:35:24.766244+0000","last_unstale":"2024-09-21T09:37:20.582448+0000","last_undegraded":"2024-09-21T09:37:20.582448+0000","last_fullsized":"2024-09-21T09:37:20.582448+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:30.932253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767587+0000","last_change":"2024-09-21T09:36:40.980366+0000","last_active":"2024-09-21T09:37:18.767587+0000","last_peered":"2024-09-21T09:37:18.767587+0000","last_clean":"2024-09-21T09:37:18.767587+0000","last_became_active":"2024-09-21T09:36:40.979656+0000","last_became_peered":"2024-09-21T09:36:40.979656+0000","last_unstale":"2024-09-21T09:37:18.767587+0000","last_undegraded":"2024-09-21T09:37:18.767587+0000","last_fullsized":"2024-09-21T09:37:18.767587+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:22.779091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767871+0000","last_change":"2024-09-21T09:36:39.885841+0000","last_active":"2024-09-21T09:37:18.767871+0000","last_peered":"2024-09-21T09:37:18.767871+0000","last_clean":"2024-09-21T09:37:18.767871+0000","last_became_active":"2024-09-21T09:36:39.885556+0000","last_became_peered":"2024-09-21T09:36:39.885556+0000","last_unstale":"2024-09-21T09:37:18.767871+0000","last_undegraded":"2024-09-21T09:37:18.767871+0000","last_fullsized":"2024-09-21T09:37:18.767871+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:14:14.747580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.229814+0000","last_change":"2024-09-21T09:35:31.069545+0000","last_active":"2024-09-21T09:37:20.229814+0000","last_peered":"2024-09-21T09:37:20.229814+0000","last_clean":"2024-09-21T09:37:20.229814+0000","last_became_active":"2024-09-21T09:35:24.766462+0000","last_became_peered":"2024-09-21T09:35:24.766462+0000","last_unstale":"2024-09-21T09:37:20.229814+0000","last_undegraded":"2024-09-21T09:37:20.229814+0000","last_fullsized":"2024-09-21T09:37:20.229814+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:48.059267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.769166+0000","last_change":"2024-09-21T09:36:40.084429+0000","last_active":"2024-09-21T09:37:18.769166+0000","last_peered":"2024-09-21T09:37:18.769166+0000","last_clean":"2024-09-21T09:37:18.769166+0000","last_became_active":"2024-09-21T09:36:40.084124+0000","last_became_peered":"2024-09-21T09:36:40.084124+0000","last_unstale":"2024-09-21T09:37:18.769166+0000","last_undegraded":"2024-09-21T09:37:18.769166+0000","last_fullsized":"2024-09-21T09:37:18.769166+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:13:42.274683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764216+0000","last_change":"2024-09-21T09:36:39.886972+0000","last_active":"2024-09-21T09:37:18.764216+0000","last_peered":"2024-09-21T09:37:18.764216+0000","last_clean":"2024-09-21T09:37:18.764216+0000","last_became_active":"2024-09-21T09:36:39.885739+0000","last_became_peered":"2024-09-21T09:36:39.885739+0000","last_unstale":"2024-09-21T09:37:18.764216+0000","last_undegraded":"2024-09-21T09:37:18.764216+0000","last_fullsized":"2024-09-21T09:37:18.764216+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:03.951390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:20.582862+0000","last_change":"2024-09-21T09:35:30.789220+0000","last_active":"2024-09-21T09:37:20.582862+0000","last_peered":"2024-09-21T09:37:20.582862+0000","last_clean":"2024-09-21T09:37:20.582862+0000","last_became_active":"2024-09-21T09:35:24.767340+0000","last_became_peered":"2024-09-21T09:35:24.767340+0000","last_unstale":"2024-09-21T09:37:20.582862+0000","last_undegraded":"2024-09-21T09:37:20.582862+0000","last_fullsized":"2024-09-21T09:37:20.582862+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:48.181309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036756400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763118+0000","last_change":"2024-09-21T09:36:40.933241+0000","last_active":"2024-09-21T09:37:18.763118+0000","last_peered":"2024-09-21T09:37:18.763118+0000","last_clean":"2024-09-21T09:37:18.763118+0000","last_became_active":"2024-09-21T09:36:40.932905+0000","last_became_peered":"2024-09-21T09:36:40.932905+0000","last_unstale":"2024-09-21T09:37:18.763118+0000","last_undegraded":"2024-09-21T09:37:18.763118+0000","last_fullsized":"2024-09-21T09:37:18.763118+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:07.761909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768806+0000","last_change":"2024-09-21T09:36:39.882383+0000","last_active":"2024-09-21T09:37:18.768806+0000","last_peered":"2024-09-21T09:37:18.768806+0000","last_clean":"2024-09-21T09:37:18.768806+0000","last_became_active":"2024-09-21T09:36:39.881756+0000","last_became_peered":"2024-09-21T09:36:39.881756+0000","last_unstale":"2024-09-21T09:37:18.768806+0000","last_undegraded":"2024-09-21T09:37:18.768806+0000","last_fullsized":"2024-09-21T09:37:18.768806+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:02:27.240205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.767540+0000","last_change":"2024-09-21T09:35:30.789192+0000","last_active":"2024-09-21T09:37:19.767540+0000","last_peered":"2024-09-21T09:37:19.767540+0000","last_clean":"2024-09-21T09:37:19.767540+0000","last_became_active":"2024-09-21T09:35:24.754596+0000","last_became_peered":"2024-09-21T09:35:24.754596+0000","last_unstale":"2024-09-21T09:37:19.767540+0000","last_undegraded":"2024-09-21T09:37:19.767540+0000","last_fullsized":"2024-09-21T09:37:19.767540+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:39.865349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765772+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:37:18.765772+0000","last_peered":"2024-09-21T09:37:18.765772+0000","last_clean":"2024-09-21T09:37:18.765772+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:37:18.765772+0000","last_undegraded":"2024-09-21T09:37:18.765772+0000","last_fullsized":"2024-09-21T09:37:18.765772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766300+0000","last_change":"2024-09-21T09:36:40.177744+0000","last_active":"2024-09-21T09:37:18.766300+0000","last_peered":"2024-09-21T09:37:18.766300+0000","last_clean":"2024-09-21T09:37:18.766300+0000","last_became_active":"2024-09-21T09:36:40.177500+0000","last_became_peered":"2024-09-21T09:36:40.177500+0000","last_unstale":"2024-09-21T09:37:18.766300+0000","last_undegraded":"2024-09-21T09:37:18.766300+0000","last_fullsized":"2024-09-21T09:37:18.766300+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:35.517184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766015+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:37:18.766015+0000","last_peered":"2024-09-21T09:37:18.766015+0000","last_clean":"2024-09-21T09:37:18.766015+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:37:18.766015+0000","last_undegraded":"2024-09-21T09:37:18.766015+0000","last_fullsized":"2024-09-21T09:37:18.766015+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767608+0000","last_change":"2024-09-21T09:36:40.980042+0000","last_active":"2024-09-21T09:37:18.767608+0000","last_peered":"2024-09-21T09:37:18.767608+0000","last_clean":"2024-09-21T09:37:18.767608+0000","last_became_active":"2024-09-21T09:36:40.979616+0000","last_became_peered":"2024-09-21T09:36:40.979616+0000","last_unstale":"2024-09-21T09:37:18.767608+0000","last_undegraded":"2024-09-21T09:37:18.767608+0000","last_fullsized":"2024-09-21T09:37:18.767608+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:00.254599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.905554+0000","last_change":"2024-09-21T09:36:39.880134+0000","last_active":"2024-09-21T09:37:18.905554+0000","last_peered":"2024-09-21T09:37:18.905554+0000","last_clean":"2024-09-21T09:37:18.905554+0000","last_became_active":"2024-09-21T09:36:39.879853+0000","last_became_peered":"2024-09-21T09:36:39.879853+0000","last_unstale":"2024-09-21T09:37:18.905554+0000","last_undegraded":"2024-09-21T09:37:18.905554+0000","last_fullsized":"2024-09-21T09:37:18.905554+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:42.799198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770194+0000","last_change":"2024-09-21T09:36:39.879888+0000","last_active":"2024-09-21T09:37:18.770194+0000","last_peered":"2024-09-21T09:37:18.770194+0000","last_clean":"2024-09-21T09:37:18.770194+0000","last_became_active":"2024-09-21T09:36:39.879622+0000","last_became_peered":"2024-09-21T09:36:39.879622+0000","last_unstale":"2024-09-21T09:37:18.770194+0000","last_undegraded":"2024-09-21T09:37:18.770194+0000","last_fullsized":"2024-09-21T09:37:18.770194+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:02:23.355344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763310+0000","last_change":"2024-09-21T09:36:40.979593+0000","last_active":"2024-09-21T09:37:18.763310+0000","last_peered":"2024-09-21T09:37:18.763310+0000","last_clean":"2024-09-21T09:37:18.763310+0000","last_became_active":"2024-09-21T09:36:40.979205+0000","last_became_peered":"2024-09-21T09:36:40.979205+0000","last_unstale":"2024-09-21T09:37:18.763310+0000","last_undegraded":"2024-09-21T09:37:18.763310+0000","last_fullsized":"2024-09-21T09:37:18.763310+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:25.107383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768866+0000","last_change":"2024-09-21T09:36:39.880706+0000","last_active":"2024-09-21T09:37:18.768866+0000","last_peered":"2024-09-21T09:37:18.768866+0000","last_clean":"2024-09-21T09:37:18.768866+0000","last_became_active":"2024-09-21T09:36:39.880094+0000","last_became_peered":"2024-09-21T09:36:39.880094+0000","last_unstale":"2024-09-21T09:37:18.768866+0000","last_undegraded":"2024-09-21T09:37:18.768866+0000","last_fullsized":"2024-09-21T09:37:18.768866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:46.918541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763070+0000","last_change":"2024-09-21T09:36:40.979477+0000","last_active":"2024-09-21T09:37:18.763070+0000","last_peered":"2024-09-21T09:37:18.763070+0000","last_clean":"2024-09-21T09:37:18.763070+0000","last_became_active":"2024-09-21T09:36:40.979165+0000","last_became_peered":"2024-09-21T09:36:40.979165+0000","last_unstale":"2024-09-21T09:37:18.763070+0000","last_undegraded":"2024-09-21T09:37:18.763070+0000","last_fullsized":"2024-09-21T09:37:18.763070+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:14.737992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.891466+0000","last_change":"2024-09-21T09:37:19.891466+0000","last_active":"2024-09-21T09:37:19.891466+0000","last_peered":"2024-09-21T09:37:19.891466+0000","last_clean":"2024-09-21T09:37:19.891466+0000","last_became_active":"2024-09-21T09:37:19.891039+0000","last_became_peered":"2024-09-21T09:37:19.891039+0000","last_unstale":"2024-09-21T09:37:19.891466+0000","last_undegraded":"2024-09-21T09:37:19.891466+0000","last_fullsized":"2024-09-21T09:37:19.891466+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:13.728638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763142+0000","last_change":"2024-09-21T09:36:39.886829+0000","last_active":"2024-09-21T09:37:18.763142+0000","last_peered":"2024-09-21T09:37:18.763142+0000","last_clean":"2024-09-21T09:37:18.763142+0000","last_became_active":"2024-09-21T09:36:39.886574+0000","last_became_peered":"2024-09-21T09:36:39.886574+0000","last_unstale":"2024-09-21T09:37:18.763142+0000","last_undegraded":"2024-09-21T09:37:18.763142+0000","last_fullsized":"2024-09-21T09:37:18.763142+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:22.122621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768586+0000","last_change":"2024-09-21T09:36:40.084441+0000","last_active":"2024-09-21T09:37:18.768586+0000","last_peered":"2024-09-21T09:37:18.768586+0000","last_clean":"2024-09-21T09:37:18.768586+0000","last_became_active":"2024-09-21T09:36:40.084159+0000","last_became_peered":"2024-09-21T09:36:40.084159+0000","last_unstale":"2024-09-21T09:37:18.768586+0000","last_undegraded":"2024-09-21T09:37:18.768586+0000","last_fullsized":"2024-09-21T09:37:18.768586+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:06.332538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.767537+0000","last_change":"2024-09-21T09:36:39.883492+0000","last_active":"2024-09-21T09:37:18.767537+0000","last_peered":"2024-09-21T09:37:18.767537+0000","last_clean":"2024-09-21T09:37:18.767537+0000","last_became_active":"2024-09-21T09:36:39.883094+0000","last_became_peered":"2024-09-21T09:36:39.883094+0000","last_unstale":"2024-09-21T09:37:18.767537+0000","last_undegraded":"2024-09-21T09:37:18.767537+0000","last_fullsized":"2024-09-21T09:37:18.767537+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:48:13.399373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764136+0000","last_change":"2024-09-21T09:36:40.979475+0000","last_active":"2024-09-21T09:37:18.764136+0000","last_peered":"2024-09-21T09:37:18.764136+0000","last_clean":"2024-09-21T09:37:18.764136+0000","last_became_active":"2024-09-21T09:36:40.979165+0000","last_became_peered":"2024-09-21T09:36:40.979165+0000","last_unstale":"2024-09-21T09:37:18.764136+0000","last_undegraded":"2024-09-21T09:37:18.764136+0000","last_fullsized":"2024-09-21T09:37:18.764136+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:44.963935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765822+0000","last_change":"2024-09-21T09:36:39.882983+0000","last_active":"2024-09-21T09:37:18.765822+0000","last_peered":"2024-09-21T09:37:18.765822+0000","last_clean":"2024-09-21T09:37:18.765822+0000","last_became_active":"2024-09-21T09:36:39.882708+0000","last_became_peered":"2024-09-21T09:36:39.882708+0000","last_unstale":"2024-09-21T09:37:18.765822+0000","last_undegraded":"2024-09-21T09:37:18.765822+0000","last_fullsized":"2024-09-21T09:37:18.765822+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:56.046813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.762901+0000","last_change":"2024-09-21T09:36:40.176237+0000","last_active":"2024-09-21T09:37:18.762901+0000","last_peered":"2024-09-21T09:37:18.762901+0000","last_clean":"2024-09-21T09:37:18.762901+0000","last_became_active":"2024-09-21T09:36:40.175943+0000","last_became_peered":"2024-09-21T09:36:40.175943+0000","last_unstale":"2024-09-21T09:37:18.762901+0000","last_undegraded":"2024-09-21T09:37:18.762901+0000","last_fullsized":"2024-09-21T09:37:18.762901+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:15.513812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.769165+0000","last_change":"2024-09-21T09:36:39.886552+0000","last_active":"2024-09-21T09:37:18.769165+0000","last_peered":"2024-09-21T09:37:18.769165+0000","last_clean":"2024-09-21T09:37:18.769165+0000","last_became_active":"2024-09-21T09:36:39.886299+0000","last_became_peered":"2024-09-21T09:36:39.886299+0000","last_unstale":"2024-09-21T09:37:18.769165+0000","last_undegraded":"2024-09-21T09:37:18.769165+0000","last_fullsized":"2024-09-21T09:37:18.769165+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:57.230097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.766255+0000","last_change":"2024-09-21T09:36:40.177700+0000","last_active":"2024-09-21T09:37:18.766255+0000","last_peered":"2024-09-21T09:37:18.766255+0000","last_clean":"2024-09-21T09:37:18.766255+0000","last_became_active":"2024-09-21T09:36:40.177425+0000","last_became_peered":"2024-09-21T09:36:40.177425+0000","last_unstale":"2024-09-21T09:37:18.766255+0000","last_undegraded":"2024-09-21T09:37:18.766255+0000","last_fullsized":"2024-09-21T09:37:18.766255+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:36:14.013782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770089+0000","last_change":"2024-09-21T09:36:39.881146+0000","last_active":"2024-09-21T09:37:18.770089+0000","last_peered":"2024-09-21T09:37:18.770089+0000","last_clean":"2024-09-21T09:37:18.770089+0000","last_became_active":"2024-09-21T09:36:39.880857+0000","last_became_peered":"2024-09-21T09:36:39.880857+0000","last_unstale":"2024-09-21T09:37:18.770089+0000","last_undegraded":"2024-09-21T09:37:18.770089+0000","last_fullsized":"2024-09-21T09:37:18.770089+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:44.106709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.765656+0000","last_change":"2024-09-21T09:36:39.897283+0000","last_active":"2024-09-21T09:37:18.765656+0000","last_peered":"2024-09-21T09:37:18.765656+0000","last_clean":"2024-09-21T09:37:18.765656+0000","last_became_active":"2024-09-21T09:36:39.897012+0000","last_became_peered":"2024-09-21T09:36:39.897012+0000","last_unstale":"2024-09-21T09:37:18.765656+0000","last_undegraded":"2024-09-21T09:37:18.765656+0000","last_fullsized":"2024-09-21T09:37:18.765656+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:55.761167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.770232+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:37:18.770232+0000","last_peered":"2024-09-21T09:37:18.770232+0000","last_clean":"2024-09-21T09:37:18.770232+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:37:18.770232+0000","last_undegraded":"2024-09-21T09:37:18.770232+0000","last_fullsized":"2024-09-21T09:37:18.770232+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.763378+0000","last_change":"2024-09-21T09:36:40.933183+0000","last_active":"2024-09-21T09:37:18.763378+0000","last_peered":"2024-09-21T09:37:18.763378+0000","last_clean":"2024-09-21T09:37:18.763378+0000","last_became_active":"2024-09-21T09:36:40.932869+0000","last_became_peered":"2024-09-21T09:36:40.932869+0000","last_unstale":"2024-09-21T09:37:18.763378+0000","last_undegraded":"2024-09-21T09:37:18.763378+0000","last_fullsized":"2024-09-21T09:37:18.763378+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:40:27.920189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.764072+0000","last_change":"2024-09-21T09:36:39.882084+0000","last_active":"2024-09-21T09:37:18.764072+0000","last_peered":"2024-09-21T09:37:18.764072+0000","last_clean":"2024-09-21T09:37:18.764072+0000","last_became_active":"2024-09-21T09:36:39.881520+0000","last_became_peered":"2024-09-21T09:36:39.881520+0000","last_unstale":"2024-09-21T09:37:18.764072+0000","last_undegraded":"2024-09-21T09:37:18.764072+0000","last_fullsized":"2024-09-21T09:37:18.764072+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:46:47.489475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-21T09:37:19.774504+0000","last_change":"2024-09-21T09:37:19.774504+0000","last_active":"2024-09-21T09:37:19.774504+0000","last_peered":"2024-09-21T09:37:19.774504+0000","last_clean":"2024-09-21T09:37:19.774504+0000","last_became_active":"2024-09-21T09:37:19.774089+0000","last_became_peered":"2024-09-21T09:37:19.774089+0000","last_unstale":"2024-09-21T09:37:19.774504+0000","last_undegraded":"2024-09-21T09:37:19.774504+0000","last_fullsized":"2024-09-21T09:37:19.774504+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:26:09.377745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-21T09:37:18.768785+0000","last_change":"2024-09-21T09:36:39.880733+0000","last_active":"2024-09-21T09:37:18.768785+0000","last_peered":"2024-09-21T09:37:18.768785+0000","last_clean":"2024-09-21T09:37:18.768785+0000","last_became_active":"2024-09-21T09:36:39.880476+0000","last_became_peered":"2024-09-21T09:36:39.880476+0000","last_unstale":"2024-09-21T09:37:18.768785+0000","last_undegraded":"2024-09-21T09:37:18.768785+0000","last_fullsized":"2024-09-21T09:37:18.768785+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:15:59.915859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234021,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28364,"kb_used_data":1344,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716788,"statfs":{"total":95995035648,"available":95965990912,"internally_reserved":0,"allocated":1376256,"data_stored":925421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.38,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.70599999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.51200000000000001,"15min":0.51000000000000001},"min":{"1min":0.37,"5min":0.215,"15min":0.215},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.33600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.71299999999999997,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.498},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51100000000000001,"15min":0.50800000000000001},"min":{"1min":0.38600000000000001,"5min":0.25,"15min":0.25},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.44700000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.69999999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53400000000000003,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.66100000000000003,"5min":0.873,"15min":0.873},"last":0.59699999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55600000000000005,"15min":0.56000000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66200000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.749,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.438}]},{"osd":4,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.308,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.57099999999999995,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.56999999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.46300000000000002,"15min":0.46700000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.64800000000000002,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.622},{"interface":"front","average":{"1min":0.505,"5min":0.48699999999999999,"15min":0.48399999999999999},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.40600000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:36:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.51100000000000001,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.623,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52400000000000002,"15min":0.52000000000000002},"min":{"1min":0.39500000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.71499999999999997,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.54500000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28356,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716796,"statfs":{"total":95995035648,"available":95965999104,"internally_reserved":0,"allocated":1368064,"data_stored":925294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54500000000000004,"15min":0.53500000000000003},"min":{"1min":0.33300000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.72299999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.54700000000000004,"15min":0.54600000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69699999999999995,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.57299999999999995}]},{"osd":1,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73999999999999999,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53100000000000003,"15min":0.52100000000000002},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.745,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.63}]},{"osd":2,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54500000000000004,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68200000000000005,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56699999999999995,"15min":0.56899999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.67100000000000004,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59699999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.52900000000000003,"15min":0.52000000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67300000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.55700000000000005,"15min":0.55000000000000004},"min":{"1min":0.42399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.55800000000000005}]},{"osd":4,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.44,"15min":0.436},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.61799999999999999,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.42699999999999999,"15min":0.41499999999999998},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70199999999999996,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.497},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.46100000000000002,"15min":0.44800000000000001},"min":{"1min":0.35599999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.441}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.48199999999999998,"15min":0.47799999999999998},"min":{"1min":0.34000000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70299999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69099999999999995,"5min":1.002,"15min":1.002},"last":0.53700000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757296,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":334275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54400000000000004,"15min":0.55300000000000005},"min":{"1min":0.39000000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.52400000000000002,"15min":0.52800000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53700000000000003,"15min":0.54900000000000004},"min":{"1min":0.38800000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.50700000000000001,"15min":0.50900000000000001},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.61499999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52000000000000002,"15min":0.52600000000000002},"min":{"1min":0.40699999999999997,"5min":0.372,"15min":0.372},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.54200000000000004,"15min":0.55200000000000005},"min":{"1min":0.37,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.59399999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.497},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.376,"15min":0.35599999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.51100000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.38500000000000001,"15min":0.36899999999999999},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.53500000000000003,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.38400000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.42799999999999999,"15min":0.41699999999999998},"min":{"1min":0.28899999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65900000000000003,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45800000000000002,"15min":0.44600000000000001},"min":{"1min":0.318,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63900000000000001,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.434}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.64100000000000001,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.53900000000000003,"15min":0.55300000000000005},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.627,"5min":0.749,"15min":0.749},"last":0.46700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018932,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27784,"kb_used_data":764,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717368,"statfs":{"total":95995035648,"available":95966584832,"internally_reserved":0,"allocated":782336,"data_stored":335314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51500000000000001,"15min":0.50600000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70599999999999996,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.502},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.64600000000000002,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.67600000000000005}]},{"osd":1,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.52200000000000002,"15min":0.52700000000000002},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61299999999999999,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.54100000000000004,"15min":0.53400000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.75900000000000001,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.59199999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72299999999999998,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.51700000000000002,"15min":0.51000000000000001},"min":{"1min":0.374,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.52100000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:36:43 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.54200000000000004,"15min":0.53900000000000003},"min":{"1min":0.40600000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64900000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68899999999999995,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 21 09:37:10 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.39600000000000002,"15min":0.378},"min":{"1min":0.29199999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.318},{"interface":"front","average":{"1min":0.439,"5min":0.39200000000000002,"15min":0.36699999999999999},"min":{"1min":0.29999999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53600000000000003,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:37:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.43099999999999999,"15min":0.42099999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.60699999999999998,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.372},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.44,"15min":0.42199999999999999},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.35999999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:36:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.47999999999999998,"15min":0.47099999999999997},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.45800000000000002,"15min":0.45500000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.40400000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182457,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.376,"15min":0.39300000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.093,"5min":2.093,"15min":2.093},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.38700000000000001,"15min":0.40300000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.39700000000000002,"15min":0.41499999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.49,"5min":1.49,"15min":1.49},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.377,"15min":0.39700000000000002},"min":{"1min":0.24299999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.255}]},{"osd":2,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41499999999999998,"15min":0.443},"min":{"1min":0.27700000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.375},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40000000000000002,"15min":0.40200000000000002},"min":{"1min":0.26400000000000001,"5min":0.254,"15min":0.254},"max":{"1min":2.1299999999999999,"5min":2.1299999999999999,"15min":2.1299999999999999},"last":0.32300000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:36:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86699999999999999,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.38300000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77300000000000002,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.57899999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.59599999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.8160000000000001,"5min":1.8160000000000001,"15min":1.8160000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.60599999999999998,"15min":0.61199999999999999},"min":{"1min":0.436,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.53500000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40400000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.137,"5min":1.137,"15min":1.137},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.58499999999999996,"15min":0.58099999999999996},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.7789999999999999,"5min":1.7789999999999999,"15min":1.7789999999999999},"last":0.56599999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:36:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57699999999999996,"15min":0.58099999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76700000000000002,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.504},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81499999999999995,"5min":0.875,"15min":0.875},"last":0.55400000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":765952,"data_stored":334220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.20499999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.45900000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.29899999999999999,"5min":0.34000000000000002,"15min":0.34499999999999997},"min":{"1min":0.224,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.38600000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:37:00 2024","interfaces":[{"interface":"back","average":{"1min":0.32700000000000001,"5min":0.36299999999999999,"15min":0.372},"min":{"1min":0.24099999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.47299999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.318,"5min":0.35399999999999998,"15min":0.35799999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.42499999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.46700000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.42599999999999999,"15min":0.44600000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.495,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.41199999999999998,"15min":0.44},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.48499999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.502}]},{"osd":4,"last update":"Sat Sep 21 09:36:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51700000000000002,"15min":0.51500000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.60899999999999999,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.67200000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:37:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52200000000000002,"15min":0.52000000000000002},"min":{"1min":0.33400000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.67800000000000005,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57799999999999996,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76300000000000001,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.94999999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.56200000000000006,"15min":0.56100000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.374,"5min":0.371,"15min":0.371},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.84099999999999997}]},{"osd":7,"last update":"Sat Sep 21 09:36:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.68100000000000005,"15min":0.70199999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.747,"5min":3.7519999999999998,"15min":3.7519999999999998},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.64800000000000002,"15min":0.66500000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.76600000000000001,"5min":3.4119999999999999,"15min":3.4119999999999999},"last":0.80600000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31856,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713296,"statfs":{"total":95995035648,"available":95962415104,"internally_reserved":0,"allocated":757760,"data_stored":333350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:37:35 2024","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":0.34200000000000003,"15min":0.31},"min":{"1min":0.19900000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.36399999999999999,"15min":0.32100000000000001},"min":{"1min":0.251,"5min":0.218,"15min":0.218},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:36:56 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.376,"15min":0.36799999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.53000000000000003,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.372,"5min":0.38800000000000001,"15min":0.38300000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 21 09:37:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40300000000000002,"15min":0.41299999999999998},"min":{"1min":0.26500000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.42399999999999999,"15min":0.441},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.58499999999999996,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.499}]},{"osd":4,"last update":"Sat Sep 21 09:36:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56000000000000005,"15min":0.56399999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.57799999999999996,"15min":0.58699999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.80500000000000005,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.74199999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:37:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.56799999999999995,"15min":0.58199999999999996},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.79900000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.59899999999999998,"15min":0.60799999999999998},"min":{"1min":0.42099999999999999,"5min":0.371,"15min":0.371},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.84299999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:37:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.40300000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.752},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59799999999999998,"15min":0.61399999999999999},"min":{"1min":0.433,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.75800000000000001,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.65800000000000003}]},{"osd":7,"last update":"Sat Sep 21 09:36:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59199999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.88100000000000001,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84199999999999997,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.80900000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738440,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32444,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712708,"statfs":{"total":95995035648,"available":95961812992,"internally_reserved":0,"allocated":1359872,"data_stored":924576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:37:37 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.35599999999999998,"15min":0.32500000000000001},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":0.48599999999999999,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.37,"15min":0.33600000000000002},"min":{"1min":0.22800000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.53900000000000003,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:37:01 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.38800000000000001,"15min":0.39000000000000001},"min":{"1min":0.23899999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.504,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.372},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.38300000000000001,"15min":0.376},"min":{"1min":0.26700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.52700000000000002,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:37:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.42299999999999999,"15min":0.43099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52300000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.379,"5min":0.41399999999999998,"15min":0.42899999999999999},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:36:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58299999999999996,"15min":0.58799999999999997},"min":{"1min":0.41999999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.73999999999999999,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:37:09 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58399999999999996},"min":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.437,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83599999999999997,"5min":1.04,"15min":1.04},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:37:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.59199999999999997,"15min":0.60599999999999998},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.77900000000000003,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61299999999999999,"15min":0.623},"min":{"1min":0.438,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.434}]},{"osd":7,"last update":"Sat Sep 21 09:36:54 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.60399999999999998,"15min":0.60899999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76800000000000002,"5min":0.874,"15min":0.874},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60799999999999998,"15min":0.60899999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.875,"5min":1.075,"15min":1.075},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:37:43.262 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:37:43.262 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:37:43.263 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:37:43.263 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:37:43.461 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:43 smithi191 ceph-mon[33301]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:43 smithi191 ceph-mon[33301]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:44.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:43 smithi076 ceph-mon[41325]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:43 smithi076 ceph-mon[41325]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:43 smithi076 ceph-mon[29313]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:37:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:43 smithi076 ceph-mon[29313]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:45.472 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:37:45.473 INFO:teuthology.orchestra.run.smithi076.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:37:46.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:45 smithi076 ceph-mon[29313]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:46.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:45 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/561047127' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:37:46.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:45 smithi076 ceph-mon[41325]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:46.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:45 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/561047127' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:37:46.051 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:37:46.052 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:37:46.061 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-21T09:37:46.061 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:37:46.082 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:37:46.082 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:37:46.083 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-21T09:37:46.083 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-21T09:37:46.085 INFO:tasks.rados:Beginning rados... 2024-09-21T09:37:46.085 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:37:46.085 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:37:46.085 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:37:46.086 INFO:tasks.rados:joining rados 2024-09-21T09:37:46.086 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:37:46.086 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T09:37:46.086 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi076.front.sepia.ceph.com 2024-09-21T09:37:46.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-21T09:37:46.087 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-21T09:37:46.087 INFO:tasks.rados:starting run 0 out of 1 2024-09-21T09:37:46.087 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-21T09:37:46.087 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_0 16 2024-09-21T09:37:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:45 smithi191 ceph-mon[33301]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:45 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/561047127' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:37:46.299 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:46.304 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:48.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:47 smithi076 ceph-mon[29313]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:48.101 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:47 smithi076 ceph-mon[41325]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:47 smithi191 ceph-mon[33301]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:48 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:48 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:49.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:37:48] "GET /metrics HTTP/1.1" 200 33710 "" "Prometheus/2.43.0" 2024-09-21T09:37:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:48 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:37:49.419 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:37:49] ENGINE Bus STOPPING 2024-09-21T09:37:49.420 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:37:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:37:49.420 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:37:49] ENGINE Bus STOPPED 2024-09-21T09:37:49.420 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:37:49] ENGINE Bus STARTING 2024-09-21T09:37:49.797 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:37:49] ENGINE Serving on http://:::9283 2024-09-21T09:37:49.797 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:37:49] ENGINE Bus STARTED 2024-09-21T09:37:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:49 smithi191 ceph-mon[33301]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:49 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/384876791' entity='client.admin' 2024-09-21T09:37:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:49 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-mon[29313]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/384876791' entity='client.admin' 2024-09-21T09:37:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-mon[41325]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/384876791' entity='client.admin' 2024-09-21T09:37:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:49 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:37:50.331 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T09:37:50.528 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2654216753' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[29313]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:50.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2654216753' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:50.948 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:50 smithi076 ceph-mon[41325]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:50.975 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_0' created 2024-09-21T09:37:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:50 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:37:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:50 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:37:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:50 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:37:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:50 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2654216753' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:50 smithi191 ceph-mon[33301]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-21T09:37:51.936 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:51 smithi076 ceph-mon[29313]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:51.936 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:51 smithi076 ceph-mon[29313]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T09:37:51.936 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:51 smithi076 ceph-mon[29313]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:37:51.937 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:51 smithi076 ceph-mon[41325]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:51.937 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:51 smithi076 ceph-mon[41325]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T09:37:51.937 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:51 smithi076 ceph-mon[41325]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:37:51.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-21T09:37:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:51 smithi191 ceph-mon[33301]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:51 smithi191 ceph-mon[33301]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-21T09:37:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:51 smithi191 ceph-mon[33301]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:37:52.985 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:52 smithi076 ceph-mon[29313]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:37:52.986 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:52 smithi076 ceph-mon[29313]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:52.986 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:52 smithi076 ceph-mon[41325]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:37:52.986 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:52 smithi076 ceph-mon[41325]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:53.035 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:53.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:52 smithi191 ceph-mon[33301]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:37:53.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:52 smithi191 ceph-mon[33301]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:54.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:53 smithi076 ceph-mon[29313]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:37:54.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:53 smithi076 ceph-mon[29313]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:37:54.345 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:53 smithi076 ceph-mon[41325]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:37:54.345 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:53 smithi076 ceph-mon[41325]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:37:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:53 smithi191 ceph-mon[33301]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:37:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:53 smithi191 ceph-mon[33301]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:37:55.269 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:54 smithi076 ceph-mon[29313]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:55.269 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:54 smithi076 ceph-mon[41325]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:55.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-09-21T09:37:55.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:54 smithi191 ceph-mon[33301]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:55.486 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:55.981 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-21T09:37:56.223 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:55 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4264380969' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:37:56.223 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:55 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4264380969' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:37:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:55 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4264380969' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:37:57.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:56 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4264380969' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:37:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:56 smithi076 ceph-mon[41325]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:37:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:56 smithi076 ceph-mon[41325]: pgmap v100: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:56 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4264380969' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:37:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:56 smithi076 ceph-mon[29313]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:37:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:56 smithi076 ceph-mon[29313]: pgmap v100: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:56 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4264380969' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:37:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:56 smithi191 ceph-mon[33301]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:37:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:56 smithi191 ceph-mon[33301]: pgmap v100: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:57.768 INFO:teuthology.orchestra.run.smithi076.stdout:wait for mgr daemons to upgrade 2024-09-21T09:37:57.864 DEBUG:teuthology.orchestra.run.smithi076:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 100 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --pool unique_pool_0 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:adding op weight read -> 100 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:adding op weight write -> 100 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:adding op weight delete -> 50 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:adding op weight snap_create -> 50 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:adding op weight snap_remove -> 50 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:adding op weight rollback -> 50 2024-09-21T09:37:57.932 INFO:tasks.rados.rados.0.smithi076.stdout:ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable) 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout:Configuration: 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout: Number of operations: 4000 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout: Number of objects: 50 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout: Max in flight operations: 16 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout: Object size (in bytes): 4000000 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout: Write stride min: 400000 2024-09-21T09:37:57.933 INFO:tasks.rados.rados.0.smithi076.stdout: Write stride max: 800000 2024-09-21T09:37:57.992 INFO:tasks.rados.rados.0.smithi076.stdout:1: write initial oid 1 2024-09-21T09:37:57.992 INFO:tasks.rados.rados.0.smithi076.stdout:1: seq_num 0 ranges {416807=475249,1542129=474678} 2024-09-21T09:37:57.997 INFO:tasks.rados.rados.0.smithi076.stdout:1: writing smithi07685715-1 from 416807 to 892056 tid 1 2024-09-21T09:37:58.004 INFO:tasks.rados.rados.0.smithi076.stdout:1: writing smithi07685715-1 from 1542129 to 2016807 tid 2 2024-09-21T09:37:58.004 INFO:tasks.rados.rados.0.smithi076.stdout:2: write initial oid 2 2024-09-21T09:37:58.004 INFO:tasks.rados.rados.0.smithi076.stdout:2: seq_num 1 ranges {416807=475249,1542129=474678} 2024-09-21T09:37:58.009 INFO:tasks.rados.rados.0.smithi076.stdout:2: writing smithi07685715-2 from 416807 to 892056 tid 1 2024-09-21T09:37:58.016 INFO:tasks.rados.rados.0.smithi076.stdout:2: writing smithi07685715-2 from 1542129 to 2016807 tid 2 2024-09-21T09:37:58.016 INFO:tasks.rados.rados.0.smithi076.stdout:3: write initial oid 3 2024-09-21T09:37:58.016 INFO:tasks.rados.rados.0.smithi076.stdout:3: seq_num 2 ranges {433614=550498,1600611=433003} 2024-09-21T09:37:58.021 INFO:tasks.rados.rados.0.smithi076.stdout:3: writing smithi07685715-3 from 433614 to 984112 tid 1 2024-09-21T09:37:58.028 INFO:tasks.rados.rados.0.smithi076.stdout:3: writing smithi07685715-3 from 1600611 to 2033614 tid 2 2024-09-21T09:37:58.028 INFO:tasks.rados.rados.0.smithi076.stdout:4: write initial oid 4 2024-09-21T09:37:58.028 INFO:tasks.rados.rados.0.smithi076.stdout:4: seq_num 3 ranges {450421=625747,1659093=391328} 2024-09-21T09:37:58.033 INFO:tasks.rados.rados.0.smithi076.stdout:4: writing smithi07685715-4 from 450421 to 1076168 tid 1 2024-09-21T09:37:58.039 INFO:tasks.rados.rados.0.smithi076.stdout:4: writing smithi07685715-4 from 1659093 to 2050421 tid 2 2024-09-21T09:37:58.039 INFO:tasks.rados.rados.0.smithi076.stdout:5: write initial oid 5 2024-09-21T09:37:58.039 INFO:tasks.rados.rados.0.smithi076.stdout:5: seq_num 4 ranges {467228=700996,1717575=349653} 2024-09-21T09:37:58.045 INFO:tasks.rados.rados.0.smithi076.stdout:5: writing smithi07685715-5 from 467228 to 1168224 tid 1 2024-09-21T09:37:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:5: writing smithi07685715-5 from 1717575 to 2067228 tid 2 2024-09-21T09:37:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:6: write initial oid 6 2024-09-21T09:37:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:6: seq_num 5 ranges {484035=776245,2059704=24331} 2024-09-21T09:37:58.057 INFO:tasks.rados.rados.0.smithi076.stdout:6: writing smithi07685715-6 from 484035 to 1260280 tid 1 2024-09-21T09:37:58.062 INFO:tasks.rados.rados.0.smithi076.stdout:6: writing smithi07685715-6 from 2059704 to 2084035 tid 2 2024-09-21T09:37:58.062 INFO:tasks.rados.rados.0.smithi076.stdout:7: write initial oid 7 2024-09-21T09:37:58.062 INFO:tasks.rados.rados.0.smithi076.stdout:7: seq_num 6 ranges {500842=451494,1718186=382656} 2024-09-21T09:37:58.067 INFO:tasks.rados.rados.0.smithi076.stdout:7: writing smithi07685715-7 from 500842 to 952336 tid 1 2024-09-21T09:37:58.074 INFO:tasks.rados.rados.0.smithi076.stdout:7: writing smithi07685715-7 from 1718186 to 2100842 tid 2 2024-09-21T09:37:58.074 INFO:tasks.rados.rados.0.smithi076.stdout:8: write initial oid 8 2024-09-21T09:37:58.074 INFO:tasks.rados.rados.0.smithi076.stdout:8: seq_num 7 ranges {517649=526743,1776668=340981} 2024-09-21T09:37:58.080 INFO:tasks.rados.rados.0.smithi076.stdout:8: writing smithi07685715-8 from 517649 to 1044392 tid 1 2024-09-21T09:37:58.086 INFO:tasks.rados.rados.0.smithi076.stdout:8: writing smithi07685715-8 from 1776668 to 2117649 tid 2 2024-09-21T09:37:58.086 INFO:tasks.rados.rados.0.smithi076.stdout:9: write initial oid 9 2024-09-21T09:37:58.086 INFO:tasks.rados.rados.0.smithi076.stdout:9: seq_num 8 ranges {534456=718345,1951503=182953} 2024-09-21T09:37:58.092 INFO:tasks.rados.rados.0.smithi076.stdout:9: writing smithi07685715-9 from 534456 to 1252801 tid 1 2024-09-21T09:37:58.098 INFO:tasks.rados.rados.0.smithi076.stdout:9: writing smithi07685715-9 from 1951503 to 2134456 tid 2 2024-09-21T09:37:58.098 INFO:tasks.rados.rados.0.smithi076.stdout:10: write initial oid 10 2024-09-21T09:37:58.098 INFO:tasks.rados.rados.0.smithi076.stdout:10: seq_num 9 ranges {551263=793594,1893632=257631} 2024-09-21T09:37:58.105 INFO:tasks.rados.rados.0.smithi076.stdout:10: writing smithi07685715-10 from 551263 to 1344857 tid 1 2024-09-21T09:37:58.109 INFO:tasks.rados.rados.0.smithi076.stdout:10: writing smithi07685715-10 from 1893632 to 2151263 tid 2 2024-09-21T09:37:58.110 INFO:tasks.rados.rados.0.smithi076.stdout:11: write initial oid 11 2024-09-21T09:37:58.110 INFO:tasks.rados.rados.0.smithi076.stdout:11: seq_num 10 ranges {568070=468843,1552114=615956} 2024-09-21T09:37:58.115 INFO:tasks.rados.rados.0.smithi076.stdout:11: writing smithi07685715-11 from 568070 to 1036913 tid 1 2024-09-21T09:37:58.121 INFO:tasks.rados.rados.0.smithi076.stdout:11: writing smithi07685715-11 from 1552114 to 2168070 tid 2 2024-09-21T09:37:58.121 INFO:tasks.rados.rados.0.smithi076.stdout:12: write initial oid 12 2024-09-21T09:37:58.122 INFO:tasks.rados.rados.0.smithi076.stdout:12: seq_num 11 ranges {584877=544092,1610596=562003} 2024-09-21T09:37:58.127 INFO:tasks.rados.rados.0.smithi076.stdout:12: writing smithi07685715-12 from 584877 to 1128969 tid 1 2024-09-21T09:37:58.133 INFO:tasks.rados.rados.0.smithi076.stdout:12: writing smithi07685715-12 from 1610596 to 2172599 tid 2 2024-09-21T09:37:58.133 INFO:tasks.rados.rados.0.smithi076.stdout:13: write initial oid 13 2024-09-21T09:37:58.134 INFO:tasks.rados.rados.0.smithi076.stdout:13: seq_num 12 ranges {601684=619341,1669078=532606} 2024-09-21T09:37:58.140 INFO:tasks.rados.rados.0.smithi076.stdout:13: writing smithi07685715-13 from 601684 to 1221025 tid 1 2024-09-21T09:37:58.146 INFO:tasks.rados.rados.0.smithi076.stdout:13: writing smithi07685715-13 from 1669078 to 2201684 tid 2 2024-09-21T09:37:58.146 INFO:tasks.rados.rados.0.smithi076.stdout:14: write initial oid 14 2024-09-21T09:37:58.146 INFO:tasks.rados.rados.0.smithi076.stdout:14: seq_num 13 ranges {618491=694590,2011207=207284} 2024-09-21T09:37:58.153 INFO:tasks.rados.rados.0.smithi076.stdout:14: writing smithi07685715-14 from 618491 to 1313081 tid 1 2024-09-21T09:37:58.158 INFO:tasks.rados.rados.0.smithi076.stdout:14: writing smithi07685715-14 from 2011207 to 2218491 tid 2 2024-09-21T09:37:58.158 INFO:tasks.rados.rados.0.smithi076.stdout:15: write initial oid 15 2024-09-21T09:37:58.158 INFO:tasks.rados.rados.0.smithi076.stdout:15: seq_num 14 ranges {635298=769839,2069689=165609} 2024-09-21T09:37:58.165 INFO:tasks.rados.rados.0.smithi076.stdout:15: writing smithi07685715-15 from 635298 to 1405137 tid 1 2024-09-21T09:37:58.170 INFO:tasks.rados.rados.0.smithi076.stdout:15: writing smithi07685715-15 from 2069689 to 2235298 tid 2 2024-09-21T09:37:58.170 INFO:tasks.rados.rados.0.smithi076.stdout:16: write initial oid 16 2024-09-21T09:37:58.170 INFO:tasks.rados.rados.0.smithi076.stdout:16: seq_num 15 ranges {652105=445088,1728171=452988} 2024-09-21T09:37:58.176 INFO:tasks.rados.rados.0.smithi076.stdout:16: writing smithi07685715-16 from 652105 to 1097193 tid 1 2024-09-21T09:37:58.182 INFO:tasks.rados.rados.0.smithi076.stdout:16: writing smithi07685715-16 from 1728171 to 2181159 tid 2 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: finishing write tid 1 to smithi07685715-1 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: finishing write tid 2 to smithi07685715-1 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: finishing write tid 3 to smithi07685715-1 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: finishing write tid 4 to smithi07685715-1 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: oid 1 updating version 0 to 1 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: oid 1 updating version 1 to 2 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: oid 1 updating version 2 to 3 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:2: finishing write tid 1 to smithi07685715-2 2024-09-21T09:37:58.183 INFO:tasks.rados.rados.0.smithi076.stdout:2: finishing write tid 2 to smithi07685715-2 2024-09-21T09:37:58.184 INFO:tasks.rados.rados.0.smithi076.stdout:2: finishing write tid 3 to smithi07685715-2 2024-09-21T09:37:58.184 INFO:tasks.rados.rados.0.smithi076.stdout:1: done (15 left) 2024-09-21T09:37:58.184 INFO:tasks.rados.rados.0.smithi076.stdout:17: write initial oid 17 2024-09-21T09:37:58.184 INFO:tasks.rados.rados.0.smithi076.stdout:17: seq_num 16 ranges {668912=636690,1903006=365906} 2024-09-21T09:37:58.189 INFO:tasks.rados.rados.0.smithi076.stdout:17: writing smithi07685715-17 from 668912 to 1305602 tid 1 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:17: writing smithi07685715-17 from 1903006 to 2268912 tid 2 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:2: finishing write tid 4 to smithi07685715-2 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:2: oid 2 updating version 0 to 2 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:2: oid 2 updating version 2 to 3 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:2: oid 2 version 3 is already newer than 1 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 3 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:3: finishing write tid 1 to smithi07685715-3 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:3: finishing write tid 2 to smithi07685715-3 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:3: finishing write tid 3 to smithi07685715-3 2024-09-21T09:37:58.196 INFO:tasks.rados.rados.0.smithi076.stdout:3: finishing write tid 4 to smithi07685715-3 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:3: oid 3 updating version 0 to 1 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:3: oid 3 updating version 1 to 2 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:3: oid 3 updating version 2 to 3 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: finishing write tid 1 to smithi07685715-4 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: finishing write tid 2 to smithi07685715-4 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: finishing write tid 3 to smithi07685715-4 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: finishing write tid 4 to smithi07685715-4 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: oid 4 updating version 0 to 1 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: oid 4 updating version 1 to 2 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: oid 4 updating version 2 to 3 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 3 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-21T09:37:58.197 INFO:tasks.rados.rados.0.smithi076.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: finishing write tid 1 to smithi07685715-5 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: finishing write tid 2 to smithi07685715-5 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: finishing write tid 3 to smithi07685715-5 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: finishing write tid 4 to smithi07685715-5 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: oid 5 updating version 0 to 5 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: oid 5 version 5 is already newer than 4 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: oid 5 updating version 5 to 6 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 6 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:6: finishing write tid 1 to smithi07685715-6 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:6: finishing write tid 2 to smithi07685715-6 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:6: finishing write tid 3 to smithi07685715-6 2024-09-21T09:37:58.198 INFO:tasks.rados.rados.0.smithi076.stdout:6: finishing write tid 4 to smithi07685715-6 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:6: oid 6 updating version 0 to 1 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:6: oid 6 updating version 1 to 2 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:6: oid 6 updating version 2 to 3 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 3 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: finishing write tid 1 to smithi07685715-7 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: finishing write tid 2 to smithi07685715-7 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: finishing write tid 3 to smithi07685715-7 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: finishing write tid 4 to smithi07685715-7 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: oid 7 updating version 0 to 2 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: oid 7 updating version 2 to 3 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: oid 7 version 3 is already newer than 1 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 3 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-21T09:37:58.199 INFO:tasks.rados.rados.0.smithi076.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:8: finishing write tid 1 to smithi07685715-8 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:8: finishing write tid 2 to smithi07685715-8 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:2: done (15 left) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:3: done (14 left) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:4: done (13 left) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:5: done (12 left) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:6: done (11 left) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:7: done (10 left) 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:18: write initial oid 18 2024-09-21T09:37:58.200 INFO:tasks.rados.rados.0.smithi076.stdout:18: seq_num 17 ranges {685719=711939,1845135=440584} 2024-09-21T09:37:58.202 INFO:tasks.rados.rados.0.smithi076.stdout:18: writing smithi07685715-18 from 685719 to 1397658 tid 1 2024-09-21T09:37:58.207 INFO:tasks.rados.rados.0.smithi076.stdout:18: writing smithi07685715-18 from 1845135 to 2285719 tid 2 2024-09-21T09:37:58.207 INFO:tasks.rados.rados.0.smithi076.stdout:19: write initial oid 19 2024-09-21T09:37:58.207 INFO:tasks.rados.rados.0.smithi076.stdout:19: seq_num 18 ranges {702526=787188,1903617=398909} 2024-09-21T09:37:58.215 INFO:tasks.rados.rados.0.smithi076.stdout:19: writing smithi07685715-19 from 702526 to 1489714 tid 1 2024-09-21T09:37:58.220 INFO:tasks.rados.rados.0.smithi076.stdout:19: writing smithi07685715-19 from 1903617 to 2302526 tid 2 2024-09-21T09:37:58.220 INFO:tasks.rados.rados.0.smithi076.stdout:20: write initial oid 20 2024-09-21T09:37:58.220 INFO:tasks.rados.rados.0.smithi076.stdout:20: seq_num 19 ranges {719333=462437,1962099=357234} 2024-09-21T09:37:58.226 INFO:tasks.rados.rados.0.smithi076.stdout:20: writing smithi07685715-20 from 719333 to 1181770 tid 1 2024-09-21T09:37:58.232 INFO:tasks.rados.rados.0.smithi076.stdout:20: writing smithi07685715-20 from 1962099 to 2319333 tid 2 2024-09-21T09:37:58.232 INFO:tasks.rados.rados.0.smithi076.stdout:21: write initial oid 21 2024-09-21T09:37:58.232 INFO:tasks.rados.rados.0.smithi076.stdout:21: seq_num 20 ranges {736140=537686,2020581=315559} 2024-09-21T09:37:58.239 INFO:tasks.rados.rados.0.smithi076.stdout:21: writing smithi07685715-21 from 736140 to 1273826 tid 1 2024-09-21T09:37:58.245 INFO:tasks.rados.rados.0.smithi076.stdout:21: writing smithi07685715-21 from 2020581 to 2336140 tid 2 2024-09-21T09:37:58.245 INFO:tasks.rados.rados.0.smithi076.stdout:22: write initial oid 22 2024-09-21T09:37:58.245 INFO:tasks.rados.rados.0.smithi076.stdout:22: seq_num 21 ranges {752947=612935,1962710=390237} 2024-09-21T09:37:58.252 INFO:tasks.rados.rados.0.smithi076.stdout:22: writing smithi07685715-22 from 752947 to 1365882 tid 1 2024-09-21T09:37:58.258 INFO:tasks.rados.rados.0.smithi076.stdout:22: writing smithi07685715-22 from 1962710 to 2352947 tid 2 2024-09-21T09:37:58.258 INFO:tasks.rados.rados.0.smithi076.stdout:23: write initial oid 23 2024-09-21T09:37:58.258 INFO:tasks.rados.rados.0.smithi076.stdout:23: seq_num 22 ranges {769754=688184,2021192=348562} 2024-09-21T09:37:58.266 INFO:tasks.rados.rados.0.smithi076.stdout:23: writing smithi07685715-23 from 769754 to 1457938 tid 1 2024-09-21T09:37:58.271 INFO:tasks.rados.rados.0.smithi076.stdout:23: writing smithi07685715-23 from 2021192 to 2369754 tid 2 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: finishing write tid 3 to smithi07685715-8 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: finishing write tid 4 to smithi07685715-8 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: oid 8 updating version 0 to 2 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: oid 8 updating version 2 to 3 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: oid 8 version 3 is already newer than 1 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 3 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:9: finishing write tid 1 to smithi07685715-9 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:9: finishing write tid 2 to smithi07685715-9 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:9: finishing write tid 3 to smithi07685715-9 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:8: done (15 left) 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:24: write initial oid 24 2024-09-21T09:37:58.272 INFO:tasks.rados.rados.0.smithi076.stdout:24: seq_num 23 ranges {786561=479786,1796027=467664} 2024-09-21T09:37:58.278 INFO:tasks.rados.rados.0.smithi076.stdout:24: writing smithi07685715-24 from 786561 to 1266347 tid 1 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:24: writing smithi07685715-24 from 1796027 to 2263691 tid 2 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:9: finishing write tid 4 to smithi07685715-9 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:9: oid 9 updating version 0 to 1 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:9: oid 9 updating version 1 to 2 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:9: oid 9 updating version 2 to 3 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 3 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:10: finishing write tid 1 to smithi07685715-10 2024-09-21T09:37:58.285 INFO:tasks.rados.rados.0.smithi076.stdout:10: finishing write tid 2 to smithi07685715-10 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:10: finishing write tid 3 to smithi07685715-10 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:10: finishing write tid 4 to smithi07685715-10 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:10: oid 10 updating version 0 to 6 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:10: oid 10 version 6 is already newer than 5 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:10: oid 10 version 6 is already newer than 4 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 6 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:11: finishing write tid 1 to smithi07685715-11 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:11: finishing write tid 2 to smithi07685715-11 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:11: finishing write tid 3 to smithi07685715-11 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:11: finishing write tid 4 to smithi07685715-11 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:11: oid 11 updating version 0 to 9 2024-09-21T09:37:58.286 INFO:tasks.rados.rados.0.smithi076.stdout:11: oid 11 version 9 is already newer than 8 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:11: oid 11 version 9 is already newer than 7 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 9 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:12: finishing write tid 1 to smithi07685715-12 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:12: finishing write tid 2 to smithi07685715-12 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:12: finishing write tid 3 to smithi07685715-12 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:12: finishing write tid 4 to smithi07685715-12 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:12: oid 12 updating version 0 to 5 2024-09-21T09:37:58.287 INFO:tasks.rados.rados.0.smithi076.stdout:12: oid 12 version 5 is already newer than 4 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:12: oid 12 updating version 5 to 6 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 6 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: finishing write tid 1 to smithi07685715-13 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: finishing write tid 2 to smithi07685715-13 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: finishing write tid 3 to smithi07685715-13 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: finishing write tid 4 to smithi07685715-13 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: oid 13 updating version 0 to 5 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: oid 13 updating version 5 to 6 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: oid 13 version 6 is already newer than 4 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 6 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-21T09:37:58.288 INFO:tasks.rados.rados.0.smithi076.stdout:14: finishing write tid 1 to smithi07685715-14 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:14: finishing write tid 2 to smithi07685715-14 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:14: finishing write tid 3 to smithi07685715-14 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:9: done (15 left) 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:10: done (14 left) 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:11: done (13 left) 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:12: done (12 left) 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:13: done (11 left) 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:25: write initial oid 25 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:25: seq_num 24 ranges {403368=555035,1454509=727675} 2024-09-21T09:37:58.289 INFO:tasks.rados.rados.0.smithi076.stdout:25: writing smithi07685715-25 from 403368 to 958403 tid 1 2024-09-21T09:37:58.296 INFO:tasks.rados.rados.0.smithi076.stdout:25: writing smithi07685715-25 from 1454509 to 2182184 tid 2 2024-09-21T09:37:58.296 INFO:tasks.rados.rados.0.smithi076.stdout:26: write initial oid 26 2024-09-21T09:37:58.296 INFO:tasks.rados.rados.0.smithi076.stdout:26: seq_num 25 ranges {420175=630284,1796638=471333} 2024-09-21T09:37:58.301 INFO:tasks.rados.rados.0.smithi076.stdout:26: writing smithi07685715-26 from 420175 to 1050459 tid 1 2024-09-21T09:37:58.308 INFO:tasks.rados.rados.0.smithi076.stdout:26: writing smithi07685715-26 from 1796638 to 2267971 tid 2 2024-09-21T09:37:58.308 INFO:tasks.rados.rados.0.smithi076.stdout:27: write initial oid 27 2024-09-21T09:37:58.308 INFO:tasks.rados.rados.0.smithi076.stdout:27: seq_num 26 ranges {436982=705533,1855120=581862} 2024-09-21T09:37:58.314 INFO:tasks.rados.rados.0.smithi076.stdout:27: writing smithi07685715-27 from 436982 to 1142515 tid 1 2024-09-21T09:37:58.322 INFO:tasks.rados.rados.0.smithi076.stdout:27: writing smithi07685715-27 from 1855120 to 2436982 tid 2 2024-09-21T09:37:58.322 INFO:tasks.rados.rados.0.smithi076.stdout:28: write initial oid 28 2024-09-21T09:37:58.322 INFO:tasks.rados.rados.0.smithi076.stdout:28: seq_num 27 ranges {453789=780782,1913602=475002} 2024-09-21T09:37:58.329 INFO:tasks.rados.rados.0.smithi076.stdout:28: writing smithi07685715-28 from 453789 to 1234571 tid 1 2024-09-21T09:37:58.335 INFO:tasks.rados.rados.0.smithi076.stdout:28: writing smithi07685715-28 from 1913602 to 2388604 tid 2 2024-09-21T09:37:58.335 INFO:tasks.rados.rados.0.smithi076.stdout:29: write initial oid 29 2024-09-21T09:37:58.336 INFO:tasks.rados.rados.0.smithi076.stdout:29: seq_num 28 ranges {470596=456031,1572084=618660} 2024-09-21T09:37:58.340 INFO:tasks.rados.rados.0.smithi076.stdout:29: writing smithi07685715-29 from 470596 to 926627 tid 1 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:29: writing smithi07685715-29 from 1572084 to 2190744 tid 2 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:14: finishing write tid 4 to smithi07685715-14 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:14: oid 14 updating version 0 to 9 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:14: oid 14 version 9 is already newer than 7 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:14: oid 14 version 9 is already newer than 8 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 9 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:37:58.348 INFO:tasks.rados.rados.0.smithi076.stdout:15: finishing write tid 1 to smithi07685715-15 2024-09-21T09:37:58.349 INFO:tasks.rados.rados.0.smithi076.stdout:15: finishing write tid 2 to smithi07685715-15 2024-09-21T09:37:58.349 INFO:tasks.rados.rados.0.smithi076.stdout:14: done (15 left) 2024-09-21T09:37:58.349 INFO:tasks.rados.rados.0.smithi076.stdout:30: write initial oid 30 2024-09-21T09:37:58.349 INFO:tasks.rados.rados.0.smithi076.stdout:30: seq_num 29 ranges {487403=531280,1514213=478671} 2024-09-21T09:37:58.353 INFO:tasks.rados.rados.0.smithi076.stdout:30: writing smithi07685715-30 from 487403 to 1018683 tid 1 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:30: writing smithi07685715-30 from 1514213 to 1992884 tid 2 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:15: finishing write tid 3 to smithi07685715-15 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:15: finishing write tid 4 to smithi07685715-15 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:15: oid 15 updating version 0 to 1 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:15: oid 15 updating version 1 to 3 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:15: oid 15 version 3 is already newer than 2 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 3 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-21T09:37:58.359 INFO:tasks.rados.rados.0.smithi076.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:37:58.360 INFO:tasks.rados.rados.0.smithi076.stdout:16: finishing write tid 1 to smithi07685715-16 2024-09-21T09:37:58.360 INFO:tasks.rados.rados.0.smithi076.stdout:15: done (15 left) 2024-09-21T09:37:58.360 INFO:tasks.rados.rados.0.smithi076.stdout:31: write initial oid 31 2024-09-21T09:37:58.360 INFO:tasks.rados.rados.0.smithi076.stdout:31: seq_num 30 ranges {504210=606529,1572695=622329} 2024-09-21T09:37:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:57 smithi191 ceph-mon[33301]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:37:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:57 smithi191 ceph-mon[33301]: Cluster is now healthy 2024-09-21T09:37:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:57 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3625324184' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:37:58.364 INFO:tasks.rados.rados.0.smithi076.stdout:31: writing smithi07685715-31 from 504210 to 1110739 tid 1 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:31: writing smithi07685715-31 from 1572695 to 2195024 tid 2 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: finishing write tid 2 to smithi07685715-16 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: finishing write tid 3 to smithi07685715-16 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: finishing write tid 4 to smithi07685715-16 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: oid 16 updating version 0 to 3 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: oid 16 version 3 is already newer than 1 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: oid 16 version 3 is already newer than 2 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 3 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-21T09:37:58.371 INFO:tasks.rados.rados.0.smithi076.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:37:58.372 INFO:tasks.rados.rados.0.smithi076.stdout:17: finishing write tid 1 to smithi07685715-17 2024-09-21T09:37:58.372 INFO:tasks.rados.rados.0.smithi076.stdout:17: finishing write tid 2 to smithi07685715-17 2024-09-21T09:37:58.372 INFO:tasks.rados.rados.0.smithi076.stdout:16: done (15 left) 2024-09-21T09:37:58.372 INFO:tasks.rados.rados.0.smithi076.stdout:32: write initial oid 32 2024-09-21T09:37:58.372 INFO:tasks.rados.rados.0.smithi076.stdout:32: seq_num 31 ranges {521017=798131,1747530=482340} 2024-09-21T09:37:58.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:57 smithi076 ceph-mon[29313]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:37:58.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:57 smithi076 ceph-mon[29313]: Cluster is now healthy 2024-09-21T09:37:58.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:57 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3625324184' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:37:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:57 smithi076 ceph-mon[41325]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:37:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:57 smithi076 ceph-mon[41325]: Cluster is now healthy 2024-09-21T09:37:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:57 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3625324184' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:37:58.378 INFO:tasks.rados.rados.0.smithi076.stdout:32: writing smithi07685715-32 from 521017 to 1319148 tid 1 2024-09-21T09:37:58.383 INFO:tasks.rados.rados.0.smithi076.stdout:32: writing smithi07685715-32 from 1747530 to 2229870 tid 2 2024-09-21T09:37:58.383 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.383 INFO:tasks.rados.rados.0.smithi076.stdout:17: finishing write tid 3 to smithi07685715-17 2024-09-21T09:37:58.383 INFO:tasks.rados.rados.0.smithi076.stdout:17: finishing write tid 4 to smithi07685715-17 2024-09-21T09:37:58.383 INFO:tasks.rados.rados.0.smithi076.stdout:17: oid 17 updating version 0 to 10 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:17: oid 17 updating version 10 to 12 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:17: oid 17 version 12 is already newer than 11 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 12 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:18: finishing write tid 1 to smithi07685715-18 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:18: finishing write tid 2 to smithi07685715-18 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:17: done (15 left) 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:33: write initial oid 33 2024-09-21T09:37:58.384 INFO:tasks.rados.rados.0.smithi076.stdout:33: seq_num 32 ranges {537824=473380,1806012=625998} 2024-09-21T09:37:58.388 INFO:tasks.rados.rados.0.smithi076.stdout:33: writing smithi07685715-33 from 537824 to 1011204 tid 1 2024-09-21T09:37:58.397 INFO:tasks.rados.rados.0.smithi076.stdout:33: writing smithi07685715-33 from 1806012 to 2432010 tid 2 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:18: finishing write tid 3 to smithi07685715-18 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:18: finishing write tid 4 to smithi07685715-18 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:18: oid 18 updating version 0 to 1 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:18: oid 18 updating version 1 to 3 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:18: oid 18 version 3 is already newer than 2 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 3 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:19: finishing write tid 1 to smithi07685715-19 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:19: finishing write tid 2 to smithi07685715-19 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:19: finishing write tid 3 to smithi07685715-19 2024-09-21T09:37:58.398 INFO:tasks.rados.rados.0.smithi076.stdout:19: finishing write tid 4 to smithi07685715-19 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:19: oid 19 updating version 0 to 1 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:19: oid 19 updating version 1 to 3 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:19: oid 19 version 3 is already newer than 2 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 3 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: finishing write tid 1 to smithi07685715-20 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: finishing write tid 2 to smithi07685715-20 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: finishing write tid 3 to smithi07685715-20 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: finishing write tid 4 to smithi07685715-20 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: oid 20 updating version 0 to 1 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: oid 20 updating version 1 to 3 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: oid 20 version 3 is already newer than 2 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 3 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-21T09:37:58.399 INFO:tasks.rados.rados.0.smithi076.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: finishing write tid 1 to smithi07685715-21 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: finishing write tid 2 to smithi07685715-21 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: finishing write tid 3 to smithi07685715-21 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: finishing write tid 4 to smithi07685715-21 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: oid 21 updating version 0 to 10 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: oid 21 updating version 10 to 11 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: oid 21 updating version 11 to 12 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 12 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:22: finishing write tid 1 to smithi07685715-22 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:22: finishing write tid 2 to smithi07685715-22 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:22: finishing write tid 3 to smithi07685715-22 2024-09-21T09:37:58.400 INFO:tasks.rados.rados.0.smithi076.stdout:22: finishing write tid 4 to smithi07685715-22 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:22: oid 22 updating version 0 to 5 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:22: oid 22 updating version 5 to 6 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:22: oid 22 version 6 is already newer than 4 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: finishing write tid 1 to smithi07685715-23 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: finishing write tid 2 to smithi07685715-23 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: finishing write tid 3 to smithi07685715-23 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: finishing write tid 4 to smithi07685715-23 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: oid 23 updating version 0 to 6 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: oid 23 version 6 is already newer than 4 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: oid 23 version 6 is already newer than 5 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 6 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-21T09:37:58.401 INFO:tasks.rados.rados.0.smithi076.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: finishing write tid 1 to smithi07685715-24 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: finishing write tid 2 to smithi07685715-24 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: finishing write tid 3 to smithi07685715-24 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: finishing write tid 4 to smithi07685715-24 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: oid 24 updating version 0 to 3 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: oid 24 version 3 is already newer than 2 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: oid 24 version 3 is already newer than 1 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 3 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:25: finishing write tid 1 to smithi07685715-25 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:25: finishing write tid 2 to smithi07685715-25 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:25: finishing write tid 3 to smithi07685715-25 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:25: finishing write tid 4 to smithi07685715-25 2024-09-21T09:37:58.402 INFO:tasks.rados.rados.0.smithi076.stdout:25: oid 25 updating version 0 to 3 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:25: oid 25 version 3 is already newer than 1 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:25: oid 25 version 3 is already newer than 2 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 3 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: finishing write tid 1 to smithi07685715-26 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: finishing write tid 2 to smithi07685715-26 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: finishing write tid 3 to smithi07685715-26 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: finishing write tid 4 to smithi07685715-26 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: oid 26 updating version 0 to 8 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: oid 26 version 8 is already newer than 7 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: oid 26 updating version 8 to 9 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 9 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:27: finishing write tid 1 to smithi07685715-27 2024-09-21T09:37:58.403 INFO:tasks.rados.rados.0.smithi076.stdout:27: finishing write tid 2 to smithi07685715-27 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:27: finishing write tid 3 to smithi07685715-27 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:27: finishing write tid 4 to smithi07685715-27 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:27: oid 27 updating version 0 to 8 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:27: oid 27 updating version 8 to 9 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:27: oid 27 version 9 is already newer than 7 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 9 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: finishing write tid 1 to smithi07685715-28 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: finishing write tid 2 to smithi07685715-28 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: finishing write tid 3 to smithi07685715-28 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: finishing write tid 4 to smithi07685715-28 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: oid 28 updating version 0 to 7 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: oid 28 updating version 7 to 9 2024-09-21T09:37:58.404 INFO:tasks.rados.rados.0.smithi076.stdout:28: oid 28 version 9 is already newer than 8 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 9 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:29: finishing write tid 1 to smithi07685715-29 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:29: finishing write tid 2 to smithi07685715-29 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:29: finishing write tid 3 to smithi07685715-29 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:18: done (15 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:19: done (14 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:20: done (13 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:21: done (12 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:22: done (11 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:23: done (10 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:24: done (9 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:25: done (8 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:26: done (7 left) 2024-09-21T09:37:58.405 INFO:tasks.rados.rados.0.smithi076.stdout:27: done (6 left) 2024-09-21T09:37:58.406 INFO:tasks.rados.rados.0.smithi076.stdout:28: done (5 left) 2024-09-21T09:37:58.406 INFO:tasks.rados.rados.0.smithi076.stdout:34: write initial oid 34 2024-09-21T09:37:58.406 INFO:tasks.rados.rados.0.smithi076.stdout:34: seq_num 33 ranges {554631=548629,1748141=486009} 2024-09-21T09:37:58.406 INFO:tasks.rados.rados.0.smithi076.stdout:34: writing smithi07685715-34 from 554631 to 1103260 tid 1 2024-09-21T09:37:58.409 INFO:tasks.rados.rados.0.smithi076.stdout:34: writing smithi07685715-34 from 1748141 to 2234150 tid 2 2024-09-21T09:37:58.409 INFO:tasks.rados.rados.0.smithi076.stdout:35: write initial oid 35 2024-09-21T09:37:58.409 INFO:tasks.rados.rados.0.smithi076.stdout:35: seq_num 34 ranges {571438=623878,1806623=629667} 2024-09-21T09:37:58.415 INFO:tasks.rados.rados.0.smithi076.stdout:35: writing smithi07685715-35 from 571438 to 1195316 tid 1 2024-09-21T09:37:58.422 INFO:tasks.rados.rados.0.smithi076.stdout:35: writing smithi07685715-35 from 1806623 to 2436290 tid 2 2024-09-21T09:37:58.422 INFO:tasks.rados.rados.0.smithi076.stdout:36: write initial oid 36 2024-09-21T09:37:58.422 INFO:tasks.rados.rados.0.smithi076.stdout:36: seq_num 35 ranges {588245=699127,1865105=489678} 2024-09-21T09:37:58.429 INFO:tasks.rados.rados.0.smithi076.stdout:36: writing smithi07685715-36 from 588245 to 1287372 tid 1 2024-09-21T09:37:58.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-21T09:37:58.435 INFO:tasks.rados.rados.0.smithi076.stdout:36: writing smithi07685715-36 from 1865105 to 2354783 tid 2 2024-09-21T09:37:58.435 INFO:tasks.rados.rados.0.smithi076.stdout:37: write initial oid 37 2024-09-21T09:37:58.435 INFO:tasks.rados.rados.0.smithi076.stdout:37: seq_num 36 ranges {605052=774376,1923587=633336} 2024-09-21T09:37:58.442 INFO:tasks.rados.rados.0.smithi076.stdout:37: writing smithi07685715-37 from 605052 to 1379428 tid 1 2024-09-21T09:37:58.449 INFO:tasks.rados.rados.0.smithi076.stdout:37: writing smithi07685715-37 from 1923587 to 2556923 tid 2 2024-09-21T09:37:58.449 INFO:tasks.rados.rados.0.smithi076.stdout:38: write initial oid 38 2024-09-21T09:37:58.449 INFO:tasks.rados.rados.0.smithi076.stdout:38: seq_num 37 ranges {621859=449625,1865716=756143} 2024-09-21T09:37:58.455 INFO:tasks.rados.rados.0.smithi076.stdout:38: writing smithi07685715-38 from 621859 to 1071484 tid 1 2024-09-21T09:37:58.464 INFO:tasks.rados.rados.0.smithi076.stdout:38: writing smithi07685715-38 from 1865716 to 2621859 tid 2 2024-09-21T09:37:58.464 INFO:tasks.rados.rados.0.smithi076.stdout:39: write initial oid 39 2024-09-21T09:37:58.464 INFO:tasks.rados.rados.0.smithi076.stdout:39: seq_num 38 ranges {638666=524874,1924198=637005} 2024-09-21T09:37:58.470 INFO:tasks.rados.rados.0.smithi076.stdout:39: writing smithi07685715-39 from 638666 to 1163540 tid 1 2024-09-21T09:37:58.478 INFO:tasks.rados.rados.0.smithi076.stdout:39: writing smithi07685715-39 from 1924198 to 2561203 tid 2 2024-09-21T09:37:58.478 INFO:tasks.rados.rados.0.smithi076.stdout:40: write initial oid 40 2024-09-21T09:37:58.478 INFO:tasks.rados.rados.0.smithi076.stdout:40: seq_num 39 ranges {655473=716476,2099033=556440} 2024-09-21T09:37:58.485 INFO:tasks.rados.rados.0.smithi076.stdout:40: writing smithi07685715-40 from 655473 to 1371949 tid 1 2024-09-21T09:37:58.492 INFO:tasks.rados.rados.0.smithi076.stdout:40: writing smithi07685715-40 from 2099033 to 2655473 tid 2 2024-09-21T09:37:58.492 INFO:tasks.rados.rados.0.smithi076.stdout:41: write initial oid 41 2024-09-21T09:37:58.492 INFO:tasks.rados.rados.0.smithi076.stdout:41: seq_num 40 ranges {672280=791725,2157515=514765} 2024-09-21T09:37:58.500 INFO:tasks.rados.rados.0.smithi076.stdout:41: writing smithi07685715-41 from 672280 to 1464005 tid 1 2024-09-21T09:37:58.507 INFO:tasks.rados.rados.0.smithi076.stdout:41: writing smithi07685715-41 from 2157515 to 2672280 tid 2 2024-09-21T09:37:58.507 INFO:tasks.rados.rados.0.smithi076.stdout:42: write initial oid 42 2024-09-21T09:37:58.507 INFO:tasks.rados.rados.0.smithi076.stdout:42: seq_num 41 ranges {689087=466974,1699644=784332} 2024-09-21T09:37:58.513 INFO:tasks.rados.rados.0.smithi076.stdout:42: writing smithi07685715-42 from 689087 to 1156061 tid 1 2024-09-21T09:37:58.520 INFO:tasks.rados.rados.0.smithi076.stdout:42: writing smithi07685715-42 from 1699644 to 2483976 tid 2 2024-09-21T09:37:58.521 INFO:tasks.rados.rados.0.smithi076.stdout:43: write initial oid 43 2024-09-21T09:37:58.521 INFO:tasks.rados.rados.0.smithi076.stdout:43: seq_num 42 ranges {705894=542223,1758126=644343} 2024-09-21T09:37:58.527 INFO:tasks.rados.rados.0.smithi076.stdout:43: writing smithi07685715-43 from 705894 to 1248117 tid 1 2024-09-21T09:37:58.534 INFO:tasks.rados.rados.0.smithi076.stdout:43: writing smithi07685715-43 from 1758126 to 2402469 tid 2 2024-09-21T09:37:58.534 INFO:tasks.rados.rados.0.smithi076.stdout:44: write initial oid 44 2024-09-21T09:37:58.534 INFO:tasks.rados.rados.0.smithi076.stdout:44: seq_num 43 ranges {722701=617472,1816608=788001} 2024-09-21T09:37:58.541 INFO:tasks.rados.rados.0.smithi076.stdout:44: writing smithi07685715-44 from 722701 to 1340173 tid 1 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:44: writing smithi07685715-44 from 1816608 to 2604609 tid 2 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:29: finishing write tid 4 to smithi07685715-29 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:29: oid 29 updating version 0 to 5 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:29: oid 29 updating version 5 to 6 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:29: oid 29 version 6 is already newer than 4 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 6 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:30: finishing write tid 1 to smithi07685715-30 2024-09-21T09:37:58.549 INFO:tasks.rados.rados.0.smithi076.stdout:30: finishing write tid 2 to smithi07685715-30 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: finishing write tid 3 to smithi07685715-30 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: finishing write tid 4 to smithi07685715-30 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: oid 30 updating version 0 to 11 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: oid 30 version 11 is already newer than 10 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: oid 30 updating version 11 to 12 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 12 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:31: finishing write tid 1 to smithi07685715-31 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:31: finishing write tid 2 to smithi07685715-31 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:29: done (15 left) 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:30: done (14 left) 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:45: write initial oid 45 2024-09-21T09:37:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:45: seq_num 44 ranges {739508=692721,1875090=648012} 2024-09-21T09:37:58.556 INFO:tasks.rados.rados.0.smithi076.stdout:45: writing smithi07685715-45 from 739508 to 1432229 tid 1 2024-09-21T09:37:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:45: writing smithi07685715-45 from 1875090 to 2523102 tid 2 2024-09-21T09:37:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:46: write initial oid 46 2024-09-21T09:37:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:46: seq_num 45 ranges {756315=767970,1933572=791670} 2024-09-21T09:37:58.570 INFO:tasks.rados.rados.0.smithi076.stdout:46: writing smithi07685715-46 from 756315 to 1524285 tid 1 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:46: writing smithi07685715-46 from 1933572 to 2725242 tid 2 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: finishing write tid 3 to smithi07685715-31 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: finishing write tid 4 to smithi07685715-31 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: oid 31 updating version 0 to 5 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: oid 31 updating version 5 to 6 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: oid 31 version 6 is already newer than 4 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 6 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:32: finishing write tid 1 to smithi07685715-32 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:31: done (15 left) 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:47: write initial oid 47 2024-09-21T09:37:58.578 INFO:tasks.rados.rados.0.smithi076.stdout:47: seq_num 46 ranges {773122=559572,1992054=651681} 2024-09-21T09:37:58.585 INFO:tasks.rados.rados.0.smithi076.stdout:47: writing smithi07685715-47 from 773122 to 1332694 tid 1 2024-09-21T09:37:58.592 INFO:tasks.rados.rados.0.smithi076.stdout:47: writing smithi07685715-47 from 1992054 to 2643735 tid 2 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: finishing write tid 2 to smithi07685715-32 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: finishing write tid 3 to smithi07685715-32 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: finishing write tid 4 to smithi07685715-32 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: oid 32 updating version 0 to 8 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: oid 32 version 8 is already newer than 7 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: oid 32 updating version 8 to 9 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 9 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:33: finishing write tid 1 to smithi07685715-33 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:33: finishing write tid 2 to smithi07685715-33 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:33: finishing write tid 3 to smithi07685715-33 2024-09-21T09:37:58.593 INFO:tasks.rados.rados.0.smithi076.stdout:33: finishing write tid 4 to smithi07685715-33 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:33: oid 33 updating version 0 to 5 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:33: oid 33 updating version 5 to 6 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:33: oid 33 version 6 is already newer than 4 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 6 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: finishing write tid 1 to smithi07685715-34 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: finishing write tid 2 to smithi07685715-34 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: finishing write tid 3 to smithi07685715-34 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: finishing write tid 4 to smithi07685715-34 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: oid 34 updating version 0 to 2 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: oid 34 updating version 2 to 3 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:34: oid 34 version 3 is already newer than 1 2024-09-21T09:37:58.594 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 3 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:35: finishing write tid 1 to smithi07685715-35 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:35: finishing write tid 2 to smithi07685715-35 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:35: finishing write tid 3 to smithi07685715-35 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:32: done (15 left) 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:33: done (14 left) 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:34: done (13 left) 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:48: write initial oid 48 2024-09-21T09:37:58.595 INFO:tasks.rados.rados.0.smithi076.stdout:48: seq_num 47 ranges {789929=634821,2050536=739393} 2024-09-21T09:37:58.600 INFO:tasks.rados.rados.0.smithi076.stdout:48: writing smithi07685715-48 from 789929 to 1424750 tid 1 2024-09-21T09:37:58.608 INFO:tasks.rados.rados.0.smithi076.stdout:48: writing smithi07685715-48 from 2050536 to 2789929 tid 2 2024-09-21T09:37:58.608 INFO:tasks.rados.rados.0.smithi076.stdout:49: write initial oid 49 2024-09-21T09:37:58.608 INFO:tasks.rados.rados.0.smithi076.stdout:49: seq_num 48 ranges {406736=710070,1709018=655350} 2024-09-21T09:37:58.614 INFO:tasks.rados.rados.0.smithi076.stdout:49: writing smithi07685715-49 from 406736 to 1116806 tid 1 2024-09-21T09:37:58.621 INFO:tasks.rados.rados.0.smithi076.stdout:49: writing smithi07685715-49 from 1709018 to 2364368 tid 2 2024-09-21T09:37:58.621 INFO:tasks.rados.rados.0.smithi076.stdout:50: write initial oid 50 2024-09-21T09:37:58.621 INFO:tasks.rados.rados.0.smithi076.stdout:50: seq_num 49 ranges {423543=785319,1767500=799008} 2024-09-21T09:37:58.628 INFO:tasks.rados.rados.0.smithi076.stdout:50: writing smithi07685715-50 from 423543 to 1208862 tid 1 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:50: writing smithi07685715-50 from 1767500 to 2566508 tid 2 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 16 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:35: finishing write tid 4 to smithi07685715-35 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:35: oid 35 updating version 0 to 15 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:35: oid 35 version 15 is already newer than 14 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:35: oid 35 version 15 is already newer than 13 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 15 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:37:58.636 INFO:tasks.rados.rados.0.smithi076.stdout:36: finishing write tid 1 to smithi07685715-36 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:35: done (15 left) 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:51: write oid 2 current snap is 0 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: finishing write tid 2 to smithi07685715-36 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: finishing write tid 3 to smithi07685715-36 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: finishing write tid 4 to smithi07685715-36 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: oid 36 updating version 0 to 5 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: oid 36 version 5 is already newer than 4 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: oid 36 updating version 5 to 6 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 6 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:37: finishing write tid 1 to smithi07685715-37 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:37: finishing write tid 2 to smithi07685715-37 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:37: finishing write tid 3 to smithi07685715-37 2024-09-21T09:37:58.637 INFO:tasks.rados.rados.0.smithi076.stdout:51: seq_num 50 ranges {440350=460568,1309629=542666,2323973=516377} 2024-09-21T09:37:58.641 INFO:tasks.rados.rados.0.smithi076.stdout:51: writing smithi07685715-2 from 440350 to 900918 tid 1 2024-09-21T09:37:58.645 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:37:58.646 INFO:tasks.rados.rados.0.smithi076.stdout:51: writing smithi07685715-2 from 1309629 to 1852295 tid 2 2024-09-21T09:37:58.652 INFO:tasks.rados.rados.0.smithi076.stdout:51: writing smithi07685715-2 from 2323973 to 2840350 tid 3 2024-09-21T09:37:58.652 INFO:tasks.rados.rados.0.smithi076.stdout:36: done (15 left) 2024-09-21T09:37:58.652 INFO:tasks.rados.rados.0.smithi076.stdout:52: rollback oid 14 current snap is 0 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:52: done (15 left) 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:53: write oid 19 current snap is 0 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:37: finishing write tid 4 to smithi07685715-37 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:37: oid 37 updating version 0 to 8 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:37: oid 37 updating version 8 to 9 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:37: oid 37 version 9 is already newer than 7 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 9 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:38: finishing write tid 1 to smithi07685715-38 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:38: finishing write tid 2 to smithi07685715-38 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:38: finishing write tid 3 to smithi07685715-38 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:38: finishing write tid 4 to smithi07685715-38 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:38: oid 38 updating version 0 to 15 2024-09-21T09:37:58.653 INFO:tasks.rados.rados.0.smithi076.stdout:38: oid 38 version 15 is already newer than 14 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:38: oid 38 version 15 is already newer than 13 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 15 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: finishing write tid 1 to smithi07685715-39 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: finishing write tid 2 to smithi07685715-39 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: finishing write tid 3 to smithi07685715-39 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: finishing write tid 4 to smithi07685715-39 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: oid 39 updating version 0 to 17 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: oid 39 version 17 is already newer than 16 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: oid 39 updating version 17 to 18 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 18 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:40: finishing write tid 1 to smithi07685715-40 2024-09-21T09:37:58.654 INFO:tasks.rados.rados.0.smithi076.stdout:40: finishing write tid 2 to smithi07685715-40 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:40: finishing write tid 3 to smithi07685715-40 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:40: finishing write tid 4 to smithi07685715-40 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:40: oid 40 updating version 0 to 10 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:40: oid 40 updating version 10 to 11 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:40: oid 40 updating version 11 to 12 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 12 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: finishing write tid 1 to smithi07685715-41 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: finishing write tid 2 to smithi07685715-41 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: finishing write tid 3 to smithi07685715-41 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: finishing write tid 4 to smithi07685715-41 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: oid 41 updating version 0 to 13 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: oid 41 updating version 13 to 14 2024-09-21T09:37:58.655 INFO:tasks.rados.rados.0.smithi076.stdout:41: oid 41 updating version 14 to 15 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 15 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: finishing write tid 1 to smithi07685715-42 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: finishing write tid 2 to smithi07685715-42 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: finishing write tid 3 to smithi07685715-42 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: finishing write tid 4 to smithi07685715-42 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: oid 42 updating version 0 to 5 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: oid 42 updating version 5 to 6 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: oid 42 version 6 is already newer than 4 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 6 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:37:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:53: seq_num 51 ranges {457157=535817,1768111=402677} 2024-09-21T09:37:58.657 INFO:tasks.rados.rados.0.smithi076.stdout:53: writing smithi07685715-19 from 457157 to 992974 tid 1 2024-09-21T09:37:58.664 INFO:tasks.rados.rados.0.smithi076.stdout:53: writing smithi07685715-19 from 1768111 to 2170788 tid 2 2024-09-21T09:37:58.664 INFO:tasks.rados.rados.0.smithi076.stdout:37: done (15 left) 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:38: done (14 left) 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:39: done (13 left) 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:40: done (12 left) 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:41: done (11 left) 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:42: done (10 left) 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:54: write oid 22 current snap is 0 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:43: finishing write tid 1 to smithi07685715-43 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:43: finishing write tid 2 to smithi07685715-43 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:43: finishing write tid 3 to smithi07685715-43 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:43: finishing write tid 4 to smithi07685715-43 2024-09-21T09:37:58.665 INFO:tasks.rados.rados.0.smithi076.stdout:43: oid 43 updating version 0 to 6 2024-09-21T09:37:58.666 INFO:tasks.rados.rados.0.smithi076.stdout:43: oid 43 version 6 is already newer than 5 2024-09-21T09:37:58.666 INFO:tasks.rados.rados.0.smithi076.stdout:43: oid 43 version 6 is already newer than 4 2024-09-21T09:37:58.666 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 6 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-21T09:37:58.666 INFO:tasks.rados.rados.0.smithi076.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:37:58.666 INFO:tasks.rados.rados.0.smithi076.stdout:44: finishing write tid 1 to smithi07685715-44 2024-09-21T09:37:58.666 INFO:tasks.rados.rados.0.smithi076.stdout:54: seq_num 52 ranges {473964=611066,1826593=546335,2778819=95145} 2024-09-21T09:37:58.670 INFO:tasks.rados.rados.0.smithi076.stdout:54: writing smithi07685715-22 from 473964 to 1085030 tid 1 2024-09-21T09:37:58.678 INFO:tasks.rados.rados.0.smithi076.stdout:54: writing smithi07685715-22 from 1826593 to 2372928 tid 2 2024-09-21T09:37:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:54: writing smithi07685715-22 from 2778819 to 2873964 tid 3 2024-09-21T09:37:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:43: done (10 left) 2024-09-21T09:37:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:55: delete oid 31 current snap is 0 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: finishing write tid 2 to smithi07685715-44 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: finishing write tid 3 to smithi07685715-44 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: finishing write tid 4 to smithi07685715-44 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: oid 44 updating version 0 to 4 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: oid 44 updating version 4 to 6 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: oid 44 version 6 is already newer than 5 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 6 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:45: finishing write tid 1 to smithi07685715-45 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:45: finishing write tid 2 to smithi07685715-45 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:45: finishing write tid 3 to smithi07685715-45 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:45: finishing write tid 4 to smithi07685715-45 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:45: oid 45 updating version 0 to 5 2024-09-21T09:37:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:45: oid 45 updating version 5 to 6 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:45: oid 45 version 6 is already newer than 4 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 6 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: finishing write tid 1 to smithi07685715-46 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: finishing write tid 2 to smithi07685715-46 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: finishing write tid 3 to smithi07685715-46 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: finishing write tid 4 to smithi07685715-46 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: oid 46 updating version 0 to 20 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: oid 46 updating version 20 to 21 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: oid 46 version 21 is already newer than 19 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 21 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:47: finishing write tid 1 to smithi07685715-47 2024-09-21T09:37:58.686 INFO:tasks.rados.rados.0.smithi076.stdout:47: finishing write tid 2 to smithi07685715-47 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:47: finishing write tid 3 to smithi07685715-47 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:47: finishing write tid 4 to smithi07685715-47 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:47: oid 47 updating version 0 to 15 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:47: oid 47 version 15 is already newer than 14 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:47: oid 47 version 15 is already newer than 13 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 15 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: finishing write tid 1 to smithi07685715-48 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: finishing write tid 2 to smithi07685715-48 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: finishing write tid 3 to smithi07685715-48 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: finishing write tid 4 to smithi07685715-48 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: oid 48 updating version 0 to 17 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: oid 48 updating version 17 to 18 2024-09-21T09:37:58.687 INFO:tasks.rados.rados.0.smithi076.stdout:48: oid 48 version 18 is already newer than 16 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 18 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: finishing write tid 1 to smithi07685715-49 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: finishing write tid 2 to smithi07685715-49 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: finishing write tid 3 to smithi07685715-49 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: finishing write tid 4 to smithi07685715-49 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: oid 49 updating version 0 to 19 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: oid 49 updating version 19 to 21 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: oid 49 version 21 is already newer than 20 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 21 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:50: finishing write tid 1 to smithi07685715-50 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:50: finishing write tid 2 to smithi07685715-50 2024-09-21T09:37:58.688 INFO:tasks.rados.rados.0.smithi076.stdout:50: finishing write tid 3 to smithi07685715-50 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:50: finishing write tid 4 to smithi07685715-50 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:50: oid 50 updating version 0 to 7 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:50: oid 50 updating version 7 to 8 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:50: oid 50 updating version 8 to 9 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 9 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: finishing write tid 1 to smithi07685715-2 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: finishing write tid 2 to smithi07685715-2 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: finishing write tid 3 to smithi07685715-2 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: finishing write tid 4 to smithi07685715-2 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: finishing write tid 5 to smithi07685715-2 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: oid 2 updating version 0 to 9 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: oid 2 version 9 is already newer than 7 2024-09-21T09:37:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:51: oid 2 updating version 9 to 10 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:51: oid 2 version 10 is already newer than 8 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 10 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:51: left oid 2 (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: finishing write tid 1 to smithi07685715-19 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: finishing write tid 2 to smithi07685715-19 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: finishing write tid 3 to smithi07685715-19 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: finishing write tid 4 to smithi07685715-19 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: oid 19 updating version 0 to 10 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: oid 19 updating version 10 to 12 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: oid 19 version 12 is already newer than 11 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 12 (ObjNum 51 snap 0 seq_num 51) dirty exists 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:53: left oid 19 (ObjNum 51 snap 0 seq_num 51) 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:54: finishing write tid 1 to smithi07685715-22 2024-09-21T09:37:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:54: finishing write tid 2 to smithi07685715-22 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:44: done (10 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:45: done (9 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:46: done (8 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:47: done (7 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:48: done (6 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:49: done (5 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:50: done (4 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:51: done (3 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:53: done (2 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:55: done (1 left) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:56: read oid 15 snap -1 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:57: delete oid 13 current snap is 0 2024-09-21T09:37:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:54: finishing write tid 3 to smithi07685715-22 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: finishing write tid 4 to smithi07685715-22 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: finishing write tid 5 to smithi07685715-22 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: oid 22 updating version 0 to 19 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: oid 22 version 19 is already newer than 18 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: oid 22 version 19 is already newer than 16 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: oid 22 version 19 is already newer than 17 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 19 (ObjNum 52 snap 0 seq_num 52) dirty exists 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: left oid 22 (ObjNum 52 snap 0 seq_num 52) 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:54: done (2 left) 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:57: done (1 left) 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:58: read oid 3 snap -1 2024-09-21T09:37:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:37:58.693 INFO:tasks.rados.rados.0.smithi076.stdout:59: read oid 28 snap -1 2024-09-21T09:37:58.693 INFO:tasks.rados.rados.0.smithi076.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:37:58.693 INFO:tasks.rados.rados.0.smithi076.stdout:60: read oid 34 snap -1 2024-09-21T09:37:58.693 INFO:tasks.rados.rados.0.smithi076.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:37:58.693 INFO:tasks.rados.rados.0.smithi076.stdout:61: delete oid 2 current snap is 0 2024-09-21T09:37:58.693 INFO:tasks.rados.rados.0.smithi076.stdout:61: done (4 left) 2024-09-21T09:37:58.694 INFO:tasks.rados.rados.0.smithi076.stdout:62: write oid 7 current snap is 0 2024-09-21T09:37:58.694 INFO:tasks.rados.rados.0.smithi076.stdout:62: seq_num 56 ranges {541192=628415,1660521=553673,2888511=52681} 2024-09-21T09:37:58.694 INFO:tasks.rados.rados.0.smithi076.stdout:62: writing smithi07685715-7 from 541192 to 1169607 tid 1 2024-09-21T09:37:58.698 INFO:tasks.rados.rados.0.smithi076.stdout:62: writing smithi07685715-7 from 1660521 to 2214194 tid 2 2024-09-21T09:37:58.702 INFO:tasks.rados.rados.0.smithi076.stdout:62: writing smithi07685715-7 from 2888511 to 2941192 tid 3 2024-09-21T09:37:58.702 INFO:tasks.rados.rados.0.smithi076.stdout:63: write oid 23 current snap is 0 2024-09-21T09:37:58.702 INFO:tasks.rados.rados.0.smithi076.stdout:63: seq_num 57 ranges {557999=703664,1719003=413684,2632287=325712} 2024-09-21T09:37:58.709 INFO:tasks.rados.rados.0.smithi076.stdout:63: writing smithi07685715-23 from 557999 to 1261663 tid 1 2024-09-21T09:37:58.714 INFO:tasks.rados.rados.0.smithi076.stdout:63: writing smithi07685715-23 from 1719003 to 2132687 tid 2 2024-09-21T09:37:58.718 INFO:tasks.rados.rados.0.smithi076.stdout:63: writing smithi07685715-23 from 2632287 to 2957999 tid 3 2024-09-21T09:37:58.718 INFO:tasks.rados.rados.0.smithi076.stdout:64: write oid 38 current snap is 0 2024-09-21T09:37:58.719 INFO:tasks.rados.rados.0.smithi076.stdout:64: seq_num 58 ranges {574806=778913,2061132=557342} 2024-09-21T09:37:58.725 INFO:tasks.rados.rados.0.smithi076.stdout:64: writing smithi07685715-38 from 574806 to 1353719 tid 1 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:64: writing smithi07685715-38 from 2061132 to 2618474 tid 2 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:65: rollback oid 25 current snap is 0 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:65: done (7 left) 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:66: read oid 29 snap -1 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:66: expect (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:67: write oid 44 current snap is 0 2024-09-21T09:37:58.733 INFO:tasks.rados.rados.0.smithi076.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:37:58.807 INFO:tasks.rados.rados.0.smithi076.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:37:58.892 INFO:tasks.rados.rados.0.smithi076.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:37:58.961 INFO:tasks.rados.rados.0.smithi076.stdout:62: finishing write tid 1 to smithi07685715-7 2024-09-21T09:37:58.961 INFO:tasks.rados.rados.0.smithi076.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: finishing write tid 2 to smithi07685715-7 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: finishing write tid 3 to smithi07685715-7 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: finishing write tid 4 to smithi07685715-7 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: finishing write tid 5 to smithi07685715-7 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: oid 7 updating version 0 to 18 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: oid 7 version 18 is already newer than 16 2024-09-21T09:37:59.040 INFO:tasks.rados.rados.0.smithi076.stdout:62: oid 7 updating version 18 to 19 2024-09-21T09:37:59.041 INFO:tasks.rados.rados.0.smithi076.stdout:62: oid 7 version 19 is already newer than 17 2024-09-21T09:37:59.041 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 19 (ObjNum 56 snap 0 seq_num 56) dirty exists 2024-09-21T09:37:59.041 INFO:tasks.rados.rados.0.smithi076.stdout:62: left oid 7 (ObjNum 56 snap 0 seq_num 56) 2024-09-21T09:37:59.041 INFO:tasks.rados.rados.0.smithi076.stdout:63: finishing write tid 1 to smithi07685715-23 2024-09-21T09:37:59.041 INFO:tasks.rados.rados.0.smithi076.stdout:67: seq_num 59 ranges {591613=454162,1719614=417353,2570780=420833} 2024-09-21T09:37:59.045 INFO:tasks.rados.rados.0.smithi076.stdout:67: writing smithi07685715-44 from 591613 to 1045775 tid 1 2024-09-21T09:37:59.051 INFO:tasks.rados.rados.0.smithi076.stdout:67: writing smithi07685715-44 from 1719614 to 2136967 tid 2 2024-09-21T09:37:59.056 INFO:tasks.rados.rados.0.smithi076.stdout:67: writing smithi07685715-44 from 2570780 to 2991613 tid 3 2024-09-21T09:37:59.056 INFO:tasks.rados.rados.0.smithi076.stdout:56: done (8 left) 2024-09-21T09:37:59.056 INFO:tasks.rados.rados.0.smithi076.stdout:58: done (7 left) 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:59: done (6 left) 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:60: done (5 left) 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:62: done (4 left) 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:68: write oid 11 current snap is 0 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:63: finishing write tid 2 to smithi07685715-23 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:63: finishing write tid 3 to smithi07685715-23 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:63: finishing write tid 4 to smithi07685715-23 2024-09-21T09:37:59.057 INFO:tasks.rados.rados.0.smithi076.stdout:68: seq_num 60 ranges {608420=529411,1778096=561011,2881850=126570} 2024-09-21T09:37:59.062 INFO:tasks.rados.rados.0.smithi076.stdout:68: writing smithi07685715-11 from 608420 to 1137831 tid 1 2024-09-21T09:37:59.067 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3625324184' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:37:59.067 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-mon[29313]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:37:59.067 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-mon[29313]: pgmap v102: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:59.067 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:37:58] "GET /metrics HTTP/1.1" 200 33710 "" "Prometheus/2.43.0" 2024-09-21T09:37:59.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3625324184' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:37:59.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-mon[41325]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:37:59.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:37:58 smithi076 ceph-mon[41325]: pgmap v102: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:59.069 INFO:tasks.rados.rados.0.smithi076.stdout:68: writing smithi07685715-11 from 1778096 to 2339107 tid 2 2024-09-21T09:37:59.073 INFO:tasks.rados.rados.0.smithi076.stdout:68: writing smithi07685715-11 from 2881850 to 3008420 tid 3 2024-09-21T09:37:59.073 INFO:tasks.rados.rados.0.smithi076.stdout:69: write oid 26 current snap is 0 2024-09-21T09:37:59.073 INFO:tasks.rados.rados.0.smithi076.stdout:63: finishing write tid 5 to smithi07685715-23 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:63: oid 23 updating version 0 to 25 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:63: oid 23 version 25 is already newer than 22 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:63: oid 23 version 25 is already newer than 24 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:63: oid 23 version 25 is already newer than 23 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 25 (ObjNum 57 snap 0 seq_num 57) dirty exists 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:63: left oid 23 (ObjNum 57 snap 0 seq_num 57) 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:64: finishing write tid 1 to smithi07685715-38 2024-09-21T09:37:59.074 INFO:tasks.rados.rados.0.smithi076.stdout:69: seq_num 61 ranges {625227=721013,1952931=704669} 2024-09-21T09:37:59.080 INFO:tasks.rados.rados.0.smithi076.stdout:69: writing smithi07685715-26 from 625227 to 1346240 tid 1 2024-09-21T09:37:59.088 INFO:tasks.rados.rados.0.smithi076.stdout:69: writing smithi07685715-26 from 1952931 to 2657600 tid 2 2024-09-21T09:37:59.088 INFO:tasks.rados.rados.0.smithi076.stdout:63: done (5 left) 2024-09-21T09:37:59.088 INFO:tasks.rados.rados.0.smithi076.stdout:70: write oid 22 current snap is 0 2024-09-21T09:37:59.088 INFO:tasks.rados.rados.0.smithi076.stdout:64: finishing write tid 2 to smithi07685715-38 2024-09-21T09:37:59.088 INFO:tasks.rados.rados.0.smithi076.stdout:70: seq_num 62 ranges {642034=796262,1895060=564680} 2024-09-21T09:37:59.096 INFO:tasks.rados.rados.0.smithi076.stdout:70: writing smithi07685715-22 from 642034 to 1438296 tid 1 2024-09-21T09:37:59.101 INFO:tasks.rados.rados.0.smithi076.stdout:70: writing smithi07685715-22 from 1895060 to 2459740 tid 2 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:71: snap_create 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:64: finishing write tid 3 to smithi07685715-38 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:64: finishing write tid 4 to smithi07685715-38 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:64: oid 38 updating version 0 to 21 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:64: oid 38 version 21 is already newer than 20 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:64: oid 38 updating version 21 to 22 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 22 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:64: left oid 38 (ObjNum 58 snap 0 seq_num 58) 2024-09-21T09:37:59.102 INFO:tasks.rados.rados.0.smithi076.stdout:66: expect (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: finishing write tid 1 to smithi07685715-44 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: finishing write tid 2 to smithi07685715-44 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: finishing write tid 3 to smithi07685715-44 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: finishing write tid 4 to smithi07685715-44 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: finishing write tid 5 to smithi07685715-44 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: oid 44 updating version 0 to 8 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: oid 44 updating version 8 to 10 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: oid 44 version 10 is already newer than 7 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: oid 44 version 10 is already newer than 9 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 10 (ObjNum 59 snap 0 seq_num 59) dirty exists 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:67: left oid 44 (ObjNum 59 snap 0 seq_num 59) 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:68: finishing write tid 1 to smithi07685715-11 2024-09-21T09:37:59.182 INFO:tasks.rados.rados.0.smithi076.stdout:68: finishing write tid 2 to smithi07685715-11 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: finishing write tid 3 to smithi07685715-11 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: finishing write tid 4 to smithi07685715-11 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: finishing write tid 5 to smithi07685715-11 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: oid 11 updating version 0 to 22 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: oid 11 updating version 22 to 23 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: oid 11 updating version 23 to 24 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: oid 11 updating version 24 to 25 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 25 (ObjNum 60 snap 0 seq_num 60) dirty exists 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:68: left oid 11 (ObjNum 60 snap 0 seq_num 60) 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:69: finishing write tid 1 to smithi07685715-26 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:69: finishing write tid 2 to smithi07685715-26 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:69: finishing write tid 3 to smithi07685715-26 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:69: finishing write tid 4 to smithi07685715-26 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:69: oid 26 updating version 0 to 10 2024-09-21T09:37:59.183 INFO:tasks.rados.rados.0.smithi076.stdout:69: oid 26 updating version 10 to 12 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:69: oid 26 version 12 is already newer than 11 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 12 (ObjNum 61 snap 0 seq_num 61) dirty exists 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:69: left oid 26 (ObjNum 61 snap 0 seq_num 61) 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: finishing write tid 1 to smithi07685715-22 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: finishing write tid 2 to smithi07685715-22 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: finishing write tid 3 to smithi07685715-22 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: finishing write tid 4 to smithi07685715-22 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: oid 22 updating version 0 to 22 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: oid 22 version 22 is already newer than 21 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: oid 22 version 22 is already newer than 20 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 22 (ObjNum 62 snap 0 seq_num 62) dirty exists 2024-09-21T09:37:59.184 INFO:tasks.rados.rados.0.smithi076.stdout:70: left oid 22 (ObjNum 62 snap 0 seq_num 62) 2024-09-21T09:37:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:58 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3625324184' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:37:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:58 smithi191 ceph-mon[33301]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:37:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:37:58 smithi191 ceph-mon[33301]: pgmap v102: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:64: done (6 left) 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:66: done (5 left) 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:67: done (4 left) 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:68: done (3 left) 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:69: done (2 left) 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:70: done (1 left) 2024-09-21T09:37:59.998 INFO:tasks.rados.rados.0.smithi076.stdout:71: done (0 left) 2024-09-21T09:37:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:72: write oid 50 current snap is 1 2024-09-21T09:37:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:72: seq_num 63 ranges {658841=471511,1553542=708338,2964119=94722} 2024-09-21T09:38:00.004 INFO:tasks.rados.rados.0.smithi076.stdout:72: writing smithi07685715-50 from 658841 to 1130352 tid 1 2024-09-21T09:38:00.010 INFO:tasks.rados.rados.0.smithi076.stdout:72: writing smithi07685715-50 from 1553542 to 2261880 tid 2 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:72: writing smithi07685715-50 from 2964119 to 3058841 tid 3 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:73: rollback oid 38 current snap is 1 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 0 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:72: finishing write tid 1 to smithi07685715-50 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:72: finishing write tid 2 to smithi07685715-50 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:74: write oid 33 current snap is 1 2024-09-21T09:38:00.015 INFO:tasks.rados.rados.0.smithi076.stdout:74: seq_num 64 ranges {675648=546760,2012024=568349} 2024-09-21T09:38:00.021 INFO:tasks.rados.rados.0.smithi076.stdout:74: writing smithi07685715-33 from 675648 to 1222408 tid 1 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:74: writing smithi07685715-33 from 2012024 to 2580373 tid 2 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:75: rollback oid 37 current snap is 1 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 0 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:72: finishing write tid 3 to smithi07685715-50 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:73: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:73: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:38:00.030 INFO:tasks.rados.rados.0.smithi076.stdout:72: finishing write tid 4 to smithi07685715-50 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: finishing write tid 5 to smithi07685715-50 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: oid 50 updating version 0 to 13 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: oid 50 version 13 is already newer than 12 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: oid 50 version 13 is already newer than 11 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: oid 50 updating version 13 to 14 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 14 (ObjNum 63 snap 1 seq_num 63) dirty exists 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: left oid 50 (ObjNum 63 snap 1 seq_num 63) 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:73: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 26 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:74: finishing write tid 1 to smithi07685715-33 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:72: done (3 left) 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:73: done (2 left) 2024-09-21T09:38:00.031 INFO:tasks.rados.rados.0.smithi076.stdout:76: snap_create 2024-09-21T09:38:00.032 INFO:tasks.rados.rados.0.smithi076.stdout:75: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:38:00.032 INFO:tasks.rados.rados.0.smithi076.stdout:75: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:75: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 13 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: finishing write tid 2 to smithi07685715-33 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: finishing write tid 3 to smithi07685715-33 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: finishing write tid 4 to smithi07685715-33 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: oid 33 updating version 0 to 8 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: oid 33 updating version 8 to 10 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: oid 33 version 10 is already newer than 9 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 10 (ObjNum 64 snap 1 seq_num 64) dirty exists 2024-09-21T09:38:00.033 INFO:tasks.rados.rados.0.smithi076.stdout:74: left oid 33 (ObjNum 64 snap 1 seq_num 64) 2024-09-21T09:38:01.004 INFO:tasks.rados.rados.0.smithi076.stdout:74: done (2 left) 2024-09-21T09:38:01.004 INFO:tasks.rados.rados.0.smithi076.stdout:75: done (1 left) 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi076.stdout:76: done (0 left) 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi076.stdout:77: read oid 33 snap -1 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi076.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-09-21T09:38:01.005 INFO:tasks.rados.rados.0.smithi076.stdout:78: snap_create 2024-09-21T09:38:01.008 INFO:tasks.rados.rados.0.smithi076.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-09-21T09:38:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:01 smithi191 ceph-mon[33301]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:38:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:01 smithi191 ceph-mon[33301]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:01 smithi191 ceph-mon[33301]: pgmap v104: 89 pgs: 89 active+clean; 114 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 11 MiB/s wr, 66 op/s 2024-09-21T09:38:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:01 smithi076 ceph-mon[29313]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:38:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:01 smithi076 ceph-mon[29313]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:01 smithi076 ceph-mon[29313]: pgmap v104: 89 pgs: 89 active+clean; 114 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 11 MiB/s wr, 66 op/s 2024-09-21T09:38:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:01 smithi076 ceph-mon[41325]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:38:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:01 smithi076 ceph-mon[41325]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:01 smithi076 ceph-mon[41325]: pgmap v104: 89 pgs: 89 active+clean; 114 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 11 MiB/s wr, 66 op/s 2024-09-21T09:38:02.005 INFO:tasks.rados.rados.0.smithi076.stdout:77: done (1 left) 2024-09-21T09:38:02.005 INFO:tasks.rados.rados.0.smithi076.stdout:78: done (0 left) 2024-09-21T09:38:02.005 INFO:tasks.rados.rados.0.smithi076.stdout:79: rollback oid 9 current snap is 3 2024-09-21T09:38:02.005 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 0 2024-09-21T09:38:02.005 INFO:tasks.rados.rados.0.smithi076.stdout:80: snap_create 2024-09-21T09:38:02.011 INFO:tasks.rados.rados.0.smithi076.stdout:79: finishing rollback tid 0 to smithi07685715-9 2024-09-21T09:38:02.012 INFO:tasks.rados.rados.0.smithi076.stdout:79: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:38:02.012 INFO:tasks.rados.rados.0.smithi076.stdout:79: finishing rollback tid 2 to smithi07685715-9 2024-09-21T09:38:02.012 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 16 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:38:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:02 smithi191 ceph-mon[33301]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:38:02.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:02 smithi076 ceph-mon[29313]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:38:02.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:02 smithi076 ceph-mon[41325]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:38:03.006 INFO:tasks.rados.rados.0.smithi076.stdout:79: done (1 left) 2024-09-21T09:38:03.006 INFO:tasks.rados.rados.0.smithi076.stdout:80: done (0 left) 2024-09-21T09:38:03.007 INFO:tasks.rados.rados.0.smithi076.stdout:81: read oid 26 snap -1 2024-09-21T09:38:03.007 INFO:tasks.rados.rados.0.smithi076.stdout:81: expect (ObjNum 61 snap 0 seq_num 61) 2024-09-21T09:38:03.007 INFO:tasks.rados.rados.0.smithi076.stdout:82: read oid 36 snap -1 2024-09-21T09:38:03.007 INFO:tasks.rados.rados.0.smithi076.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:38:03.007 INFO:tasks.rados.rados.0.smithi076.stdout:83: snap_create 2024-09-21T09:38:03.010 INFO:tasks.rados.rados.0.smithi076.stdout:81: expect (ObjNum 61 snap 0 seq_num 61) 2024-09-21T09:38:03.155 INFO:tasks.rados.rados.0.smithi076.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:38:03.273 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:03 smithi076 ceph-mon[41325]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:38:03.273 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:03 smithi076 ceph-mon[41325]: pgmap v107: 89 pgs: 89 active+clean; 114 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 14 MiB/s wr, 85 op/s 2024-09-21T09:38:03.273 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:03 smithi076 ceph-mon[29313]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:38:03.273 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:03 smithi076 ceph-mon[29313]: pgmap v107: 89 pgs: 89 active+clean; 114 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 14 MiB/s wr, 85 op/s 2024-09-21T09:38:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:03 smithi191 ceph-mon[33301]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:38:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:03 smithi191 ceph-mon[33301]: pgmap v107: 89 pgs: 89 active+clean; 114 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 14 MiB/s wr, 85 op/s 2024-09-21T09:38:04.007 INFO:tasks.rados.rados.0.smithi076.stdout:81: done (2 left) 2024-09-21T09:38:04.008 INFO:tasks.rados.rados.0.smithi076.stdout:82: done (1 left) 2024-09-21T09:38:04.008 INFO:tasks.rados.rados.0.smithi076.stdout:83: done (0 left) 2024-09-21T09:38:04.008 INFO:tasks.rados.rados.0.smithi076.stdout:84: delete oid 7 current snap is 5 2024-09-21T09:38:04.009 INFO:tasks.rados.rados.0.smithi076.stdout:84: done (0 left) 2024-09-21T09:38:04.009 INFO:tasks.rados.rados.0.smithi076.stdout:85: write oid 48 current snap is 5 2024-09-21T09:38:04.010 INFO:tasks.rados.rados.0.smithi076.stdout:85: seq_num 66 ranges {709262=697258,2012635=572018,3046388=62874} 2024-09-21T09:38:04.018 INFO:tasks.rados.rados.0.smithi076.stdout:85: writing smithi07685715-48 from 709262 to 1406520 tid 1 2024-09-21T09:38:04.025 INFO:tasks.rados.rados.0.smithi076.stdout:85: writing smithi07685715-48 from 2012635 to 2584653 tid 2 2024-09-21T09:38:04.029 INFO:tasks.rados.rados.0.smithi076.stdout:85: writing smithi07685715-48 from 3046388 to 3109262 tid 3 2024-09-21T09:38:04.029 INFO:tasks.rados.rados.0.smithi076.stdout:86: snap_create 2024-09-21T09:38:04.029 INFO:tasks.rados.rados.0.smithi076.stdout:85: finishing write tid 1 to smithi07685715-48 2024-09-21T09:38:04.029 INFO:tasks.rados.rados.0.smithi076.stdout:85: finishing write tid 2 to smithi07685715-48 2024-09-21T09:38:04.030 INFO:tasks.rados.rados.0.smithi076.stdout:85: finishing write tid 3 to smithi07685715-48 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: finishing write tid 4 to smithi07685715-48 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: finishing write tid 5 to smithi07685715-48 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: oid 48 updating version 0 to 28 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: oid 48 version 28 is already newer than 27 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: oid 48 updating version 28 to 30 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: oid 48 version 30 is already newer than 29 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 30 (ObjNum 66 snap 5 seq_num 66) dirty exists 2024-09-21T09:38:04.031 INFO:tasks.rados.rados.0.smithi076.stdout:85: left oid 48 (ObjNum 66 snap 5 seq_num 66) 2024-09-21T09:38:04.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:04 smithi076 ceph-mon[41325]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:38:04.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:04 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:04.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:04 smithi076 ceph-mon[41325]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:38:04.290 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:04 smithi076 ceph-mon[29313]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:38:04.290 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:04 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:04.290 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:04 smithi076 ceph-mon[29313]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:38:04.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:04 smithi191 ceph-mon[33301]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:38:04.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:04 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:04.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:04 smithi191 ceph-mon[33301]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:38:05.030 INFO:tasks.rados.rados.0.smithi076.stdout:85: done (1 left) 2024-09-21T09:38:05.030 INFO:tasks.rados.rados.0.smithi076.stdout:86: done (0 left) 2024-09-21T09:38:05.030 INFO:tasks.rados.rados.0.smithi076.stdout:87: rollback oid 10 current snap is 6 2024-09-21T09:38:05.030 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 2 2024-09-21T09:38:05.030 INFO:tasks.rados.rados.0.smithi076.stdout:88: snap_create 2024-09-21T09:38:05.034 INFO:tasks.rados.rados.0.smithi076.stdout:87: finishing rollback tid 0 to smithi07685715-10 2024-09-21T09:38:05.035 INFO:tasks.rados.rados.0.smithi076.stdout:87: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:38:05.035 INFO:tasks.rados.rados.0.smithi076.stdout:87: finishing rollback tid 2 to smithi07685715-10 2024-09-21T09:38:05.035 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 32 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:38:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:05 smithi191 ceph-mon[33301]: pgmap v110: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 909 KiB/s wr, 11 op/s 2024-09-21T09:38:05.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:05 smithi076 ceph-mon[29313]: pgmap v110: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 909 KiB/s wr, 11 op/s 2024-09-21T09:38:05.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:05 smithi076 ceph-mon[41325]: pgmap v110: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 909 KiB/s wr, 11 op/s 2024-09-21T09:38:06.038 INFO:tasks.rados.rados.0.smithi076.stdout:87: done (1 left) 2024-09-21T09:38:06.038 INFO:tasks.rados.rados.0.smithi076.stdout:88: done (0 left) 2024-09-21T09:38:06.038 INFO:tasks.rados.rados.0.smithi076.stdout:89: delete oid 23 current snap is 7 2024-09-21T09:38:06.042 INFO:tasks.rados.rados.0.smithi076.stdout:89: done (0 left) 2024-09-21T09:38:06.042 INFO:tasks.rados.rados.0.smithi076.stdout:90: read oid 27 snap 3 2024-09-21T09:38:06.042 INFO:tasks.rados.rados.0.smithi076.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:06.042 INFO:tasks.rados.rados.0.smithi076.stdout:91: write oid 12 current snap is 7 2024-09-21T09:38:06.042 INFO:tasks.rados.rados.0.smithi076.stdout:91: seq_num 68 ranges {742876=447756,1729599=575687,3101234=41642} 2024-09-21T09:38:06.048 INFO:tasks.rados.rados.0.smithi076.stdout:91: writing smithi07685715-12 from 742876 to 1190632 tid 1 2024-09-21T09:38:06.055 INFO:tasks.rados.rados.0.smithi076.stdout:91: writing smithi07685715-12 from 1729599 to 2305286 tid 2 2024-09-21T09:38:06.060 INFO:tasks.rados.rados.0.smithi076.stdout:91: writing smithi07685715-12 from 3101234 to 3142876 tid 3 2024-09-21T09:38:06.060 INFO:tasks.rados.rados.0.smithi076.stdout:92: snap_create 2024-09-21T09:38:06.060 INFO:tasks.rados.rados.0.smithi076.stdout:91: finishing write tid 1 to smithi07685715-12 2024-09-21T09:38:06.061 INFO:tasks.rados.rados.0.smithi076.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: finishing write tid 2 to smithi07685715-12 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: finishing write tid 3 to smithi07685715-12 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: finishing write tid 4 to smithi07685715-12 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: finishing write tid 5 to smithi07685715-12 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: oid 12 updating version 0 to 19 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: oid 12 updating version 19 to 21 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: oid 12 version 21 is already newer than 20 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:91: oid 12 version 21 is already newer than 18 2024-09-21T09:38:06.146 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 21 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-09-21T09:38:06.147 INFO:tasks.rados.rados.0.smithi076.stdout:91: left oid 12 (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:06.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:06 smithi191 ceph-mon[33301]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:38:06.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:06 smithi076 ceph-mon[29313]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:38:06.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:06 smithi076 ceph-mon[41325]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:38:07.041 INFO:tasks.rados.rados.0.smithi076.stdout:90: done (2 left) 2024-09-21T09:38:07.042 INFO:tasks.rados.rados.0.smithi076.stdout:91: done (1 left) 2024-09-21T09:38:07.042 INFO:tasks.rados.rados.0.smithi076.stdout:92: done (0 left) 2024-09-21T09:38:07.042 INFO:tasks.rados.rados.0.smithi076.stdout:93: snap_remove snap 7 2024-09-21T09:38:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:07 smithi191 ceph-mon[33301]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:38:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:07 smithi191 ceph-mon[33301]: pgmap v113: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 909 KiB/s wr, 11 op/s 2024-09-21T09:38:07.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:07 smithi076 ceph-mon[29313]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:38:07.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:07 smithi076 ceph-mon[29313]: pgmap v113: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 909 KiB/s wr, 11 op/s 2024-09-21T09:38:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:07 smithi076 ceph-mon[41325]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:38:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:07 smithi076 ceph-mon[41325]: pgmap v113: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 909 KiB/s wr, 11 op/s 2024-09-21T09:38:08.045 INFO:tasks.rados.rados.0.smithi076.stdout:93: done (0 left) 2024-09-21T09:38:08.045 INFO:tasks.rados.rados.0.smithi076.stdout:94: write oid 15 current snap is 8 2024-09-21T09:38:08.045 INFO:tasks.rados.rados.0.smithi076.stdout:94: seq_num 69 ranges {759683=639358,1904434=719345,3128657=31026} 2024-09-21T09:38:08.053 INFO:tasks.rados.rados.0.smithi076.stdout:94: writing smithi07685715-15 from 759683 to 1399041 tid 1 2024-09-21T09:38:08.059 INFO:tasks.rados.rados.0.smithi076.stdout:94: writing smithi07685715-15 from 1904434 to 2623779 tid 2 2024-09-21T09:38:08.063 INFO:tasks.rados.rados.0.smithi076.stdout:94: writing smithi07685715-15 from 3128657 to 3159683 tid 3 2024-09-21T09:38:08.063 INFO:tasks.rados.rados.0.smithi076.stdout:95: snap_create 2024-09-21T09:38:08.063 INFO:tasks.rados.rados.0.smithi076.stdout:94: finishing write tid 1 to smithi07685715-15 2024-09-21T09:38:08.063 INFO:tasks.rados.rados.0.smithi076.stdout:94: finishing write tid 2 to smithi07685715-15 2024-09-21T09:38:08.064 INFO:tasks.rados.rados.0.smithi076.stdout:94: finishing write tid 3 to smithi07685715-15 2024-09-21T09:38:08.064 INFO:tasks.rados.rados.0.smithi076.stdout:94: finishing write tid 4 to smithi07685715-15 2024-09-21T09:38:08.064 INFO:tasks.rados.rados.0.smithi076.stdout:94: finishing write tid 5 to smithi07685715-15 2024-09-21T09:38:08.064 INFO:tasks.rados.rados.0.smithi076.stdout:94: oid 15 updating version 0 to 11 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi076.stdout:94: oid 15 version 11 is already newer than 9 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi076.stdout:94: oid 15 version 11 is already newer than 8 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi076.stdout:94: oid 15 version 11 is already newer than 10 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 11 (ObjNum 69 snap 8 seq_num 69) dirty exists 2024-09-21T09:38:08.065 INFO:tasks.rados.rados.0.smithi076.stdout:94: left oid 15 (ObjNum 69 snap 8 seq_num 69) 2024-09-21T09:38:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:08 smithi191 ceph-mon[33301]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:38:08.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:08 smithi076 ceph-mon[29313]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:38:08.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:08 smithi076 ceph-mon[41325]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:38:09.057 INFO:tasks.rados.rados.0.smithi076.stdout:94: done (1 left) 2024-09-21T09:38:09.057 INFO:tasks.rados.rados.0.smithi076.stdout:95: done (0 left) 2024-09-21T09:38:09.057 INFO:tasks.rados.rados.0.smithi076.stdout:96: rollback oid 42 current snap is 9 2024-09-21T09:38:09.057 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 5 2024-09-21T09:38:09.057 INFO:tasks.rados.rados.0.smithi076.stdout:97: write oid 3 current snap is 9 2024-09-21T09:38:09.057 INFO:tasks.rados.rados.0.smithi076.stdout:97: seq_num 70 ranges {776490=714607,2246563=579356} 2024-09-21T09:38:09.065 INFO:tasks.rados.rados.0.smithi076.stdout:97: writing smithi07685715-3 from 776490 to 1491097 tid 1 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:97: writing smithi07685715-3 from 2246563 to 2825919 tid 2 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:98: delete oid 4 current snap is 9 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:96: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:96: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:96: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 10 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:38:09.073 INFO:tasks.rados.rados.0.smithi076.stdout:97: finishing write tid 1 to smithi07685715-3 2024-09-21T09:38:09.075 INFO:tasks.rados.rados.0.smithi076.stdout:96: done (2 left) 2024-09-21T09:38:09.075 INFO:tasks.rados.rados.0.smithi076.stdout:98: done (1 left) 2024-09-21T09:38:09.075 INFO:tasks.rados.rados.0.smithi076.stdout:99: read oid 46 snap -1 2024-09-21T09:38:09.075 INFO:tasks.rados.rados.0.smithi076.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:09.075 INFO:tasks.rados.rados.0.smithi076.stdout:100: write oid 36 current snap is 9 2024-09-21T09:38:09.076 INFO:tasks.rados.rados.0.smithi076.stdout:100: seq_num 72 ranges {410104=465105,1563527=583025,2810926=399178} 2024-09-21T09:38:09.080 INFO:tasks.rados.rados.0.smithi076.stdout:100: writing smithi07685715-36 from 410104 to 875209 tid 1 2024-09-21T09:38:09.087 INFO:tasks.rados.rados.0.smithi076.stdout:100: writing smithi07685715-36 from 1563527 to 2146552 tid 2 2024-09-21T09:38:09.093 INFO:tasks.rados.rados.0.smithi076.stdout:100: writing smithi07685715-36 from 2810926 to 3210104 tid 3 2024-09-21T09:38:09.093 INFO:tasks.rados.rados.0.smithi076.stdout:101: snap_remove snap 4 2024-09-21T09:38:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:09 smithi191 ceph-mon[33301]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:38:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:09 smithi191 ceph-mon[33301]: pgmap v116: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:09.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:09 smithi076 ceph-mon[29313]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:38:09.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:09 smithi076 ceph-mon[29313]: pgmap v116: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:09 smithi076 ceph-mon[41325]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:38:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:09 smithi076 ceph-mon[41325]: pgmap v116: 89 pgs: 89 active+clean; 118 MiB data, 409 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:09.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:38:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:38:08] "GET /metrics HTTP/1.1" 200 36062 "" "Prometheus/2.43.0" 2024-09-21T09:38:10.062 INFO:tasks.rados.rados.0.smithi076.stdout:101: done (3 left) 2024-09-21T09:38:10.062 INFO:tasks.rados.rados.0.smithi076.stdout:102: write oid 41 current snap is 9 2024-09-21T09:38:10.062 INFO:tasks.rados.rados.0.smithi076.stdout:102: seq_num 73 ranges {426911=540354,1622009=726683,3121996=104915} 2024-09-21T09:38:10.067 INFO:tasks.rados.rados.0.smithi076.stdout:102: writing smithi07685715-41 from 426911 to 967265 tid 1 2024-09-21T09:38:10.075 INFO:tasks.rados.rados.0.smithi076.stdout:102: writing smithi07685715-41 from 1622009 to 2348692 tid 2 2024-09-21T09:38:10.080 INFO:tasks.rados.rados.0.smithi076.stdout:102: writing smithi07685715-41 from 3121996 to 3226911 tid 3 2024-09-21T09:38:10.080 INFO:tasks.rados.rados.0.smithi076.stdout:103: snap_remove snap 5 2024-09-21T09:38:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:10 smithi191 ceph-mon[33301]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:38:10.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:10 smithi076 ceph-mon[29313]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:38:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:10 smithi076 ceph-mon[41325]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi076.stdout:103: done (4 left) 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi076.stdout:104: delete oid 15 current snap is 9 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi076.stdout:97: finishing write tid 2 to smithi07685715-3 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi076.stdout:97: finishing write tid 3 to smithi07685715-3 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi076.stdout:97: finishing write tid 4 to smithi07685715-3 2024-09-21T09:38:11.067 INFO:tasks.rados.rados.0.smithi076.stdout:97: oid 3 updating version 0 to 6 2024-09-21T09:38:11.068 INFO:tasks.rados.rados.0.smithi076.stdout:97: oid 3 updating version 6 to 7 2024-09-21T09:38:11.068 INFO:tasks.rados.rados.0.smithi076.stdout:97: oid 3 version 7 is already newer than 5 2024-09-21T09:38:11.068 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 7 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-09-21T09:38:11.068 INFO:tasks.rados.rados.0.smithi076.stdout:97: left oid 3 (ObjNum 70 snap 9 seq_num 70) 2024-09-21T09:38:11.068 INFO:tasks.rados.rados.0.smithi076.stdout:100: finishing write tid 1 to smithi07685715-36 2024-09-21T09:38:11.068 INFO:tasks.rados.rados.0.smithi076.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: finishing write tid 2 to smithi07685715-36 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: finishing write tid 3 to smithi07685715-36 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: finishing write tid 4 to smithi07685715-36 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: finishing write tid 5 to smithi07685715-36 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: oid 36 updating version 0 to 21 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: oid 36 version 21 is already newer than 19 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: oid 36 version 21 is already newer than 20 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: oid 36 version 21 is already newer than 18 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 21 (ObjNum 72 snap 9 seq_num 72) dirty exists 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:100: left oid 36 (ObjNum 72 snap 9 seq_num 72) 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:102: finishing write tid 1 to smithi07685715-41 2024-09-21T09:38:11.167 INFO:tasks.rados.rados.0.smithi076.stdout:102: finishing write tid 2 to smithi07685715-41 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: finishing write tid 3 to smithi07685715-41 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:97: done (4 left) 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:99: done (3 left) 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:100: done (2 left) 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:104: done (1 left) 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:105: write oid 49 current snap is 9 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: finishing write tid 4 to smithi07685715-41 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: finishing write tid 5 to smithi07685715-41 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: oid 41 updating version 0 to 26 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: oid 41 version 26 is already newer than 25 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: oid 41 version 26 is already newer than 24 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: oid 41 updating version 26 to 27 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 27 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:102: left oid 41 (ObjNum 73 snap 9 seq_num 73) 2024-09-21T09:38:11.168 INFO:tasks.rados.rados.0.smithi076.stdout:105: seq_num 75 ranges {460525=690852,1622620=730352,3060489=200036} 2024-09-21T09:38:11.172 INFO:tasks.rados.rados.0.smithi076.stdout:105: writing smithi07685715-49 from 460525 to 1151377 tid 1 2024-09-21T09:38:11.179 INFO:tasks.rados.rados.0.smithi076.stdout:105: writing smithi07685715-49 from 1622620 to 2352972 tid 2 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:105: writing smithi07685715-49 from 3060489 to 3260525 tid 3 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:102: done (1 left) 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:106: rollback oid 3 current snap is 9 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 2 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:105: finishing write tid 1 to smithi07685715-49 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:105: finishing write tid 2 to smithi07685715-49 2024-09-21T09:38:11.185 INFO:tasks.rados.rados.0.smithi076.stdout:107: delete oid 10 current snap is 9 2024-09-21T09:38:11.187 INFO:tasks.rados.rados.0.smithi076.stdout:107: done (2 left) 2024-09-21T09:38:11.187 INFO:tasks.rados.rados.0.smithi076.stdout:108: rollback oid 36 current snap is 9 2024-09-21T09:38:11.187 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 2 2024-09-21T09:38:11.187 INFO:tasks.rados.rados.0.smithi076.stdout:109: write oid 13 current snap is 9 2024-09-21T09:38:11.188 INFO:tasks.rados.rados.0.smithi076.stdout:109: seq_num 77 ranges {494139=557703,1455937=734021,2948041=346098} 2024-09-21T09:38:11.193 INFO:tasks.rados.rados.0.smithi076.stdout:109: writing smithi07685715-13 from 494139 to 1051842 tid 1 2024-09-21T09:38:11.199 INFO:tasks.rados.rados.0.smithi076.stdout:109: writing smithi07685715-13 from 1455937 to 2189958 tid 2 2024-09-21T09:38:11.206 INFO:tasks.rados.rados.0.smithi076.stdout:109: writing smithi07685715-13 from 2948041 to 3294139 tid 3 2024-09-21T09:38:11.206 INFO:tasks.rados.rados.0.smithi076.stdout:110: write oid 24 current snap is 9 2024-09-21T09:38:11.206 INFO:tasks.rados.rados.0.smithi076.stdout:105: finishing write tid 3 to smithi07685715-49 2024-09-21T09:38:11.206 INFO:tasks.rados.rados.0.smithi076.stdout:110: seq_num 78 ranges {510946=632952,1798066=477679,2742758=457217} 2024-09-21T09:38:11.212 INFO:tasks.rados.rados.0.smithi076.stdout:110: writing smithi07685715-24 from 510946 to 1143898 tid 1 2024-09-21T09:38:11.218 INFO:tasks.rados.rados.0.smithi076.stdout:110: writing smithi07685715-24 from 1798066 to 2275745 tid 2 2024-09-21T09:38:11.224 INFO:tasks.rados.rados.0.smithi076.stdout:110: writing smithi07685715-24 from 2742758 to 3199975 tid 3 2024-09-21T09:38:11.224 INFO:tasks.rados.rados.0.smithi076.stdout:111: snap_create 2024-09-21T09:38:11.224 INFO:tasks.rados.rados.0.smithi076.stdout:105: finishing write tid 4 to smithi07685715-49 2024-09-21T09:38:11.224 INFO:tasks.rados.rados.0.smithi076.stdout:106: finishing rollback tid 0 to smithi07685715-3 2024-09-21T09:38:11.224 INFO:tasks.rados.rados.0.smithi076.stdout:106: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:105: finishing write tid 5 to smithi07685715-49 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:105: oid 49 updating version 0 to 40 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:105: oid 49 updating version 40 to 42 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:105: oid 49 version 42 is already newer than 41 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:105: oid 49 updating version 42 to 43 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 43 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:105: left oid 49 (ObjNum 75 snap 9 seq_num 75) 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:106: finishing rollback tid 2 to smithi07685715-3 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 14 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:108: finishing rollback tid 0 to smithi07685715-36 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:108: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:108: finishing rollback tid 2 to smithi07685715-36 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 32 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:109: finishing write tid 1 to smithi07685715-13 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:109: finishing write tid 2 to smithi07685715-13 2024-09-21T09:38:11.225 INFO:tasks.rados.rados.0.smithi076.stdout:109: finishing write tid 3 to smithi07685715-13 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: finishing write tid 4 to smithi07685715-13 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: finishing write tid 5 to smithi07685715-13 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: oid 13 updating version 0 to 16 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: oid 13 version 16 is already newer than 15 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: oid 13 version 16 is already newer than 13 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: oid 13 version 16 is already newer than 14 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 16 (ObjNum 77 snap 9 seq_num 77) dirty exists 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:109: left oid 13 (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:110: finishing write tid 1 to smithi07685715-24 2024-09-21T09:38:11.226 INFO:tasks.rados.rados.0.smithi076.stdout:110: finishing write tid 2 to smithi07685715-24 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: finishing write tid 3 to smithi07685715-24 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: finishing write tid 4 to smithi07685715-24 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: finishing write tid 5 to smithi07685715-24 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: oid 24 updating version 0 to 7 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: oid 24 version 7 is already newer than 6 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: oid 24 version 7 is already newer than 5 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: oid 24 updating version 7 to 8 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 8 (ObjNum 78 snap 9 seq_num 78) dirty exists 2024-09-21T09:38:11.227 INFO:tasks.rados.rados.0.smithi076.stdout:110: left oid 24 (ObjNum 78 snap 9 seq_num 78) 2024-09-21T09:38:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:11 smithi191 ceph-mon[33301]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:38:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:11 smithi191 ceph-mon[33301]: pgmap v119: 89 pgs: 89 active+clean; 125 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:38:11.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:11 smithi076 ceph-mon[29313]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:38:11.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:11 smithi076 ceph-mon[29313]: pgmap v119: 89 pgs: 89 active+clean; 125 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:38:11.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:11 smithi076 ceph-mon[41325]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:38:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:11 smithi076 ceph-mon[41325]: pgmap v119: 89 pgs: 89 active+clean; 125 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:105: done (5 left) 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:106: done (4 left) 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:108: done (3 left) 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:109: done (2 left) 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:110: done (1 left) 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:111: done (0 left) 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:112: write oid 47 current snap is 10 2024-09-21T09:38:12.078 INFO:tasks.rados.rados.0.smithi076.stdout:112: seq_num 79 ranges {527753=708201,1856548=737690,3286534=41219} 2024-09-21T09:38:12.084 INFO:tasks.rados.rados.0.smithi076.stdout:112: writing smithi07685715-47 from 527753 to 1235954 tid 1 2024-09-21T09:38:12.093 INFO:tasks.rados.rados.0.smithi076.stdout:112: writing smithi07685715-47 from 1856548 to 2594238 tid 2 2024-09-21T09:38:12.098 INFO:tasks.rados.rados.0.smithi076.stdout:112: writing smithi07685715-47 from 3286534 to 3327753 tid 3 2024-09-21T09:38:12.098 INFO:tasks.rados.rados.0.smithi076.stdout:113: delete oid 17 current snap is 10 2024-09-21T09:38:12.098 INFO:tasks.rados.rados.0.smithi076.stdout:112: finishing write tid 1 to smithi07685715-47 2024-09-21T09:38:12.098 INFO:tasks.rados.rados.0.smithi076.stdout:112: finishing write tid 2 to smithi07685715-47 2024-09-21T09:38:12.099 INFO:tasks.rados.rados.0.smithi076.stdout:112: finishing write tid 3 to smithi07685715-47 2024-09-21T09:38:12.099 INFO:tasks.rados.rados.0.smithi076.stdout:112: finishing write tid 4 to smithi07685715-47 2024-09-21T09:38:12.100 INFO:tasks.rados.rados.0.smithi076.stdout:113: done (1 left) 2024-09-21T09:38:12.100 INFO:tasks.rados.rados.0.smithi076.stdout:114: snap_remove snap 2 2024-09-21T09:38:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:12 smithi191 ceph-mon[33301]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:38:12.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:12 smithi076 ceph-mon[29313]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:38:12.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:12 smithi076 ceph-mon[41325]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:38:13.082 INFO:tasks.rados.rados.0.smithi076.stdout:114: done (1 left) 2024-09-21T09:38:13.083 INFO:tasks.rados.rados.0.smithi076.stdout:115: write oid 1 current snap is 10 2024-09-21T09:38:13.083 INFO:tasks.rados.rados.0.smithi076.stdout:115: seq_num 81 ranges {561367=458699,1573512=741359,2941380=419987} 2024-09-21T09:38:13.087 INFO:tasks.rados.rados.0.smithi076.stdout:115: writing smithi07685715-1 from 561367 to 1020066 tid 1 2024-09-21T09:38:13.095 INFO:tasks.rados.rados.0.smithi076.stdout:115: writing smithi07685715-1 from 1573512 to 2314871 tid 2 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:115: writing smithi07685715-1 from 2941380 to 3361367 tid 3 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:116: read oid 20 snap -1 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:116: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:117: rollback oid 41 current snap is 10 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 1 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:112: finishing write tid 5 to smithi07685715-47 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:112: oid 47 updating version 0 to 47 2024-09-21T09:38:13.102 INFO:tasks.rados.rados.0.smithi076.stdout:112: oid 47 version 47 is already newer than 45 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:112: oid 47 version 47 is already newer than 46 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:112: oid 47 updating version 47 to 48 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 48 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:112: left oid 47 (ObjNum 79 snap 10 seq_num 79) 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:115: finishing write tid 1 to smithi07685715-1 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:115: finishing write tid 2 to smithi07685715-1 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:112: done (3 left) 2024-09-21T09:38:13.103 INFO:tasks.rados.rados.0.smithi076.stdout:118: delete oid 38 current snap is 10 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:118: done (3 left) 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:119: read oid 13 snap -1 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:119: expect (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:120: read oid 14 snap -1 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:120: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:121: read oid 45 snap -1 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:121: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:38:13.104 INFO:tasks.rados.rados.0.smithi076.stdout:122: snap_remove snap 9 2024-09-21T09:38:13.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:13 smithi191 ceph-mon[33301]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:38:13.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:13 smithi191 ceph-mon[33301]: pgmap v122: 89 pgs: 89 active+clean; 125 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:38:13.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:13 smithi076 ceph-mon[29313]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:38:13.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:13 smithi076 ceph-mon[29313]: pgmap v122: 89 pgs: 89 active+clean; 125 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:38:13.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:13 smithi076 ceph-mon[41325]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:38:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:13 smithi076 ceph-mon[41325]: pgmap v122: 89 pgs: 89 active+clean; 125 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:122: done (6 left) 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:123: write oid 2 current snap is 10 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:117: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:117: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:117: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 35 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:38:13.486 INFO:tasks.rados.rados.0.smithi076.stdout:116: expect (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:38:13.574 INFO:tasks.rados.rados.0.smithi076.stdout:115: finishing write tid 3 to smithi07685715-1 2024-09-21T09:38:13.574 INFO:tasks.rados.rados.0.smithi076.stdout:115: finishing write tid 4 to smithi07685715-1 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:115: finishing write tid 5 to smithi07685715-1 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:115: oid 1 updating version 0 to 28 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:115: oid 1 updating version 28 to 29 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:115: oid 1 version 29 is already newer than 27 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:115: oid 1 updating version 29 to 30 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 30 (ObjNum 81 snap 10 seq_num 81) dirty exists 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:115: left oid 1 (ObjNum 81 snap 10 seq_num 81) 2024-09-21T09:38:13.575 INFO:tasks.rados.rados.0.smithi076.stdout:120: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:38:13.656 INFO:tasks.rados.rados.0.smithi076.stdout:121: expect (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:38:13.752 INFO:tasks.rados.rados.0.smithi076.stdout:119: expect (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:123: seq_num 83 ranges {594981=609197,1974123=745028,3279873=115108} 2024-09-21T09:38:13.873 INFO:tasks.rados.rados.0.smithi076.stdout:123: writing smithi07685715-2 from 594981 to 1204178 tid 1 2024-09-21T09:38:13.882 INFO:tasks.rados.rados.0.smithi076.stdout:123: writing smithi07685715-2 from 1974123 to 2719151 tid 2 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:123: writing smithi07685715-2 from 3279873 to 3394981 tid 3 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:115: done (6 left) 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:116: done (5 left) 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:117: done (4 left) 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:119: done (3 left) 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:120: done (2 left) 2024-09-21T09:38:13.886 INFO:tasks.rados.rados.0.smithi076.stdout:121: done (1 left) 2024-09-21T09:38:13.887 INFO:tasks.rados.rados.0.smithi076.stdout:124: write oid 22 current snap is 10 2024-09-21T09:38:13.887 INFO:tasks.rados.rados.0.smithi076.stdout:123: finishing write tid 1 to smithi07685715-2 2024-09-21T09:38:13.887 INFO:tasks.rados.rados.0.smithi076.stdout:124: seq_num 84 ranges {611788=400799,1748958=488686,2907296=504492} 2024-09-21T09:38:13.891 INFO:tasks.rados.rados.0.smithi076.stdout:124: writing smithi07685715-22 from 611788 to 1012587 tid 1 2024-09-21T09:38:13.898 INFO:tasks.rados.rados.0.smithi076.stdout:124: writing smithi07685715-22 from 1748958 to 2237644 tid 2 2024-09-21T09:38:13.905 INFO:tasks.rados.rados.0.smithi076.stdout:124: writing smithi07685715-22 from 2907296 to 3411788 tid 3 2024-09-21T09:38:13.905 INFO:tasks.rados.rados.0.smithi076.stdout:125: write oid 35 current snap is 10 2024-09-21T09:38:13.905 INFO:tasks.rados.rados.0.smithi076.stdout:125: seq_num 85 ranges {628595=476048,1807440=632344,2934719=452474} 2024-09-21T09:38:13.911 INFO:tasks.rados.rados.0.smithi076.stdout:125: writing smithi07685715-35 from 628595 to 1104643 tid 1 2024-09-21T09:38:13.918 INFO:tasks.rados.rados.0.smithi076.stdout:125: writing smithi07685715-35 from 1807440 to 2439784 tid 2 2024-09-21T09:38:13.924 INFO:tasks.rados.rados.0.smithi076.stdout:125: writing smithi07685715-35 from 2934719 to 3387193 tid 3 2024-09-21T09:38:13.924 INFO:tasks.rados.rados.0.smithi076.stdout:126: snap_create 2024-09-21T09:38:13.924 INFO:tasks.rados.rados.0.smithi076.stdout:123: finishing write tid 2 to smithi07685715-2 2024-09-21T09:38:13.924 INFO:tasks.rados.rados.0.smithi076.stdout:123: finishing write tid 3 to smithi07685715-2 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: finishing write tid 4 to smithi07685715-2 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: finishing write tid 5 to smithi07685715-2 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: oid 2 updating version 0 to 18 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: oid 2 version 18 is already newer than 17 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: oid 2 updating version 18 to 19 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: oid 2 updating version 19 to 20 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 20 (ObjNum 83 snap 10 seq_num 83) dirty exists 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:123: left oid 2 (ObjNum 83 snap 10 seq_num 83) 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: finishing write tid 1 to smithi07685715-22 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: finishing write tid 2 to smithi07685715-22 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: finishing write tid 3 to smithi07685715-22 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: finishing write tid 4 to smithi07685715-22 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: finishing write tid 5 to smithi07685715-22 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: oid 22 updating version 0 to 42 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: oid 22 version 42 is already newer than 40 2024-09-21T09:38:13.925 INFO:tasks.rados.rados.0.smithi076.stdout:124: oid 22 version 42 is already newer than 39 2024-09-21T09:38:13.926 INFO:tasks.rados.rados.0.smithi076.stdout:124: oid 22 version 42 is already newer than 41 2024-09-21T09:38:13.926 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 42 (ObjNum 84 snap 10 seq_num 84) dirty exists 2024-09-21T09:38:13.926 INFO:tasks.rados.rados.0.smithi076.stdout:124: left oid 22 (ObjNum 84 snap 10 seq_num 84) 2024-09-21T09:38:13.926 INFO:tasks.rados.rados.0.smithi076.stdout:125: finishing write tid 1 to smithi07685715-35 2024-09-21T09:38:13.926 INFO:tasks.rados.rados.0.smithi076.stdout:125: finishing write tid 2 to smithi07685715-35 2024-09-21T09:38:13.926 INFO:tasks.rados.rados.0.smithi076.stdout:125: finishing write tid 3 to smithi07685715-35 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: finishing write tid 4 to smithi07685715-35 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: finishing write tid 5 to smithi07685715-35 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: oid 35 updating version 0 to 34 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: oid 35 updating version 34 to 37 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: oid 35 version 37 is already newer than 35 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: oid 35 version 37 is already newer than 36 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 37 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-09-21T09:38:13.927 INFO:tasks.rados.rados.0.smithi076.stdout:125: left oid 35 (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:38:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:14 smithi191 ceph-mon[33301]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:38:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:14 smithi191 ceph-mon[33301]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:38:14.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:14 smithi076 ceph-mon[29313]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:38:14.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:14 smithi076 ceph-mon[29313]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:38:14.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:14 smithi076 ceph-mon[41325]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:38:14.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:14 smithi076 ceph-mon[41325]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:38:14.489 INFO:tasks.rados.rados.0.smithi076.stdout:123: done (3 left) 2024-09-21T09:38:14.489 INFO:tasks.rados.rados.0.smithi076.stdout:124: done (2 left) 2024-09-21T09:38:14.489 INFO:tasks.rados.rados.0.smithi076.stdout:125: done (1 left) 2024-09-21T09:38:14.489 INFO:tasks.rados.rados.0.smithi076.stdout:126: done (0 left) 2024-09-21T09:38:14.489 INFO:tasks.rados.rados.0.smithi076.stdout:127: write oid 32 current snap is 11 2024-09-21T09:38:14.490 INFO:tasks.rados.rados.0.smithi076.stdout:127: seq_num 86 ranges {645402=551297,1749569=492355,2845789=599613} 2024-09-21T09:38:14.495 INFO:tasks.rados.rados.0.smithi076.stdout:127: writing smithi07685715-32 from 645402 to 1196699 tid 1 2024-09-21T09:38:14.502 INFO:tasks.rados.rados.0.smithi076.stdout:127: writing smithi07685715-32 from 1749569 to 2241924 tid 2 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:127: writing smithi07685715-32 from 2845789 to 3445402 tid 3 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:128: read oid 35 snap -1 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:128: expect (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:129: read oid 11 snap -1 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:129: expect (ObjNum 60 snap 0 seq_num 60) 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:130: write oid 42 current snap is 11 2024-09-21T09:38:14.509 INFO:tasks.rados.rados.0.smithi076.stdout:130: seq_num 87 ranges {662209=626546,1808051=636013,2873212=588997} 2024-09-21T09:38:14.516 INFO:tasks.rados.rados.0.smithi076.stdout:130: writing smithi07685715-42 from 662209 to 1288755 tid 1 2024-09-21T09:38:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:130: writing smithi07685715-42 from 1808051 to 2444064 tid 2 2024-09-21T09:38:14.529 INFO:tasks.rados.rados.0.smithi076.stdout:130: writing smithi07685715-42 from 2873212 to 3462209 tid 3 2024-09-21T09:38:14.530 INFO:tasks.rados.rados.0.smithi076.stdout:131: snap_remove snap 10 2024-09-21T09:38:15.494 INFO:tasks.rados.rados.0.smithi076.stdout:127: finishing write tid 1 to smithi07685715-32 2024-09-21T09:38:15.494 INFO:tasks.rados.rados.0.smithi076.stdout:131: done (4 left) 2024-09-21T09:38:15.494 INFO:tasks.rados.rados.0.smithi076.stdout:132: read oid 16 snap -1 2024-09-21T09:38:15.495 INFO:tasks.rados.rados.0.smithi076.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:38:15.495 INFO:tasks.rados.rados.0.smithi076.stdout:133: snap_remove snap 3 2024-09-21T09:38:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:15 smithi191 ceph-mon[33301]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:38:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:15 smithi191 ceph-mon[33301]: pgmap v126: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-21T09:38:15.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:15 smithi076 ceph-mon[29313]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:38:15.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:15 smithi076 ceph-mon[29313]: pgmap v126: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-21T09:38:15.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:15 smithi076 ceph-mon[41325]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:38:15.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:15 smithi076 ceph-mon[41325]: pgmap v126: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-21T09:38:16.495 INFO:tasks.rados.rados.0.smithi076.stdout:133: done (5 left) 2024-09-21T09:38:16.495 INFO:tasks.rados.rados.0.smithi076.stdout:134: write oid 20 current snap is 11 2024-09-21T09:38:16.495 INFO:tasks.rados.rados.0.smithi076.stdout:134: seq_num 88 ranges {679016=701795,1866533=496024,2900635=402643} 2024-09-21T09:38:16.504 INFO:tasks.rados.rados.0.smithi076.stdout:134: writing smithi07685715-20 from 679016 to 1380811 tid 1 2024-09-21T09:38:16.510 INFO:tasks.rados.rados.0.smithi076.stdout:134: writing smithi07685715-20 from 1866533 to 2362557 tid 2 2024-09-21T09:38:16.515 INFO:tasks.rados.rados.0.smithi076.stdout:134: writing smithi07685715-20 from 2900635 to 3303278 tid 3 2024-09-21T09:38:16.515 INFO:tasks.rados.rados.0.smithi076.stdout:135: read oid 10 snap -1 2024-09-21T09:38:16.515 INFO:tasks.rados.rados.0.smithi076.stdout:135: expect deleted 2024-09-21T09:38:16.516 INFO:tasks.rados.rados.0.smithi076.stdout:136: write oid 33 current snap is 11 2024-09-21T09:38:16.516 INFO:tasks.rados.rados.0.smithi076.stdout:127: finishing write tid 2 to smithi07685715-32 2024-09-21T09:38:16.516 INFO:tasks.rados.rados.0.smithi076.stdout:127: finishing write tid 3 to smithi07685715-32 2024-09-21T09:38:16.516 INFO:tasks.rados.rados.0.smithi076.stdout:136: seq_num 89 ranges {695823=777044,1925015=639682,3328058=167765} 2024-09-21T09:38:16.523 INFO:tasks.rados.rados.0.smithi076.stdout:136: writing smithi07685715-33 from 695823 to 1472867 tid 1 2024-09-21T09:38:16.529 INFO:tasks.rados.rados.0.smithi076.stdout:136: writing smithi07685715-33 from 1925015 to 2564697 tid 2 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:136: writing smithi07685715-33 from 3328058 to 3495823 tid 3 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:137: delete oid 1 current snap is 11 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:127: finishing write tid 4 to smithi07685715-32 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:127: finishing write tid 5 to smithi07685715-32 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:127: oid 32 updating version 0 to 15 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:127: oid 32 updating version 15 to 18 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:127: oid 32 version 18 is already newer than 17 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:127: oid 32 version 18 is already newer than 16 2024-09-21T09:38:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 18 (ObjNum 86 snap 11 seq_num 86) dirty exists 2024-09-21T09:38:16.536 INFO:tasks.rados.rados.0.smithi076.stdout:127: left oid 32 (ObjNum 86 snap 11 seq_num 86) 2024-09-21T09:38:16.536 INFO:tasks.rados.rados.0.smithi076.stdout:128: expect (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi076.stdout:130: finishing write tid 1 to smithi07685715-42 2024-09-21T09:38:16.711 INFO:tasks.rados.rados.0.smithi076.stdout:129: expect (ObjNum 60 snap 0 seq_num 60) 2024-09-21T09:38:16.853 INFO:tasks.rados.rados.0.smithi076.stdout:130: finishing write tid 2 to smithi07685715-42 2024-09-21T09:38:16.853 INFO:tasks.rados.rados.0.smithi076.stdout:130: finishing write tid 3 to smithi07685715-42 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:127: done (8 left) 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:128: done (7 left) 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:129: done (6 left) 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:137: done (5 left) 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:138: snap_create 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:130: finishing write tid 4 to smithi07685715-42 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:130: finishing write tid 5 to smithi07685715-42 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:130: oid 42 updating version 0 to 18 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:130: oid 42 updating version 18 to 21 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:130: oid 42 version 21 is already newer than 20 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:130: oid 42 version 21 is already newer than 19 2024-09-21T09:38:16.854 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 21 (ObjNum 87 snap 11 seq_num 87) dirty exists 2024-09-21T09:38:16.855 INFO:tasks.rados.rados.0.smithi076.stdout:130: left oid 42 (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:16.855 INFO:tasks.rados.rados.0.smithi076.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:38:16.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:16 smithi191 ceph-mon[33301]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:38:16.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:16 smithi076 ceph-mon[29313]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:38:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:16 smithi076 ceph-mon[41325]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:38:16.934 INFO:tasks.rados.rados.0.smithi076.stdout:134: finishing write tid 1 to smithi07685715-20 2024-09-21T09:38:16.934 INFO:tasks.rados.rados.0.smithi076.stdout:134: finishing write tid 2 to smithi07685715-20 2024-09-21T09:38:16.934 INFO:tasks.rados.rados.0.smithi076.stdout:134: finishing write tid 3 to smithi07685715-20 2024-09-21T09:38:16.934 INFO:tasks.rados.rados.0.smithi076.stdout:134: finishing write tid 4 to smithi07685715-20 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:134: finishing write tid 5 to smithi07685715-20 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:134: oid 20 updating version 0 to 17 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:134: oid 20 version 17 is already newer than 16 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:134: oid 20 updating version 17 to 18 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:134: oid 20 version 18 is already newer than 15 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 18 (ObjNum 88 snap 11 seq_num 88) dirty exists 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:134: left oid 20 (ObjNum 88 snap 11 seq_num 88) 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: finishing write tid 1 to smithi07685715-33 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: finishing write tid 2 to smithi07685715-33 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: finishing write tid 3 to smithi07685715-33 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: finishing write tid 4 to smithi07685715-33 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: finishing write tid 5 to smithi07685715-33 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: oid 33 updating version 0 to 12 2024-09-21T09:38:16.935 INFO:tasks.rados.rados.0.smithi076.stdout:136: oid 33 updating version 12 to 15 2024-09-21T09:38:16.936 INFO:tasks.rados.rados.0.smithi076.stdout:136: oid 33 version 15 is already newer than 14 2024-09-21T09:38:16.936 INFO:tasks.rados.rados.0.smithi076.stdout:136: oid 33 version 15 is already newer than 13 2024-09-21T09:38:16.936 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 15 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-09-21T09:38:16.936 INFO:tasks.rados.rados.0.smithi076.stdout:136: left oid 33 (ObjNum 89 snap 11 seq_num 89) 2024-09-21T09:38:17.513 INFO:tasks.rados.rados.0.smithi076.stdout:130: done (5 left) 2024-09-21T09:38:17.514 INFO:tasks.rados.rados.0.smithi076.stdout:132: done (4 left) 2024-09-21T09:38:17.514 INFO:tasks.rados.rados.0.smithi076.stdout:134: done (3 left) 2024-09-21T09:38:17.514 INFO:tasks.rados.rados.0.smithi076.stdout:135: done (2 left) 2024-09-21T09:38:17.514 INFO:tasks.rados.rados.0.smithi076.stdout:136: done (1 left) 2024-09-21T09:38:17.514 INFO:tasks.rados.rados.0.smithi076.stdout:138: done (0 left) 2024-09-21T09:38:17.514 INFO:tasks.rados.rados.0.smithi076.stdout:139: snap_remove snap 1 2024-09-21T09:38:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:17 smithi191 ceph-mon[33301]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:38:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:17 smithi191 ceph-mon[33301]: pgmap v129: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-21T09:38:17.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:17 smithi076 ceph-mon[29313]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:38:17.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:17 smithi076 ceph-mon[29313]: pgmap v129: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-21T09:38:17.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:17 smithi076 ceph-mon[41325]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:38:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:17 smithi076 ceph-mon[41325]: pgmap v129: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-21T09:38:18.490 INFO:tasks.rados.rados.0.smithi076.stdout:139: done (0 left) 2024-09-21T09:38:18.490 INFO:tasks.rados.rados.0.smithi076.stdout:140: snap_create 2024-09-21T09:38:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:18 smithi191 ceph-mon[33301]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:38:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:18 smithi191 ceph-mon[33301]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:38:18.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:18 smithi076 ceph-mon[29313]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:38:18.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:18 smithi076 ceph-mon[29313]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:38:18.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:18 smithi076 ceph-mon[41325]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:38:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:18 smithi076 ceph-mon[41325]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:38:19.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:38:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:38:18] "GET /metrics HTTP/1.1" 200 36062 "" "Prometheus/2.43.0" 2024-09-21T09:38:19.490 INFO:tasks.rados.rados.0.smithi076.stdout:140: done (0 left) 2024-09-21T09:38:19.490 INFO:tasks.rados.rados.0.smithi076.stdout:141: read oid 31 snap -1 2024-09-21T09:38:19.491 INFO:tasks.rados.rados.0.smithi076.stdout:141: expect deleted 2024-09-21T09:38:19.491 INFO:tasks.rados.rados.0.smithi076.stdout:142: read oid 13 snap -1 2024-09-21T09:38:19.491 INFO:tasks.rados.rados.0.smithi076.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:19.491 INFO:tasks.rados.rados.0.smithi076.stdout:143: read oid 7 snap -1 2024-09-21T09:38:19.491 INFO:tasks.rados.rados.0.smithi076.stdout:143: expect deleted 2024-09-21T09:38:19.491 INFO:tasks.rados.rados.0.smithi076.stdout:144: snap_create 2024-09-21T09:38:19.495 INFO:tasks.rados.rados.0.smithi076.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-09-21T09:38:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:19 smithi191 ceph-mon[33301]: pgmap v132: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:19 smithi191 ceph-mon[33301]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:38:19.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:19 smithi076 ceph-mon[29313]: pgmap v132: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:19.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:19.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:19 smithi076 ceph-mon[29313]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:38:19.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:19 smithi076 ceph-mon[41325]: pgmap v132: 89 pgs: 89 active+clean; 146 MiB data, 446 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:19 smithi076 ceph-mon[41325]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:38:20.491 INFO:tasks.rados.rados.0.smithi076.stdout:141: done (3 left) 2024-09-21T09:38:20.491 INFO:tasks.rados.rados.0.smithi076.stdout:142: done (2 left) 2024-09-21T09:38:20.491 INFO:tasks.rados.rados.0.smithi076.stdout:143: done (1 left) 2024-09-21T09:38:20.491 INFO:tasks.rados.rados.0.smithi076.stdout:144: done (0 left) 2024-09-21T09:38:20.492 INFO:tasks.rados.rados.0.smithi076.stdout:145: snap_create 2024-09-21T09:38:21.493 INFO:tasks.rados.rados.0.smithi076.stdout:145: done (0 left) 2024-09-21T09:38:21.493 INFO:tasks.rados.rados.0.smithi076.stdout:146: write oid 9 current snap is 15 2024-09-21T09:38:21.493 INFO:tasks.rados.rados.0.smithi076.stdout:146: seq_num 91 ranges {729437=527542,1925626=643351,3266551=262886} 2024-09-21T09:38:21.500 INFO:tasks.rados.rados.0.smithi076.stdout:146: writing smithi07685715-9 from 729437 to 1256979 tid 1 2024-09-21T09:38:21.508 INFO:tasks.rados.rados.0.smithi076.stdout:146: writing smithi07685715-9 from 1925626 to 2568977 tid 2 2024-09-21T09:38:21.514 INFO:tasks.rados.rados.0.smithi076.stdout:146: writing smithi07685715-9 from 3266551 to 3529437 tid 3 2024-09-21T09:38:21.514 INFO:tasks.rados.rados.0.smithi076.stdout:147: rollback oid 14 current snap is 15 2024-09-21T09:38:21.514 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 13 2024-09-21T09:38:21.514 INFO:tasks.rados.rados.0.smithi076.stdout:146: finishing write tid 1 to smithi07685715-9 2024-09-21T09:38:21.514 INFO:tasks.rados.rados.0.smithi076.stdout:146: finishing write tid 2 to smithi07685715-9 2024-09-21T09:38:21.515 INFO:tasks.rados.rados.0.smithi076.stdout:148: write oid 6 current snap is 15 2024-09-21T09:38:21.515 INFO:tasks.rados.rados.0.smithi076.stdout:148: seq_num 92 ranges {746244=719144,2100461=503362,3126680=419564} 2024-09-21T09:38:21.522 INFO:tasks.rados.rados.0.smithi076.stdout:148: writing smithi07685715-6 from 746244 to 1465388 tid 1 2024-09-21T09:38:21.529 INFO:tasks.rados.rados.0.smithi076.stdout:148: writing smithi07685715-6 from 2100461 to 2603823 tid 2 2024-09-21T09:38:21.535 INFO:tasks.rados.rados.0.smithi076.stdout:148: writing smithi07685715-6 from 3126680 to 3546244 tid 3 2024-09-21T09:38:21.535 INFO:tasks.rados.rados.0.smithi076.stdout:149: read oid 42 snap -1 2024-09-21T09:38:21.535 INFO:tasks.rados.rados.0.smithi076.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:150: snap_create 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:147: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: finishing write tid 3 to smithi07685715-9 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: finishing write tid 4 to smithi07685715-9 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: finishing write tid 5 to smithi07685715-9 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: oid 9 updating version 0 to 40 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: oid 9 version 40 is already newer than 39 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: oid 9 version 40 is already newer than 38 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: oid 9 version 40 is already newer than 37 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 40 (ObjNum 91 snap 15 seq_num 91) dirty exists 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:146: left oid 9 (ObjNum 91 snap 15 seq_num 91) 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:147: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:147: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:38:21.536 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 74 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:38:21.537 INFO:tasks.rados.rados.0.smithi076.stdout:148: finishing write tid 1 to smithi07685715-6 2024-09-21T09:38:21.537 INFO:tasks.rados.rados.0.smithi076.stdout:148: finishing write tid 2 to smithi07685715-6 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: finishing write tid 3 to smithi07685715-6 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: finishing write tid 4 to smithi07685715-6 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: finishing write tid 5 to smithi07685715-6 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: oid 6 updating version 0 to 15 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: oid 6 version 15 is already newer than 13 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: oid 6 version 15 is already newer than 14 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: oid 6 version 15 is already newer than 12 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 15 (ObjNum 92 snap 15 seq_num 92) dirty exists 2024-09-21T09:38:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:148: left oid 6 (ObjNum 92 snap 15 seq_num 92) 2024-09-21T09:38:21.540 INFO:tasks.rados.rados.0.smithi076.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:21 smithi191 ceph-mon[33301]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:38:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:21 smithi191 ceph-mon[33301]: pgmap v135: 89 pgs: 89 active+clean; 148 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:21.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:21 smithi076 ceph-mon[29313]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:38:21.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:21 smithi076 ceph-mon[29313]: pgmap v135: 89 pgs: 89 active+clean; 148 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:21.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:21 smithi076 ceph-mon[41325]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:38:21.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:21 smithi076 ceph-mon[41325]: pgmap v135: 89 pgs: 89 active+clean; 148 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:22.503 INFO:tasks.rados.rados.0.smithi076.stdout:146: done (4 left) 2024-09-21T09:38:22.503 INFO:tasks.rados.rados.0.smithi076.stdout:147: done (3 left) 2024-09-21T09:38:22.503 INFO:tasks.rados.rados.0.smithi076.stdout:148: done (2 left) 2024-09-21T09:38:22.503 INFO:tasks.rados.rados.0.smithi076.stdout:149: done (1 left) 2024-09-21T09:38:22.503 INFO:tasks.rados.rados.0.smithi076.stdout:150: done (0 left) 2024-09-21T09:38:22.503 INFO:tasks.rados.rados.0.smithi076.stdout:151: write oid 46 current snap is 16 2024-09-21T09:38:22.504 INFO:tasks.rados.rados.0.smithi076.stdout:151: seq_num 93 ranges {763051=794393,2158943=647020,3437750=125301} 2024-09-21T09:38:22.511 INFO:tasks.rados.rados.0.smithi076.stdout:151: writing smithi07685715-46 from 763051 to 1557444 tid 1 2024-09-21T09:38:22.518 INFO:tasks.rados.rados.0.smithi076.stdout:151: writing smithi07685715-46 from 2158943 to 2805963 tid 2 2024-09-21T09:38:22.523 INFO:tasks.rados.rados.0.smithi076.stdout:151: writing smithi07685715-46 from 3437750 to 3563051 tid 3 2024-09-21T09:38:22.523 INFO:tasks.rados.rados.0.smithi076.stdout:152: read oid 25 snap -1 2024-09-21T09:38:22.523 INFO:tasks.rados.rados.0.smithi076.stdout:152: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:38:22.523 INFO:tasks.rados.rados.0.smithi076.stdout:153: write oid 50 current snap is 16 2024-09-21T09:38:22.523 INFO:tasks.rados.rados.0.smithi076.stdout:151: finishing write tid 1 to smithi07685715-46 2024-09-21T09:38:22.523 INFO:tasks.rados.rados.0.smithi076.stdout:151: finishing write tid 2 to smithi07685715-46 2024-09-21T09:38:22.524 INFO:tasks.rados.rados.0.smithi076.stdout:153: seq_num 94 ranges {779858=469642,1817425=507031,2781526=586628} 2024-09-21T09:38:22.530 INFO:tasks.rados.rados.0.smithi076.stdout:153: writing smithi07685715-50 from 779858 to 1249500 tid 1 2024-09-21T09:38:22.536 INFO:tasks.rados.rados.0.smithi076.stdout:153: writing smithi07685715-50 from 1817425 to 2324456 tid 2 2024-09-21T09:38:22.542 INFO:tasks.rados.rados.0.smithi076.stdout:153: writing smithi07685715-50 from 2781526 to 3368154 tid 3 2024-09-21T09:38:22.542 INFO:tasks.rados.rados.0.smithi076.stdout:154: read oid 6 snap 12 2024-09-21T09:38:22.542 INFO:tasks.rados.rados.0.smithi076.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:38:22.542 INFO:tasks.rados.rados.0.smithi076.stdout:155: snap_remove snap 8 2024-09-21T09:38:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:22 smithi191 ceph-mon[33301]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:38:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:22 smithi076 ceph-mon[29313]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:38:22.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:22 smithi076 ceph-mon[41325]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:38:23.499 INFO:tasks.rados.rados.0.smithi076.stdout:155: done (4 left) 2024-09-21T09:38:23.499 INFO:tasks.rados.rados.0.smithi076.stdout:156: snap_remove snap 6 2024-09-21T09:38:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:23 smithi191 ceph-mon[33301]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:38:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:23 smithi191 ceph-mon[33301]: pgmap v138: 89 pgs: 89 active+clean; 148 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:23 smithi191 ceph-mon[33301]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:38:23.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:23 smithi076 ceph-mon[41325]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:38:23.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:23 smithi076 ceph-mon[41325]: pgmap v138: 89 pgs: 89 active+clean; 148 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:23.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:23 smithi076 ceph-mon[41325]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:38:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:23 smithi076 ceph-mon[29313]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:38:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:23 smithi076 ceph-mon[29313]: pgmap v138: 89 pgs: 89 active+clean; 148 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 778 KiB/s wr, 7 op/s 2024-09-21T09:38:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:23 smithi076 ceph-mon[29313]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:156: done (4 left) 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:157: delete oid 16 current snap is 16 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: finishing write tid 3 to smithi07685715-46 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: finishing write tid 4 to smithi07685715-46 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: finishing write tid 5 to smithi07685715-46 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: oid 46 updating version 0 to 51 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: oid 46 version 51 is already newer than 50 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: oid 46 version 51 is already newer than 49 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: oid 46 version 51 is already newer than 48 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 51 (ObjNum 93 snap 16 seq_num 93) dirty exists 2024-09-21T09:38:24.492 INFO:tasks.rados.rados.0.smithi076.stdout:151: left oid 46 (ObjNum 93 snap 16 seq_num 93) 2024-09-21T09:38:24.493 INFO:tasks.rados.rados.0.smithi076.stdout:152: expect (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:38:24.577 INFO:tasks.rados.rados.0.smithi076.stdout:153: finishing write tid 1 to smithi07685715-50 2024-09-21T09:38:24.577 INFO:tasks.rados.rados.0.smithi076.stdout:153: finishing write tid 2 to smithi07685715-50 2024-09-21T09:38:24.577 INFO:tasks.rados.rados.0.smithi076.stdout:151: done (4 left) 2024-09-21T09:38:24.577 INFO:tasks.rados.rados.0.smithi076.stdout:152: done (3 left) 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:157: done (2 left) 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:158: write oid 32 current snap is 16 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: finishing write tid 3 to smithi07685715-50 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: finishing write tid 4 to smithi07685715-50 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: finishing write tid 5 to smithi07685715-50 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: oid 50 updating version 0 to 49 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: oid 50 version 49 is already newer than 48 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: oid 50 version 49 is already newer than 46 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: oid 50 version 49 is already newer than 47 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 49 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:153: left oid 50 (ObjNum 94 snap 16 seq_num 94) 2024-09-21T09:38:24.578 INFO:tasks.rados.rados.0.smithi076.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:38:24.659 INFO:tasks.rados.rados.0.smithi076.stdout:158: seq_num 96 ranges {413472=620140,1818036=510700,3120019=493453} 2024-09-21T09:38:24.665 INFO:tasks.rados.rados.0.smithi076.stdout:158: writing smithi07685715-32 from 413472 to 1033612 tid 1 2024-09-21T09:38:24.673 INFO:tasks.rados.rados.0.smithi076.stdout:158: writing smithi07685715-32 from 1818036 to 2328736 tid 2 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:158: writing smithi07685715-32 from 3120019 to 3613472 tid 3 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:153: done (2 left) 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:154: done (1 left) 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:159: read oid 41 snap -1 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:159: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:160: delete oid 28 current snap is 16 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:158: finishing write tid 1 to smithi07685715-32 2024-09-21T09:38:24.681 INFO:tasks.rados.rados.0.smithi076.stdout:158: finishing write tid 2 to smithi07685715-32 2024-09-21T09:38:24.683 INFO:tasks.rados.rados.0.smithi076.stdout:160: done (2 left) 2024-09-21T09:38:24.683 INFO:tasks.rados.rados.0.smithi076.stdout:161: write oid 44 current snap is 16 2024-09-21T09:38:24.683 INFO:tasks.rados.rados.0.smithi076.stdout:161: seq_num 98 ranges {447086=770638,1935000=798016,3458512=188574} 2024-09-21T09:38:24.690 INFO:tasks.rados.rados.0.smithi076.stdout:161: writing smithi07685715-44 from 447086 to 1217724 tid 1 2024-09-21T09:38:24.699 INFO:tasks.rados.rados.0.smithi076.stdout:161: writing smithi07685715-44 from 1935000 to 2733016 tid 2 2024-09-21T09:38:24.704 INFO:tasks.rados.rados.0.smithi076.stdout:161: writing smithi07685715-44 from 3458512 to 3647086 tid 3 2024-09-21T09:38:24.704 INFO:tasks.rados.rados.0.smithi076.stdout:162: snap_remove snap 12 2024-09-21T09:38:25.500 INFO:tasks.rados.rados.0.smithi076.stdout:162: done (3 left) 2024-09-21T09:38:25.501 INFO:tasks.rados.rados.0.smithi076.stdout:163: rollback oid 15 current snap is 16 2024-09-21T09:38:25.501 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 0 2024-09-21T09:38:25.501 INFO:tasks.rados.rados.0.smithi076.stdout:164: read oid 21 snap -1 2024-09-21T09:38:25.501 INFO:tasks.rados.rados.0.smithi076.stdout:164: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:38:25.503 INFO:tasks.rados.rados.0.smithi076.stdout:165: read oid 17 snap 13 2024-09-21T09:38:25.503 INFO:tasks.rados.rados.0.smithi076.stdout:165: expect deleted 2024-09-21T09:38:25.503 INFO:tasks.rados.rados.0.smithi076.stdout:166: read oid 18 snap 13 2024-09-21T09:38:25.503 INFO:tasks.rados.rados.0.smithi076.stdout:166: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:38:25.503 INFO:tasks.rados.rados.0.smithi076.stdout:167: write oid 24 current snap is 16 2024-09-21T09:38:25.503 INFO:tasks.rados.rados.0.smithi076.stdout:167: seq_num 99 ranges {463893=562240,1593482=658027,2685935=559341} 2024-09-21T09:38:25.507 INFO:tasks.rados.rados.0.smithi076.stdout:167: writing smithi07685715-24 from 463893 to 1026133 tid 1 2024-09-21T09:38:25.514 INFO:tasks.rados.rados.0.smithi076.stdout:167: writing smithi07685715-24 from 1593482 to 2251509 tid 2 2024-09-21T09:38:25.520 INFO:tasks.rados.rados.0.smithi076.stdout:167: writing smithi07685715-24 from 2685935 to 3245276 tid 3 2024-09-21T09:38:25.520 INFO:tasks.rados.rados.0.smithi076.stdout:168: write oid 3 current snap is 16 2024-09-21T09:38:25.520 INFO:tasks.rados.rados.0.smithi076.stdout:158: finishing write tid 3 to smithi07685715-32 2024-09-21T09:38:25.520 INFO:tasks.rados.rados.0.smithi076.stdout:168: seq_num 100 ranges {480700=637489,1651964=401685,2713358=770613} 2024-09-21T09:38:25.526 INFO:tasks.rados.rados.0.smithi076.stdout:168: writing smithi07685715-3 from 480700 to 1118189 tid 1 2024-09-21T09:38:25.531 INFO:tasks.rados.rados.0.smithi076.stdout:168: writing smithi07685715-3 from 1651964 to 2053649 tid 2 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:168: writing smithi07685715-3 from 2713358 to 3483971 tid 3 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:169: write oid 26 current snap is 16 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:158: finishing write tid 4 to smithi07685715-32 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:158: finishing write tid 5 to smithi07685715-32 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:158: oid 32 updating version 0 to 33 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:158: oid 32 version 33 is already newer than 30 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:158: oid 32 version 33 is already newer than 31 2024-09-21T09:38:25.540 INFO:tasks.rados.rados.0.smithi076.stdout:158: oid 32 version 33 is already newer than 32 2024-09-21T09:38:25.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 33 (ObjNum 96 snap 16 seq_num 96) dirty exists 2024-09-21T09:38:25.541 INFO:tasks.rados.rados.0.smithi076.stdout:158: left oid 32 (ObjNum 96 snap 16 seq_num 96) 2024-09-21T09:38:25.541 INFO:tasks.rados.rados.0.smithi076.stdout:169: seq_num 101 ranges {497507=712738,1710446=661696,3140781=556726} 2024-09-21T09:38:25.546 INFO:tasks.rados.rados.0.smithi076.stdout:169: writing smithi07685715-26 from 497507 to 1210245 tid 1 2024-09-21T09:38:25.553 INFO:tasks.rados.rados.0.smithi076.stdout:169: writing smithi07685715-26 from 1710446 to 2372142 tid 2 2024-09-21T09:38:25.560 INFO:tasks.rados.rados.0.smithi076.stdout:169: writing smithi07685715-26 from 3140781 to 3697507 tid 3 2024-09-21T09:38:25.561 INFO:tasks.rados.rados.0.smithi076.stdout:158: done (9 left) 2024-09-21T09:38:25.561 INFO:tasks.rados.rados.0.smithi076.stdout:170: write oid 14 current snap is 16 2024-09-21T09:38:25.561 INFO:tasks.rados.rados.0.smithi076.stdout:159: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: finishing write tid 1 to smithi07685715-44 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: finishing write tid 2 to smithi07685715-44 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: finishing write tid 3 to smithi07685715-44 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: finishing write tid 4 to smithi07685715-44 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: finishing write tid 5 to smithi07685715-44 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: oid 44 updating version 0 to 22 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: oid 44 updating version 22 to 24 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: oid 44 version 24 is already newer than 23 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: oid 44 version 24 is already newer than 21 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 24 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-09-21T09:38:25.663 INFO:tasks.rados.rados.0.smithi076.stdout:161: left oid 44 (ObjNum 98 snap 16 seq_num 98) 2024-09-21T09:38:25.664 INFO:tasks.rados.rados.0.smithi076.stdout:163: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:38:25.664 INFO:tasks.rados.rados.0.smithi076.stdout:163: finishing rollback tid 2 to smithi07685715-15 2024-09-21T09:38:25.664 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 29 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-21T09:38:25.664 INFO:tasks.rados.rados.0.smithi076.stdout:170: seq_num 102 ranges {514314=787987,1768928=405354,2768204=509510} 2024-09-21T09:38:25.670 INFO:tasks.rados.rados.0.smithi076.stdout:170: writing smithi07685715-14 from 514314 to 1302301 tid 1 2024-09-21T09:38:25.675 INFO:tasks.rados.rados.0.smithi076.stdout:170: writing smithi07685715-14 from 1768928 to 2174282 tid 2 2024-09-21T09:38:25.681 INFO:tasks.rados.rados.0.smithi076.stdout:170: writing smithi07685715-14 from 2768204 to 3277714 tid 3 2024-09-21T09:38:25.681 INFO:tasks.rados.rados.0.smithi076.stdout:159: done (9 left) 2024-09-21T09:38:25.682 INFO:tasks.rados.rados.0.smithi076.stdout:161: done (8 left) 2024-09-21T09:38:25.682 INFO:tasks.rados.rados.0.smithi076.stdout:163: done (7 left) 2024-09-21T09:38:25.682 INFO:tasks.rados.rados.0.smithi076.stdout:171: rollback oid 22 current snap is 16 2024-09-21T09:38:25.682 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 15 2024-09-21T09:38:25.682 INFO:tasks.rados.rados.0.smithi076.stdout:164: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:38:25.779 INFO:tasks.rados.rados.0.smithi076.stdout:166: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:38:25.862 INFO:tasks.rados.rados.0.smithi076.stdout:167: finishing write tid 1 to smithi07685715-24 2024-09-21T09:38:25.862 INFO:tasks.rados.rados.0.smithi076.stdout:167: finishing write tid 2 to smithi07685715-24 2024-09-21T09:38:25.862 INFO:tasks.rados.rados.0.smithi076.stdout:167: finishing write tid 3 to smithi07685715-24 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:164: done (7 left) 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:165: done (6 left) 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:166: done (5 left) 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:172: rollback oid 9 current snap is 16 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 14 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: finishing write tid 4 to smithi07685715-24 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: finishing write tid 5 to smithi07685715-24 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: oid 24 updating version 0 to 21 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: oid 24 updating version 21 to 22 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: oid 24 updating version 22 to 23 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: oid 24 version 23 is already newer than 20 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 23 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:167: left oid 24 (ObjNum 99 snap 16 seq_num 99) 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:168: finishing write tid 1 to smithi07685715-3 2024-09-21T09:38:25.863 INFO:tasks.rados.rados.0.smithi076.stdout:168: finishing write tid 2 to smithi07685715-3 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: finishing write tid 3 to smithi07685715-3 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: finishing write tid 4 to smithi07685715-3 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: finishing write tid 5 to smithi07685715-3 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: oid 3 updating version 0 to 29 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: oid 3 version 29 is already newer than 28 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: oid 3 version 29 is already newer than 26 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: oid 3 version 29 is already newer than 27 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 29 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: left oid 3 (ObjNum 100 snap 16 seq_num 100) 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:169: finishing write tid 1 to smithi07685715-26 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:169: finishing write tid 2 to smithi07685715-26 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:169: finishing write tid 3 to smithi07685715-26 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:169: finishing write tid 4 to smithi07685715-26 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:167: done (5 left) 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:168: done (4 left) 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:173: write oid 29 current snap is 16 2024-09-21T09:38:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:173: seq_num 103 ranges {531121=463236,1711057=665365,3079274=651847} 2024-09-21T09:38:25.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:25 smithi191 ceph-mon[33301]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:38:25.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:25 smithi191 ceph-mon[33301]: pgmap v141: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2024-09-21T09:38:25.867 INFO:tasks.rados.rados.0.smithi076.stdout:173: writing smithi07685715-29 from 531121 to 994357 tid 1 2024-09-21T09:38:25.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:25 smithi076 ceph-mon[29313]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:38:25.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:25 smithi076 ceph-mon[29313]: pgmap v141: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2024-09-21T09:38:25.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:25 smithi076 ceph-mon[41325]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:38:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:25 smithi076 ceph-mon[41325]: pgmap v141: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2024-09-21T09:38:25.875 INFO:tasks.rados.rados.0.smithi076.stdout:173: writing smithi07685715-29 from 1711057 to 2376422 tid 2 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:173: writing smithi07685715-29 from 3079274 to 3731121 tid 3 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:174: read oid 2 snap -1 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:174: expect (ObjNum 83 snap 10 seq_num 83) 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:175: read oid 6 snap -1 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:175: expect (ObjNum 92 snap 15 seq_num 92) 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:176: rollback oid 7 current snap is 16 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 15 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:177: write oid 45 current snap is 16 2024-09-21T09:38:25.884 INFO:tasks.rados.rados.0.smithi076.stdout:177: seq_num 104 ranges {547928=538485,1769539=409023,2706697=532054} 2024-09-21T09:38:25.890 INFO:tasks.rados.rados.0.smithi076.stdout:177: writing smithi07685715-45 from 547928 to 1086413 tid 1 2024-09-21T09:38:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:177: writing smithi07685715-45 from 1769539 to 2178562 tid 2 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:177: writing smithi07685715-45 from 2706697 to 3238751 tid 3 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:178: read oid 23 snap 11 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:178: expect deleted 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:179: rollback oid 44 current snap is 16 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 14 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:169: finishing write tid 5 to smithi07685715-26 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:169: oid 26 updating version 0 to 52 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:169: oid 26 version 52 is already newer than 51 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:169: oid 26 version 52 is already newer than 50 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:169: oid 26 updating version 52 to 53 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 53 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:169: left oid 26 (ObjNum 101 snap 16 seq_num 101) 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:170: finishing write tid 1 to smithi07685715-14 2024-09-21T09:38:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:170: finishing write tid 2 to smithi07685715-14 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:169: done (10 left) 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:180: rollback oid 30 current snap is 16 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 11 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:171: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:171: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:171: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 66 (ObjNum 84 snap 10 seq_num 84) dirty exists 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:170: finishing write tid 3 to smithi07685715-14 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:171: done (10 left) 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:181: read oid 46 snap 14 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:181: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:182: read oid 26 snap -1 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:182: expect (ObjNum 101 snap 16 seq_num 101) 2024-09-21T09:38:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:183: write oid 31 current snap is 16 2024-09-21T09:38:25.907 INFO:tasks.rados.rados.0.smithi076.stdout:183: seq_num 105 ranges {564735=613734,1828021=669034,3134120=630615} 2024-09-21T09:38:25.911 INFO:tasks.rados.rados.0.smithi076.stdout:183: writing smithi07685715-31 from 564735 to 1178469 tid 1 2024-09-21T09:38:25.919 INFO:tasks.rados.rados.0.smithi076.stdout:183: writing smithi07685715-31 from 1828021 to 2497055 tid 2 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:183: writing smithi07685715-31 from 3134120 to 3764735 tid 3 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:184: write oid 19 current snap is 16 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: finishing write tid 4 to smithi07685715-14 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: finishing write tid 5 to smithi07685715-14 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: oid 14 updating version 0 to 97 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: oid 14 version 97 is already newer than 94 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: oid 14 version 97 is already newer than 96 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: oid 14 version 97 is already newer than 95 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 97 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-09-21T09:38:25.927 INFO:tasks.rados.rados.0.smithi076.stdout:170: left oid 14 (ObjNum 102 snap 16 seq_num 102) 2024-09-21T09:38:25.928 INFO:tasks.rados.rados.0.smithi076.stdout:172: finishing rollback tid 0 to smithi07685715-9 2024-09-21T09:38:25.928 INFO:tasks.rados.rados.0.smithi076.stdout:172: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:38:25.928 INFO:tasks.rados.rados.0.smithi076.stdout:172: finishing rollback tid 2 to smithi07685715-9 2024-09-21T09:38:25.928 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 57 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:38:25.928 INFO:tasks.rados.rados.0.smithi076.stdout:173: finishing write tid 1 to smithi07685715-29 2024-09-21T09:38:25.928 INFO:tasks.rados.rados.0.smithi076.stdout:184: seq_num 106 ranges {581542=688983,1886503=412692,2761543=670951} 2024-09-21T09:38:25.933 INFO:tasks.rados.rados.0.smithi076.stdout:184: writing smithi07685715-19 from 581542 to 1270525 tid 1 2024-09-21T09:38:25.939 INFO:tasks.rados.rados.0.smithi076.stdout:184: writing smithi07685715-19 from 1886503 to 2299195 tid 2 2024-09-21T09:38:25.946 INFO:tasks.rados.rados.0.smithi076.stdout:184: writing smithi07685715-19 from 2761543 to 3432494 tid 3 2024-09-21T09:38:25.946 INFO:tasks.rados.rados.0.smithi076.stdout:170: done (13 left) 2024-09-21T09:38:25.946 INFO:tasks.rados.rados.0.smithi076.stdout:172: done (12 left) 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:185: delete oid 15 current snap is 16 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: finishing write tid 2 to smithi07685715-29 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:176: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 28 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: finishing write tid 3 to smithi07685715-29 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: finishing write tid 4 to smithi07685715-29 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: finishing write tid 5 to smithi07685715-29 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: oid 29 updating version 0 to 29 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: oid 29 version 29 is already newer than 27 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: oid 29 version 29 is already newer than 28 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: oid 29 version 29 is already newer than 26 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 29 (ObjNum 103 snap 16 seq_num 103) dirty exists 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:173: left oid 29 (ObjNum 103 snap 16 seq_num 103) 2024-09-21T09:38:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:174: expect (ObjNum 83 snap 10 seq_num 83) 2024-09-21T09:38:26.074 INFO:tasks.rados.rados.0.smithi076.stdout:177: finishing write tid 1 to smithi07685715-45 2024-09-21T09:38:26.074 INFO:tasks.rados.rados.0.smithi076.stdout:175: expect (ObjNum 92 snap 15 seq_num 92) 2024-09-21T09:38:26.263 INFO:tasks.rados.rados.0.smithi076.stdout:177: finishing write tid 2 to smithi07685715-45 2024-09-21T09:38:26.263 INFO:tasks.rados.rados.0.smithi076.stdout:179: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:179: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:173: done (12 left) 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:174: done (11 left) 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:175: done (10 left) 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:176: done (9 left) 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:178: done (8 left) 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:185: done (7 left) 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:186: delete oid 25 current snap is 16 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:179: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 31 (ObjNum 59 snap 0 seq_num 59) dirty exists 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:180: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:180: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:180: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 92 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:177: finishing write tid 3 to smithi07685715-45 2024-09-21T09:38:26.264 INFO:tasks.rados.rados.0.smithi076.stdout:181: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:38:26.360 INFO:tasks.rados.rados.0.smithi076.stdout:182: expect (ObjNum 101 snap 16 seq_num 101) 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: finishing write tid 4 to smithi07685715-45 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: finishing write tid 5 to smithi07685715-45 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: oid 45 updating version 0 to 33 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: oid 45 version 33 is already newer than 32 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: oid 45 updating version 33 to 34 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: oid 45 version 34 is already newer than 31 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 34 (ObjNum 104 snap 16 seq_num 104) dirty exists 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:177: left oid 45 (ObjNum 104 snap 16 seq_num 104) 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:183: finishing write tid 1 to smithi07685715-31 2024-09-21T09:38:26.591 INFO:tasks.rados.rados.0.smithi076.stdout:183: finishing write tid 2 to smithi07685715-31 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:177: done (7 left) 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:179: done (6 left) 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:180: done (5 left) 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:181: done (4 left) 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:182: done (3 left) 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:186: done (2 left) 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:187: snap_create 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: finishing write tid 3 to smithi07685715-31 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: finishing write tid 4 to smithi07685715-31 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: finishing write tid 5 to smithi07685715-31 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: oid 31 updating version 0 to 37 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: oid 31 updating version 37 to 38 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: oid 31 version 38 is already newer than 36 2024-09-21T09:38:26.592 INFO:tasks.rados.rados.0.smithi076.stdout:183: oid 31 updating version 38 to 39 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 39 (ObjNum 105 snap 16 seq_num 105) dirty exists 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:183: left oid 31 (ObjNum 105 snap 16 seq_num 105) 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: finishing write tid 1 to smithi07685715-19 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: finishing write tid 2 to smithi07685715-19 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: finishing write tid 3 to smithi07685715-19 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: finishing write tid 4 to smithi07685715-19 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: finishing write tid 5 to smithi07685715-19 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: oid 19 updating version 0 to 42 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: oid 19 version 42 is already newer than 41 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: oid 19 updating version 42 to 43 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: oid 19 updating version 43 to 44 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 44 (ObjNum 106 snap 16 seq_num 106) dirty exists 2024-09-21T09:38:26.593 INFO:tasks.rados.rados.0.smithi076.stdout:184: left oid 19 (ObjNum 106 snap 16 seq_num 106) 2024-09-21T09:38:26.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:26 smithi191 ceph-mon[33301]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:38:26.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:26 smithi076 ceph-mon[29313]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:38:26.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:26 smithi076 ceph-mon[41325]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:183: done (2 left) 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:184: done (1 left) 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:187: done (0 left) 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:188: read oid 42 snap -1 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:189: rollback oid 30 current snap is 17 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 14 2024-09-21T09:38:27.504 INFO:tasks.rados.rados.0.smithi076.stdout:190: delete oid 20 current snap is 17 2024-09-21T09:38:27.508 INFO:tasks.rados.rados.0.smithi076.stdout:189: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:38:27.509 INFO:tasks.rados.rados.0.smithi076.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:189: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:189: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 96 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:188: done (2 left) 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:189: done (1 left) 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:190: done (0 left) 2024-09-21T09:38:27.687 INFO:tasks.rados.rados.0.smithi076.stdout:191: snap_remove snap 0 2024-09-21T09:38:27.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:27 smithi191 ceph-mon[33301]: pgmap v143: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:38:27.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:27 smithi076 ceph-mon[29313]: pgmap v143: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:38:27.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:27 smithi076 ceph-mon[41325]: pgmap v143: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:38:28.492 INFO:tasks.rados.rados.0.smithi076.stdout:191: done (0 left) 2024-09-21T09:38:28.493 INFO:tasks.rados.rados.0.smithi076.stdout:192: snap_remove snap 15 2024-09-21T09:38:28.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:28 smithi191 ceph-mon[33301]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:38:28.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:28 smithi191 ceph-mon[33301]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:38:28.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:28 smithi076 ceph-mon[29313]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:38:28.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:28 smithi076 ceph-mon[29313]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:38:28.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:28 smithi076 ceph-mon[41325]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:38:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:28 smithi076 ceph-mon[41325]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:38:29.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:38:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:38:28] "GET /metrics HTTP/1.1" 200 36062 "" "Prometheus/2.43.0" 2024-09-21T09:38:29.499 INFO:tasks.rados.rados.0.smithi076.stdout:192: done (0 left) 2024-09-21T09:38:29.499 INFO:tasks.rados.rados.0.smithi076.stdout:193: read oid 1 snap 11 2024-09-21T09:38:29.499 INFO:tasks.rados.rados.0.smithi076.stdout:193: expect deleted 2024-09-21T09:38:29.499 INFO:tasks.rados.rados.0.smithi076.stdout:194: snap_create 2024-09-21T09:38:29.716 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:29 smithi076 ceph-mon[29313]: pgmap v146: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:38:29.716 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:29 smithi076 ceph-mon[29313]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:38:29.716 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:29 smithi076 ceph-mon[41325]: pgmap v146: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:38:29.716 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:29 smithi076 ceph-mon[41325]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:38:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:29 smithi191 ceph-mon[33301]: pgmap v146: 89 pgs: 89 active+clean; 153 MiB data, 484 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:38:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:29 smithi191 ceph-mon[33301]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:38:30.494 INFO:tasks.rados.rados.0.smithi076.stdout:193: done (1 left) 2024-09-21T09:38:30.494 INFO:tasks.rados.rados.0.smithi076.stdout:194: done (0 left) 2024-09-21T09:38:30.494 INFO:tasks.rados.rados.0.smithi076.stdout:195: delete oid 45 current snap is 18 2024-09-21T09:38:30.498 INFO:tasks.rados.rados.0.smithi076.stdout:195: done (0 left) 2024-09-21T09:38:30.499 INFO:tasks.rados.rados.0.smithi076.stdout:196: read oid 12 snap 13 2024-09-21T09:38:30.499 INFO:tasks.rados.rados.0.smithi076.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:30.499 INFO:tasks.rados.rados.0.smithi076.stdout:197: rollback oid 44 current snap is 18 2024-09-21T09:38:30.499 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 17 2024-09-21T09:38:30.499 INFO:tasks.rados.rados.0.smithi076.stdout:198: snap_remove snap 14 2024-09-21T09:38:30.623 INFO:teuthology.orchestra.run.smithi076.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:31.421 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:198: done (2 left) 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:199: delete oid 24 current snap is 18 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:197: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:197: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:197: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 41 (ObjNum 59 snap 0 seq_num 59) dirty exists 2024-09-21T09:38:31.504 INFO:tasks.rados.rados.0.smithi076.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-09-21T09:38:31.667 INFO:tasks.rados.rados.0.smithi076.stdout:196: done (2 left) 2024-09-21T09:38:31.667 INFO:tasks.rados.rados.0.smithi076.stdout:197: done (1 left) 2024-09-21T09:38:31.667 INFO:tasks.rados.rados.0.smithi076.stdout:199: done (0 left) 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:200: read oid 9 snap 13 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:201: rollback oid 45 current snap is 18 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 16 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:202: rollback oid 14 current snap is 18 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 13 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:203: write oid 34 current snap is 18 2024-09-21T09:38:31.668 INFO:tasks.rados.rados.0.smithi076.stdout:203: seq_num 112 ranges {682384=456830,1721042=423699,2809728=454936,3731357=151027} 2024-09-21T09:38:31.674 INFO:tasks.rados.rados.0.smithi076.stdout:203: writing smithi07685715-34 from 682384 to 1139214 tid 1 2024-09-21T09:38:31.679 INFO:tasks.rados.rados.0.smithi076.stdout:203: writing smithi07685715-34 from 1721042 to 2144741 tid 2 2024-09-21T09:38:31.686 INFO:tasks.rados.rados.0.smithi076.stdout:203: writing smithi07685715-34 from 2809728 to 3264664 tid 3 2024-09-21T09:38:31.689 INFO:tasks.rados.rados.0.smithi076.stdout:203: writing smithi07685715-34 from 3731357 to 3882384 tid 4 2024-09-21T09:38:31.689 INFO:tasks.rados.rados.0.smithi076.stdout:204: snap_create 2024-09-21T09:38:31.689 INFO:tasks.rados.rados.0.smithi076.stdout:201: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:38:31.689 INFO:tasks.rados.rados.0.smithi076.stdout:202: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:38:31.690 INFO:tasks.rados.rados.0.smithi076.stdout:202: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:38:31.690 INFO:tasks.rados.rados.0.smithi076.stdout:202: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:38:31.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 116 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:38:31.690 INFO:tasks.rados.rados.0.smithi076.stdout:201: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:38:31.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 47 (ObjNum 104 snap 16 seq_num 104) dirty exists 2024-09-21T09:38:31.690 INFO:tasks.rados.rados.0.smithi076.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:38:31.759 INFO:tasks.rados.rados.0.smithi076.stdout:203: finishing write tid 1 to smithi07685715-34 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: finishing write tid 2 to smithi07685715-34 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: finishing write tid 3 to smithi07685715-34 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: finishing write tid 4 to smithi07685715-34 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: finishing write tid 5 to smithi07685715-34 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: finishing write tid 6 to smithi07685715-34 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: oid 34 updating version 0 to 11 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: oid 34 version 11 is already newer than 10 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: oid 34 updating version 11 to 12 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: oid 34 version 12 is already newer than 9 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: oid 34 version 12 is already newer than 8 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 12 (ObjNum 112 snap 18 seq_num 112) dirty exists 2024-09-21T09:38:31.760 INFO:tasks.rados.rados.0.smithi076.stdout:203: left oid 34 (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:38:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:31 smithi191 ceph-mon[33301]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:38:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:31 smithi191 ceph-mon[33301]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:31 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:31.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:31 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:31.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:31 smithi191 ceph-mon[33301]: pgmap v149: 89 pgs: 89 active+clean; 138 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-21T09:38:31.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[29313]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[29313]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[29313]: pgmap v149: 89 pgs: 89 active+clean; 138 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[41325]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[41325]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:31 smithi076 ceph-mon[41325]: pgmap v149: 89 pgs: 89 active+clean; 138 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-21T09:38:32.043 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:200: done (4 left) 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:201: done (3 left) 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:202: done (2 left) 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:203: done (1 left) 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:204: done (0 left) 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:205: write oid 49 current snap is 19 2024-09-21T09:38:32.628 INFO:tasks.rados.rados.0.smithi076.stdout:205: seq_num 113 ranges {699191=532079,1779524=567357,2837151=666208} 2024-09-21T09:38:32.634 INFO:tasks.rados.rados.0.smithi076.stdout:205: writing smithi07685715-49 from 699191 to 1231270 tid 1 2024-09-21T09:38:32.642 INFO:tasks.rados.rados.0.smithi076.stdout:205: writing smithi07685715-49 from 1779524 to 2346881 tid 2 2024-09-21T09:38:32.648 INFO:tasks.rados.rados.0.smithi076.stdout:205: writing smithi07685715-49 from 2837151 to 3503359 tid 3 2024-09-21T09:38:32.648 INFO:tasks.rados.rados.0.smithi076.stdout:206: snap_remove snap 18 2024-09-21T09:38:32.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:32 smithi076 ceph-mon[29313]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:38:32.798 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:32 smithi076 ceph-mon[41325]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:38:32.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:32 smithi191 ceph-mon[33301]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:206: done (1 left) 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:207: rollback oid 12 current snap is 19 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 11 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: finishing write tid 1 to smithi07685715-49 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: finishing write tid 2 to smithi07685715-49 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: finishing write tid 3 to smithi07685715-49 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: finishing write tid 4 to smithi07685715-49 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: finishing write tid 5 to smithi07685715-49 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: oid 49 updating version 0 to 121 2024-09-21T09:38:33.493 INFO:tasks.rados.rados.0.smithi076.stdout:205: oid 49 version 121 is already newer than 118 2024-09-21T09:38:33.494 INFO:tasks.rados.rados.0.smithi076.stdout:205: oid 49 version 121 is already newer than 119 2024-09-21T09:38:33.494 INFO:tasks.rados.rados.0.smithi076.stdout:205: oid 49 version 121 is already newer than 120 2024-09-21T09:38:33.494 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 121 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-09-21T09:38:33.494 INFO:tasks.rados.rados.0.smithi076.stdout:205: left oid 49 (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:33.494 INFO:tasks.rados.rados.0.smithi076.stdout:205: done (1 left) 2024-09-21T09:38:33.494 INFO:tasks.rados.rados.0.smithi076.stdout:208: snap_remove snap 13 2024-09-21T09:38:33.864 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:33 smithi076 ceph-mon[29313]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:38:33.864 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:33 smithi076 ceph-mon[29313]: pgmap v152: 89 pgs: 89 active+clean; 138 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-21T09:38:33.864 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:33 smithi076 ceph-mon[29313]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:38:33.864 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:33 smithi076 ceph-mon[41325]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:38:33.864 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:33 smithi076 ceph-mon[41325]: pgmap v152: 89 pgs: 89 active+clean; 138 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-21T09:38:33.864 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:33 smithi076 ceph-mon[41325]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:38:33.883 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:33 smithi191 ceph-mon[33301]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:38:33.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:33 smithi191 ceph-mon[33301]: pgmap v152: 89 pgs: 89 active+clean; 138 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 4.2 MiB/s wr, 36 op/s 2024-09-21T09:38:33.884 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:33 smithi191 ceph-mon[33301]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:38:34.337 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:38:34.506 INFO:tasks.rados.rados.0.smithi076.stdout:208: done (1 left) 2024-09-21T09:38:34.506 INFO:tasks.rados.rados.0.smithi076.stdout:209: delete oid 21 current snap is 19 2024-09-21T09:38:34.507 INFO:tasks.rados.rados.0.smithi076.stdout:207: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:38:34.507 INFO:tasks.rados.rados.0.smithi076.stdout:207: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:38:34.507 INFO:tasks.rados.rados.0.smithi076.stdout:207: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:38:34.507 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 75 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-09-21T09:38:34.508 INFO:tasks.rados.rados.0.smithi076.stdout:207: done (1 left) 2024-09-21T09:38:34.508 INFO:tasks.rados.rados.0.smithi076.stdout:209: done (0 left) 2024-09-21T09:38:34.508 INFO:tasks.rados.rados.0.smithi076.stdout:210: read oid 19 snap -1 2024-09-21T09:38:34.508 INFO:tasks.rados.rados.0.smithi076.stdout:210: expect (ObjNum 106 snap 16 seq_num 106) 2024-09-21T09:38:34.508 INFO:tasks.rados.rados.0.smithi076.stdout:211: snap_remove snap 11 2024-09-21T09:38:34.792 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[29313]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:34.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:34 smithi076 ceph-mon[41325]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (2m) 77s ago 3m 24.5M - 0.25.0 c8568f914cd2 d930ef8d31ba 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (2m) 0s ago 2m 85.2M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (108s) 77s ago 107s 13.9M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (105s) 0s ago 105s 14.5M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,8765 running (6m) 0s ago 6m 439M - 18.2.4-871-g6a1bdf85 0adab276c97e a91e8d898a8a 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:9283,8765 running (8m) 77s ago 8m 483M - 18.2.4-871-g6a1bdf85 0adab276c97e 3f03f8aa752c 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (8m) 77s ago 8m 46.3M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:38:34.886 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (7m) 0s ago 7m 41.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (7m) 77s ago 7m 35.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (2m) 77s ago 2m 14.4M - 1.5.0 0da6a335fe13 6dbd51c32fb5 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (2m) 0s ago 2m 15.2M - 1.5.0 0da6a335fe13 9722666fd438 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (6m) 77s ago 6m 51.0M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (6m) 77s ago 6m 49.9M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (5m) 77s ago 5m 44.1M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (5m) 77s ago 5m 50.0M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (5m) 0s ago 5m 58.0M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (4m) 0s ago 4m 61.0M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (4m) 0s ago 4m 50.1M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (3m) 0s ago 3m 50.9M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:38:34.887 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (117s) 0s ago 3m 43.1M - 2.43.0 a07b618ecd1d 8726c547d40a 2024-09-21T09:38:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:34 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:34 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:34 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:34 smithi191 ceph-mon[33301]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:38:35.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 15 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:38:35.219 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:38:35.504 INFO:tasks.rados.rados.0.smithi076.stdout:211: done (1 left) 2024-09-21T09:38:35.505 INFO:tasks.rados.rados.0.smithi076.stdout:212: delete oid 2 current snap is 19 2024-09-21T09:38:35.505 INFO:tasks.rados.rados.0.smithi076.stdout:210: expect (ObjNum 106 snap 16 seq_num 106) 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "", 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "", 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:38:35.520 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:38:35.725 INFO:tasks.rados.rados.0.smithi076.stdout:210: done (1 left) 2024-09-21T09:38:35.725 INFO:tasks.rados.rados.0.smithi076.stdout:212: done (0 left) 2024-09-21T09:38:35.725 INFO:tasks.rados.rados.0.smithi076.stdout:213: snap_create 2024-09-21T09:38:35.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[29313]: from='client.14811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[29313]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[29313]: pgmap v155: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/2406389634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[29313]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[41325]: from='client.14811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[41325]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[41325]: pgmap v155: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/2406389634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:38:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:35 smithi076 ceph-mon[41325]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:38:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:35 smithi191 ceph-mon[33301]: from='client.14811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:35 smithi191 ceph-mon[33301]: from='client.14817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:35 smithi191 ceph-mon[33301]: pgmap v155: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-09-21T09:38:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:35 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2406389634' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:38:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:35 smithi191 ceph-mon[33301]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:38:36.642 INFO:tasks.rados.rados.0.smithi076.stdout:213: done (0 left) 2024-09-21T09:38:36.642 INFO:tasks.rados.rados.0.smithi076.stdout:214: delete oid 46 current snap is 20 2024-09-21T09:38:36.644 INFO:tasks.rados.rados.0.smithi076.stdout:214: done (0 left) 2024-09-21T09:38:36.644 INFO:tasks.rados.rados.0.smithi076.stdout:215: write oid 16 current snap is 20 2024-09-21T09:38:36.644 INFO:tasks.rados.rados.0.smithi076.stdout:215: seq_num 117 ranges {766419=549428,2013452=574695,3346843=427649} 2024-09-21T09:38:36.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: writing smithi07685715-16 from 766419 to 1315847 tid 1 2024-09-21T09:38:36.658 INFO:tasks.rados.rados.0.smithi076.stdout:215: writing smithi07685715-16 from 2013452 to 2588147 tid 2 2024-09-21T09:38:36.665 INFO:tasks.rados.rados.0.smithi076.stdout:215: writing smithi07685715-16 from 3346843 to 3774492 tid 3 2024-09-21T09:38:36.665 INFO:tasks.rados.rados.0.smithi076.stdout:216: read oid 30 snap -1 2024-09-21T09:38:36.665 INFO:tasks.rados.rados.0.smithi076.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:36.665 INFO:tasks.rados.rados.0.smithi076.stdout:217: snap_remove snap 16 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:36 smithi191 ceph-mon[33301]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:38:37.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[29313]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:38:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:38:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:38:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:36 smithi076 ceph-mon[41325]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:38:37.650 INFO:tasks.rados.rados.0.smithi076.stdout:215: finishing write tid 1 to smithi07685715-16 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: finishing write tid 2 to smithi07685715-16 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: finishing write tid 3 to smithi07685715-16 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: finishing write tid 4 to smithi07685715-16 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: finishing write tid 5 to smithi07685715-16 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: oid 16 updating version 0 to 92 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: oid 16 version 92 is already newer than 90 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: oid 16 version 92 is already newer than 91 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: oid 16 version 92 is already newer than 89 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 92 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:215: left oid 16 (ObjNum 117 snap 20 seq_num 117) 2024-09-21T09:38:37.651 INFO:tasks.rados.rados.0.smithi076.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:37.734 INFO:tasks.rados.rados.0.smithi076.stdout:215: done (2 left) 2024-09-21T09:38:37.734 INFO:tasks.rados.rados.0.smithi076.stdout:216: done (1 left) 2024-09-21T09:38:37.734 INFO:tasks.rados.rados.0.smithi076.stdout:217: done (0 left) 2024-09-21T09:38:37.734 INFO:tasks.rados.rados.0.smithi076.stdout:218: snap_remove snap 17 2024-09-21T09:38:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:37 smithi191 ceph-mon[33301]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:37 smithi191 ceph-mon[33301]: pgmap v158: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-09-21T09:38:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:37 smithi191 ceph-mon[33301]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:38:38.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:37 smithi076 ceph-mon[41325]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:38.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:37 smithi076 ceph-mon[41325]: pgmap v158: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-09-21T09:38:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:37 smithi076 ceph-mon[41325]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:38:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:37 smithi076 ceph-mon[29313]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:38:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:37 smithi076 ceph-mon[29313]: pgmap v158: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-09-21T09:38:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:37 smithi076 ceph-mon[29313]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:38:38.495 INFO:tasks.rados.rados.0.smithi076.stdout:218: done (0 left) 2024-09-21T09:38:38.496 INFO:tasks.rados.rados.0.smithi076.stdout:219: write oid 11 current snap is 20 2024-09-21T09:38:38.496 INFO:tasks.rados.rados.0.smithi076.stdout:219: seq_num 118 ranges {783226=624677,2071934=434706,2974266=638921} 2024-09-21T09:38:38.504 INFO:tasks.rados.rados.0.smithi076.stdout:219: writing smithi07685715-11 from 783226 to 1407903 tid 1 2024-09-21T09:38:38.510 INFO:tasks.rados.rados.0.smithi076.stdout:219: writing smithi07685715-11 from 2071934 to 2506640 tid 2 2024-09-21T09:38:38.517 INFO:tasks.rados.rados.0.smithi076.stdout:219: writing smithi07685715-11 from 2974266 to 3613187 tid 3 2024-09-21T09:38:38.517 INFO:tasks.rados.rados.0.smithi076.stdout:220: rollback oid 12 current snap is 20 2024-09-21T09:38:38.517 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 19 2024-09-21T09:38:38.517 INFO:tasks.rados.rados.0.smithi076.stdout:219: finishing write tid 1 to smithi07685715-11 2024-09-21T09:38:38.517 INFO:tasks.rados.rados.0.smithi076.stdout:219: finishing write tid 2 to smithi07685715-11 2024-09-21T09:38:38.517 INFO:tasks.rados.rados.0.smithi076.stdout:221: snap_create 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:220: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:220: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:219: finishing write tid 3 to smithi07685715-11 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:220: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 95 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:219: finishing write tid 4 to smithi07685715-11 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:219: finishing write tid 5 to smithi07685715-11 2024-09-21T09:38:38.520 INFO:tasks.rados.rados.0.smithi076.stdout:219: oid 11 updating version 0 to 98 2024-09-21T09:38:38.521 INFO:tasks.rados.rados.0.smithi076.stdout:219: oid 11 version 98 is already newer than 97 2024-09-21T09:38:38.521 INFO:tasks.rados.rados.0.smithi076.stdout:219: oid 11 version 98 is already newer than 95 2024-09-21T09:38:38.521 INFO:tasks.rados.rados.0.smithi076.stdout:219: oid 11 version 98 is already newer than 96 2024-09-21T09:38:38.521 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 98 (ObjNum 118 snap 20 seq_num 118) dirty exists 2024-09-21T09:38:38.521 INFO:tasks.rados.rados.0.smithi076.stdout:219: left oid 11 (ObjNum 118 snap 20 seq_num 118) 2024-09-21T09:38:39.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:38:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:38:38] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.43.0" 2024-09-21T09:38:39.504 INFO:tasks.rados.rados.0.smithi076.stdout:219: done (2 left) 2024-09-21T09:38:39.504 INFO:tasks.rados.rados.0.smithi076.stdout:220: done (1 left) 2024-09-21T09:38:39.504 INFO:tasks.rados.rados.0.smithi076.stdout:221: done (0 left) 2024-09-21T09:38:39.504 INFO:tasks.rados.rados.0.smithi076.stdout:222: delete oid 28 current snap is 21 2024-09-21T09:38:39.507 INFO:tasks.rados.rados.0.smithi076.stdout:222: done (0 left) 2024-09-21T09:38:39.507 INFO:tasks.rados.rados.0.smithi076.stdout:223: snap_create 2024-09-21T09:38:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:39 smithi191 ceph-mon[33301]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:38:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:39 smithi191 ceph-mon[33301]: pgmap v161: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:39.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:39 smithi076 ceph-mon[41325]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:38:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:39 smithi076 ceph-mon[41325]: pgmap v161: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:39 smithi076 ceph-mon[29313]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:38:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:39 smithi076 ceph-mon[29313]: pgmap v161: 89 pgs: 89 active+clean; 147 MiB data, 471 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:223: done (0 left) 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:224: read oid 49 snap -1 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:225: read oid 23 snap 21 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:225: expect deleted 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:226: write oid 7 current snap is 22 2024-09-21T09:38:40.512 INFO:tasks.rados.rados.0.smithi076.stdout:226: seq_num 120 ranges {416840=775175,1672545=344295} 2024-09-21T09:38:40.518 INFO:tasks.rados.rados.0.smithi076.stdout:226: writing smithi07685715-7 from 416840 to 1192015 tid 1 2024-09-21T09:38:40.523 INFO:tasks.rados.rados.0.smithi076.stdout:226: writing smithi07685715-7 from 1672545 to 2016840 tid 2 2024-09-21T09:38:40.523 INFO:tasks.rados.rados.0.smithi076.stdout:227: snap_create 2024-09-21T09:38:40.525 INFO:tasks.rados.rados.0.smithi076.stdout:226: finishing write tid 1 to smithi07685715-7 2024-09-21T09:38:40.526 INFO:tasks.rados.rados.0.smithi076.stdout:226: finishing write tid 2 to smithi07685715-7 2024-09-21T09:38:40.526 INFO:tasks.rados.rados.0.smithi076.stdout:226: finishing write tid 3 to smithi07685715-7 2024-09-21T09:38:40.526 INFO:tasks.rados.rados.0.smithi076.stdout:226: finishing write tid 4 to smithi07685715-7 2024-09-21T09:38:40.526 INFO:tasks.rados.rados.0.smithi076.stdout:226: oid 7 updating version 0 to 126 2024-09-21T09:38:40.526 INFO:tasks.rados.rados.0.smithi076.stdout:226: oid 7 updating version 126 to 127 2024-09-21T09:38:40.527 INFO:tasks.rados.rados.0.smithi076.stdout:226: oid 7 version 127 is already newer than 125 2024-09-21T09:38:40.527 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 127 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-09-21T09:38:40.527 INFO:tasks.rados.rados.0.smithi076.stdout:226: left oid 7 (ObjNum 120 snap 22 seq_num 120) 2024-09-21T09:38:40.527 INFO:tasks.rados.rados.0.smithi076.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:40 smithi191 ceph-mon[33301]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:38:40.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:40 smithi076 ceph-mon[41325]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:38:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:40 smithi076 ceph-mon[29313]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:224: done (3 left) 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:225: done (2 left) 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:226: done (1 left) 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:227: done (0 left) 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:228: rollback oid 47 current snap is 23 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 19 2024-09-21T09:38:41.517 INFO:tasks.rados.rados.0.smithi076.stdout:229: delete oid 28 current snap is 23 2024-09-21T09:38:41.521 INFO:tasks.rados.rados.0.smithi076.stdout:228: finishing rollback tid 0 to smithi07685715-47 2024-09-21T09:38:41.521 INFO:tasks.rados.rados.0.smithi076.stdout:228: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:38:41.521 INFO:tasks.rados.rados.0.smithi076.stdout:228: finishing rollback tid 2 to smithi07685715-47 2024-09-21T09:38:41.521 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 146 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-09-21T09:38:41.522 INFO:tasks.rados.rados.0.smithi076.stdout:228: done (1 left) 2024-09-21T09:38:41.522 INFO:tasks.rados.rados.0.smithi076.stdout:229: done (0 left) 2024-09-21T09:38:41.522 INFO:tasks.rados.rados.0.smithi076.stdout:230: snap_remove snap 21 2024-09-21T09:38:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:41 smithi191 ceph-mon[33301]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:38:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:41 smithi191 ceph-mon[33301]: pgmap v164: 89 pgs: 89 active+clean; 110 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:41 smithi076 ceph-mon[29313]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:38:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:41 smithi076 ceph-mon[29313]: pgmap v164: 89 pgs: 89 active+clean; 110 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:41 smithi076 ceph-mon[41325]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:38:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:41 smithi076 ceph-mon[41325]: pgmap v164: 89 pgs: 89 active+clean; 110 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:42.523 INFO:tasks.rados.rados.0.smithi076.stdout:230: done (0 left) 2024-09-21T09:38:42.523 INFO:tasks.rados.rados.0.smithi076.stdout:231: write oid 40 current snap is 23 2024-09-21T09:38:42.523 INFO:tasks.rados.rados.0.smithi076.stdout:231: seq_num 122 ranges {450454=642026,1505862=544592} 2024-09-21T09:38:42.528 INFO:tasks.rados.rados.0.smithi076.stdout:231: writing smithi07685715-40 from 450454 to 1092480 tid 1 2024-09-21T09:38:42.534 INFO:tasks.rados.rados.0.smithi076.stdout:231: writing smithi07685715-40 from 1505862 to 2050454 tid 2 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:232: read oid 49 snap -1 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:232: expect (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:233: read oid 27 snap -1 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:233: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:234: read oid 17 snap -1 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:234: expect deleted 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:235: delete oid 2 current snap is 23 2024-09-21T09:38:42.535 INFO:tasks.rados.rados.0.smithi076.stdout:231: finishing write tid 1 to smithi07685715-40 2024-09-21T09:38:42.537 INFO:tasks.rados.rados.0.smithi076.stdout:234: done (4 left) 2024-09-21T09:38:42.537 INFO:tasks.rados.rados.0.smithi076.stdout:235: done (3 left) 2024-09-21T09:38:42.537 INFO:tasks.rados.rados.0.smithi076.stdout:236: write oid 8 current snap is 23 2024-09-21T09:38:42.537 INFO:tasks.rados.rados.0.smithi076.stdout:236: seq_num 124 ranges {484068=792524,1906473=177595} 2024-09-21T09:38:42.544 INFO:tasks.rados.rados.0.smithi076.stdout:236: writing smithi07685715-8 from 484068 to 1276592 tid 1 2024-09-21T09:38:42.549 INFO:tasks.rados.rados.0.smithi076.stdout:236: writing smithi07685715-8 from 1906473 to 2084068 tid 2 2024-09-21T09:38:42.549 INFO:tasks.rados.rados.0.smithi076.stdout:237: write oid 22 current snap is 23 2024-09-21T09:38:42.549 INFO:tasks.rados.rados.0.smithi076.stdout:237: seq_num 125 ranges {500875=467773,1564955=535920} 2024-09-21T09:38:42.554 INFO:tasks.rados.rados.0.smithi076.stdout:237: writing smithi07685715-22 from 500875 to 968648 tid 1 2024-09-21T09:38:42.560 INFO:tasks.rados.rados.0.smithi076.stdout:237: writing smithi07685715-22 from 1564955 to 2100875 tid 2 2024-09-21T09:38:42.561 INFO:tasks.rados.rados.0.smithi076.stdout:238: snap_create 2024-09-21T09:38:42.561 INFO:tasks.rados.rados.0.smithi076.stdout:233: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: finishing write tid 2 to smithi07685715-40 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: finishing write tid 3 to smithi07685715-40 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: finishing write tid 4 to smithi07685715-40 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: oid 40 updating version 0 to 95 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: oid 40 version 95 is already newer than 94 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: oid 40 updating version 95 to 96 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 96 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:231: left oid 40 (ObjNum 122 snap 23 seq_num 122) 2024-09-21T09:38:42.638 INFO:tasks.rados.rados.0.smithi076.stdout:232: expect (ObjNum 113 snap 19 seq_num 113) 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: finishing write tid 1 to smithi07685715-8 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: finishing write tid 2 to smithi07685715-8 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: finishing write tid 3 to smithi07685715-8 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: finishing write tid 4 to smithi07685715-8 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: oid 8 updating version 0 to 152 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: oid 8 version 152 is already newer than 151 2024-09-21T09:38:42.856 INFO:tasks.rados.rados.0.smithi076.stdout:236: oid 8 updating version 152 to 153 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 153 (ObjNum 124 snap 23 seq_num 124) dirty exists 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:236: left oid 8 (ObjNum 124 snap 23 seq_num 124) 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: finishing write tid 1 to smithi07685715-22 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: finishing write tid 2 to smithi07685715-22 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: finishing write tid 3 to smithi07685715-22 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: finishing write tid 4 to smithi07685715-22 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: oid 22 updating version 0 to 99 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: oid 22 version 99 is already newer than 98 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: oid 22 updating version 99 to 100 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 100 (ObjNum 125 snap 23 seq_num 125) dirty exists 2024-09-21T09:38:42.857 INFO:tasks.rados.rados.0.smithi076.stdout:237: left oid 22 (ObjNum 125 snap 23 seq_num 125) 2024-09-21T09:38:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:42 smithi191 ceph-mon[33301]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:38:42.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:42 smithi076 ceph-mon[41325]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:38:42.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:42 smithi076 ceph-mon[29313]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:231: done (5 left) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:232: done (4 left) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:233: done (3 left) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:236: done (2 left) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:237: done (1 left) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:238: done (0 left) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:239: read oid 44 snap -1 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:239: expect (ObjNum 59 snap 0 seq_num 59) 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:240: write oid 22 current snap is 24 2024-09-21T09:38:43.496 INFO:tasks.rados.rados.0.smithi076.stdout:240: seq_num 126 ranges {517682=543022,1623437=494245} 2024-09-21T09:38:43.501 INFO:tasks.rados.rados.0.smithi076.stdout:240: writing smithi07685715-22 from 517682 to 1060704 tid 1 2024-09-21T09:38:43.507 INFO:tasks.rados.rados.0.smithi076.stdout:240: writing smithi07685715-22 from 1623437 to 2117682 tid 2 2024-09-21T09:38:43.508 INFO:tasks.rados.rados.0.smithi076.stdout:241: snap_create 2024-09-21T09:38:43.509 INFO:tasks.rados.rados.0.smithi076.stdout:239: expect (ObjNum 59 snap 0 seq_num 59) 2024-09-21T09:38:43.641 INFO:tasks.rados.rados.0.smithi076.stdout:240: finishing write tid 1 to smithi07685715-22 2024-09-21T09:38:43.641 INFO:tasks.rados.rados.0.smithi076.stdout:240: finishing write tid 2 to smithi07685715-22 2024-09-21T09:38:43.641 INFO:tasks.rados.rados.0.smithi076.stdout:240: finishing write tid 3 to smithi07685715-22 2024-09-21T09:38:43.641 INFO:tasks.rados.rados.0.smithi076.stdout:240: finishing write tid 4 to smithi07685715-22 2024-09-21T09:38:43.641 INFO:tasks.rados.rados.0.smithi076.stdout:240: oid 22 updating version 0 to 103 2024-09-21T09:38:43.642 INFO:tasks.rados.rados.0.smithi076.stdout:240: oid 22 updating version 103 to 104 2024-09-21T09:38:43.642 INFO:tasks.rados.rados.0.smithi076.stdout:240: oid 22 version 104 is already newer than 102 2024-09-21T09:38:43.642 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 104 (ObjNum 126 snap 24 seq_num 126) dirty exists 2024-09-21T09:38:43.642 INFO:tasks.rados.rados.0.smithi076.stdout:240: left oid 22 (ObjNum 126 snap 24 seq_num 126) 2024-09-21T09:38:43.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:43 smithi191 ceph-mon[33301]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:38:43.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:43 smithi191 ceph-mon[33301]: pgmap v167: 89 pgs: 89 active+clean; 110 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:43.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:43 smithi191 ceph-mon[33301]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:38:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:43 smithi076 ceph-mon[29313]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:38:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:43 smithi076 ceph-mon[29313]: pgmap v167: 89 pgs: 89 active+clean; 110 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:43 smithi076 ceph-mon[29313]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:38:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:43 smithi076 ceph-mon[41325]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:38:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:43 smithi076 ceph-mon[41325]: pgmap v167: 89 pgs: 89 active+clean; 110 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 5 op/s 2024-09-21T09:38:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:43 smithi076 ceph-mon[41325]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:38:44.500 INFO:tasks.rados.rados.0.smithi076.stdout:239: done (2 left) 2024-09-21T09:38:44.500 INFO:tasks.rados.rados.0.smithi076.stdout:240: done (1 left) 2024-09-21T09:38:44.500 INFO:tasks.rados.rados.0.smithi076.stdout:241: done (0 left) 2024-09-21T09:38:44.500 INFO:tasks.rados.rados.0.smithi076.stdout:242: rollback oid 28 current snap is 25 2024-09-21T09:38:44.500 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 28 to 24 2024-09-21T09:38:44.500 INFO:tasks.rados.rados.0.smithi076.stdout:243: delete oid 29 current snap is 25 2024-09-21T09:38:44.502 INFO:tasks.rados.rados.0.smithi076.stdout:243: done (1 left) 2024-09-21T09:38:44.502 INFO:tasks.rados.rados.0.smithi076.stdout:244: rollback oid 26 current snap is 25 2024-09-21T09:38:44.502 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 24 2024-09-21T09:38:44.502 INFO:tasks.rados.rados.0.smithi076.stdout:245: rollback oid 16 current snap is 25 2024-09-21T09:38:44.502 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 20 2024-09-21T09:38:44.502 INFO:tasks.rados.rados.0.smithi076.stdout:246: snap_create 2024-09-21T09:38:44.503 INFO:tasks.rados.rados.0.smithi076.stdout:242: finishing rollback tid 1 to smithi07685715-28 2024-09-21T09:38:44.503 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 0 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-09-21T09:38:44.505 INFO:tasks.rados.rados.0.smithi076.stdout:245: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:38:44.506 INFO:tasks.rados.rados.0.smithi076.stdout:244: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:38:44.506 INFO:tasks.rados.rados.0.smithi076.stdout:245: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:38:44.506 INFO:tasks.rados.rados.0.smithi076.stdout:244: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:38:44.506 INFO:tasks.rados.rados.0.smithi076.stdout:245: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:38:44.506 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 108 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-09-21T09:38:44.506 INFO:tasks.rados.rados.0.smithi076.stdout:244: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:38:44.507 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 99 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-09-21T09:38:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:38:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:38:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:38:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:44 smithi191 ceph-mon[33301]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:38:45.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:45.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[29313]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:38:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:44 smithi076 ceph-mon[41325]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:38:45.501 INFO:tasks.rados.rados.0.smithi076.stdout:242: done (3 left) 2024-09-21T09:38:45.501 INFO:tasks.rados.rados.0.smithi076.stdout:244: done (2 left) 2024-09-21T09:38:45.501 INFO:tasks.rados.rados.0.smithi076.stdout:245: done (1 left) 2024-09-21T09:38:45.501 INFO:tasks.rados.rados.0.smithi076.stdout:246: done (0 left) 2024-09-21T09:38:45.501 INFO:tasks.rados.rados.0.smithi076.stdout:247: snap_create 2024-09-21T09:38:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:45 smithi191 ceph-mon[33301]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi191 2024-09-21T09:38:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:45 smithi191 ceph-mon[33301]: pgmap v170: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:38:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:45 smithi191 ceph-mon[33301]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:38:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:45 smithi076 ceph-mon[29313]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi191 2024-09-21T09:38:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:45 smithi076 ceph-mon[29313]: pgmap v170: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:38:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:45 smithi076 ceph-mon[29313]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:38:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:45 smithi076 ceph-mon[41325]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi191 2024-09-21T09:38:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:45 smithi076 ceph-mon[41325]: pgmap v170: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:38:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:45 smithi076 ceph-mon[41325]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:38:46.506 INFO:tasks.rados.rados.0.smithi076.stdout:247: done (0 left) 2024-09-21T09:38:46.506 INFO:tasks.rados.rados.0.smithi076.stdout:248: write oid 48 current snap is 27 2024-09-21T09:38:46.506 INFO:tasks.rados.rados.0.smithi076.stdout:248: seq_num 128 ranges {551296=693520,2024048=127248} 2024-09-21T09:38:46.511 INFO:tasks.rados.rados.0.smithi076.stdout:248: writing smithi07685715-48 from 551296 to 1244816 tid 1 2024-09-21T09:38:46.517 INFO:tasks.rados.rados.0.smithi076.stdout:248: writing smithi07685715-48 from 2024048 to 2151296 tid 2 2024-09-21T09:38:46.517 INFO:tasks.rados.rados.0.smithi076.stdout:249: write oid 44 current snap is 27 2024-09-21T09:38:46.517 INFO:tasks.rados.rados.0.smithi076.stdout:249: seq_num 129 ranges {568103=768769,2082530=85573} 2024-09-21T09:38:46.524 INFO:tasks.rados.rados.0.smithi076.stdout:249: writing smithi07685715-44 from 568103 to 1336872 tid 1 2024-09-21T09:38:46.529 INFO:tasks.rados.rados.0.smithi076.stdout:249: writing smithi07685715-44 from 2082530 to 2168103 tid 2 2024-09-21T09:38:46.529 INFO:tasks.rados.rados.0.smithi076.stdout:250: snap_remove snap 26 2024-09-21T09:38:47.509 INFO:tasks.rados.rados.0.smithi076.stdout:250: done (2 left) 2024-09-21T09:38:47.509 INFO:tasks.rados.rados.0.smithi076.stdout:251: write oid 8 current snap is 27 2024-09-21T09:38:47.509 INFO:tasks.rados.rados.0.smithi076.stdout:248: finishing write tid 1 to smithi07685715-48 2024-09-21T09:38:47.509 INFO:tasks.rados.rados.0.smithi076.stdout:251: seq_num 130 ranges {584910=560371,1857365=327545} 2024-09-21T09:38:47.514 INFO:tasks.rados.rados.0.smithi076.stdout:251: writing smithi07685715-8 from 584910 to 1145281 tid 1 2024-09-21T09:38:47.521 INFO:tasks.rados.rados.0.smithi076.stdout:251: writing smithi07685715-8 from 1857365 to 2184910 tid 2 2024-09-21T09:38:47.521 INFO:tasks.rados.rados.0.smithi076.stdout:252: snap_create 2024-09-21T09:38:47.521 INFO:tasks.rados.rados.0.smithi076.stdout:248: finishing write tid 2 to smithi07685715-48 2024-09-21T09:38:47.521 INFO:tasks.rados.rados.0.smithi076.stdout:248: finishing write tid 3 to smithi07685715-48 2024-09-21T09:38:47.521 INFO:tasks.rados.rados.0.smithi076.stdout:248: finishing write tid 4 to smithi07685715-48 2024-09-21T09:38:47.521 INFO:tasks.rados.rados.0.smithi076.stdout:248: oid 48 updating version 0 to 156 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:248: oid 48 updating version 156 to 157 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:248: oid 48 version 157 is already newer than 155 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 157 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:248: left oid 48 (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: finishing write tid 1 to smithi07685715-44 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: finishing write tid 2 to smithi07685715-44 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: finishing write tid 3 to smithi07685715-44 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: finishing write tid 4 to smithi07685715-44 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: oid 44 updating version 0 to 61 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: oid 44 version 61 is already newer than 60 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:249: oid 44 version 61 is already newer than 59 2024-09-21T09:38:47.522 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 61 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-09-21T09:38:47.523 INFO:tasks.rados.rados.0.smithi076.stdout:249: left oid 44 (ObjNum 129 snap 27 seq_num 129) 2024-09-21T09:38:47.523 INFO:tasks.rados.rados.0.smithi076.stdout:251: finishing write tid 1 to smithi07685715-8 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:251: finishing write tid 2 to smithi07685715-8 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:251: finishing write tid 3 to smithi07685715-8 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:251: finishing write tid 4 to smithi07685715-8 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:251: oid 8 updating version 0 to 162 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:251: oid 8 version 162 is already newer than 159 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:251: oid 8 updating version 162 to 163 2024-09-21T09:38:47.524 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 163 (ObjNum 130 snap 27 seq_num 130) dirty exists 2024-09-21T09:38:47.525 INFO:tasks.rados.rados.0.smithi076.stdout:251: left oid 8 (ObjNum 130 snap 27 seq_num 130) 2024-09-21T09:38:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:47 smithi191 ceph-mon[33301]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:38:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:47 smithi191 ceph-mon[33301]: pgmap v173: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:38:47.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:47 smithi076 ceph-mon[41325]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:38:47.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:47 smithi076 ceph-mon[41325]: pgmap v173: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:38:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:47 smithi076 ceph-mon[29313]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:38:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:47 smithi076 ceph-mon[29313]: pgmap v173: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-09-21T09:38:48.497 INFO:tasks.rados.rados.0.smithi076.stdout:248: done (3 left) 2024-09-21T09:38:48.497 INFO:tasks.rados.rados.0.smithi076.stdout:249: done (2 left) 2024-09-21T09:38:48.497 INFO:tasks.rados.rados.0.smithi076.stdout:251: done (1 left) 2024-09-21T09:38:48.497 INFO:tasks.rados.rados.0.smithi076.stdout:252: done (0 left) 2024-09-21T09:38:48.497 INFO:tasks.rados.rados.0.smithi076.stdout:253: snap_remove snap 27 2024-09-21T09:38:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:48 smithi191 ceph-mon[33301]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:38:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:48 smithi191 ceph-mon[33301]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:38:48.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:48 smithi076 ceph-mon[41325]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:38:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:48 smithi076 ceph-mon[41325]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:38:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:48 smithi076 ceph-mon[29313]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:38:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:48 smithi076 ceph-mon[29313]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:38:49.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:38:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:38:48] "GET /metrics HTTP/1.1" 200 36063 "" "Prometheus/2.43.0" 2024-09-21T09:38:49.498 INFO:tasks.rados.rados.0.smithi076.stdout:253: done (0 left) 2024-09-21T09:38:49.498 INFO:tasks.rados.rados.0.smithi076.stdout:254: read oid 29 snap -1 2024-09-21T09:38:49.498 INFO:tasks.rados.rados.0.smithi076.stdout:254: expect deleted 2024-09-21T09:38:49.498 INFO:tasks.rados.rados.0.smithi076.stdout:255: read oid 15 snap -1 2024-09-21T09:38:49.498 INFO:tasks.rados.rados.0.smithi076.stdout:255: expect deleted 2024-09-21T09:38:49.498 INFO:tasks.rados.rados.0.smithi076.stdout:256: delete oid 36 current snap is 28 2024-09-21T09:38:49.503 INFO:tasks.rados.rados.0.smithi076.stdout:254: done (2 left) 2024-09-21T09:38:49.503 INFO:tasks.rados.rados.0.smithi076.stdout:255: done (1 left) 2024-09-21T09:38:49.503 INFO:tasks.rados.rados.0.smithi076.stdout:256: done (0 left) 2024-09-21T09:38:49.503 INFO:tasks.rados.rados.0.smithi076.stdout:257: delete oid 44 current snap is 28 2024-09-21T09:38:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:257: done (0 left) 2024-09-21T09:38:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:258: read oid 29 snap -1 2024-09-21T09:38:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:258: expect deleted 2024-09-21T09:38:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:259: write oid 19 current snap is 28 2024-09-21T09:38:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:259: seq_num 133 ranges {635331=786118,1916458=318873} 2024-09-21T09:38:49.516 INFO:tasks.rados.rados.0.smithi076.stdout:259: writing smithi07685715-19 from 635331 to 1421449 tid 1 2024-09-21T09:38:49.521 INFO:tasks.rados.rados.0.smithi076.stdout:259: writing smithi07685715-19 from 1916458 to 2235331 tid 2 2024-09-21T09:38:49.521 INFO:tasks.rados.rados.0.smithi076.stdout:260: snap_remove snap 19 2024-09-21T09:38:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:49 smithi191 ceph-mon[33301]: pgmap v176: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:49 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:49 smithi191 ceph-mon[33301]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:38:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:49 smithi076 ceph-mon[41325]: pgmap v176: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:49 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:49 smithi076 ceph-mon[41325]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:38:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:49 smithi076 ceph-mon[29313]: pgmap v176: 89 pgs: 89 active+clean; 116 MiB data, 429 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:49 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:38:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:49 smithi076 ceph-mon[29313]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:38:50.517 INFO:tasks.rados.rados.0.smithi076.stdout:260: done (2 left) 2024-09-21T09:38:50.517 INFO:tasks.rados.rados.0.smithi076.stdout:261: write oid 36 current snap is 28 2024-09-21T09:38:50.517 INFO:tasks.rados.rados.0.smithi076.stdout:261: seq_num 134 ranges {652138=461367,1574940=677198} 2024-09-21T09:38:50.524 INFO:tasks.rados.rados.0.smithi076.stdout:261: writing smithi07685715-36 from 652138 to 1113505 tid 1 2024-09-21T09:38:50.530 INFO:tasks.rados.rados.0.smithi076.stdout:261: writing smithi07685715-36 from 1574940 to 2252138 tid 2 2024-09-21T09:38:50.530 INFO:tasks.rados.rados.0.smithi076.stdout:262: read oid 2 snap -1 2024-09-21T09:38:50.530 INFO:tasks.rados.rados.0.smithi076.stdout:262: expect deleted 2024-09-21T09:38:50.530 INFO:tasks.rados.rados.0.smithi076.stdout:263: write oid 26 current snap is 28 2024-09-21T09:38:50.530 INFO:tasks.rados.rados.0.smithi076.stdout:263: seq_num 135 ranges {668945=536616,1633422=491363} 2024-09-21T09:38:50.536 INFO:tasks.rados.rados.0.smithi076.stdout:263: writing smithi07685715-26 from 668945 to 1205561 tid 1 2024-09-21T09:38:50.542 INFO:tasks.rados.rados.0.smithi076.stdout:263: writing smithi07685715-26 from 1633422 to 2124785 tid 2 2024-09-21T09:38:50.542 INFO:tasks.rados.rados.0.smithi076.stdout:264: read oid 37 snap -1 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:264: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:265: read oid 43 snap -1 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:265: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:266: write oid 8 current snap is 28 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: finishing write tid 1 to smithi07685715-19 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: finishing write tid 2 to smithi07685715-19 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: finishing write tid 3 to smithi07685715-19 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: finishing write tid 4 to smithi07685715-19 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: oid 19 updating version 0 to 66 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: oid 19 updating version 66 to 67 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: oid 19 updating version 67 to 68 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 68 (ObjNum 133 snap 28 seq_num 133) dirty exists 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:259: left oid 19 (ObjNum 133 snap 28 seq_num 133) 2024-09-21T09:38:50.543 INFO:tasks.rados.rados.0.smithi076.stdout:266: seq_num 136 ranges {685752=611865,1975551=310201} 2024-09-21T09:38:50.550 INFO:tasks.rados.rados.0.smithi076.stdout:266: writing smithi07685715-8 from 685752 to 1297617 tid 1 2024-09-21T09:38:50.556 INFO:tasks.rados.rados.0.smithi076.stdout:266: writing smithi07685715-8 from 1975551 to 2285752 tid 2 2024-09-21T09:38:50.556 INFO:tasks.rados.rados.0.smithi076.stdout:258: done (7 left) 2024-09-21T09:38:50.556 INFO:tasks.rados.rados.0.smithi076.stdout:259: done (6 left) 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:267: rollback oid 17 current snap is 28 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 24 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: finishing write tid 1 to smithi07685715-36 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: finishing write tid 2 to smithi07685715-36 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:262: done (6 left) 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:268: write oid 6 current snap is 28 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: finishing write tid 3 to smithi07685715-36 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: finishing write tid 4 to smithi07685715-36 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: oid 36 updating version 0 to 82 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: oid 36 updating version 82 to 84 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: oid 36 version 84 is already newer than 83 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 84 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:261: left oid 36 (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:263: finishing write tid 1 to smithi07685715-26 2024-09-21T09:38:50.557 INFO:tasks.rados.rados.0.smithi076.stdout:268: seq_num 137 ranges {702559=403467,1750386=495032} 2024-09-21T09:38:50.562 INFO:tasks.rados.rados.0.smithi076.stdout:268: writing smithi07685715-6 from 702559 to 1106026 tid 1 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:268: writing smithi07685715-6 from 1750386 to 2245418 tid 2 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:261: done (6 left) 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:269: read oid 7 snap -1 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:269: expect (ObjNum 120 snap 22 seq_num 120) 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:270: rollback oid 23 current snap is 28 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 24 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: finishing write tid 2 to smithi07685715-26 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: finishing write tid 3 to smithi07685715-26 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:271: snap_create 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: finishing write tid 4 to smithi07685715-26 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: oid 26 updating version 0 to 106 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: oid 26 version 106 is already newer than 105 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: oid 26 updating version 106 to 107 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 107 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-09-21T09:38:50.569 INFO:tasks.rados.rados.0.smithi076.stdout:263: left oid 26 (ObjNum 135 snap 28 seq_num 135) 2024-09-21T09:38:50.570 INFO:tasks.rados.rados.0.smithi076.stdout:265: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:38:50.656 INFO:tasks.rados.rados.0.smithi076.stdout:264: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:38:50.737 INFO:tasks.rados.rados.0.smithi076.stdout:266: finishing write tid 1 to smithi07685715-8 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:267: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 0 (ObjNum 161 snap 0 seq_num 864406624) dirty dne 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: finishing write tid 2 to smithi07685715-8 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: finishing write tid 3 to smithi07685715-8 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: finishing write tid 4 to smithi07685715-8 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: oid 8 updating version 0 to 171 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: oid 8 updating version 171 to 172 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: oid 8 version 172 is already newer than 170 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 172 (ObjNum 136 snap 28 seq_num 136) dirty exists 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:266: left oid 8 (ObjNum 136 snap 28 seq_num 136) 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:268: finishing write tid 1 to smithi07685715-6 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:270: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:38:50.738 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: finishing write tid 2 to smithi07685715-6 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: finishing write tid 3 to smithi07685715-6 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: finishing write tid 4 to smithi07685715-6 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: oid 6 updating version 0 to 68 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: oid 6 updating version 68 to 70 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: oid 6 version 70 is already newer than 69 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 70 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:268: left oid 6 (ObjNum 137 snap 28 seq_num 137) 2024-09-21T09:38:50.739 INFO:tasks.rados.rados.0.smithi076.stdout:269: expect (ObjNum 120 snap 22 seq_num 120) 2024-09-21T09:38:51.528 INFO:tasks.rados.rados.0.smithi076.stdout:263: done (8 left) 2024-09-21T09:38:51.528 INFO:tasks.rados.rados.0.smithi076.stdout:264: done (7 left) 2024-09-21T09:38:51.528 INFO:tasks.rados.rados.0.smithi076.stdout:265: done (6 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:266: done (5 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:267: done (4 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:268: done (3 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:269: done (2 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:270: done (1 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:271: done (0 left) 2024-09-21T09:38:51.529 INFO:tasks.rados.rados.0.smithi076.stdout:272: snap_create 2024-09-21T09:38:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:51 smithi191 ceph-mon[33301]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:38:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:51 smithi191 ceph-mon[33301]: pgmap v179: 89 pgs: 89 active+clean; 119 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 896 KiB/s wr, 5 op/s 2024-09-21T09:38:51.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:51 smithi076 ceph-mon[41325]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:38:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:51 smithi076 ceph-mon[41325]: pgmap v179: 89 pgs: 89 active+clean; 119 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 896 KiB/s wr, 5 op/s 2024-09-21T09:38:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:51 smithi076 ceph-mon[29313]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:38:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:51 smithi076 ceph-mon[29313]: pgmap v179: 89 pgs: 89 active+clean; 119 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 896 KiB/s wr, 5 op/s 2024-09-21T09:38:52.524 INFO:tasks.rados.rados.0.smithi076.stdout:272: done (0 left) 2024-09-21T09:38:52.524 INFO:tasks.rados.rados.0.smithi076.stdout:273: write oid 31 current snap is 30 2024-09-21T09:38:52.524 INFO:tasks.rados.rados.0.smithi076.stdout:273: seq_num 138 ranges {719366=478716,1808868=510498} 2024-09-21T09:38:52.531 INFO:tasks.rados.rados.0.smithi076.stdout:273: writing smithi07685715-31 from 719366 to 1198082 tid 1 2024-09-21T09:38:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:273: writing smithi07685715-31 from 1808868 to 2319366 tid 2 2024-09-21T09:38:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:274: delete oid 38 current snap is 30 2024-09-21T09:38:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:273: finishing write tid 1 to smithi07685715-31 2024-09-21T09:38:52.539 INFO:tasks.rados.rados.0.smithi076.stdout:274: done (1 left) 2024-09-21T09:38:52.539 INFO:tasks.rados.rados.0.smithi076.stdout:275: rollback oid 6 current snap is 30 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 25 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:276: snap_create 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:273: finishing write tid 2 to smithi07685715-31 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:273: finishing write tid 3 to smithi07685715-31 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:273: finishing write tid 4 to smithi07685715-31 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:273: oid 31 updating version 0 to 74 2024-09-21T09:38:52.540 INFO:tasks.rados.rados.0.smithi076.stdout:273: oid 31 version 74 is already newer than 73 2024-09-21T09:38:52.541 INFO:tasks.rados.rados.0.smithi076.stdout:273: oid 31 version 74 is already newer than 72 2024-09-21T09:38:52.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 74 (ObjNum 138 snap 30 seq_num 138) dirty exists 2024-09-21T09:38:52.541 INFO:tasks.rados.rados.0.smithi076.stdout:273: left oid 31 (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:38:52.543 INFO:tasks.rados.rados.0.smithi076.stdout:275: finishing rollback tid 0 to smithi07685715-6 2024-09-21T09:38:52.544 INFO:tasks.rados.rados.0.smithi076.stdout:275: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:38:52.544 INFO:tasks.rados.rados.0.smithi076.stdout:275: finishing rollback tid 2 to smithi07685715-6 2024-09-21T09:38:52.544 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 74 (ObjNum 92 snap 15 seq_num 92) dirty exists 2024-09-21T09:38:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:52 smithi191 ceph-mon[33301]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:38:52.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:52 smithi076 ceph-mon[29313]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:38:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:52 smithi076 ceph-mon[41325]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:38:53.499 INFO:tasks.rados.rados.0.smithi076.stdout:273: done (2 left) 2024-09-21T09:38:53.499 INFO:tasks.rados.rados.0.smithi076.stdout:275: done (1 left) 2024-09-21T09:38:53.500 INFO:tasks.rados.rados.0.smithi076.stdout:276: done (0 left) 2024-09-21T09:38:53.500 INFO:tasks.rados.rados.0.smithi076.stdout:277: write oid 19 current snap is 31 2024-09-21T09:38:53.500 INFO:tasks.rados.rados.0.smithi076.stdout:277: seq_num 140 ranges {752980=629214,1809479=543501} 2024-09-21T09:38:53.507 INFO:tasks.rados.rados.0.smithi076.stdout:277: writing smithi07685715-19 from 752980 to 1382194 tid 1 2024-09-21T09:38:53.512 INFO:tasks.rados.rados.0.smithi076.stdout:277: writing smithi07685715-19 from 1809479 to 2352980 tid 2 2024-09-21T09:38:53.512 INFO:tasks.rados.rados.0.smithi076.stdout:278: snap_remove snap 29 2024-09-21T09:38:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:53 smithi191 ceph-mon[33301]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:38:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:53 smithi191 ceph-mon[33301]: pgmap v182: 89 pgs: 89 active+clean; 119 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 896 KiB/s wr, 5 op/s 2024-09-21T09:38:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:53 smithi191 ceph-mon[33301]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:38:53.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:53 smithi076 ceph-mon[29313]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:38:53.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:53 smithi076 ceph-mon[29313]: pgmap v182: 89 pgs: 89 active+clean; 119 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 896 KiB/s wr, 5 op/s 2024-09-21T09:38:53.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:53 smithi076 ceph-mon[29313]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:38:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:53 smithi076 ceph-mon[41325]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:38:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:53 smithi076 ceph-mon[41325]: pgmap v182: 89 pgs: 89 active+clean; 119 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 896 KiB/s wr, 5 op/s 2024-09-21T09:38:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:53 smithi076 ceph-mon[41325]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: finishing write tid 1 to smithi07685715-19 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: finishing write tid 2 to smithi07685715-19 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: finishing write tid 3 to smithi07685715-19 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: finishing write tid 4 to smithi07685715-19 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: oid 19 updating version 0 to 78 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: oid 19 version 78 is already newer than 77 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: oid 19 version 78 is already newer than 76 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 78 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: left oid 19 (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:38:54.501 INFO:tasks.rados.rados.0.smithi076.stdout:277: done (1 left) 2024-09-21T09:38:54.502 INFO:tasks.rados.rados.0.smithi076.stdout:278: done (0 left) 2024-09-21T09:38:54.502 INFO:tasks.rados.rados.0.smithi076.stdout:279: delete oid 6 current snap is 31 2024-09-21T09:38:54.506 INFO:tasks.rados.rados.0.smithi076.stdout:279: done (0 left) 2024-09-21T09:38:54.506 INFO:tasks.rados.rados.0.smithi076.stdout:280: snap_remove snap 24 2024-09-21T09:38:55.506 INFO:tasks.rados.rados.0.smithi076.stdout:280: done (0 left) 2024-09-21T09:38:55.506 INFO:tasks.rados.rados.0.smithi076.stdout:281: rollback oid 49 current snap is 31 2024-09-21T09:38:55.506 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 22 2024-09-21T09:38:55.506 INFO:tasks.rados.rados.0.smithi076.stdout:282: snap_remove snap 25 2024-09-21T09:38:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:55 smithi191 ceph-mon[33301]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:38:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:55 smithi191 ceph-mon[33301]: pgmap v185: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:38:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:55 smithi076 ceph-mon[29313]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:38:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:55 smithi076 ceph-mon[29313]: pgmap v185: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:38:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:55 smithi076 ceph-mon[41325]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:38:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:55 smithi076 ceph-mon[41325]: pgmap v185: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:38:56.507 INFO:tasks.rados.rados.0.smithi076.stdout:281: finishing rollback tid 0 to smithi07685715-49 2024-09-21T09:38:56.507 INFO:tasks.rados.rados.0.smithi076.stdout:281: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:38:56.507 INFO:tasks.rados.rados.0.smithi076.stdout:281: finishing rollback tid 2 to smithi07685715-49 2024-09-21T09:38:56.507 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 176 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-09-21T09:38:56.507 INFO:tasks.rados.rados.0.smithi076.stdout:281: done (1 left) 2024-09-21T09:38:56.507 INFO:tasks.rados.rados.0.smithi076.stdout:282: done (0 left) 2024-09-21T09:38:56.508 INFO:tasks.rados.rados.0.smithi076.stdout:283: read oid 19 snap -1 2024-09-21T09:38:56.508 INFO:tasks.rados.rados.0.smithi076.stdout:283: expect (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:38:56.508 INFO:tasks.rados.rados.0.smithi076.stdout:284: snap_create 2024-09-21T09:38:56.515 INFO:tasks.rados.rados.0.smithi076.stdout:283: expect (ObjNum 140 snap 31 seq_num 140) 2024-09-21T09:38:56.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:56 smithi191 ceph-mon[33301]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:38:56.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:56 smithi076 ceph-mon[29313]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:38:56.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:56 smithi076 ceph-mon[41325]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:38:57.510 INFO:tasks.rados.rados.0.smithi076.stdout:283: done (1 left) 2024-09-21T09:38:57.510 INFO:tasks.rados.rados.0.smithi076.stdout:284: done (0 left) 2024-09-21T09:38:57.510 INFO:tasks.rados.rados.0.smithi076.stdout:285: snap_create 2024-09-21T09:38:57.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:57 smithi191 ceph-mon[33301]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:38:57.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:57 smithi191 ceph-mon[33301]: pgmap v188: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:38:57.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:57 smithi076 ceph-mon[29313]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:38:57.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:57 smithi076 ceph-mon[29313]: pgmap v188: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:38:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:57 smithi076 ceph-mon[41325]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:38:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:57 smithi076 ceph-mon[41325]: pgmap v188: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.5 MiB/s wr, 15 op/s 2024-09-21T09:38:58.501 INFO:tasks.rados.rados.0.smithi076.stdout:285: done (0 left) 2024-09-21T09:38:58.501 INFO:tasks.rados.rados.0.smithi076.stdout:286: snap_create 2024-09-21T09:38:58.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:58 smithi191 ceph-mon[33301]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:38:58.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:58 smithi191 ceph-mon[33301]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:38:58.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:58 smithi076 ceph-mon[29313]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:38:58.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:58 smithi076 ceph-mon[29313]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:38:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:58 smithi076 ceph-mon[41325]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:38:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:58 smithi076 ceph-mon[41325]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:38:59.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:38:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:38:58] "GET /metrics HTTP/1.1" 200 36063 "" "Prometheus/2.43.0" 2024-09-21T09:38:59.502 INFO:tasks.rados.rados.0.smithi076.stdout:286: done (0 left) 2024-09-21T09:38:59.502 INFO:tasks.rados.rados.0.smithi076.stdout:287: write oid 13 current snap is 34 2024-09-21T09:38:59.502 INFO:tasks.rados.rados.0.smithi076.stdout:287: seq_num 142 ranges {786594=779712,2326443=60151} 2024-09-21T09:38:59.510 INFO:tasks.rados.rados.0.smithi076.stdout:287: writing smithi07685715-13 from 786594 to 1566306 tid 1 2024-09-21T09:38:59.515 INFO:tasks.rados.rados.0.smithi076.stdout:287: writing smithi07685715-13 from 2326443 to 2386594 tid 2 2024-09-21T09:38:59.515 INFO:tasks.rados.rados.0.smithi076.stdout:288: write oid 29 current snap is 34 2024-09-21T09:38:59.515 INFO:tasks.rados.rados.0.smithi076.stdout:288: seq_num 143 ranges {403401=454961,1584925=506039} 2024-09-21T09:38:59.519 INFO:tasks.rados.rados.0.smithi076.stdout:288: writing smithi07685715-29 from 403401 to 858362 tid 1 2024-09-21T09:38:59.527 INFO:tasks.rados.rados.0.smithi076.stdout:288: writing smithi07685715-29 from 1584925 to 2090964 tid 2 2024-09-21T09:38:59.527 INFO:tasks.rados.rados.0.smithi076.stdout:289: read oid 30 snap -1 2024-09-21T09:38:59.527 INFO:tasks.rados.rados.0.smithi076.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:59.527 INFO:tasks.rados.rados.0.smithi076.stdout:290: delete oid 22 current snap is 34 2024-09-21T09:38:59.527 INFO:tasks.rados.rados.0.smithi076.stdout:287: finishing write tid 1 to smithi07685715-13 2024-09-21T09:38:59.527 INFO:tasks.rados.rados.0.smithi076.stdout:287: finishing write tid 2 to smithi07685715-13 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:287: finishing write tid 3 to smithi07685715-13 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:287: finishing write tid 4 to smithi07685715-13 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:287: oid 13 updating version 0 to 29 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:287: oid 13 updating version 29 to 30 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:287: oid 13 updating version 30 to 31 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 31 (ObjNum 142 snap 34 seq_num 142) dirty exists 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:287: left oid 13 (ObjNum 142 snap 34 seq_num 142) 2024-09-21T09:38:59.528 INFO:tasks.rados.rados.0.smithi076.stdout:288: finishing write tid 1 to smithi07685715-29 2024-09-21T09:38:59.530 INFO:tasks.rados.rados.0.smithi076.stdout:288: finishing write tid 2 to smithi07685715-29 2024-09-21T09:38:59.532 INFO:tasks.rados.rados.0.smithi076.stdout:288: finishing write tid 3 to smithi07685715-29 2024-09-21T09:38:59.532 INFO:tasks.rados.rados.0.smithi076.stdout:288: finishing write tid 4 to smithi07685715-29 2024-09-21T09:38:59.532 INFO:tasks.rados.rados.0.smithi076.stdout:288: oid 29 updating version 0 to 63 2024-09-21T09:38:59.532 INFO:tasks.rados.rados.0.smithi076.stdout:288: oid 29 updating version 63 to 64 2024-09-21T09:38:59.532 INFO:tasks.rados.rados.0.smithi076.stdout:288: oid 29 version 64 is already newer than 62 2024-09-21T09:38:59.532 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 64 (ObjNum 143 snap 34 seq_num 143) dirty exists 2024-09-21T09:38:59.533 INFO:tasks.rados.rados.0.smithi076.stdout:288: left oid 29 (ObjNum 143 snap 34 seq_num 143) 2024-09-21T09:38:59.533 INFO:tasks.rados.rados.0.smithi076.stdout:287: done (3 left) 2024-09-21T09:38:59.533 INFO:tasks.rados.rados.0.smithi076.stdout:288: done (2 left) 2024-09-21T09:38:59.533 INFO:tasks.rados.rados.0.smithi076.stdout:290: done (1 left) 2024-09-21T09:38:59.533 INFO:tasks.rados.rados.0.smithi076.stdout:291: write oid 8 current snap is 34 2024-09-21T09:38:59.533 INFO:tasks.rados.rados.0.smithi076.stdout:291: seq_num 145 ranges {437015=721812,1701889=509708} 2024-09-21T09:38:59.538 INFO:tasks.rados.rados.0.smithi076.stdout:291: writing smithi07685715-8 from 437015 to 1158827 tid 1 2024-09-21T09:38:59.544 INFO:tasks.rados.rados.0.smithi076.stdout:291: writing smithi07685715-8 from 1701889 to 2211597 tid 2 2024-09-21T09:38:59.544 INFO:tasks.rados.rados.0.smithi076.stdout:292: delete oid 12 current snap is 34 2024-09-21T09:38:59.544 INFO:tasks.rados.rados.0.smithi076.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:38:59.626 INFO:tasks.rados.rados.0.smithi076.stdout:291: finishing write tid 1 to smithi07685715-8 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: finishing write tid 2 to smithi07685715-8 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: finishing write tid 3 to smithi07685715-8 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: finishing write tid 4 to smithi07685715-8 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: oid 8 updating version 0 to 189 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: oid 8 updating version 189 to 190 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: oid 8 version 190 is already newer than 188 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 190 (ObjNum 145 snap 34 seq_num 145) dirty exists 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: left oid 8 (ObjNum 145 snap 34 seq_num 145) 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:289: done (2 left) 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:291: done (1 left) 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:292: done (0 left) 2024-09-21T09:38:59.627 INFO:tasks.rados.rados.0.smithi076.stdout:293: write oid 28 current snap is 34 2024-09-21T09:38:59.628 INFO:tasks.rados.rados.0.smithi076.stdout:293: seq_num 147 ranges {470629=472310,1418853=513377} 2024-09-21T09:38:59.631 INFO:tasks.rados.rados.0.smithi076.stdout:293: writing smithi07685715-28 from 470629 to 942939 tid 1 2024-09-21T09:38:59.637 INFO:tasks.rados.rados.0.smithi076.stdout:293: writing smithi07685715-28 from 1418853 to 1932230 tid 2 2024-09-21T09:38:59.637 INFO:tasks.rados.rados.0.smithi076.stdout:294: write oid 26 current snap is 34 2024-09-21T09:38:59.637 INFO:tasks.rados.rados.0.smithi076.stdout:293: finishing write tid 1 to smithi07685715-28 2024-09-21T09:38:59.637 INFO:tasks.rados.rados.0.smithi076.stdout:294: seq_num 148 ranges {487436=547559,1760982=657035} 2024-09-21T09:38:59.642 INFO:tasks.rados.rados.0.smithi076.stdout:294: writing smithi07685715-26 from 487436 to 1034995 tid 1 2024-09-21T09:38:59.650 INFO:tasks.rados.rados.0.smithi076.stdout:294: writing smithi07685715-26 from 1760982 to 2418017 tid 2 2024-09-21T09:38:59.650 INFO:tasks.rados.rados.0.smithi076.stdout:295: rollback oid 7 current snap is 34 2024-09-21T09:38:59.650 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 30 2024-09-21T09:38:59.650 INFO:tasks.rados.rados.0.smithi076.stdout:293: finishing write tid 2 to smithi07685715-28 2024-09-21T09:38:59.650 INFO:tasks.rados.rados.0.smithi076.stdout:293: finishing write tid 3 to smithi07685715-28 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:293: finishing write tid 4 to smithi07685715-28 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:293: oid 28 updating version 0 to 192 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:293: oid 28 updating version 192 to 193 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:293: oid 28 version 193 is already newer than 191 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 193 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:293: left oid 28 (ObjNum 147 snap 34 seq_num 147) 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:293: done (2 left) 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:296: rollback oid 10 current snap is 34 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 20 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:294: finishing write tid 1 to smithi07685715-26 2024-09-21T09:38:59.651 INFO:tasks.rados.rados.0.smithi076.stdout:297: delete oid 8 current snap is 34 2024-09-21T09:38:59.652 INFO:tasks.rados.rados.0.smithi076.stdout:297: done (3 left) 2024-09-21T09:38:59.652 INFO:tasks.rados.rados.0.smithi076.stdout:298: snap_remove snap 28 2024-09-21T09:38:59.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:59 smithi191 ceph-mon[33301]: pgmap v191: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:59.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:38:59 smithi191 ceph-mon[33301]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:38:59.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:59 smithi076 ceph-mon[29313]: pgmap v191: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:59.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:38:59 smithi076 ceph-mon[29313]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:38:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:59 smithi076 ceph-mon[41325]: pgmap v191: 89 pgs: 89 active+clean; 123 MiB data, 448 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:38:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:38:59 smithi076 ceph-mon[41325]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:39:00.528 INFO:tasks.rados.rados.0.smithi076.stdout:298: done (3 left) 2024-09-21T09:39:00.528 INFO:tasks.rados.rados.0.smithi076.stdout:299: snap_remove snap 32 2024-09-21T09:39:01.535 INFO:tasks.rados.rados.0.smithi076.stdout:294: finishing write tid 2 to smithi07685715-26 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:295: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:294: finishing write tid 3 to smithi07685715-26 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:295: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:294: finishing write tid 4 to smithi07685715-26 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:294: oid 26 updating version 0 to 115 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:294: oid 26 updating version 115 to 117 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:294: oid 26 version 117 is already newer than 116 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 117 (ObjNum 148 snap 34 seq_num 148) dirty exists 2024-09-21T09:39:01.536 INFO:tasks.rados.rados.0.smithi076.stdout:294: left oid 26 (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:295: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 132 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:296: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 0 (ObjNum 49 snap 0 seq_num 1682140544) dirty dne 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:294: done (3 left) 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:295: done (2 left) 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:296: done (1 left) 2024-09-21T09:39:01.537 INFO:tasks.rados.rados.0.smithi076.stdout:299: done (0 left) 2024-09-21T09:39:01.538 INFO:tasks.rados.rados.0.smithi076.stdout:300: rollback oid 36 current snap is 34 2024-09-21T09:39:01.538 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 30 2024-09-21T09:39:01.538 INFO:tasks.rados.rados.0.smithi076.stdout:301: snap_create 2024-09-21T09:39:01.541 INFO:tasks.rados.rados.0.smithi076.stdout:300: finishing rollback tid 0 to smithi07685715-36 2024-09-21T09:39:01.541 INFO:tasks.rados.rados.0.smithi076.stdout:300: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:39:01.541 INFO:tasks.rados.rados.0.smithi076.stdout:300: finishing rollback tid 2 to smithi07685715-36 2024-09-21T09:39:01.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 92 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-09-21T09:39:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:01 smithi191 ceph-mon[33301]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:39:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:01 smithi191 ceph-mon[33301]: pgmap v194: 89 pgs: 89 active+clean; 129 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:01 smithi076 ceph-mon[29313]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:39:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:01 smithi076 ceph-mon[29313]: pgmap v194: 89 pgs: 89 active+clean; 129 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:01 smithi076 ceph-mon[41325]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:39:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:01 smithi076 ceph-mon[41325]: pgmap v194: 89 pgs: 89 active+clean; 129 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:02.539 INFO:tasks.rados.rados.0.smithi076.stdout:300: done (1 left) 2024-09-21T09:39:02.539 INFO:tasks.rados.rados.0.smithi076.stdout:301: done (0 left) 2024-09-21T09:39:02.539 INFO:tasks.rados.rados.0.smithi076.stdout:302: write oid 3 current snap is 35 2024-09-21T09:39:02.539 INFO:tasks.rados.rados.0.smithi076.stdout:302: seq_num 150 ranges {521050=698057,1877946=643104} 2024-09-21T09:39:02.545 INFO:tasks.rados.rados.0.smithi076.stdout:302: writing smithi07685715-3 from 521050 to 1219107 tid 1 2024-09-21T09:39:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:302: writing smithi07685715-3 from 1877946 to 2521050 tid 2 2024-09-21T09:39:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:303: read oid 25 snap 30 2024-09-21T09:39:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:303: expect deleted 2024-09-21T09:39:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:304: snap_create 2024-09-21T09:39:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:302: finishing write tid 1 to smithi07685715-3 2024-09-21T09:39:02.555 INFO:tasks.rados.rados.0.smithi076.stdout:302: finishing write tid 2 to smithi07685715-3 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:302: finishing write tid 3 to smithi07685715-3 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:302: finishing write tid 4 to smithi07685715-3 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:302: oid 3 updating version 0 to 41 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:302: oid 3 updating version 41 to 43 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:302: oid 3 version 43 is already newer than 42 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 43 (ObjNum 150 snap 35 seq_num 150) dirty exists 2024-09-21T09:39:02.556 INFO:tasks.rados.rados.0.smithi076.stdout:302: left oid 3 (ObjNum 150 snap 35 seq_num 150) 2024-09-21T09:39:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:02 smithi191 ceph-mon[33301]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:39:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:02 smithi076 ceph-mon[29313]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:39:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:02 smithi076 ceph-mon[41325]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:39:03.502 INFO:tasks.rados.rados.0.smithi076.stdout:302: done (2 left) 2024-09-21T09:39:03.502 INFO:tasks.rados.rados.0.smithi076.stdout:303: done (1 left) 2024-09-21T09:39:03.502 INFO:tasks.rados.rados.0.smithi076.stdout:304: done (0 left) 2024-09-21T09:39:03.502 INFO:tasks.rados.rados.0.smithi076.stdout:305: snap_remove snap 34 2024-09-21T09:39:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:03 smithi191 ceph-mon[33301]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:39:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:03 smithi191 ceph-mon[33301]: pgmap v197: 89 pgs: 89 active+clean; 129 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:03 smithi191 ceph-mon[33301]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:39:03.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:03 smithi076 ceph-mon[41325]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:39:03.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:03 smithi076 ceph-mon[41325]: pgmap v197: 89 pgs: 89 active+clean; 129 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:03.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:03 smithi076 ceph-mon[41325]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:39:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:03 smithi076 ceph-mon[29313]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:39:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:03 smithi076 ceph-mon[29313]: pgmap v197: 89 pgs: 89 active+clean; 129 MiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:39:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:03 smithi076 ceph-mon[29313]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:39:04.503 INFO:tasks.rados.rados.0.smithi076.stdout:305: done (0 left) 2024-09-21T09:39:04.503 INFO:tasks.rados.rados.0.smithi076.stdout:306: read oid 30 snap -1 2024-09-21T09:39:04.503 INFO:tasks.rados.rados.0.smithi076.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:39:04.503 INFO:tasks.rados.rados.0.smithi076.stdout:307: write oid 9 current snap is 36 2024-09-21T09:39:04.503 INFO:tasks.rados.rados.0.smithi076.stdout:307: seq_num 151 ranges {537857=773306,1936428=520715} 2024-09-21T09:39:04.509 INFO:tasks.rados.rados.0.smithi076.stdout:307: writing smithi07685715-9 from 537857 to 1311163 tid 1 2024-09-21T09:39:04.516 INFO:tasks.rados.rados.0.smithi076.stdout:307: writing smithi07685715-9 from 1936428 to 2457143 tid 2 2024-09-21T09:39:04.516 INFO:tasks.rados.rados.0.smithi076.stdout:308: delete oid 1 current snap is 36 2024-09-21T09:39:04.517 INFO:tasks.rados.rados.0.smithi076.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:39:04.602 INFO:tasks.rados.rados.0.smithi076.stdout:307: finishing write tid 1 to smithi07685715-9 2024-09-21T09:39:04.603 INFO:tasks.rados.rados.0.smithi076.stdout:306: done (2 left) 2024-09-21T09:39:04.603 INFO:tasks.rados.rados.0.smithi076.stdout:308: done (1 left) 2024-09-21T09:39:04.603 INFO:tasks.rados.rados.0.smithi076.stdout:309: write oid 27 current snap is 36 2024-09-21T09:39:04.603 INFO:tasks.rados.rados.0.smithi076.stdout:307: finishing write tid 2 to smithi07685715-9 2024-09-21T09:39:04.603 INFO:tasks.rados.rados.0.smithi076.stdout:309: seq_num 153 ranges {571471=640157,1653392=524384} 2024-09-21T09:39:04.609 INFO:tasks.rados.rados.0.smithi076.stdout:309: writing smithi07685715-27 from 571471 to 1211628 tid 1 2024-09-21T09:39:04.614 INFO:tasks.rados.rados.0.smithi076.stdout:309: writing smithi07685715-27 from 1653392 to 2177776 tid 2 2024-09-21T09:39:04.614 INFO:tasks.rados.rados.0.smithi076.stdout:310: snap_create 2024-09-21T09:39:04.614 INFO:tasks.rados.rados.0.smithi076.stdout:307: finishing write tid 3 to smithi07685715-9 2024-09-21T09:39:04.614 INFO:tasks.rados.rados.0.smithi076.stdout:307: finishing write tid 4 to smithi07685715-9 2024-09-21T09:39:04.614 INFO:tasks.rados.rados.0.smithi076.stdout:307: oid 9 updating version 0 to 127 2024-09-21T09:39:04.615 INFO:tasks.rados.rados.0.smithi076.stdout:307: oid 9 updating version 127 to 129 2024-09-21T09:39:04.615 INFO:tasks.rados.rados.0.smithi076.stdout:307: oid 9 version 129 is already newer than 128 2024-09-21T09:39:04.615 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 129 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-09-21T09:39:04.615 INFO:tasks.rados.rados.0.smithi076.stdout:307: left oid 9 (ObjNum 151 snap 36 seq_num 151) 2024-09-21T09:39:04.615 INFO:tasks.rados.rados.0.smithi076.stdout:309: finishing write tid 1 to smithi07685715-27 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: finishing write tid 2 to smithi07685715-27 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: finishing write tid 3 to smithi07685715-27 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: finishing write tid 4 to smithi07685715-27 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: oid 27 updating version 0 to 124 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: oid 27 version 124 is already newer than 122 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: oid 27 version 124 is already newer than 123 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 124 (ObjNum 153 snap 36 seq_num 153) dirty exists 2024-09-21T09:39:04.617 INFO:tasks.rados.rados.0.smithi076.stdout:309: left oid 27 (ObjNum 153 snap 36 seq_num 153) 2024-09-21T09:39:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:04 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:04 smithi191 ceph-mon[33301]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:39:04.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:04 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:04.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:04 smithi076 ceph-mon[29313]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:39:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:04 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:04 smithi076 ceph-mon[41325]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:39:05.548 INFO:tasks.rados.rados.0.smithi076.stdout:307: done (2 left) 2024-09-21T09:39:05.548 INFO:tasks.rados.rados.0.smithi076.stdout:309: done (1 left) 2024-09-21T09:39:05.548 INFO:tasks.rados.rados.0.smithi076.stdout:310: done (0 left) 2024-09-21T09:39:05.548 INFO:tasks.rados.rados.0.smithi076.stdout:311: snap_create 2024-09-21T09:39:05.813 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:05 smithi076 ceph-mon[29313]: pgmap v200: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 928 KiB/s wr, 6 op/s 2024-09-21T09:39:05.814 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:05 smithi076 ceph-mon[41325]: pgmap v200: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 928 KiB/s wr, 6 op/s 2024-09-21T09:39:05.814 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:39:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:05 smithi191 ceph-mon[33301]: pgmap v200: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 928 KiB/s wr, 6 op/s 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (2m) 30s ago 3m 26.6M - 0.25.0 c8568f914cd2 d930ef8d31ba 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (3m) 32s ago 3m 85.2M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (2m) 30s ago 2m 14.6M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (2m) 32s ago 2m 14.5M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,8765 running (7m) 32s ago 7m 439M - 18.2.4-871-g6a1bdf85 0adab276c97e a91e8d898a8a 2024-09-21T09:39:06.341 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:9283,8765 running (9m) 30s ago 9m 487M - 18.2.4-871-g6a1bdf85 0adab276c97e 3f03f8aa752c 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (9m) 30s ago 9m 48.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (7m) 32s ago 7m 41.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (7m) 30s ago 7m 39.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (3m) 30s ago 3m 14.8M - 1.5.0 0da6a335fe13 6dbd51c32fb5 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (3m) 32s ago 3m 15.2M - 1.5.0 0da6a335fe13 9722666fd438 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (7m) 30s ago 7m 62.4M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (6m) 30s ago 6m 58.9M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (6m) 30s ago 6m 62.6M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (5m) 30s ago 5m 56.7M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (5m) 32s ago 5m 58.0M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (5m) 32s ago 5m 61.0M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (4m) 32s ago 4m 50.1M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (4m) 32s ago 4m 50.9M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:39:06.342 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (2m) 32s ago 3m 43.1M - 2.43.0 a07b618ecd1d 8726c547d40a 2024-09-21T09:39:06.549 INFO:tasks.rados.rados.0.smithi076.stdout:311: done (0 left) 2024-09-21T09:39:06.549 INFO:tasks.rados.rados.0.smithi076.stdout:312: read oid 35 snap -1 2024-09-21T09:39:06.550 INFO:tasks.rados.rados.0.smithi076.stdout:312: expect (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:39:06.550 INFO:tasks.rados.rados.0.smithi076.stdout:313: delete oid 29 current snap is 38 2024-09-21T09:39:06.554 INFO:tasks.rados.rados.0.smithi076.stdout:313: done (1 left) 2024-09-21T09:39:06.554 INFO:tasks.rados.rados.0.smithi076.stdout:314: snap_remove snap 22 2024-09-21T09:39:06.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:06 smithi076 ceph-mon[29313]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:39:06.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:06 smithi076 ceph-mon[29313]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:06 smithi076 ceph-mon[41325]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:39:06.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:06 smithi076 ceph-mon[41325]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:39:06.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 15 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:39:06.661 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:39:06.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:06 smithi191 ceph-mon[33301]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:39:06.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:06 smithi191 ceph-mon[33301]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "0/2 daemons upgraded", 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 image on host smithi191", 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:39:06.929 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:39:07.560 INFO:tasks.rados.rados.0.smithi076.stdout:314: done (1 left) 2024-09-21T09:39:07.560 INFO:tasks.rados.rados.0.smithi076.stdout:315: snap_remove snap 30 2024-09-21T09:39:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:07 smithi191 ceph-mon[33301]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:07 smithi191 ceph-mon[33301]: from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:07 smithi191 ceph-mon[33301]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:39:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:07 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3123264229' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:39:07.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:07 smithi191 ceph-mon[33301]: pgmap v203: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 929 KiB/s wr, 6 op/s 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[29313]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[29313]: from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[29313]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3123264229' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[29313]: pgmap v203: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 929 KiB/s wr, 6 op/s 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[41325]: from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[41325]: from='client.24752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[41325]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3123264229' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:39:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:07 smithi076 ceph-mon[41325]: pgmap v203: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 929 KiB/s wr, 6 op/s 2024-09-21T09:39:08.504 INFO:tasks.rados.rados.0.smithi076.stdout:315: done (1 left) 2024-09-21T09:39:08.504 INFO:tasks.rados.rados.0.smithi076.stdout:316: write oid 47 current snap is 38 2024-09-21T09:39:08.504 INFO:tasks.rados.rados.0.smithi076.stdout:316: seq_num 155 ranges {605085=790655,2170356=434729} 2024-09-21T09:39:08.513 INFO:tasks.rados.rados.0.smithi076.stdout:316: writing smithi07685715-47 from 605085 to 1395740 tid 1 2024-09-21T09:39:08.520 INFO:tasks.rados.rados.0.smithi076.stdout:316: writing smithi07685715-47 from 2170356 to 2605085 tid 2 2024-09-21T09:39:08.520 INFO:tasks.rados.rados.0.smithi076.stdout:317: snap_create 2024-09-21T09:39:08.520 INFO:tasks.rados.rados.0.smithi076.stdout:312: expect (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:39:08.683 INFO:tasks.rados.rados.0.smithi076.stdout:316: finishing write tid 1 to smithi07685715-47 2024-09-21T09:39:08.683 INFO:tasks.rados.rados.0.smithi076.stdout:316: finishing write tid 2 to smithi07685715-47 2024-09-21T09:39:08.683 INFO:tasks.rados.rados.0.smithi076.stdout:316: finishing write tid 3 to smithi07685715-47 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi076.stdout:316: finishing write tid 4 to smithi07685715-47 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi076.stdout:316: oid 47 updating version 0 to 214 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi076.stdout:316: oid 47 updating version 214 to 216 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi076.stdout:316: oid 47 version 216 is already newer than 215 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 216 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-09-21T09:39:08.684 INFO:tasks.rados.rados.0.smithi076.stdout:316: left oid 47 (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:08 smithi191 ceph-mon[33301]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:08 smithi191 ceph-mon[33301]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:39:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:08 smithi191 ceph-mon[33301]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:39:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-mon[29313]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-mon[29313]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:39:08.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-mon[29313]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:39:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-mon[41325]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-mon[41325]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:39:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-mon[41325]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:39:09.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:39:08] "GET /metrics HTTP/1.1" 200 36063 "" "Prometheus/2.43.0" 2024-09-21T09:39:09.503 INFO:tasks.rados.rados.0.smithi076.stdout:312: done (2 left) 2024-09-21T09:39:09.504 INFO:tasks.rados.rados.0.smithi076.stdout:316: done (1 left) 2024-09-21T09:39:09.504 INFO:tasks.rados.rados.0.smithi076.stdout:317: done (0 left) 2024-09-21T09:39:09.504 INFO:tasks.rados.rados.0.smithi076.stdout:318: write oid 7 current snap is 39 2024-09-21T09:39:09.504 INFO:tasks.rados.rados.0.smithi076.stdout:318: seq_num 156 ranges {621892=465904,1712485=671711} 2024-09-21T09:39:09.509 INFO:tasks.rados.rados.0.smithi076.stdout:318: writing smithi07685715-7 from 621892 to 1087796 tid 1 2024-09-21T09:39:09.517 INFO:tasks.rados.rados.0.smithi076.stdout:318: writing smithi07685715-7 from 1712485 to 2384196 tid 2 2024-09-21T09:39:09.517 INFO:tasks.rados.rados.0.smithi076.stdout:319: rollback oid 50 current snap is 39 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 36 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:318: finishing write tid 1 to smithi07685715-7 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:320: read oid 36 snap -1 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:321: rollback oid 44 current snap is 39 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 37 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:322: write oid 17 current snap is 39 2024-09-21T09:39:09.518 INFO:tasks.rados.rados.0.smithi076.stdout:322: seq_num 157 ranges {638699=541153,1770967=531722} 2024-09-21T09:39:09.525 INFO:tasks.rados.rados.0.smithi076.stdout:322: writing smithi07685715-17 from 638699 to 1179852 tid 1 2024-09-21T09:39:09.532 INFO:tasks.rados.rados.0.smithi076.stdout:322: writing smithi07685715-17 from 1770967 to 2302689 tid 2 2024-09-21T09:39:09.532 INFO:tasks.rados.rados.0.smithi076.stdout:323: snap_create 2024-09-21T09:39:09.532 INFO:tasks.rados.rados.0.smithi076.stdout:321: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:39:09.532 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 64 (ObjNum 49 snap 0 seq_num 1681955422) dirty dne 2024-09-21T09:39:09.532 INFO:tasks.rados.rados.0.smithi076.stdout:318: finishing write tid 2 to smithi07685715-7 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:318: finishing write tid 3 to smithi07685715-7 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:318: finishing write tid 4 to smithi07685715-7 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:318: oid 7 updating version 0 to 142 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:318: oid 7 updating version 142 to 144 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:318: oid 7 version 144 is already newer than 143 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 144 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:318: left oid 7 (ObjNum 156 snap 39 seq_num 156) 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:322: finishing write tid 1 to smithi07685715-17 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:319: finishing rollback tid 0 to smithi07685715-50 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:319: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:319: finishing rollback tid 2 to smithi07685715-50 2024-09-21T09:39:09.533 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 100 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-09-21T09:39:09.534 INFO:tasks.rados.rados.0.smithi076.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:09.606 INFO:tasks.rados.rados.0.smithi076.stdout:322: finishing write tid 2 to smithi07685715-17 2024-09-21T09:39:09.606 INFO:tasks.rados.rados.0.smithi076.stdout:322: finishing write tid 3 to smithi07685715-17 2024-09-21T09:39:09.606 INFO:tasks.rados.rados.0.smithi076.stdout:322: finishing write tid 4 to smithi07685715-17 2024-09-21T09:39:09.606 INFO:tasks.rados.rados.0.smithi076.stdout:322: oid 17 updating version 0 to 145 2024-09-21T09:39:09.606 INFO:tasks.rados.rados.0.smithi076.stdout:322: oid 17 updating version 145 to 147 2024-09-21T09:39:09.606 INFO:tasks.rados.rados.0.smithi076.stdout:322: oid 17 version 147 is already newer than 146 2024-09-21T09:39:09.607 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 147 (ObjNum 157 snap 39 seq_num 157) dirty exists 2024-09-21T09:39:09.607 INFO:tasks.rados.rados.0.smithi076.stdout:322: left oid 17 (ObjNum 157 snap 39 seq_num 157) 2024-09-21T09:39:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:09 smithi191 ceph-mon[33301]: pgmap v206: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:09 smithi191 ceph-mon[33301]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:39:09.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:09 smithi076 ceph-mon[29313]: pgmap v206: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:09.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:09 smithi076 ceph-mon[29313]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:39:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:09 smithi076 ceph-mon[41325]: pgmap v206: 89 pgs: 89 active+clean; 132 MiB data, 470 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:09 smithi076 ceph-mon[41325]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:318: done (5 left) 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:319: done (4 left) 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:320: done (3 left) 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:321: done (2 left) 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:322: done (1 left) 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:323: done (0 left) 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:324: write oid 40 current snap is 40 2024-09-21T09:39:10.510 INFO:tasks.rados.rados.0.smithi076.stdout:324: seq_num 158 ranges {655506=616402,1829449=675380} 2024-09-21T09:39:10.516 INFO:tasks.rados.rados.0.smithi076.stdout:324: writing smithi07685715-40 from 655506 to 1271908 tid 1 2024-09-21T09:39:10.523 INFO:tasks.rados.rados.0.smithi076.stdout:324: writing smithi07685715-40 from 1829449 to 2504829 tid 2 2024-09-21T09:39:10.523 INFO:tasks.rados.rados.0.smithi076.stdout:325: snap_create 2024-09-21T09:39:10.523 INFO:tasks.rados.rados.0.smithi076.stdout:324: finishing write tid 1 to smithi07685715-40 2024-09-21T09:39:10.527 INFO:tasks.rados.rados.0.smithi076.stdout:324: finishing write tid 2 to smithi07685715-40 2024-09-21T09:39:10.527 INFO:tasks.rados.rados.0.smithi076.stdout:324: finishing write tid 3 to smithi07685715-40 2024-09-21T09:39:10.527 INFO:tasks.rados.rados.0.smithi076.stdout:324: finishing write tid 4 to smithi07685715-40 2024-09-21T09:39:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:324: oid 40 updating version 0 to 139 2024-09-21T09:39:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:324: oid 40 updating version 139 to 140 2024-09-21T09:39:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:324: oid 40 version 140 is already newer than 138 2024-09-21T09:39:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 140 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-09-21T09:39:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:324: left oid 40 (ObjNum 158 snap 40 seq_num 158) 2024-09-21T09:39:11.513 INFO:tasks.rados.rados.0.smithi076.stdout:324: done (1 left) 2024-09-21T09:39:11.513 INFO:tasks.rados.rados.0.smithi076.stdout:325: done (0 left) 2024-09-21T09:39:11.513 INFO:tasks.rados.rados.0.smithi076.stdout:326: write oid 33 current snap is 41 2024-09-21T09:39:11.513 INFO:tasks.rados.rados.0.smithi076.stdout:326: seq_num 159 ranges {672313=691651,1887931=419038} 2024-09-21T09:39:11.520 INFO:tasks.rados.rados.0.smithi076.stdout:326: writing smithi07685715-33 from 672313 to 1363964 tid 1 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:326: writing smithi07685715-33 from 1887931 to 2306969 tid 2 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:327: read oid 18 snap 23 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:327: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:328: read oid 48 snap 40 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:329: write oid 49 current snap is 41 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:326: finishing write tid 1 to smithi07685715-33 2024-09-21T09:39:11.526 INFO:tasks.rados.rados.0.smithi076.stdout:329: seq_num 160 ranges {689120=483253,1946413=679049} 2024-09-21T09:39:11.532 INFO:tasks.rados.rados.0.smithi076.stdout:329: writing smithi07685715-49 from 689120 to 1172373 tid 1 2024-09-21T09:39:11.540 INFO:tasks.rados.rados.0.smithi076.stdout:329: writing smithi07685715-49 from 1946413 to 2625462 tid 2 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:330: snap_create 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: finishing write tid 2 to smithi07685715-33 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: finishing write tid 3 to smithi07685715-33 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: finishing write tid 4 to smithi07685715-33 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: oid 33 updating version 0 to 38 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: oid 33 version 38 is already newer than 37 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: oid 33 updating version 38 to 39 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 39 (ObjNum 159 snap 41 seq_num 159) dirty exists 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:326: left oid 33 (ObjNum 159 snap 41 seq_num 159) 2024-09-21T09:39:11.541 INFO:tasks.rados.rados.0.smithi076.stdout:327: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:39:11.613 INFO:tasks.rados.rados.0.smithi076.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:11.735 INFO:tasks.rados.rados.0.smithi076.stdout:329: finishing write tid 1 to smithi07685715-49 2024-09-21T09:39:11.735 INFO:tasks.rados.rados.0.smithi076.stdout:329: finishing write tid 2 to smithi07685715-49 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:329: finishing write tid 3 to smithi07685715-49 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:329: finishing write tid 4 to smithi07685715-49 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:329: oid 49 updating version 0 to 220 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:329: oid 49 version 220 is already newer than 219 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:329: oid 49 version 220 is already newer than 218 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 220 (ObjNum 160 snap 41 seq_num 160) dirty exists 2024-09-21T09:39:11.736 INFO:tasks.rados.rados.0.smithi076.stdout:329: left oid 49 (ObjNum 160 snap 41 seq_num 160) 2024-09-21T09:39:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:11 smithi191 ceph-mon[33301]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:39:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:11 smithi191 ceph-mon[33301]: pgmap v209: 89 pgs: 89 active+clean; 134 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:39:11.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:11 smithi076 ceph-mon[29313]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:39:11.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:11 smithi076 ceph-mon[29313]: pgmap v209: 89 pgs: 89 active+clean; 134 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:39:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:11 smithi076 ceph-mon[41325]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:39:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:11 smithi076 ceph-mon[41325]: pgmap v209: 89 pgs: 89 active+clean; 134 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:39:12.515 INFO:tasks.rados.rados.0.smithi076.stdout:326: done (4 left) 2024-09-21T09:39:12.516 INFO:tasks.rados.rados.0.smithi076.stdout:327: done (3 left) 2024-09-21T09:39:12.516 INFO:tasks.rados.rados.0.smithi076.stdout:328: done (2 left) 2024-09-21T09:39:12.516 INFO:tasks.rados.rados.0.smithi076.stdout:329: done (1 left) 2024-09-21T09:39:12.516 INFO:tasks.rados.rados.0.smithi076.stdout:330: done (0 left) 2024-09-21T09:39:12.516 INFO:tasks.rados.rados.0.smithi076.stdout:331: snap_remove snap 39 2024-09-21T09:39:12.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:12 smithi191 ceph-mon[33301]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:39:12.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:12 smithi076 ceph-mon[29313]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:39:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:12 smithi076 ceph-mon[41325]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:39:13.505 INFO:tasks.rados.rados.0.smithi076.stdout:331: done (0 left) 2024-09-21T09:39:13.505 INFO:tasks.rados.rados.0.smithi076.stdout:332: rollback oid 33 current snap is 42 2024-09-21T09:39:13.505 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 38 2024-09-21T09:39:13.505 INFO:tasks.rados.rados.0.smithi076.stdout:333: snap_remove snap 33 2024-09-21T09:39:13.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:13 smithi191 ceph-mon[33301]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:39:13.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:13 smithi191 ceph-mon[33301]: pgmap v212: 89 pgs: 89 active+clean; 134 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:39:13.805 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:13 smithi191 ceph-mon[33301]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:39:13.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:13 smithi076 ceph-mon[29313]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:39:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:13 smithi076 ceph-mon[29313]: pgmap v212: 89 pgs: 89 active+clean; 134 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:39:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:13 smithi076 ceph-mon[29313]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:39:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:13 smithi076 ceph-mon[41325]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:39:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:13 smithi076 ceph-mon[41325]: pgmap v212: 89 pgs: 89 active+clean; 134 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 839 KiB/s wr, 9 op/s 2024-09-21T09:39:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:13 smithi076 ceph-mon[41325]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:39:14.505 INFO:tasks.rados.rados.0.smithi076.stdout:332: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:39:14.505 INFO:tasks.rados.rados.0.smithi076.stdout:332: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:39:14.505 INFO:tasks.rados.rados.0.smithi076.stdout:332: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:39:14.505 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 45 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-09-21T09:39:14.506 INFO:tasks.rados.rados.0.smithi076.stdout:332: done (1 left) 2024-09-21T09:39:14.506 INFO:tasks.rados.rados.0.smithi076.stdout:333: done (0 left) 2024-09-21T09:39:14.506 INFO:tasks.rados.rados.0.smithi076.stdout:334: rollback oid 38 current snap is 42 2024-09-21T09:39:14.506 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 37 2024-09-21T09:39:14.506 INFO:tasks.rados.rados.0.smithi076.stdout:335: write oid 48 current snap is 42 2024-09-21T09:39:14.506 INFO:tasks.rados.rados.0.smithi076.stdout:335: seq_num 161 ranges {705927=558502,2004895=422707} 2024-09-21T09:39:14.513 INFO:tasks.rados.rados.0.smithi076.stdout:335: writing smithi07685715-48 from 705927 to 1264429 tid 1 2024-09-21T09:39:14.520 INFO:tasks.rados.rados.0.smithi076.stdout:335: writing smithi07685715-48 from 2004895 to 2427602 tid 2 2024-09-21T09:39:14.520 INFO:tasks.rados.rados.0.smithi076.stdout:336: snap_create 2024-09-21T09:39:14.520 INFO:tasks.rados.rados.0.smithi076.stdout:334: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:39:14.520 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 0 (ObjNum 97 snap 0 seq_num 1682394782) dirty dne 2024-09-21T09:39:14.520 INFO:tasks.rados.rados.0.smithi076.stdout:335: finishing write tid 1 to smithi07685715-48 2024-09-21T09:39:14.522 INFO:tasks.rados.rados.0.smithi076.stdout:335: finishing write tid 2 to smithi07685715-48 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:335: finishing write tid 3 to smithi07685715-48 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:335: finishing write tid 4 to smithi07685715-48 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:335: oid 48 updating version 0 to 228 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:335: oid 48 updating version 228 to 232 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:335: oid 48 version 232 is already newer than 231 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 232 (ObjNum 161 snap 42 seq_num 161) dirty exists 2024-09-21T09:39:14.523 INFO:tasks.rados.rados.0.smithi076.stdout:335: left oid 48 (ObjNum 161 snap 42 seq_num 161) 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: Upgrade: Updating mgr.x 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: Deploying daemon mgr.x on smithi191 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[29313]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: Upgrade: Updating mgr.x 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:15.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:15.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:15.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:15.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:15.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: Deploying daemon mgr.x on smithi191 2024-09-21T09:39:15.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:14 smithi076 ceph-mon[41325]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: Upgrade: Updating mgr.x 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: Deploying daemon mgr.x on smithi191 2024-09-21T09:39:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:14 smithi191 ceph-mon[33301]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:39:15.511 INFO:tasks.rados.rados.0.smithi076.stdout:334: done (2 left) 2024-09-21T09:39:15.511 INFO:tasks.rados.rados.0.smithi076.stdout:335: done (1 left) 2024-09-21T09:39:15.511 INFO:tasks.rados.rados.0.smithi076.stdout:336: done (0 left) 2024-09-21T09:39:15.511 INFO:tasks.rados.rados.0.smithi076.stdout:337: read oid 36 snap -1 2024-09-21T09:39:15.511 INFO:tasks.rados.rados.0.smithi076.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:15.511 INFO:tasks.rados.rados.0.smithi076.stdout:338: delete oid 35 current snap is 43 2024-09-21T09:39:15.513 INFO:tasks.rados.rados.0.smithi076.stdout:338: done (1 left) 2024-09-21T09:39:15.513 INFO:tasks.rados.rados.0.smithi076.stdout:339: rollback oid 35 current snap is 43 2024-09-21T09:39:15.514 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 35 2024-09-21T09:39:15.514 INFO:tasks.rados.rados.0.smithi076.stdout:340: write oid 25 current snap is 43 2024-09-21T09:39:15.514 INFO:tasks.rados.rados.0.smithi076.stdout:340: seq_num 163 ranges {739541=709000,2121859=426376} 2024-09-21T09:39:15.521 INFO:tasks.rados.rados.0.smithi076.stdout:340: writing smithi07685715-25 from 739541 to 1448541 tid 1 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:340: writing smithi07685715-25 from 2121859 to 2548235 tid 2 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:341: rollback oid 29 current snap is 43 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 40 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:339: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:339: finishing rollback tid 2 to smithi07685715-35 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 107 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-09-21T09:39:15.528 INFO:tasks.rados.rados.0.smithi076.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:15.599 INFO:tasks.rados.rados.0.smithi076.stdout:340: finishing write tid 1 to smithi07685715-25 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:341: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 66 (ObjNum 49 snap 0 seq_num 864375664) dirty dne 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: finishing write tid 2 to smithi07685715-25 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: finishing write tid 3 to smithi07685715-25 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: finishing write tid 4 to smithi07685715-25 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: oid 25 updating version 0 to 48 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: oid 25 updating version 48 to 50 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: oid 25 version 50 is already newer than 49 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 50 (ObjNum 163 snap 43 seq_num 163) dirty exists 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: left oid 25 (ObjNum 163 snap 43 seq_num 163) 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:337: done (3 left) 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:339: done (2 left) 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:340: done (1 left) 2024-09-21T09:39:15.600 INFO:tasks.rados.rados.0.smithi076.stdout:341: done (0 left) 2024-09-21T09:39:15.601 INFO:tasks.rados.rados.0.smithi076.stdout:342: delete oid 6 current snap is 43 2024-09-21T09:39:15.601 INFO:tasks.rados.rados.0.smithi076.stdout:342: done (0 left) 2024-09-21T09:39:15.601 INFO:tasks.rados.rados.0.smithi076.stdout:343: delete oid 3 current snap is 43 2024-09-21T09:39:15.602 INFO:tasks.rados.rados.0.smithi076.stdout:343: done (0 left) 2024-09-21T09:39:15.603 INFO:tasks.rados.rados.0.smithi076.stdout:344: write oid 7 current snap is 43 2024-09-21T09:39:15.603 INFO:tasks.rados.rados.0.smithi076.stdout:344: seq_num 166 ranges {789962=534747,1780952=690056} 2024-09-21T09:39:15.609 INFO:tasks.rados.rados.0.smithi076.stdout:344: writing smithi07685715-7 from 789962 to 1324709 tid 1 2024-09-21T09:39:15.616 INFO:tasks.rados.rados.0.smithi076.stdout:344: writing smithi07685715-7 from 1780952 to 2471008 tid 2 2024-09-21T09:39:15.616 INFO:tasks.rados.rados.0.smithi076.stdout:345: read oid 31 snap -1 2024-09-21T09:39:15.617 INFO:tasks.rados.rados.0.smithi076.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:15.617 INFO:tasks.rados.rados.0.smithi076.stdout:346: write oid 12 current snap is 43 2024-09-21T09:39:15.617 INFO:tasks.rados.rados.0.smithi076.stdout:344: finishing write tid 1 to smithi07685715-7 2024-09-21T09:39:15.617 INFO:tasks.rados.rados.0.smithi076.stdout:346: seq_num 167 ranges {406769=609996,1439434=433714,2303522=503247} 2024-09-21T09:39:15.621 INFO:tasks.rados.rados.0.smithi076.stdout:346: writing smithi07685715-12 from 406769 to 1016765 tid 1 2024-09-21T09:39:15.626 INFO:tasks.rados.rados.0.smithi076.stdout:346: writing smithi07685715-12 from 1439434 to 1873148 tid 2 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:346: writing smithi07685715-12 from 2303522 to 2806769 tid 3 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:347: rollback oid 36 current snap is 43 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 40 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:344: finishing write tid 2 to smithi07685715-7 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:344: finishing write tid 3 to smithi07685715-7 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:344: finishing write tid 4 to smithi07685715-7 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:344: oid 7 updating version 0 to 151 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:344: oid 7 updating version 151 to 152 2024-09-21T09:39:15.632 INFO:tasks.rados.rados.0.smithi076.stdout:344: oid 7 updating version 152 to 153 2024-09-21T09:39:15.633 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 153 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-21T09:39:15.633 INFO:tasks.rados.rados.0.smithi076.stdout:344: left oid 7 (ObjNum 166 snap 43 seq_num 166) 2024-09-21T09:39:15.633 INFO:tasks.rados.rados.0.smithi076.stdout:345: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:15.717 INFO:tasks.rados.rados.0.smithi076.stdout:346: finishing write tid 1 to smithi07685715-12 2024-09-21T09:39:15.717 INFO:tasks.rados.rados.0.smithi076.stdout:346: finishing write tid 2 to smithi07685715-12 2024-09-21T09:39:15.717 INFO:tasks.rados.rados.0.smithi076.stdout:344: done (3 left) 2024-09-21T09:39:15.717 INFO:tasks.rados.rados.0.smithi076.stdout:345: done (2 left) 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:348: snap_create 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: finishing write tid 3 to smithi07685715-12 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: finishing write tid 4 to smithi07685715-12 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:347: finishing rollback tid 0 to smithi07685715-36 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: finishing write tid 5 to smithi07685715-12 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: oid 12 updating version 0 to 151 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: oid 12 version 151 is already newer than 148 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: oid 12 version 151 is already newer than 150 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: oid 12 version 151 is already newer than 149 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 151 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:346: left oid 12 (ObjNum 167 snap 43 seq_num 167) 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:347: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:347: finishing rollback tid 2 to smithi07685715-36 2024-09-21T09:39:15.718 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 108 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-09-21T09:39:15.749 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:15 smithi191 systemd[1]: Stopping Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:16.034 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:15 smithi191 ceph-mon[33301]: pgmap v215: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:39:16.035 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:15 smithi191 ceph-mon[33301]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:39:16.035 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:15 smithi191 podman[69452]: 2024-09-21 09:39:15.74755648 +0000 UTC m=+0.143228051 container died a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T09:39:16.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:15 smithi076 ceph-mon[29313]: pgmap v215: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:39:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:15 smithi076 ceph-mon[29313]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:39:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:15 smithi076 ceph-mon[41325]: pgmap v215: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:39:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:15 smithi076 ceph-mon[41325]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:39:16.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 podman[69452]: 2024-09-21 09:39:16.199516179 +0000 UTC m=+0.595187750 container cleanup a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:39:16.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 bash[69452]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:39:16.513 INFO:tasks.rados.rados.0.smithi076.stdout:346: done (2 left) 2024-09-21T09:39:16.513 INFO:tasks.rados.rados.0.smithi076.stdout:347: done (1 left) 2024-09-21T09:39:16.513 INFO:tasks.rados.rados.0.smithi076.stdout:348: done (0 left) 2024-09-21T09:39:16.513 INFO:tasks.rados.rados.0.smithi076.stdout:349: write oid 13 current snap is 44 2024-09-21T09:39:16.513 INFO:tasks.rados.rados.0.smithi076.stdout:349: seq_num 168 ranges {423576=401598,1497916=693725} 2024-09-21T09:39:16.517 INFO:tasks.rados.rados.0.smithi076.stdout:349: writing smithi07685715-13 from 423576 to 825174 tid 1 2024-09-21T09:39:16.525 INFO:tasks.rados.rados.0.smithi076.stdout:349: writing smithi07685715-13 from 1497916 to 2191641 tid 2 2024-09-21T09:39:16.525 INFO:tasks.rados.rados.0.smithi076.stdout:350: write oid 44 current snap is 44 2024-09-21T09:39:16.525 INFO:tasks.rados.rados.0.smithi076.stdout:350: seq_num 169 ranges {440383=476847,1556398=437383,2474721=365662} 2024-09-21T09:39:16.529 INFO:tasks.rados.rados.0.smithi076.stdout:350: writing smithi07685715-44 from 440383 to 917230 tid 1 2024-09-21T09:39:16.535 INFO:tasks.rados.rados.0.smithi076.stdout:350: writing smithi07685715-44 from 1556398 to 1993781 tid 2 2024-09-21T09:39:16.540 INFO:tasks.rados.rados.0.smithi076.stdout:350: writing smithi07685715-44 from 2474721 to 2840383 tid 3 2024-09-21T09:39:16.540 INFO:tasks.rados.rados.0.smithi076.stdout:351: write oid 21 current snap is 44 2024-09-21T09:39:16.540 INFO:tasks.rados.rados.0.smithi076.stdout:349: finishing write tid 1 to smithi07685715-13 2024-09-21T09:39:16.541 INFO:tasks.rados.rados.0.smithi076.stdout:349: finishing write tid 2 to smithi07685715-13 2024-09-21T09:39:16.541 INFO:tasks.rados.rados.0.smithi076.stdout:351: seq_num 170 ranges {457190=552096,1614880=581041,2785791=71399} 2024-09-21T09:39:16.546 INFO:tasks.rados.rados.0.smithi076.stdout:351: writing smithi07685715-21 from 457190 to 1009286 tid 1 2024-09-21T09:39:16.552 INFO:tasks.rados.rados.0.smithi076.stdout:351: writing smithi07685715-21 from 1614880 to 2195921 tid 2 2024-09-21T09:39:16.556 INFO:tasks.rados.rados.0.smithi076.stdout:351: writing smithi07685715-21 from 2785791 to 2857190 tid 3 2024-09-21T09:39:16.556 INFO:tasks.rados.rados.0.smithi076.stdout:352: write oid 14 current snap is 44 2024-09-21T09:39:16.556 INFO:tasks.rados.rados.0.smithi076.stdout:352: seq_num 171 ranges {473997=627345,1673362=441052,2529567=344430} 2024-09-21T09:39:16.562 INFO:tasks.rados.rados.0.smithi076.stdout:352: writing smithi07685715-14 from 473997 to 1101342 tid 1 2024-09-21T09:39:16.567 INFO:tasks.rados.rados.0.smithi076.stdout:352: writing smithi07685715-14 from 1673362 to 2114414 tid 2 2024-09-21T09:39:16.572 INFO:tasks.rados.rados.0.smithi076.stdout:352: writing smithi07685715-14 from 2529567 to 2873997 tid 3 2024-09-21T09:39:16.572 INFO:tasks.rados.rados.0.smithi076.stdout:353: snap_create 2024-09-21T09:39:16.572 INFO:tasks.rados.rados.0.smithi076.stdout:349: finishing write tid 3 to smithi07685715-13 2024-09-21T09:39:16.572 INFO:tasks.rados.rados.0.smithi076.stdout:349: finishing write tid 4 to smithi07685715-13 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:349: oid 13 updating version 0 to 44 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:349: oid 13 version 44 is already newer than 43 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:349: oid 13 updating version 44 to 45 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 45 (ObjNum 168 snap 44 seq_num 168) dirty exists 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:349: left oid 13 (ObjNum 168 snap 44 seq_num 168) 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: finishing write tid 1 to smithi07685715-44 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: finishing write tid 2 to smithi07685715-44 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: finishing write tid 3 to smithi07685715-44 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: finishing write tid 4 to smithi07685715-44 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: finishing write tid 5 to smithi07685715-44 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: oid 44 updating version 0 to 99 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: oid 44 updating version 99 to 100 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: oid 44 version 100 is already newer than 98 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:350: oid 44 version 100 is already newer than 97 2024-09-21T09:39:16.573 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 100 (ObjNum 169 snap 44 seq_num 169) dirty exists 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:350: left oid 44 (ObjNum 169 snap 44 seq_num 169) 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: finishing write tid 1 to smithi07685715-21 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: finishing write tid 2 to smithi07685715-21 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: finishing write tid 3 to smithi07685715-21 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: finishing write tid 4 to smithi07685715-21 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: finishing write tid 5 to smithi07685715-21 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: oid 21 updating version 0 to 109 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: oid 21 updating version 109 to 111 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: oid 21 version 111 is already newer than 110 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: oid 21 version 111 is already newer than 108 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 111 (ObjNum 170 snap 44 seq_num 170) dirty exists 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:351: left oid 21 (ObjNum 170 snap 44 seq_num 170) 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:352: finishing write tid 1 to smithi07685715-14 2024-09-21T09:39:16.574 INFO:tasks.rados.rados.0.smithi076.stdout:352: finishing write tid 2 to smithi07685715-14 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: finishing write tid 3 to smithi07685715-14 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: finishing write tid 4 to smithi07685715-14 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: finishing write tid 5 to smithi07685715-14 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: oid 14 updating version 0 to 157 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: oid 14 version 157 is already newer than 155 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: oid 14 updating version 157 to 158 2024-09-21T09:39:16.575 INFO:tasks.rados.rados.0.smithi076.stdout:352: oid 14 version 158 is already newer than 156 2024-09-21T09:39:16.576 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 158 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-09-21T09:39:16.576 INFO:tasks.rados.rados.0.smithi076.stdout:352: left oid 14 (ObjNum 171 snap 44 seq_num 171) 2024-09-21T09:39:16.862 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 podman[69469]: 2024-09-21 09:39:16.518222437 +0000 UTC m=+0.762095514 container remove a91e8d898a8ad1d987e6ecfd708e4d8c6b67e4367a229ba12d4963bb2f5c3583 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T09:39:16.862 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:39:16.862 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:39:16.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 systemd[1]: Stopped Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:16.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service: Consumed 10.978s CPU time. 2024-09-21T09:39:17.220 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:16 smithi191 systemd[1]: Starting Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:17.220 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 podman[69605]: 2024-09-21 09:39:16.983489585 +0000 UTC m=+0.016072216 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:17.220 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 podman[69605]: 2024-09-21 09:39:17.102506602 +0000 UTC m=+0.135089228 container create 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:349: done (4 left) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:350: done (3 left) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:351: done (2 left) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:352: done (1 left) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:353: done (0 left) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:354: rollback oid 19 current snap is 45 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 38 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:355: read oid 36 snap -1 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:356: write oid 27 current snap is 45 2024-09-21T09:39:17.519 INFO:tasks.rados.rados.0.smithi076.stdout:356: seq_num 172 ranges {490804=702594,1615491=584710,2724284=166520} 2024-09-21T09:39:17.525 INFO:tasks.rados.rados.0.smithi076.stdout:356: writing smithi07685715-27 from 490804 to 1193398 tid 1 2024-09-21T09:39:17.531 INFO:tasks.rados.rados.0.smithi076.stdout:356: writing smithi07685715-27 from 1615491 to 2200201 tid 2 2024-09-21T09:39:17.535 INFO:tasks.rados.rados.0.smithi076.stdout:356: writing smithi07685715-27 from 2724284 to 2890804 tid 3 2024-09-21T09:39:17.536 INFO:tasks.rados.rados.0.smithi076.stdout:357: snap_create 2024-09-21T09:39:17.536 INFO:tasks.rados.rados.0.smithi076.stdout:354: finishing rollback tid 0 to smithi07685715-19 2024-09-21T09:39:17.536 INFO:tasks.rados.rados.0.smithi076.stdout:354: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:39:17.536 INFO:tasks.rados.rados.0.smithi076.stdout:354: finishing rollback tid 2 to smithi07685715-19 2024-09-21T09:39:17.536 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 104 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:39:17.536 INFO:tasks.rados.rados.0.smithi076.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:17.611 INFO:tasks.rados.rados.0.smithi076.stdout:356: finishing write tid 1 to smithi07685715-27 2024-09-21T09:39:17.611 INFO:tasks.rados.rados.0.smithi076.stdout:356: finishing write tid 2 to smithi07685715-27 2024-09-21T09:39:17.611 INFO:tasks.rados.rados.0.smithi076.stdout:356: finishing write tid 3 to smithi07685715-27 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: finishing write tid 4 to smithi07685715-27 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: finishing write tid 5 to smithi07685715-27 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: oid 27 updating version 0 to 152 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: oid 27 version 152 is already newer than 150 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: oid 27 updating version 152 to 153 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: oid 27 version 153 is already newer than 151 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 153 (ObjNum 172 snap 45 seq_num 172) dirty exists 2024-09-21T09:39:17.612 INFO:tasks.rados.rados.0.smithi076.stdout:356: left oid 27 (ObjNum 172 snap 45 seq_num 172) 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:17 smithi191 ceph-mon[33301]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:17 smithi191 ceph-mon[33301]: pgmap v218: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 podman[69605]: 2024-09-21 09:39:17.338385682 +0000 UTC m=+0.370968314 container init 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 podman[69605]: 2024-09-21 09:39:17.343614506 +0000 UTC m=+0.376197141 container start 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 ceph-mgr[69643]: -- 172.21.15.191:0/3802026359 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e8d2aa50e0 con 0x55e8d2a83000 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 bash[69605]: 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 2024-09-21T09:39:17.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:17 smithi191 systemd[1]: Started Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:17 smithi076 ceph-mon[29313]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:39:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:17 smithi076 ceph-mon[29313]: pgmap v218: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:39:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:17 smithi076 ceph-mon[41325]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:39:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:17 smithi076 ceph-mon[41325]: pgmap v218: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:39:18.507 INFO:tasks.rados.rados.0.smithi076.stdout:354: done (3 left) 2024-09-21T09:39:18.507 INFO:tasks.rados.rados.0.smithi076.stdout:355: done (2 left) 2024-09-21T09:39:18.507 INFO:tasks.rados.rados.0.smithi076.stdout:356: done (1 left) 2024-09-21T09:39:18.507 INFO:tasks.rados.rados.0.smithi076.stdout:357: done (0 left) 2024-09-21T09:39:18.507 INFO:tasks.rados.rados.0.smithi076.stdout:358: write oid 28 current snap is 46 2024-09-21T09:39:18.507 INFO:tasks.rados.rados.0.smithi076.stdout:358: seq_num 173 ranges {507611=777843,2073973=444721} 2024-09-21T09:39:18.513 INFO:tasks.rados.rados.0.smithi076.stdout:358: writing smithi07685715-28 from 507611 to 1285454 tid 1 2024-09-21T09:39:18.521 INFO:tasks.rados.rados.0.smithi076.stdout:358: writing smithi07685715-28 from 2073973 to 2518694 tid 2 2024-09-21T09:39:18.521 INFO:tasks.rados.rados.0.smithi076.stdout:359: snap_create 2024-09-21T09:39:18.523 INFO:tasks.rados.rados.0.smithi076.stdout:358: finishing write tid 1 to smithi07685715-28 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: finishing write tid 2 to smithi07685715-28 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: finishing write tid 3 to smithi07685715-28 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: finishing write tid 4 to smithi07685715-28 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: oid 28 updating version 0 to 234 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: oid 28 updating version 234 to 236 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: oid 28 version 236 is already newer than 235 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 236 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-09-21T09:39:18.524 INFO:tasks.rados.rados.0.smithi076.stdout:358: left oid 28 (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:18 smithi191 ceph-mon[33301]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:39:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:18 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:18.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:18 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:18.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:18 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:18.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:18 smithi191 ceph-mon[33301]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[29313]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[29313]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[41325]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:18.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:18.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-mon[41325]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:39:19.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:18 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:39:18] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.43.0" 2024-09-21T09:39:19.525 INFO:tasks.rados.rados.0.smithi076.stdout:358: done (1 left) 2024-09-21T09:39:19.526 INFO:tasks.rados.rados.0.smithi076.stdout:359: done (0 left) 2024-09-21T09:39:19.526 INFO:tasks.rados.rados.0.smithi076.stdout:360: snap_remove snap 45 2024-09-21T09:39:19.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:19 smithi191 ceph-mon[33301]: pgmap v221: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:19.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:19 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:19 smithi076 ceph-mon[29313]: pgmap v221: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:19 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:19 smithi076 ceph-mon[41325]: pgmap v221: 89 pgs: 89 active+clean; 141 MiB data, 494 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:19 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:20.527 INFO:tasks.rados.rados.0.smithi076.stdout:360: done (0 left) 2024-09-21T09:39:20.527 INFO:tasks.rados.rados.0.smithi076.stdout:361: read oid 32 snap -1 2024-09-21T09:39:20.527 INFO:tasks.rados.rados.0.smithi076.stdout:361: expect (ObjNum 96 snap 16 seq_num 96) 2024-09-21T09:39:20.527 INFO:tasks.rados.rados.0.smithi076.stdout:362: read oid 10 snap 43 2024-09-21T09:39:20.528 INFO:tasks.rados.rados.0.smithi076.stdout:362: expect deleted 2024-09-21T09:39:20.528 INFO:tasks.rados.rados.0.smithi076.stdout:363: read oid 26 snap -1 2024-09-21T09:39:20.528 INFO:tasks.rados.rados.0.smithi076.stdout:363: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:20.528 INFO:tasks.rados.rados.0.smithi076.stdout:364: read oid 2 snap -1 2024-09-21T09:39:20.528 INFO:tasks.rados.rados.0.smithi076.stdout:364: expect deleted 2024-09-21T09:39:20.528 INFO:tasks.rados.rados.0.smithi076.stdout:365: delete oid 40 current snap is 47 2024-09-21T09:39:20.531 INFO:tasks.rados.rados.0.smithi076.stdout:362: done (4 left) 2024-09-21T09:39:20.531 INFO:tasks.rados.rados.0.smithi076.stdout:365: done (3 left) 2024-09-21T09:39:20.531 INFO:tasks.rados.rados.0.smithi076.stdout:366: read oid 28 snap -1 2024-09-21T09:39:20.531 INFO:tasks.rados.rados.0.smithi076.stdout:366: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:20.532 INFO:tasks.rados.rados.0.smithi076.stdout:367: rollback oid 30 current snap is 47 2024-09-21T09:39:20.532 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 38 2024-09-21T09:39:20.532 INFO:tasks.rados.rados.0.smithi076.stdout:368: rollback oid 7 current snap is 47 2024-09-21T09:39:20.532 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 43 2024-09-21T09:39:20.532 INFO:tasks.rados.rados.0.smithi076.stdout:369: write oid 46 current snap is 47 2024-09-21T09:39:20.532 INFO:tasks.rados.rados.0.smithi076.stdout:369: seq_num 175 ranges {541225=644694,1907290=448390} 2024-09-21T09:39:20.538 INFO:tasks.rados.rados.0.smithi076.stdout:369: writing smithi07685715-46 from 541225 to 1185919 tid 1 2024-09-21T09:39:20.545 INFO:tasks.rados.rados.0.smithi076.stdout:369: writing smithi07685715-46 from 1907290 to 2355680 tid 2 2024-09-21T09:39:20.545 INFO:tasks.rados.rados.0.smithi076.stdout:370: write oid 42 current snap is 47 2024-09-21T09:39:20.545 INFO:tasks.rados.rados.0.smithi076.stdout:368: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:39:20.545 INFO:tasks.rados.rados.0.smithi076.stdout:368: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:39:20.546 INFO:tasks.rados.rados.0.smithi076.stdout:370: seq_num 176 ranges {558032=719943,1849419=592048} 2024-09-21T09:39:20.551 INFO:tasks.rados.rados.0.smithi076.stdout:370: writing smithi07685715-42 from 558032 to 1277975 tid 1 2024-09-21T09:39:20.558 INFO:tasks.rados.rados.0.smithi076.stdout:370: writing smithi07685715-42 from 1849419 to 2441467 tid 2 2024-09-21T09:39:20.558 INFO:tasks.rados.rados.0.smithi076.stdout:371: read oid 6 snap -1 2024-09-21T09:39:20.558 INFO:tasks.rados.rados.0.smithi076.stdout:371: expect deleted 2024-09-21T09:39:20.558 INFO:tasks.rados.rados.0.smithi076.stdout:372: delete oid 17 current snap is 47 2024-09-21T09:39:20.558 INFO:tasks.rados.rados.0.smithi076.stdout:368: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:39:20.559 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 162 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-21T09:39:20.559 INFO:tasks.rados.rados.0.smithi076.stdout:367: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:39:20.559 INFO:tasks.rados.rados.0.smithi076.stdout:367: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:39:20.559 INFO:tasks.rados.rados.0.smithi076.stdout:361: expect (ObjNum 96 snap 16 seq_num 96) 2024-09-21T09:39:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:366: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:20 smithi191 ceph-mon[33301]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:39:20.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:20 smithi076 ceph-mon[29313]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:39:20.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:20 smithi076 ceph-mon[41325]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:39:20.882 INFO:tasks.rados.rados.0.smithi076.stdout:367: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:39:20.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 242 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:39:20.882 INFO:tasks.rados.rados.0.smithi076.stdout:369: finishing write tid 1 to smithi07685715-46 2024-09-21T09:39:20.882 INFO:tasks.rados.rados.0.smithi076.stdout:363: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:369: finishing write tid 2 to smithi07685715-46 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:369: finishing write tid 3 to smithi07685715-46 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:361: done (9 left) 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:363: done (8 left) 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:364: done (7 left) 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:366: done (6 left) 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:367: done (5 left) 2024-09-21T09:39:21.040 INFO:tasks.rados.rados.0.smithi076.stdout:368: done (4 left) 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:372: done (3 left) 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:373: write oid 25 current snap is 47 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:369: finishing write tid 4 to smithi07685715-46 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:369: oid 46 updating version 0 to 114 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:369: oid 46 version 114 is already newer than 112 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:369: oid 46 version 114 is already newer than 113 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 114 (ObjNum 175 snap 47 seq_num 175) dirty exists 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:369: left oid 46 (ObjNum 175 snap 47 seq_num 175) 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:370: finishing write tid 1 to smithi07685715-42 2024-09-21T09:39:21.041 INFO:tasks.rados.rados.0.smithi076.stdout:373: seq_num 178 ranges {591646=470441,1566383=595717,2888822=102824} 2024-09-21T09:39:21.045 INFO:tasks.rados.rados.0.smithi076.stdout:373: writing smithi07685715-25 from 591646 to 1062087 tid 1 2024-09-21T09:39:21.051 INFO:tasks.rados.rados.0.smithi076.stdout:373: writing smithi07685715-25 from 1566383 to 2162100 tid 2 2024-09-21T09:39:21.056 INFO:tasks.rados.rados.0.smithi076.stdout:373: writing smithi07685715-25 from 2888822 to 2991646 tid 3 2024-09-21T09:39:21.056 INFO:tasks.rados.rados.0.smithi076.stdout:369: done (3 left) 2024-09-21T09:39:21.056 INFO:tasks.rados.rados.0.smithi076.stdout:374: write oid 44 current snap is 47 2024-09-21T09:39:21.056 INFO:tasks.rados.rados.0.smithi076.stdout:370: finishing write tid 2 to smithi07685715-42 2024-09-21T09:39:21.056 INFO:tasks.rados.rados.0.smithi076.stdout:370: finishing write tid 3 to smithi07685715-42 2024-09-21T09:39:21.056 INFO:tasks.rados.rados.0.smithi076.stdout:374: seq_num 179 ranges {608453=545690,1624865=455728,2632598=375855} 2024-09-21T09:39:21.062 INFO:tasks.rados.rados.0.smithi076.stdout:374: writing smithi07685715-44 from 608453 to 1154143 tid 1 2024-09-21T09:39:21.067 INFO:tasks.rados.rados.0.smithi076.stdout:374: writing smithi07685715-44 from 1624865 to 2080593 tid 2 2024-09-21T09:39:21.073 INFO:tasks.rados.rados.0.smithi076.stdout:374: writing smithi07685715-44 from 2632598 to 3008453 tid 3 2024-09-21T09:39:21.073 INFO:tasks.rados.rados.0.smithi076.stdout:371: done (3 left) 2024-09-21T09:39:21.073 INFO:tasks.rados.rados.0.smithi076.stdout:375: rollback oid 33 current snap is 47 2024-09-21T09:39:21.073 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 46 2024-09-21T09:39:21.073 INFO:tasks.rados.rados.0.smithi076.stdout:370: finishing write tid 4 to smithi07685715-42 2024-09-21T09:39:21.073 INFO:tasks.rados.rados.0.smithi076.stdout:370: oid 42 updating version 0 to 36 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:370: oid 42 updating version 36 to 37 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:370: oid 42 version 37 is already newer than 35 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 37 (ObjNum 176 snap 47 seq_num 176) dirty exists 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:370: left oid 42 (ObjNum 176 snap 47 seq_num 176) 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: finishing write tid 1 to smithi07685715-25 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: finishing write tid 2 to smithi07685715-25 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: finishing write tid 3 to smithi07685715-25 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: finishing write tid 4 to smithi07685715-25 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:370: done (3 left) 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:376: delete oid 14 current snap is 47 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: finishing write tid 5 to smithi07685715-25 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: oid 25 updating version 0 to 54 2024-09-21T09:39:21.074 INFO:tasks.rados.rados.0.smithi076.stdout:373: oid 25 updating version 54 to 55 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:373: oid 25 version 55 is already newer than 52 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:373: oid 25 version 55 is already newer than 53 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 55 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:373: left oid 25 (ObjNum 178 snap 47 seq_num 178) 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:374: finishing write tid 1 to smithi07685715-44 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:374: finishing write tid 2 to smithi07685715-44 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:375: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:39:21.075 INFO:tasks.rados.rados.0.smithi076.stdout:373: done (3 left) 2024-09-21T09:39:21.076 INFO:tasks.rados.rados.0.smithi076.stdout:376: done (2 left) 2024-09-21T09:39:21.076 INFO:tasks.rados.rados.0.smithi076.stdout:377: snap_remove snap 37 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:377: done (2 left) 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:378: read oid 29 snap -1 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:378: expect deleted 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:379: delete oid 34 current snap is 47 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:375: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:374: finishing write tid 3 to smithi07685715-44 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:375: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 59 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-09-21T09:39:21.538 INFO:tasks.rados.rados.0.smithi076.stdout:374: finishing write tid 4 to smithi07685715-44 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:374: finishing write tid 5 to smithi07685715-44 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:374: oid 44 updating version 0 to 105 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:374: oid 44 version 105 is already newer than 102 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:374: oid 44 version 105 is already newer than 103 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:374: oid 44 version 105 is already newer than 104 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 105 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-09-21T09:39:21.539 INFO:tasks.rados.rados.0.smithi076.stdout:374: left oid 44 (ObjNum 179 snap 47 seq_num 179) 2024-09-21T09:39:21.541 INFO:tasks.rados.rados.0.smithi076.stdout:374: done (3 left) 2024-09-21T09:39:21.541 INFO:tasks.rados.rados.0.smithi076.stdout:375: done (2 left) 2024-09-21T09:39:21.542 INFO:tasks.rados.rados.0.smithi076.stdout:378: done (1 left) 2024-09-21T09:39:21.542 INFO:tasks.rados.rados.0.smithi076.stdout:379: done (0 left) 2024-09-21T09:39:21.542 INFO:tasks.rados.rados.0.smithi076.stdout:380: snap_create 2024-09-21T09:39:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:21 smithi191 ceph-mon[33301]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:39:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:21 smithi191 ceph-mon[33301]: pgmap v224: 89 pgs: 89 active+clean; 162 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-21T09:39:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:21 smithi076 ceph-mon[29313]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:39:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:21 smithi076 ceph-mon[29313]: pgmap v224: 89 pgs: 89 active+clean; 162 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-21T09:39:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:21 smithi076 ceph-mon[41325]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:39:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:21 smithi076 ceph-mon[41325]: pgmap v224: 89 pgs: 89 active+clean; 162 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-21T09:39:22.552 INFO:tasks.rados.rados.0.smithi076.stdout:380: done (0 left) 2024-09-21T09:39:22.552 INFO:tasks.rados.rados.0.smithi076.stdout:381: read oid 34 snap -1 2024-09-21T09:39:22.552 INFO:tasks.rados.rados.0.smithi076.stdout:381: expect deleted 2024-09-21T09:39:22.552 INFO:tasks.rados.rados.0.smithi076.stdout:382: read oid 47 snap -1 2024-09-21T09:39:22.552 INFO:tasks.rados.rados.0.smithi076.stdout:382: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:22.553 INFO:tasks.rados.rados.0.smithi076.stdout:383: write oid 25 current snap is 48 2024-09-21T09:39:22.553 INFO:tasks.rados.rados.0.smithi076.stdout:383: seq_num 182 ranges {658874=771437,2083958=603055} 2024-09-21T09:39:22.559 INFO:tasks.rados.rados.0.smithi076.stdout:383: writing smithi07685715-25 from 658874 to 1430311 tid 1 2024-09-21T09:39:22.567 INFO:tasks.rados.rados.0.smithi076.stdout:383: writing smithi07685715-25 from 2083958 to 2687013 tid 2 2024-09-21T09:39:22.567 INFO:tasks.rados.rados.0.smithi076.stdout:384: rollback oid 1 current snap is 48 2024-09-21T09:39:22.567 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 1 to 35 2024-09-21T09:39:22.567 INFO:tasks.rados.rados.0.smithi076.stdout:383: finishing write tid 1 to smithi07685715-25 2024-09-21T09:39:22.567 INFO:tasks.rados.rados.0.smithi076.stdout:382: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:22.679 INFO:tasks.rados.rados.0.smithi076.stdout:384: finishing rollback tid 1 to smithi07685715-1 2024-09-21T09:39:22.679 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 0 (ObjNum 33 snap 0 seq_num 1682338337) dirty dne 2024-09-21T09:39:22.679 INFO:tasks.rados.rados.0.smithi076.stdout:383: finishing write tid 2 to smithi07685715-25 2024-09-21T09:39:22.679 INFO:tasks.rados.rados.0.smithi076.stdout:383: finishing write tid 3 to smithi07685715-25 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:383: finishing write tid 4 to smithi07685715-25 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:383: oid 25 updating version 0 to 64 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:383: oid 25 version 64 is already newer than 63 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:383: oid 25 updating version 64 to 65 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 65 (ObjNum 182 snap 48 seq_num 182) dirty exists 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:383: left oid 25 (ObjNum 182 snap 48 seq_num 182) 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:381: done (3 left) 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:382: done (2 left) 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:383: done (1 left) 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:384: done (0 left) 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:385: write oid 11 current snap is 48 2024-09-21T09:39:22.680 INFO:tasks.rados.rados.0.smithi076.stdout:385: seq_num 183 ranges {675681=563039,1858793=746713,3025937=49744} 2024-09-21T09:39:22.685 INFO:tasks.rados.rados.0.smithi076.stdout:385: writing smithi07685715-11 from 675681 to 1238720 tid 1 2024-09-21T09:39:22.693 INFO:tasks.rados.rados.0.smithi076.stdout:385: writing smithi07685715-11 from 1858793 to 2605506 tid 2 2024-09-21T09:39:22.698 INFO:tasks.rados.rados.0.smithi076.stdout:385: writing smithi07685715-11 from 3025937 to 3075681 tid 3 2024-09-21T09:39:22.698 INFO:tasks.rados.rados.0.smithi076.stdout:386: snap_create 2024-09-21T09:39:22.698 INFO:tasks.rados.rados.0.smithi076.stdout:385: finishing write tid 1 to smithi07685715-11 2024-09-21T09:39:22.698 INFO:tasks.rados.rados.0.smithi076.stdout:385: finishing write tid 2 to smithi07685715-11 2024-09-21T09:39:22.698 INFO:tasks.rados.rados.0.smithi076.stdout:385: finishing write tid 3 to smithi07685715-11 2024-09-21T09:39:22.698 INFO:tasks.rados.rados.0.smithi076.stdout:385: finishing write tid 4 to smithi07685715-11 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:385: finishing write tid 5 to smithi07685715-11 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:385: oid 11 updating version 0 to 118 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:385: oid 11 updating version 118 to 121 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:385: oid 11 version 121 is already newer than 119 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:385: oid 11 version 121 is already newer than 120 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 121 (ObjNum 183 snap 48 seq_num 183) dirty exists 2024-09-21T09:39:22.699 INFO:tasks.rados.rados.0.smithi076.stdout:385: left oid 11 (ObjNum 183 snap 48 seq_num 183) 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: Standby manager daemon x restarted 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: Standby manager daemon x started 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:39:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:22.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:22] ENGINE Bus STARTING 2024-09-21T09:39:22.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: CherryPy Checker: 2024-09-21T09:39:22.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: The Application mounted at '' has an empty config. 2024-09-21T09:39:22.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:22.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:22] ENGINE Serving on http://:::9283 2024-09-21T09:39:22.864 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:22 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:22] ENGINE Bus STARTED 2024-09-21T09:39:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: Standby manager daemon x restarted 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: Standby manager daemon x started 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: Standby manager daemon x restarted 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: Standby manager daemon x started 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:39:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:39:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:22 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:23.548 INFO:tasks.rados.rados.0.smithi076.stdout:385: done (1 left) 2024-09-21T09:39:23.548 INFO:tasks.rados.rados.0.smithi076.stdout:386: done (0 left) 2024-09-21T09:39:23.548 INFO:tasks.rados.rados.0.smithi076.stdout:387: read oid 39 snap -1 2024-09-21T09:39:23.548 INFO:tasks.rados.rados.0.smithi076.stdout:387: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:39:23.548 INFO:tasks.rados.rados.0.smithi076.stdout:388: delete oid 22 current snap is 49 2024-09-21T09:39:23.555 INFO:tasks.rados.rados.0.smithi076.stdout:388: done (1 left) 2024-09-21T09:39:23.555 INFO:tasks.rados.rados.0.smithi076.stdout:389: rollback oid 29 current snap is 49 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 38 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:390: read oid 28 snap -1 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:390: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:391: read oid 33 snap -1 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:391: expect (ObjNum 89 snap 11 seq_num 89) 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:392: rollback oid 20 current snap is 49 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 48 2024-09-21T09:39:23.556 INFO:tasks.rados.rados.0.smithi076.stdout:393: snap_remove snap 23 2024-09-21T09:39:23.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:23 smithi191 ceph-mon[33301]: mgrmap e22: y(active, since 4m), standbys: x 2024-09-21T09:39:23.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:23 smithi191 ceph-mon[33301]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:39:23.854 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:23 smithi191 ceph-mon[33301]: pgmap v227: 89 pgs: 89 active+clean; 162 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-21T09:39:23.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:23 smithi076 ceph-mon[41325]: mgrmap e22: y(active, since 4m), standbys: x 2024-09-21T09:39:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:23 smithi076 ceph-mon[41325]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:39:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:23 smithi076 ceph-mon[41325]: pgmap v227: 89 pgs: 89 active+clean; 162 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-21T09:39:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:23 smithi076 ceph-mon[29313]: mgrmap e22: y(active, since 4m), standbys: x 2024-09-21T09:39:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:23 smithi076 ceph-mon[29313]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:39:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:23 smithi076 ceph-mon[29313]: pgmap v227: 89 pgs: 89 active+clean; 162 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-09-21T09:39:24.563 INFO:tasks.rados.rados.0.smithi076.stdout:392: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:39:24.563 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:24.564 INFO:tasks.rados.rados.0.smithi076.stdout:389: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:39:24.564 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 66 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:39:24.564 INFO:tasks.rados.rados.0.smithi076.stdout:391: expect (ObjNum 89 snap 11 seq_num 89) 2024-09-21T09:39:24.764 INFO:tasks.rados.rados.0.smithi076.stdout:387: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:39:24.850 INFO:tasks.rados.rados.0.smithi076.stdout:390: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:24.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:24 smithi191 ceph-mon[33301]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:39:24.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:24 smithi076 ceph-mon[29313]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:39:24.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:24 smithi076 ceph-mon[41325]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:387: done (5 left) 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:389: done (4 left) 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:390: done (3 left) 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:391: done (2 left) 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:392: done (1 left) 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:393: done (0 left) 2024-09-21T09:39:24.976 INFO:tasks.rados.rados.0.smithi076.stdout:394: write oid 4 current snap is 49 2024-09-21T09:39:24.977 INFO:tasks.rados.rados.0.smithi076.stdout:394: seq_num 185 ranges {709295=713537,1859404=750382} 2024-09-21T09:39:24.984 INFO:tasks.rados.rados.0.smithi076.stdout:394: writing smithi07685715-4 from 709295 to 1422832 tid 1 2024-09-21T09:39:24.990 INFO:tasks.rados.rados.0.smithi076.stdout:394: writing smithi07685715-4 from 1859404 to 2609786 tid 2 2024-09-21T09:39:24.990 INFO:tasks.rados.rados.0.smithi076.stdout:395: write oid 22 current snap is 49 2024-09-21T09:39:24.990 INFO:tasks.rados.rados.0.smithi076.stdout:395: seq_num 186 ranges {726102=788786,1917886=610393,3108206=17896} 2024-09-21T09:39:24.998 INFO:tasks.rados.rados.0.smithi076.stdout:395: writing smithi07685715-22 from 726102 to 1514888 tid 1 2024-09-21T09:39:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:395: writing smithi07685715-22 from 1917886 to 2528279 tid 2 2024-09-21T09:39:25.007 INFO:tasks.rados.rados.0.smithi076.stdout:395: writing smithi07685715-22 from 3108206 to 3126102 tid 3 2024-09-21T09:39:25.007 INFO:tasks.rados.rados.0.smithi076.stdout:396: write oid 34 current snap is 49 2024-09-21T09:39:25.007 INFO:tasks.rados.rados.0.smithi076.stdout:394: finishing write tid 1 to smithi07685715-4 2024-09-21T09:39:25.007 INFO:tasks.rados.rados.0.smithi076.stdout:396: seq_num 187 ranges {742909=464035,1976368=754051} 2024-09-21T09:39:25.013 INFO:tasks.rados.rados.0.smithi076.stdout:396: writing smithi07685715-34 from 742909 to 1206944 tid 1 2024-09-21T09:39:25.021 INFO:tasks.rados.rados.0.smithi076.stdout:396: writing smithi07685715-34 from 1976368 to 2730419 tid 2 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:397: read oid 15 snap 47 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:397: expect deleted 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:398: snap_create 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: finishing write tid 2 to smithi07685715-4 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: finishing write tid 3 to smithi07685715-4 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: finishing write tid 4 to smithi07685715-4 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: oid 4 updating version 0 to 119 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: oid 4 version 119 is already newer than 118 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: oid 4 version 119 is already newer than 117 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 119 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:394: left oid 4 (ObjNum 185 snap 49 seq_num 185) 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:395: finishing write tid 1 to smithi07685715-22 2024-09-21T09:39:25.022 INFO:tasks.rados.rados.0.smithi076.stdout:395: finishing write tid 2 to smithi07685715-22 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: finishing write tid 3 to smithi07685715-22 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: finishing write tid 4 to smithi07685715-22 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: finishing write tid 5 to smithi07685715-22 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: oid 22 updating version 0 to 169 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: oid 22 updating version 169 to 170 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: oid 22 updating version 170 to 171 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: oid 22 updating version 171 to 172 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 172 (ObjNum 186 snap 49 seq_num 186) dirty exists 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:395: left oid 22 (ObjNum 186 snap 49 seq_num 186) 2024-09-21T09:39:25.023 INFO:tasks.rados.rados.0.smithi076.stdout:396: finishing write tid 1 to smithi07685715-34 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: finishing write tid 2 to smithi07685715-34 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: finishing write tid 3 to smithi07685715-34 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: finishing write tid 4 to smithi07685715-34 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: oid 34 updating version 0 to 26 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: oid 34 version 26 is already newer than 25 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: oid 34 updating version 26 to 27 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 27 (ObjNum 187 snap 49 seq_num 187) dirty exists 2024-09-21T09:39:25.024 INFO:tasks.rados.rados.0.smithi076.stdout:396: left oid 34 (ObjNum 187 snap 49 seq_num 187) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:394: done (4 left) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:395: done (3 left) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:396: done (2 left) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:397: done (1 left) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:398: done (0 left) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:399: read oid 47 snap 46 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:399: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:400: rollback oid 24 current snap is 50 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 24 to 20 2024-09-21T09:39:25.572 INFO:tasks.rados.rados.0.smithi076.stdout:401: delete oid 8 current snap is 50 2024-09-21T09:39:25.573 INFO:tasks.rados.rados.0.smithi076.stdout:400: finishing rollback tid 1 to smithi07685715-24 2024-09-21T09:39:25.573 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 0 (ObjNum 129 snap 0 seq_num 864405272) dirty dne 2024-09-21T09:39:25.581 INFO:tasks.rados.rados.0.smithi076.stdout:400: done (2 left) 2024-09-21T09:39:25.581 INFO:tasks.rados.rados.0.smithi076.stdout:401: done (1 left) 2024-09-21T09:39:25.581 INFO:tasks.rados.rados.0.smithi076.stdout:402: snap_remove snap 20 2024-09-21T09:39:25.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:25 smithi191 ceph-mon[33301]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:39:25.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:25 smithi191 ceph-mon[33301]: pgmap v230: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.8 MiB/s wr, 26 op/s 2024-09-21T09:39:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:25 smithi076 ceph-mon[29313]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:39:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:25 smithi076 ceph-mon[29313]: pgmap v230: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.8 MiB/s wr, 26 op/s 2024-09-21T09:39:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:25 smithi076 ceph-mon[41325]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:39:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:25 smithi076 ceph-mon[41325]: pgmap v230: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.8 MiB/s wr, 26 op/s 2024-09-21T09:39:26.577 INFO:tasks.rados.rados.0.smithi076.stdout:402: done (1 left) 2024-09-21T09:39:26.577 INFO:tasks.rados.rados.0.smithi076.stdout:403: snap_create 2024-09-21T09:39:26.577 INFO:tasks.rados.rados.0.smithi076.stdout:399: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:26.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:26 smithi191 ceph-mon[33301]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:39:26.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:26 smithi076 ceph-mon[29313]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:39:26.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:26 smithi076 ceph-mon[41325]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:39:27.582 INFO:tasks.rados.rados.0.smithi076.stdout:399: done (1 left) 2024-09-21T09:39:27.582 INFO:tasks.rados.rados.0.smithi076.stdout:403: done (0 left) 2024-09-21T09:39:27.582 INFO:tasks.rados.rados.0.smithi076.stdout:404: write oid 45 current snap is 51 2024-09-21T09:39:27.583 INFO:tasks.rados.rados.0.smithi076.stdout:404: seq_num 189 ranges {776523=614533,1976979=757720} 2024-09-21T09:39:27.590 INFO:tasks.rados.rados.0.smithi076.stdout:404: writing smithi07685715-45 from 776523 to 1391056 tid 1 2024-09-21T09:39:27.598 INFO:tasks.rados.rados.0.smithi076.stdout:404: writing smithi07685715-45 from 1976979 to 2734699 tid 2 2024-09-21T09:39:27.598 INFO:tasks.rados.rados.0.smithi076.stdout:404: finishing write tid 1 to smithi07685715-45 2024-09-21T09:39:27.598 INFO:tasks.rados.rados.0.smithi076.stdout:405: write oid 9 current snap is 51 2024-09-21T09:39:27.598 INFO:tasks.rados.rados.0.smithi076.stdout:405: seq_num 190 ranges {793330=689782,2035461=617731,3101545=91785} 2024-09-21T09:39:27.606 INFO:tasks.rados.rados.0.smithi076.stdout:405: writing smithi07685715-9 from 793330 to 1483112 tid 1 2024-09-21T09:39:27.613 INFO:tasks.rados.rados.0.smithi076.stdout:405: writing smithi07685715-9 from 2035461 to 2653192 tid 2 2024-09-21T09:39:27.616 INFO:tasks.rados.rados.0.smithi076.stdout:405: writing smithi07685715-9 from 3101545 to 3193330 tid 3 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:406: write oid 21 current snap is 51 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: finishing write tid 2 to smithi07685715-45 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: finishing write tid 3 to smithi07685715-45 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: finishing write tid 4 to smithi07685715-45 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: oid 45 updating version 0 to 59 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: oid 45 version 59 is already newer than 57 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: oid 45 version 59 is already newer than 58 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 59 (ObjNum 189 snap 51 seq_num 189) dirty exists 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:404: left oid 45 (ObjNum 189 snap 51 seq_num 189) 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:405: finishing write tid 1 to smithi07685715-9 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:405: finishing write tid 2 to smithi07685715-9 2024-09-21T09:39:27.617 INFO:tasks.rados.rados.0.smithi076.stdout:406: seq_num 191 ranges {410137=481384,1410296=761389,2728968=481169} 2024-09-21T09:39:27.621 INFO:tasks.rados.rados.0.smithi076.stdout:406: writing smithi07685715-21 from 410137 to 891521 tid 1 2024-09-21T09:39:27.627 INFO:tasks.rados.rados.0.smithi076.stdout:406: writing smithi07685715-21 from 1410296 to 2171685 tid 2 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:406: writing smithi07685715-21 from 2728968 to 3210137 tid 3 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:404: done (2 left) 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:407: rollback oid 38 current snap is 51 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 46 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:405: finishing write tid 3 to smithi07685715-9 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:405: finishing write tid 4 to smithi07685715-9 2024-09-21T09:39:27.634 INFO:tasks.rados.rados.0.smithi076.stdout:405: finishing write tid 5 to smithi07685715-9 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:405: oid 9 updating version 0 to 165 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:405: oid 9 updating version 165 to 167 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:405: oid 9 version 167 is already newer than 166 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:405: oid 9 updating version 167 to 168 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 168 (ObjNum 190 snap 51 seq_num 190) dirty exists 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:405: left oid 9 (ObjNum 190 snap 51 seq_num 190) 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:406: finishing write tid 1 to smithi07685715-21 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:406: finishing write tid 2 to smithi07685715-21 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:405: done (2 left) 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:408: write oid 4 current snap is 51 2024-09-21T09:39:27.635 INFO:tasks.rados.rados.0.smithi076.stdout:408: seq_num 192 ranges {426944=556633,1468778=621400,2872744=354200} 2024-09-21T09:39:27.639 INFO:tasks.rados.rados.0.smithi076.stdout:408: writing smithi07685715-4 from 426944 to 983577 tid 1 2024-09-21T09:39:27.645 INFO:tasks.rados.rados.0.smithi076.stdout:408: writing smithi07685715-4 from 1468778 to 2090178 tid 2 2024-09-21T09:39:27.651 INFO:tasks.rados.rados.0.smithi076.stdout:408: writing smithi07685715-4 from 2872744 to 3226944 tid 3 2024-09-21T09:39:27.651 INFO:tasks.rados.rados.0.smithi076.stdout:409: snap_create 2024-09-21T09:39:27.651 INFO:tasks.rados.rados.0.smithi076.stdout:407: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: finishing write tid 3 to smithi07685715-21 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: finishing write tid 4 to smithi07685715-21 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: finishing write tid 5 to smithi07685715-21 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: oid 21 updating version 0 to 132 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: oid 21 updating version 132 to 133 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: oid 21 updating version 133 to 134 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: oid 21 version 134 is already newer than 131 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 134 (ObjNum 191 snap 51 seq_num 191) dirty exists 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:406: left oid 21 (ObjNum 191 snap 51 seq_num 191) 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:408: finishing write tid 1 to smithi07685715-4 2024-09-21T09:39:27.652 INFO:tasks.rados.rados.0.smithi076.stdout:408: finishing write tid 2 to smithi07685715-4 2024-09-21T09:39:27.653 INFO:tasks.rados.rados.0.smithi076.stdout:408: finishing write tid 3 to smithi07685715-4 2024-09-21T09:39:27.653 INFO:tasks.rados.rados.0.smithi076.stdout:408: finishing write tid 4 to smithi07685715-4 2024-09-21T09:39:27.653 INFO:tasks.rados.rados.0.smithi076.stdout:408: finishing write tid 5 to smithi07685715-4 2024-09-21T09:39:27.654 INFO:tasks.rados.rados.0.smithi076.stdout:408: oid 4 updating version 0 to 125 2024-09-21T09:39:27.654 INFO:tasks.rados.rados.0.smithi076.stdout:408: oid 4 updating version 125 to 128 2024-09-21T09:39:27.654 INFO:tasks.rados.rados.0.smithi076.stdout:408: oid 4 version 128 is already newer than 126 2024-09-21T09:39:27.654 INFO:tasks.rados.rados.0.smithi076.stdout:408: oid 4 version 128 is already newer than 127 2024-09-21T09:39:27.654 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 128 (ObjNum 192 snap 51 seq_num 192) dirty exists 2024-09-21T09:39:27.654 INFO:tasks.rados.rados.0.smithi076.stdout:408: left oid 4 (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:39:27.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:27 smithi191 ceph-mon[33301]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:39:27.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:27 smithi191 ceph-mon[33301]: pgmap v233: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.8 MiB/s wr, 26 op/s 2024-09-21T09:39:27.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:27 smithi076 ceph-mon[29313]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:39:27.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:27 smithi076 ceph-mon[29313]: pgmap v233: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.8 MiB/s wr, 26 op/s 2024-09-21T09:39:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:27 smithi076 ceph-mon[41325]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:39:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:27 smithi076 ceph-mon[41325]: pgmap v233: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.8 MiB/s wr, 26 op/s 2024-09-21T09:39:28.509 INFO:tasks.rados.rados.0.smithi076.stdout:406: done (3 left) 2024-09-21T09:39:28.509 INFO:tasks.rados.rados.0.smithi076.stdout:407: done (2 left) 2024-09-21T09:39:28.509 INFO:tasks.rados.rados.0.smithi076.stdout:408: done (1 left) 2024-09-21T09:39:28.509 INFO:tasks.rados.rados.0.smithi076.stdout:409: done (0 left) 2024-09-21T09:39:28.509 INFO:tasks.rados.rados.0.smithi076.stdout:410: snap_create 2024-09-21T09:39:28.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:28 smithi191 ceph-mon[33301]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:39:28.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:28 smithi191 ceph-mon[33301]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:39:28.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:28 smithi076 ceph-mon[29313]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:39:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:28 smithi076 ceph-mon[29313]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:39:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:28 smithi076 ceph-mon[41325]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:39:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:28 smithi076 ceph-mon[41325]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:39:29.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:39:28] "GET /metrics HTTP/1.1" 200 36056 "" "Prometheus/2.43.0" 2024-09-21T09:39:29.511 INFO:tasks.rados.rados.0.smithi076.stdout:410: done (0 left) 2024-09-21T09:39:29.511 INFO:tasks.rados.rados.0.smithi076.stdout:411: write oid 8 current snap is 53 2024-09-21T09:39:29.511 INFO:tasks.rados.rados.0.smithi076.stdout:411: seq_num 193 ranges {443751=631882,1810907=765058,3067461=176290} 2024-09-21T09:39:29.516 INFO:tasks.rados.rados.0.smithi076.stdout:411: writing smithi07685715-8 from 443751 to 1075633 tid 1 2024-09-21T09:39:29.525 INFO:tasks.rados.rados.0.smithi076.stdout:411: writing smithi07685715-8 from 1810907 to 2575965 tid 2 2024-09-21T09:39:29.530 INFO:tasks.rados.rados.0.smithi076.stdout:411: writing smithi07685715-8 from 3067461 to 3243751 tid 3 2024-09-21T09:39:29.530 INFO:tasks.rados.rados.0.smithi076.stdout:412: snap_create 2024-09-21T09:39:29.530 INFO:tasks.rados.rados.0.smithi076.stdout:411: finishing write tid 1 to smithi07685715-8 2024-09-21T09:39:29.530 INFO:tasks.rados.rados.0.smithi076.stdout:411: finishing write tid 2 to smithi07685715-8 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: finishing write tid 3 to smithi07685715-8 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: finishing write tid 4 to smithi07685715-8 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: finishing write tid 5 to smithi07685715-8 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: oid 8 updating version 0 to 275 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: oid 8 version 275 is already newer than 274 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: oid 8 updating version 275 to 277 2024-09-21T09:39:29.532 INFO:tasks.rados.rados.0.smithi076.stdout:411: oid 8 version 277 is already newer than 276 2024-09-21T09:39:29.533 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 277 (ObjNum 193 snap 53 seq_num 193) dirty exists 2024-09-21T09:39:29.533 INFO:tasks.rados.rados.0.smithi076.stdout:411: left oid 8 (ObjNum 193 snap 53 seq_num 193) 2024-09-21T09:39:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:29 smithi191 ceph-mon[33301]: pgmap v236: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:29 smithi191 ceph-mon[33301]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:39:29.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:29 smithi076 ceph-mon[29313]: pgmap v236: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:29.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:29 smithi076 ceph-mon[29313]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:39:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:29 smithi076 ceph-mon[41325]: pgmap v236: 89 pgs: 89 active+clean; 170 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:29 smithi076 ceph-mon[41325]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:39:30.512 INFO:tasks.rados.rados.0.smithi076.stdout:411: done (1 left) 2024-09-21T09:39:30.512 INFO:tasks.rados.rados.0.smithi076.stdout:412: done (0 left) 2024-09-21T09:39:30.513 INFO:tasks.rados.rados.0.smithi076.stdout:413: snap_create 2024-09-21T09:39:31.517 INFO:tasks.rados.rados.0.smithi076.stdout:413: done (0 left) 2024-09-21T09:39:31.518 INFO:tasks.rados.rados.0.smithi076.stdout:414: snap_remove snap 41 2024-09-21T09:39:31.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:31 smithi191 ceph-mon[33301]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:39:31.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:31 smithi191 ceph-mon[33301]: pgmap v239: 89 pgs: 89 active+clean; 162 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 13 op/s 2024-09-21T09:39:31.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:31 smithi076 ceph-mon[29313]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:39:31.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:31 smithi076 ceph-mon[29313]: pgmap v239: 89 pgs: 89 active+clean; 162 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 13 op/s 2024-09-21T09:39:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:31 smithi076 ceph-mon[41325]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:39:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:31 smithi076 ceph-mon[41325]: pgmap v239: 89 pgs: 89 active+clean; 162 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 13 op/s 2024-09-21T09:39:32.525 INFO:tasks.rados.rados.0.smithi076.stdout:414: done (0 left) 2024-09-21T09:39:32.526 INFO:tasks.rados.rados.0.smithi076.stdout:415: rollback oid 9 current snap is 55 2024-09-21T09:39:32.526 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 35 2024-09-21T09:39:32.526 INFO:tasks.rados.rados.0.smithi076.stdout:416: rollback oid 11 current snap is 55 2024-09-21T09:39:32.526 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 49 2024-09-21T09:39:32.526 INFO:tasks.rados.rados.0.smithi076.stdout:417: write oid 25 current snap is 55 2024-09-21T09:39:32.526 INFO:tasks.rados.rados.0.smithi076.stdout:417: seq_num 194 ranges {460558=707131,1869389=508716,3094884=165674} 2024-09-21T09:39:32.532 INFO:tasks.rados.rados.0.smithi076.stdout:417: writing smithi07685715-25 from 460558 to 1167689 tid 1 2024-09-21T09:39:32.539 INFO:tasks.rados.rados.0.smithi076.stdout:417: writing smithi07685715-25 from 1869389 to 2378105 tid 2 2024-09-21T09:39:32.544 INFO:tasks.rados.rados.0.smithi076.stdout:417: writing smithi07685715-25 from 3094884 to 3260558 tid 3 2024-09-21T09:39:32.544 INFO:tasks.rados.rados.0.smithi076.stdout:418: write oid 5 current snap is 55 2024-09-21T09:39:32.545 INFO:tasks.rados.rados.0.smithi076.stdout:415: finishing rollback tid 0 to smithi07685715-9 2024-09-21T09:39:32.545 INFO:tasks.rados.rados.0.smithi076.stdout:418: seq_num 195 ranges {477365=782380,1927871=768727,3122307=155058} 2024-09-21T09:39:32.551 INFO:tasks.rados.rados.0.smithi076.stdout:418: writing smithi07685715-5 from 477365 to 1259745 tid 1 2024-09-21T09:39:32.559 INFO:tasks.rados.rados.0.smithi076.stdout:418: writing smithi07685715-5 from 1927871 to 2696598 tid 2 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:418: writing smithi07685715-5 from 3122307 to 3277365 tid 3 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:419: read oid 6 snap -1 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:419: expect deleted 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:420: write oid 35 current snap is 55 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:415: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:415: finishing rollback tid 2 to smithi07685715-9 2024-09-21T09:39:32.563 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 174 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:39:32.564 INFO:tasks.rados.rados.0.smithi076.stdout:416: finishing rollback tid 0 to smithi07685715-11 2024-09-21T09:39:32.564 INFO:tasks.rados.rados.0.smithi076.stdout:420: seq_num 196 ranges {494172=457629,1586353=512385,2749730=544442} 2024-09-21T09:39:32.567 INFO:tasks.rados.rados.0.smithi076.stdout:420: writing smithi07685715-35 from 494172 to 951801 tid 1 2024-09-21T09:39:32.574 INFO:tasks.rados.rados.0.smithi076.stdout:420: writing smithi07685715-35 from 1586353 to 2098738 tid 2 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:420: writing smithi07685715-35 from 2749730 to 3294172 tid 3 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:415: done (5 left) 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:421: rollback oid 38 current snap is 55 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 35 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:416: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:416: finishing rollback tid 2 to smithi07685715-11 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 138 (ObjNum 183 snap 48 seq_num 183) dirty exists 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:417: finishing write tid 1 to smithi07685715-25 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:416: done (5 left) 2024-09-21T09:39:32.581 INFO:tasks.rados.rados.0.smithi076.stdout:422: write oid 9 current snap is 55 2024-09-21T09:39:32.582 INFO:tasks.rados.rados.0.smithi076.stdout:417: finishing write tid 2 to smithi07685715-25 2024-09-21T09:39:32.582 INFO:tasks.rados.rados.0.smithi076.stdout:422: seq_num 197 ranges {510979=532878,1528482=772396,3060800=250179} 2024-09-21T09:39:32.586 INFO:tasks.rados.rados.0.smithi076.stdout:422: writing smithi07685715-9 from 510979 to 1043857 tid 1 2024-09-21T09:39:32.593 INFO:tasks.rados.rados.0.smithi076.stdout:422: writing smithi07685715-9 from 1528482 to 2300878 tid 2 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:422: writing smithi07685715-9 from 3060800 to 3310979 tid 3 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:423: snap_create 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:417: finishing write tid 3 to smithi07685715-25 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:417: finishing write tid 4 to smithi07685715-25 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:417: finishing write tid 5 to smithi07685715-25 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:417: oid 25 updating version 0 to 75 2024-09-21T09:39:32.599 INFO:tasks.rados.rados.0.smithi076.stdout:417: oid 25 updating version 75 to 76 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:417: oid 25 version 76 is already newer than 74 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:417: oid 25 version 76 is already newer than 73 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 76 (ObjNum 194 snap 55 seq_num 194) dirty exists 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:417: left oid 25 (ObjNum 194 snap 55 seq_num 194) 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: finishing write tid 1 to smithi07685715-5 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: finishing write tid 2 to smithi07685715-5 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: finishing write tid 3 to smithi07685715-5 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: finishing write tid 4 to smithi07685715-5 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: finishing write tid 5 to smithi07685715-5 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: oid 5 updating version 0 to 147 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: oid 5 version 147 is already newer than 144 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: oid 5 version 147 is already newer than 145 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:418: oid 5 version 147 is already newer than 146 2024-09-21T09:39:32.600 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 147 (ObjNum 195 snap 55 seq_num 195) dirty exists 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:418: left oid 5 (ObjNum 195 snap 55 seq_num 195) 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: finishing write tid 1 to smithi07685715-35 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: finishing write tid 2 to smithi07685715-35 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:421: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 0 (ObjNum 305 snap 0 seq_num 1682337832) dirty dne 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: finishing write tid 3 to smithi07685715-35 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: finishing write tid 4 to smithi07685715-35 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: finishing write tid 5 to smithi07685715-35 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: oid 35 updating version 0 to 150 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: oid 35 updating version 150 to 151 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: oid 35 updating version 151 to 152 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: oid 35 version 152 is already newer than 149 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 152 (ObjNum 196 snap 55 seq_num 196) dirty exists 2024-09-21T09:39:32.601 INFO:tasks.rados.rados.0.smithi076.stdout:420: left oid 35 (ObjNum 196 snap 55 seq_num 196) 2024-09-21T09:39:32.602 INFO:tasks.rados.rados.0.smithi076.stdout:422: finishing write tid 1 to smithi07685715-9 2024-09-21T09:39:32.602 INFO:tasks.rados.rados.0.smithi076.stdout:422: finishing write tid 2 to smithi07685715-9 2024-09-21T09:39:32.602 INFO:tasks.rados.rados.0.smithi076.stdout:422: finishing write tid 3 to smithi07685715-9 2024-09-21T09:39:32.602 INFO:tasks.rados.rados.0.smithi076.stdout:422: finishing write tid 4 to smithi07685715-9 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:422: finishing write tid 5 to smithi07685715-9 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:422: oid 9 updating version 0 to 176 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:422: oid 9 updating version 176 to 177 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:422: oid 9 version 177 is already newer than 175 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:422: oid 9 updating version 177 to 178 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 178 (ObjNum 197 snap 55 seq_num 197) dirty exists 2024-09-21T09:39:32.603 INFO:tasks.rados.rados.0.smithi076.stdout:422: left oid 9 (ObjNum 197 snap 55 seq_num 197) 2024-09-21T09:39:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:32 smithi191 ceph-mon[33301]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:39:32.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:32 smithi076 ceph-mon[41325]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:39:32.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:32 smithi076 ceph-mon[29313]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:417: done (6 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:418: done (5 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:419: done (4 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:420: done (3 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:421: done (2 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:422: done (1 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:423: done (0 left) 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:424: rollback oid 48 current snap is 56 2024-09-21T09:39:33.511 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 36 2024-09-21T09:39:33.512 INFO:tasks.rados.rados.0.smithi076.stdout:425: snap_create 2024-09-21T09:39:33.517 INFO:tasks.rados.rados.0.smithi076.stdout:424: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:39:33.518 INFO:tasks.rados.rados.0.smithi076.stdout:424: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:39:33.518 INFO:tasks.rados.rados.0.smithi076.stdout:424: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:39:33.518 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 287 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:39:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:33 smithi191 ceph-mon[33301]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:39:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:33 smithi191 ceph-mon[33301]: pgmap v242: 89 pgs: 89 active+clean; 162 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 13 op/s 2024-09-21T09:39:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:33 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:33 smithi191 ceph-mon[33301]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:39:33.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[29313]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[29313]: pgmap v242: 89 pgs: 89 active+clean; 162 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 13 op/s 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[29313]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[41325]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[41325]: pgmap v242: 89 pgs: 89 active+clean; 162 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.8 MiB/s wr, 13 op/s 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:33 smithi076 ceph-mon[41325]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:39:34.512 INFO:tasks.rados.rados.0.smithi076.stdout:424: done (1 left) 2024-09-21T09:39:34.512 INFO:tasks.rados.rados.0.smithi076.stdout:425: done (0 left) 2024-09-21T09:39:34.513 INFO:tasks.rados.rados.0.smithi076.stdout:426: snap_remove snap 56 2024-09-21T09:39:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:34 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:34 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:34 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:34 smithi191 ceph-mon[33301]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:39:34.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[29313]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:34 smithi076 ceph-mon[41325]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:39:35.514 INFO:tasks.rados.rados.0.smithi076.stdout:426: done (0 left) 2024-09-21T09:39:35.514 INFO:tasks.rados.rados.0.smithi076.stdout:427: delete oid 27 current snap is 57 2024-09-21T09:39:35.518 INFO:tasks.rados.rados.0.smithi076.stdout:427: done (0 left) 2024-09-21T09:39:35.518 INFO:tasks.rados.rados.0.smithi076.stdout:428: rollback oid 33 current snap is 57 2024-09-21T09:39:35.518 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 51 2024-09-21T09:39:35.518 INFO:tasks.rados.rados.0.smithi076.stdout:429: read oid 45 snap -1 2024-09-21T09:39:35.518 INFO:tasks.rados.rados.0.smithi076.stdout:429: expect (ObjNum 189 snap 51 seq_num 189) 2024-09-21T09:39:35.519 INFO:tasks.rados.rados.0.smithi076.stdout:430: read oid 26 snap -1 2024-09-21T09:39:35.519 INFO:tasks.rados.rados.0.smithi076.stdout:430: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:35.519 INFO:tasks.rados.rados.0.smithi076.stdout:431: write oid 43 current snap is 57 2024-09-21T09:39:35.519 INFO:tasks.rados.rados.0.smithi076.stdout:431: seq_num 199 ranges {544593=799729,1761799=776065,2948352=396241} 2024-09-21T09:39:35.526 INFO:tasks.rados.rados.0.smithi076.stdout:431: writing smithi07685715-43 from 544593 to 1344322 tid 1 2024-09-21T09:39:35.534 INFO:tasks.rados.rados.0.smithi076.stdout:431: writing smithi07685715-43 from 1761799 to 2537864 tid 2 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:431: writing smithi07685715-43 from 2948352 to 3344593 tid 3 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:432: read oid 47 snap -1 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:432: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:433: read oid 31 snap -1 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:433: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:434: write oid 30 current snap is 57 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:428: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:428: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:39:35.539 INFO:tasks.rados.rados.0.smithi076.stdout:434: seq_num 200 ranges {561400=474978,1820281=519723,2859422=457579} 2024-09-21T09:39:35.544 INFO:tasks.rados.rados.0.smithi076.stdout:434: writing smithi07685715-30 from 561400 to 1036378 tid 1 2024-09-21T09:39:35.551 INFO:tasks.rados.rados.0.smithi076.stdout:434: writing smithi07685715-30 from 1820281 to 2340004 tid 2 2024-09-21T09:39:35.557 INFO:tasks.rados.rados.0.smithi076.stdout:434: writing smithi07685715-30 from 2859422 to 3317001 tid 3 2024-09-21T09:39:35.557 INFO:tasks.rados.rados.0.smithi076.stdout:435: write oid 21 current snap is 57 2024-09-21T09:39:35.557 INFO:tasks.rados.rados.0.smithi076.stdout:435: seq_num 201 ranges {578207=550227,1762410=779734,3286845=91362} 2024-09-21T09:39:35.563 INFO:tasks.rados.rados.0.smithi076.stdout:435: writing smithi07685715-21 from 578207 to 1128434 tid 1 2024-09-21T09:39:35.570 INFO:tasks.rados.rados.0.smithi076.stdout:435: writing smithi07685715-21 from 1762410 to 2542144 tid 2 2024-09-21T09:39:35.575 INFO:tasks.rados.rados.0.smithi076.stdout:435: writing smithi07685715-21 from 3286845 to 3378207 tid 3 2024-09-21T09:39:35.575 INFO:tasks.rados.rados.0.smithi076.stdout:436: rollback oid 1 current snap is 57 2024-09-21T09:39:35.575 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 1 to 43 2024-09-21T09:39:35.576 INFO:tasks.rados.rados.0.smithi076.stdout:437: snap_remove snap 42 2024-09-21T09:39:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:35 smithi191 ceph-mon[33301]: pgmap v245: 89 pgs: 89 active+clean; 169 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:39:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:35 smithi191 ceph-mon[33301]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:39:35.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:35 smithi076 ceph-mon[41325]: pgmap v245: 89 pgs: 89 active+clean; 169 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:39:35.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:35 smithi076 ceph-mon[41325]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:39:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:35 smithi076 ceph-mon[29313]: pgmap v245: 89 pgs: 89 active+clean; 169 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:39:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:35 smithi076 ceph-mon[29313]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:39:36.540 INFO:tasks.rados.rados.0.smithi076.stdout:437: done (9 left) 2024-09-21T09:39:36.540 INFO:tasks.rados.rados.0.smithi076.stdout:438: write oid 40 current snap is 57 2024-09-21T09:39:36.540 INFO:tasks.rados.rados.0.smithi076.stdout:438: seq_num 202 ranges {595014=625476,1820892=523392,2797915=596476} 2024-09-21T09:39:36.546 INFO:tasks.rados.rados.0.smithi076.stdout:438: writing smithi07685715-40 from 595014 to 1220490 tid 1 2024-09-21T09:39:36.553 INFO:tasks.rados.rados.0.smithi076.stdout:438: writing smithi07685715-40 from 1820892 to 2344284 tid 2 2024-09-21T09:39:36.559 INFO:tasks.rados.rados.0.smithi076.stdout:438: writing smithi07685715-40 from 2797915 to 3394391 tid 3 2024-09-21T09:39:36.559 INFO:tasks.rados.rados.0.smithi076.stdout:439: write oid 49 current snap is 57 2024-09-21T09:39:36.559 INFO:tasks.rados.rados.0.smithi076.stdout:428: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:39:36.559 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 80 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-09-21T09:39:36.559 INFO:tasks.rados.rados.0.smithi076.stdout:430: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:36.717 INFO:tasks.rados.rados.0.smithi076.stdout:429: expect (ObjNum 189 snap 51 seq_num 189) 2024-09-21T09:39:36.881 INFO:tasks.rados.rados.0.smithi076.stdout:431: finishing write tid 1 to smithi07685715-43 2024-09-21T09:39:36.881 INFO:tasks.rados.rados.0.smithi076.stdout:431: finishing write tid 2 to smithi07685715-43 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: finishing write tid 3 to smithi07685715-43 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: finishing write tid 4 to smithi07685715-43 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: finishing write tid 5 to smithi07685715-43 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: oid 43 updating version 0 to 36 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: oid 43 version 36 is already newer than 33 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: oid 43 version 36 is already newer than 34 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: oid 43 version 36 is already newer than 35 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 36 (ObjNum 199 snap 57 seq_num 199) dirty exists 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:431: left oid 43 (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:39:36.882 INFO:tasks.rados.rados.0.smithi076.stdout:433: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:39:36.967 INFO:tasks.rados.rados.0.smithi076.stdout:432: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: finishing write tid 1 to smithi07685715-30 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: finishing write tid 2 to smithi07685715-30 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: finishing write tid 3 to smithi07685715-30 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: finishing write tid 4 to smithi07685715-30 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: finishing write tid 5 to smithi07685715-30 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: oid 30 updating version 0 to 291 2024-09-21T09:39:37.088 INFO:tasks.rados.rados.0.smithi076.stdout:434: oid 30 version 291 is already newer than 290 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:434: oid 30 version 291 is already newer than 289 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:434: oid 30 updating version 291 to 292 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 292 (ObjNum 200 snap 57 seq_num 200) dirty exists 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:434: left oid 30 (ObjNum 200 snap 57 seq_num 200) 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: finishing write tid 1 to smithi07685715-21 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: finishing write tid 2 to smithi07685715-21 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: finishing write tid 3 to smithi07685715-21 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: finishing write tid 4 to smithi07685715-21 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: finishing write tid 5 to smithi07685715-21 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: oid 21 updating version 0 to 156 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: oid 21 version 156 is already newer than 154 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: oid 21 updating version 156 to 157 2024-09-21T09:39:37.089 INFO:tasks.rados.rados.0.smithi076.stdout:435: oid 21 version 157 is already newer than 155 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 157 (ObjNum 201 snap 57 seq_num 201) dirty exists 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:435: left oid 21 (ObjNum 201 snap 57 seq_num 201) 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:436: finishing rollback tid 1 to smithi07685715-1 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 0 (ObjNum 33 snap 0 seq_num 830379256) dirty dne 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: finishing write tid 1 to smithi07685715-40 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: finishing write tid 2 to smithi07685715-40 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: finishing write tid 3 to smithi07685715-40 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: finishing write tid 4 to smithi07685715-40 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: finishing write tid 5 to smithi07685715-40 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: oid 40 updating version 0 to 194 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: oid 40 version 194 is already newer than 191 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: oid 40 version 194 is already newer than 192 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:438: oid 40 version 194 is already newer than 193 2024-09-21T09:39:37.090 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 194 (ObjNum 202 snap 57 seq_num 202) dirty exists 2024-09-21T09:39:37.091 INFO:tasks.rados.rados.0.smithi076.stdout:438: left oid 40 (ObjNum 202 snap 57 seq_num 202) 2024-09-21T09:39:37.091 INFO:tasks.rados.rados.0.smithi076.stdout:439: seq_num 203 ranges {611821=700725,1879374=783403,3341691=70130} 2024-09-21T09:39:37.095 INFO:tasks.rados.rados.0.smithi076.stdout:439: writing smithi07685715-49 from 611821 to 1312546 tid 1 2024-09-21T09:39:37.102 INFO:tasks.rados.rados.0.smithi076.stdout:439: writing smithi07685715-49 from 1879374 to 2662777 tid 2 2024-09-21T09:39:37.107 INFO:tasks.rados.rados.0.smithi076.stdout:439: writing smithi07685715-49 from 3341691 to 3411821 tid 3 2024-09-21T09:39:37.107 INFO:tasks.rados.rados.0.smithi076.stdout:428: done (10 left) 2024-09-21T09:39:37.107 INFO:tasks.rados.rados.0.smithi076.stdout:429: done (9 left) 2024-09-21T09:39:37.107 INFO:tasks.rados.rados.0.smithi076.stdout:430: done (8 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:431: done (7 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:432: done (6 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:433: done (5 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:434: done (4 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:435: done (3 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:436: done (2 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:438: done (1 left) 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:440: snap_create 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:439: finishing write tid 1 to smithi07685715-49 2024-09-21T09:39:37.108 INFO:tasks.rados.rados.0.smithi076.stdout:439: finishing write tid 2 to smithi07685715-49 2024-09-21T09:39:37.109 INFO:tasks.rados.rados.0.smithi076.stdout:439: finishing write tid 3 to smithi07685715-49 2024-09-21T09:39:37.109 INFO:tasks.rados.rados.0.smithi076.stdout:439: finishing write tid 4 to smithi07685715-49 2024-09-21T09:39:37.109 INFO:tasks.rados.rados.0.smithi076.stdout:439: finishing write tid 5 to smithi07685715-49 2024-09-21T09:39:37.109 INFO:tasks.rados.rados.0.smithi076.stdout:439: oid 49 updating version 0 to 303 2024-09-21T09:39:37.110 INFO:tasks.rados.rados.0.smithi076.stdout:439: oid 49 version 303 is already newer than 302 2024-09-21T09:39:37.110 INFO:tasks.rados.rados.0.smithi076.stdout:439: oid 49 version 303 is already newer than 301 2024-09-21T09:39:37.110 INFO:tasks.rados.rados.0.smithi076.stdout:439: oid 49 version 303 is already newer than 300 2024-09-21T09:39:37.110 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 303 (ObjNum 203 snap 57 seq_num 203) dirty exists 2024-09-21T09:39:37.110 INFO:tasks.rados.rados.0.smithi076.stdout:439: left oid 49 (ObjNum 203 snap 57 seq_num 203) 2024-09-21T09:39:37.230 INFO:tasks.rados.rados.0.smithi076.stdout:439: done (1 left) 2024-09-21T09:39:37.230 INFO:tasks.rados.rados.0.smithi076.stdout:440: done (0 left) 2024-09-21T09:39:37.230 INFO:tasks.rados.rados.0.smithi076.stdout:441: snap_remove snap 46 2024-09-21T09:39:37.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:37] ENGINE Bus STOPPING 2024-09-21T09:39:37.538 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.228+0000 7f371f1c1640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T09:39:37.538 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setuser ceph since I am not root 2024-09-21T09:39:37.538 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ignoring --setgroup ceph since I am not root 2024-09-21T09:39:37.538 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.370+0000 7fb20cef2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:39:37.538 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.427+0000 7fb20cef2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.579+0000 7fb20cef2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.646+0000 7fb20cef2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.792+0000 7fb20cef2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: pgmap v248: 89 pgs: 89 active+clean; 169 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.796 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: Activating manager daemon x 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: mgrmap e23: x(active, starting, since 0.00824028s) 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:39:37.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:39:37.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: pgmap v248: 89 pgs: 89 active+clean; 169 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:37.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: Activating manager daemon x 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: mgrmap e23: x(active, starting, since 0.00824028s) 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:39:37.800 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:39:37.801 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:39:37.801 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:39:37.801 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:39:37.801 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:39:37.801 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:39:37.801 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:39:37.862 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:39:37.862 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:37] ENGINE Bus STOPPED 2024-09-21T09:39:37.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:37.640Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T09:39:37.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:37.642Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T09:39:37.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:37.642Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T09:39:37.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:37.644Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T09:39:37.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:37.644Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T09:39:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:39:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: pgmap v248: 89 pgs: 89 active+clean; 169 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.5 MiB/s wr, 7 op/s 2024-09-21T09:39:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 172.21.15.76:0/2868063214' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: Activating manager daemon x 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24326 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: mgrmap e23: x(active, starting, since 0.00824028s) 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:39:37.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:39:37.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:37 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:39:38.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:37.935+0000 7fb20cef2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:39:38.229 INFO:tasks.rados.rados.0.smithi076.stdout:441: done (0 left) 2024-09-21T09:39:38.230 INFO:tasks.rados.rados.0.smithi076.stdout:442: read oid 36 snap -1 2024-09-21T09:39:38.230 INFO:tasks.rados.rados.0.smithi076.stdout:442: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:38.230 INFO:tasks.rados.rados.0.smithi076.stdout:443: read oid 35 snap 54 2024-09-21T09:39:38.230 INFO:tasks.rados.rados.0.smithi076.stdout:443: expect (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:39:38.230 INFO:tasks.rados.rados.0.smithi076.stdout:444: snap_remove snap 51 2024-09-21T09:39:38.511 INFO:tasks.rados.rados.0.smithi076.stdout:444: done (2 left) 2024-09-21T09:39:38.512 INFO:tasks.rados.rados.0.smithi076.stdout:445: rollback oid 37 current snap is 58 2024-09-21T09:39:38.512 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 53 2024-09-21T09:39:38.512 INFO:tasks.rados.rados.0.smithi076.stdout:443: expect (ObjNum 85 snap 10 seq_num 85) 2024-09-21T09:39:38.515 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:38.256+0000 7fb20cef2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:39:38.515 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:38.320+0000 7fb20cef2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.332+0000 7f1695e2b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.332+0000 7f1695e2b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.332+0000 7f1695e2b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.332+0000 7f1695e2b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.332+0000 7f1695e2b640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.333+0000 7f1699632640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.333+0000 7f1699632640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.333+0000 7f1699632640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.333+0000 7f1699632640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.541 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:38.333+0000 7f1699632640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:39:38.651 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:39:38.714 INFO:tasks.rados.rados.0.smithi076.stdout:442: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:445: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:445: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:445: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 80 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:442: done (2 left) 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:443: done (1 left) 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:445: done (0 left) 2024-09-21T09:39:38.793 INFO:tasks.rados.rados.0.smithi076.stdout:446: snap_remove snap 50 2024-09-21T09:39:38.859 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:38] ENGINE Bus STARTING 2024-09-21T09:39:38.859 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:38] ENGINE Serving on http://:::9283 2024-09-21T09:39:38.859 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:39:38] ENGINE Bus STARTED 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: Manager daemon x is now available 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: Checking for cert/key for grafana.a 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: Migrating grafana.a cert to cert store 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: Migrating grafana.a key to cert store 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:38.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:38.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:38.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:38.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:38.861 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:38 smithi191 ceph-mon[33301]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:39:38.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:38.512+0000 7fb20cef2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: Manager daemon x is now available 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: Checking for cert/key for grafana.a 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: Migrating grafana.a cert to cert store 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: Migrating grafana.a key to cert store 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[29313]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: Manager daemon x is now available 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: Checking for cert/key for grafana.a 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: Migrating grafana.a cert to cert store 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: Migrating grafana.a key to cert store 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:39:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:38 smithi076 ceph-mon[41325]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (3m) 63s ago 4m 26.6M - 0.25.0 c8568f914cd2 d930ef8d31ba 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (3m) 6s ago 3m 88.6M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (2m) 63s ago 2m 14.6M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (2m) 6s ago 2m 14.7M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (22s) 6s ago 8m 66.5M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 8ab387e15f88 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:9283,8765 running (9m) 63s ago 9m 487M - 18.2.4-871-g6a1bdf85 0adab276c97e 3f03f8aa752c 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (9m) 63s ago 9m 48.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (8m) 6s ago 8m 39.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (8m) 63s ago 8m 39.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (3m) 63s ago 3m 14.8M - 1.5.0 0da6a335fe13 6dbd51c32fb5 2024-09-21T09:39:39.255 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (3m) 6s ago 3m 15.1M - 1.5.0 0da6a335fe13 9722666fd438 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (7m) 63s ago 7m 62.4M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (7m) 63s ago 7m 58.9M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (6m) 63s ago 6m 62.6M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (6m) 63s ago 6m 56.7M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (6m) 6s ago 6m 58.1M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (5m) 6s ago 5m 71.3M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (5m) 6s ago 5m 55.8M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (4m) 6s ago 4m 55.1M 4016M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:39:39.256 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (3m) 6s ago 4m 51.4M - 2.43.0 a07b618ecd1d 8726c547d40a 2024-09-21T09:39:39.565 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:39.494+0000 7fb20cef2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:39:39.576 INFO:tasks.rados.rados.0.smithi076.stdout:446: done (0 left) 2024-09-21T09:39:39.576 INFO:tasks.rados.rados.0.smithi076.stdout:447: write oid 49 current snap is 58 2024-09-21T09:39:39.576 INFO:tasks.rados.rados.0.smithi076.stdout:447: seq_num 204 ranges {628628=775974,1937856=527061,3252761=175867} 2024-09-21T09:39:39.584 INFO:tasks.rados.rados.0.smithi076.stdout:447: writing smithi07685715-49 from 628628 to 1404602 tid 1 2024-09-21T09:39:39.590 INFO:tasks.rados.rados.0.smithi076.stdout:447: writing smithi07685715-49 from 1937856 to 2464917 tid 2 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:447: writing smithi07685715-49 from 3252761 to 3428628 tid 3 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:447: finishing write tid 1 to smithi07685715-49 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:448: read oid 34 snap 43 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:448: expect (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:449: read oid 8 snap -1 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:449: expect (ObjNum 193 snap 53 seq_num 193) 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:450: write oid 42 current snap is 58 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:447: finishing write tid 2 to smithi07685715-49 2024-09-21T09:39:39.596 INFO:tasks.rados.rados.0.smithi076.stdout:450: seq_num 205 ranges {645435=451223,1879985=787072,3280184=165251} 2024-09-21T09:39:39.602 INFO:tasks.rados.rados.0.smithi076.stdout:450: writing smithi07685715-42 from 645435 to 1096658 tid 1 2024-09-21T09:39:39.611 INFO:tasks.rados.rados.0.smithi076.stdout:450: writing smithi07685715-42 from 1879985 to 2667057 tid 2 2024-09-21T09:39:39.613 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:39:39.614 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:39:39.615 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:39:39.615 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:39:39.615 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:39:39.615 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:39:39.616 INFO:tasks.rados.rados.0.smithi076.stdout:450: writing smithi07685715-42 from 3280184 to 3445435 tid 3 2024-09-21T09:39:39.616 INFO:tasks.rados.rados.0.smithi076.stdout:448: expect (ObjNum 112 snap 18 seq_num 112) 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: finishing write tid 3 to smithi07685715-49 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: finishing write tid 4 to smithi07685715-49 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: finishing write tid 5 to smithi07685715-49 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: oid 49 updating version 0 to 325 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: oid 49 version 325 is already newer than 324 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: oid 49 version 325 is already newer than 323 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: oid 49 updating version 325 to 326 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 326 (ObjNum 204 snap 58 seq_num 204) dirty exists 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:447: left oid 49 (ObjNum 204 snap 58 seq_num 204) 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:450: finishing write tid 1 to smithi07685715-42 2024-09-21T09:39:39.802 INFO:tasks.rados.rados.0.smithi076.stdout:449: expect (ObjNum 193 snap 53 seq_num 193) 2024-09-21T09:39:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: mgrmap e24: x(active, since 1.34246s) 2024-09-21T09:39:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: pgmap v5: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: Deploying cephadm binary to smithi076 2024-09-21T09:39:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: Deploying cephadm binary to smithi191 2024-09-21T09:39:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: from='client.24806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: pgmap v6: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:39 smithi191 ceph-mon[33301]: from='client.14919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:39.563+0000 7fb20cef2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:39:39.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: mgrmap e24: x(active, since 1.34246s) 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: pgmap v5: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: Deploying cephadm binary to smithi076 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: Deploying cephadm binary to smithi191 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: from='client.24806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: pgmap v6: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[41325]: from='client.14919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: mgrmap e24: x(active, since 1.34246s) 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: pgmap v5: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: Deploying cephadm binary to smithi076 2024-09-21T09:39:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: Deploying cephadm binary to smithi191 2024-09-21T09:39:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: from='client.24806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: pgmap v6: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:39 smithi076 ceph-mon[29313]: from='client.14919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:450: finishing write tid 2 to smithi07685715-42 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:450: finishing write tid 3 to smithi07685715-42 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:447: done (3 left) 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:448: done (2 left) 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:449: done (1 left) 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:451: snap_create 2024-09-21T09:39:39.914 INFO:tasks.rados.rados.0.smithi076.stdout:450: finishing write tid 4 to smithi07685715-42 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:450: finishing write tid 5 to smithi07685715-42 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:450: oid 42 updating version 0 to 53 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:450: oid 42 version 53 is already newer than 51 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:450: oid 42 updating version 53 to 54 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:450: oid 42 version 54 is already newer than 52 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 54 (ObjNum 205 snap 58 seq_num 205) dirty exists 2024-09-21T09:39:39.915 INFO:tasks.rados.rados.0.smithi076.stdout:450: left oid 42 (ObjNum 205 snap 58 seq_num 205) 2024-09-21T09:39:39.954 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:39:39.954 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:39:39.954 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:39:39.955 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:39:39.955 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:39:39.955 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "1/2 daemons upgraded", 2024-09-21T09:39:39.955 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "", 2024-09-21T09:39:39.955 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:39:39.955 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:39:40.567 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-mon[29313]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:39:40.567 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1071439974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:39:40.568 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:40.372+0000 7fb20cef2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:39:40.568 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:40.492+0000 7fb20cef2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:39:40.576 INFO:tasks.rados.rados.0.smithi076.stdout:450: done (1 left) 2024-09-21T09:39:40.576 INFO:tasks.rados.rados.0.smithi076.stdout:451: done (0 left) 2024-09-21T09:39:40.576 INFO:tasks.rados.rados.0.smithi076.stdout:452: snap_create 2024-09-21T09:39:40.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-mon[33301]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:39:40.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1071439974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:39:40.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-mon[33301]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:40.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-mon[29313]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-mon[41325]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:39:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1071439974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:39:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-mon[41325]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:39:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:40.642+0000 7fb20cef2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:39:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:40.710+0000 7fb20cef2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:39:41.089 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:40.818+0000 7f1672b6b640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:39:40] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: Traceback (most recent call last): 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: self._connections.run(self.expiration_interval) 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: self._run(expiration_interval) 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: s = self.context.wrap_socket( 2024-09-21T09:39:41.090 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: return self.sslsocket_class._create( 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: self.do_handshake() 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: self._sslobj.do_handshake() 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:41.091 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:39:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:39:41.183 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:40.879+0000 7fb20cef2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:39:41.183 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:40.960+0000 7fb20cef2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:39:41.183 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.021+0000 7fb20cef2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:39:41.183 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.181+0000 7fb20cef2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:39:41.469 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.252+0000 7fb20cef2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:39:41.469 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.320+0000 7fb20cef2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:39:41.469 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.406+0000 7fb20cef2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:39:41.469 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.467+0000 7fb20cef2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:39:41.579 INFO:tasks.rados.rados.0.smithi076.stdout:452: done (0 left) 2024-09-21T09:39:41.579 INFO:tasks.rados.rados.0.smithi076.stdout:453: delete oid 44 current snap is 60 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:453: done (0 left) 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:454: rollback oid 27 current snap is 60 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 35 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:455: rollback oid 7 current snap is 60 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 57 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:456: read oid 25 snap -1 2024-09-21T09:39:41.582 INFO:tasks.rados.rados.0.smithi076.stdout:456: expect (ObjNum 194 snap 55 seq_num 194) 2024-09-21T09:39:41.583 INFO:tasks.rados.rados.0.smithi076.stdout:457: read oid 48 snap -1 2024-09-21T09:39:41.583 INFO:tasks.rados.rados.0.smithi076.stdout:457: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:41.583 INFO:tasks.rados.rados.0.smithi076.stdout:458: write oid 8 current snap is 60 2024-09-21T09:39:41.583 INFO:tasks.rados.rados.0.smithi076.stdout:458: seq_num 207 ranges {679049=718074,2113302=674388,3335030=144019} 2024-09-21T09:39:41.590 INFO:tasks.rados.rados.0.smithi076.stdout:458: writing smithi07685715-8 from 679049 to 1397123 tid 1 2024-09-21T09:39:41.598 INFO:tasks.rados.rados.0.smithi076.stdout:458: writing smithi07685715-8 from 2113302 to 2787690 tid 2 2024-09-21T09:39:41.602 INFO:tasks.rados.rados.0.smithi076.stdout:458: writing smithi07685715-8 from 3335030 to 3479049 tid 3 2024-09-21T09:39:41.602 INFO:tasks.rados.rados.0.smithi076.stdout:459: snap_remove snap 55 2024-09-21T09:39:41.844 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: mgrmap e25: x(active, since 3s) 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: [21/Sep/2024:09:39:40] ENGINE Bus STARTING 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: [21/Sep/2024:09:39:40] ENGINE Serving on http://172.21.15.191:8765 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: [21/Sep/2024:09:39:40] ENGINE Serving on https://172.21.15.191:7150 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: [21/Sep/2024:09:39:40] ENGINE Bus STARTED 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: [21/Sep/2024:09:39:40] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: Traceback (most recent call last): 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._run(expiration_interval) 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:41.845 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: s = self.context.wrap_socket( 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: return self.sslsocket_class._create( 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: self.do_handshake() 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._sslobj.do_handshake() 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:41.846 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:41 smithi191 ceph-mon[33301]: pgmap v9: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:41.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:41.528+0000 7fb20cef2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: mgrmap e25: x(active, since 3s) 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: [21/Sep/2024:09:39:40] ENGINE Bus STARTING 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: [21/Sep/2024:09:39:40] ENGINE Serving on http://172.21.15.191:8765 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: [21/Sep/2024:09:39:40] ENGINE Serving on https://172.21.15.191:7150 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: [21/Sep/2024:09:39:40] ENGINE Bus STARTED 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: [21/Sep/2024:09:39:40] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:39:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: self.do_handshake() 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[41325]: pgmap v9: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: mgrmap e25: x(active, since 3s) 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:39:40] ENGINE Bus STARTING 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:39:40] ENGINE Serving on http://172.21.15.191:8765 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:39:40] ENGINE Serving on https://172.21.15.191:7150 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:39:40] ENGINE Bus STARTED 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: [21/Sep/2024:09:39:40] ENGINE Error in HTTPServer.serve 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:39:41.875 INFO:journalctl@ceph.mon.a.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: self.do_handshake() 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:39:41.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:41 smithi076 ceph-mon[29313]: pgmap v9: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:42.596 INFO:tasks.rados.rados.0.smithi076.stdout:459: done (5 left) 2024-09-21T09:39:42.596 INFO:tasks.rados.rados.0.smithi076.stdout:460: snap_remove snap 52 2024-09-21T09:39:42.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:39:42.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: Standby manager daemon y started 2024-09-21T09:39:42.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: Standby manager daemon y started 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:39:42.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:42.365+0000 7fb20cef2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:39:42] ENGINE Bus STARTING 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: CherryPy Checker: 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: The Application mounted at '' has an empty config. 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:39:42] ENGINE Serving on http://:::9283 2024-09-21T09:39:42.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: [21/Sep/2024:09:39:42] ENGINE Bus STARTED 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: Standby manager daemon y started 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3211805099' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:42 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:460: done (5 left) 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:461: snap_create 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:454: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:454: finishing rollback tid 2 to smithi07685715-27 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 204 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:455: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:455: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:455: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 200 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-21T09:39:43.515 INFO:tasks.rados.rados.0.smithi076.stdout:456: expect (ObjNum 194 snap 55 seq_num 194) 2024-09-21T09:39:43.750 INFO:tasks.rados.rados.0.smithi076.stdout:457: expect (ObjNum 128 snap 27 seq_num 128) 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi076 to 2480M 2024-09-21T09:39:43.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: pgmap v12: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:43.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:43 smithi191 ceph-mon[33301]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: finishing write tid 1 to smithi07685715-8 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: finishing write tid 2 to smithi07685715-8 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: finishing write tid 3 to smithi07685715-8 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: finishing write tid 4 to smithi07685715-8 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: finishing write tid 5 to smithi07685715-8 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: oid 8 updating version 0 to 331 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: oid 8 version 331 is already newer than 330 2024-09-21T09:39:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:458: oid 8 version 331 is already newer than 328 2024-09-21T09:39:43.871 INFO:tasks.rados.rados.0.smithi076.stdout:458: oid 8 version 331 is already newer than 329 2024-09-21T09:39:43.871 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 331 (ObjNum 207 snap 60 seq_num 207) dirty exists 2024-09-21T09:39:43.871 INFO:tasks.rados.rados.0.smithi076.stdout:458: left oid 8 (ObjNum 207 snap 60 seq_num 207) 2024-09-21T09:39:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-21T09:39:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:39:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi076 to 2480M 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: pgmap v12: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[29313]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:43.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi076 to 2480M 2024-09-21T09:39:43.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:43.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: pgmap v12: 89 pgs: 89 active+clean; 180 MiB data, 587 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:43.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:43 smithi076 ceph-mon[41325]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:454: done (5 left) 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:455: done (4 left) 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:456: done (3 left) 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:457: done (2 left) 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:458: done (1 left) 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:461: done (0 left) 2024-09-21T09:39:44.515 INFO:tasks.rados.rados.0.smithi076.stdout:462: delete oid 40 current snap is 61 2024-09-21T09:39:44.518 INFO:tasks.rados.rados.0.smithi076.stdout:462: done (0 left) 2024-09-21T09:39:44.518 INFO:tasks.rados.rados.0.smithi076.stdout:463: rollback oid 3 current snap is 61 2024-09-21T09:39:44.518 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 47 2024-09-21T09:39:44.519 INFO:tasks.rados.rados.0.smithi076.stdout:464: rollback oid 19 current snap is 61 2024-09-21T09:39:44.519 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 60 2024-09-21T09:39:44.519 INFO:tasks.rados.rados.0.smithi076.stdout:465: snap_remove snap 53 2024-09-21T09:39:45.519 INFO:tasks.rados.rados.0.smithi076.stdout:465: done (2 left) 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:466: delete oid 29 current snap is 61 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:464: finishing rollback tid 0 to smithi07685715-19 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:464: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:464: finishing rollback tid 2 to smithi07685715-19 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 122 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:463: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:39:45.520 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 53 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:39:45.525 INFO:tasks.rados.rados.0.smithi076.stdout:463: done (2 left) 2024-09-21T09:39:45.525 INFO:tasks.rados.rados.0.smithi076.stdout:464: done (1 left) 2024-09-21T09:39:45.525 INFO:tasks.rados.rados.0.smithi076.stdout:466: done (0 left) 2024-09-21T09:39:45.525 INFO:tasks.rados.rados.0.smithi076.stdout:467: snap_create 2024-09-21T09:39:45.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:45 smithi191 ceph-mon[33301]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:39:45.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:45 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:45 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:45 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:45 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:45 smithi191 ceph-mon[33301]: pgmap v15: 89 pgs: 89 active+clean; 187 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 32 op/s 2024-09-21T09:39:45.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[29313]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[29313]: pgmap v15: 89 pgs: 89 active+clean; 187 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 32 op/s 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[41325]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:45.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:45 smithi076 ceph-mon[41325]: pgmap v15: 89 pgs: 89 active+clean; 187 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 32 op/s 2024-09-21T09:39:46.534 INFO:tasks.rados.rados.0.smithi076.stdout:467: done (0 left) 2024-09-21T09:39:46.535 INFO:tasks.rados.rados.0.smithi076.stdout:468: read oid 26 snap -1 2024-09-21T09:39:46.535 INFO:tasks.rados.rados.0.smithi076.stdout:468: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:46.535 INFO:tasks.rados.rados.0.smithi076.stdout:469: delete oid 16 current snap is 62 2024-09-21T09:39:46.538 INFO:tasks.rados.rados.0.smithi076.stdout:469: done (1 left) 2024-09-21T09:39:46.538 INFO:tasks.rados.rados.0.smithi076.stdout:470: snap_remove snap 38 2024-09-21T09:39:46.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:46 smithi191 ceph-mon[33301]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:39:46.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:46 smithi076 ceph-mon[29313]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:39:46.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:46 smithi076 ceph-mon[41325]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:39:47.545 INFO:tasks.rados.rados.0.smithi076.stdout:470: done (1 left) 2024-09-21T09:39:47.545 INFO:tasks.rados.rados.0.smithi076.stdout:471: snap_remove snap 44 2024-09-21T09:39:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:47 smithi191 ceph-mon[33301]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:39:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:47 smithi191 ceph-mon[33301]: pgmap v18: 89 pgs: 89 active+clean; 187 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 32 op/s 2024-09-21T09:39:47.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:47 smithi076 ceph-mon[29313]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:39:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:47 smithi076 ceph-mon[29313]: pgmap v18: 89 pgs: 89 active+clean; 187 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 32 op/s 2024-09-21T09:39:47.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:47 smithi076 ceph-mon[41325]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:39:47.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:47 smithi076 ceph-mon[41325]: pgmap v18: 89 pgs: 89 active+clean; 187 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 32 op/s 2024-09-21T09:39:48.520 INFO:tasks.rados.rados.0.smithi076.stdout:471: done (1 left) 2024-09-21T09:39:48.520 INFO:tasks.rados.rados.0.smithi076.stdout:472: snap_create 2024-09-21T09:39:48.521 INFO:tasks.rados.rados.0.smithi076.stdout:468: expect (ObjNum 148 snap 34 seq_num 148) 2024-09-21T09:39:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:39:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:48 smithi191 ceph-mon[33301]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:39:48.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:39:48.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[29313]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:39:48.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:48.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:48.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:39:48.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-mon[41325]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:39:49.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[29568]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:39:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:39:49.518 INFO:tasks.rados.rados.0.smithi076.stdout:468: done (1 left) 2024-09-21T09:39:49.518 INFO:tasks.rados.rados.0.smithi076.stdout:472: done (0 left) 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:473: read oid 27 snap -1 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:473: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:474: rollback oid 48 current snap is 63 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 31 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:475: read oid 17 snap 35 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:475: expect deleted 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:476: read oid 14 snap -1 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:476: expect deleted 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:477: write oid 46 current snap is 63 2024-09-21T09:39:49.519 INFO:tasks.rados.rados.0.smithi076.stdout:477: seq_num 211 ranges {746277=619070,1830877=681726,2928369=617908} 2024-09-21T09:39:49.525 INFO:tasks.rados.rados.0.smithi076.stdout:477: writing smithi07685715-46 from 746277 to 1365347 tid 1 2024-09-21T09:39:49.532 INFO:tasks.rados.rados.0.smithi076.stdout:477: writing smithi07685715-46 from 1830877 to 2512603 tid 2 2024-09-21T09:39:49.537 INFO:tasks.rados.rados.0.smithi076.stdout:477: writing smithi07685715-46 from 2928369 to 3546277 tid 3 2024-09-21T09:39:49.537 INFO:tasks.rados.rados.0.smithi076.stdout:478: snap_remove snap 54 2024-09-21T09:39:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Adjusting osd_memory_target on smithi191 to 2480M 2024-09-21T09:39:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:39:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: pgmap v21: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T09:39:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:39:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:50 smithi191 ceph-mon[33301]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Adjusting osd_memory_target on smithi191 to 2480M 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: pgmap v21: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[29313]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Adjusting osd_memory_target on smithi191 to 2480M 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:50.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: pgmap v21: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:39:50.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:50 smithi076 ceph-mon[41325]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:478: done (5 left) 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:479: read oid 4 snap -1 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:479: expect (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:480: rollback oid 7 current snap is 63 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 57 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:474: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:39:50.521 INFO:tasks.rados.rados.0.smithi076.stdout:474: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:39:50.522 INFO:tasks.rados.rados.0.smithi076.stdout:474: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:39:50.522 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 373 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-09-21T09:39:50.522 INFO:tasks.rados.rados.0.smithi076.stdout:473: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:39:50.602 INFO:tasks.rados.rados.0.smithi076.stdout:477: finishing write tid 1 to smithi07685715-46 2024-09-21T09:39:50.602 INFO:tasks.rados.rados.0.smithi076.stdout:477: finishing write tid 2 to smithi07685715-46 2024-09-21T09:39:50.602 INFO:tasks.rados.rados.0.smithi076.stdout:473: done (6 left) 2024-09-21T09:39:50.602 INFO:tasks.rados.rados.0.smithi076.stdout:474: done (5 left) 2024-09-21T09:39:50.602 INFO:tasks.rados.rados.0.smithi076.stdout:475: done (4 left) 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:476: done (3 left) 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:481: rollback oid 14 current snap is 63 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 57 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: finishing write tid 3 to smithi07685715-46 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: finishing write tid 4 to smithi07685715-46 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: finishing write tid 5 to smithi07685715-46 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: oid 46 updating version 0 to 225 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: oid 46 version 225 is already newer than 223 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: oid 46 version 225 is already newer than 224 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: oid 46 version 225 is already newer than 222 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 225 (ObjNum 211 snap 63 seq_num 211) dirty exists 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:477: left oid 46 (ObjNum 211 snap 63 seq_num 211) 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:480: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:39:50.603 INFO:tasks.rados.rados.0.smithi076.stdout:480: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:39:50.604 INFO:tasks.rados.rados.0.smithi076.stdout:480: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:39:50.604 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 220 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-21T09:39:50.604 INFO:tasks.rados.rados.0.smithi076.stdout:479: expect (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:481: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 166 (ObjNum 129 snap 0 seq_num 864408856) dirty dne 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:477: done (3 left) 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:479: done (2 left) 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:480: done (1 left) 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:481: done (0 left) 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:482: rollback oid 42 current snap is 63 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 59 2024-09-21T09:39:50.733 INFO:tasks.rados.rados.0.smithi076.stdout:483: snap_remove snap 31 2024-09-21T09:39:51.531 INFO:tasks.rados.rados.0.smithi076.stdout:483: done (1 left) 2024-09-21T09:39:51.531 INFO:tasks.rados.rados.0.smithi076.stdout:484: delete oid 16 current snap is 63 2024-09-21T09:39:51.531 INFO:tasks.rados.rados.0.smithi076.stdout:482: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:39:51.531 INFO:tasks.rados.rados.0.smithi076.stdout:482: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:39:51.531 INFO:tasks.rados.rados.0.smithi076.stdout:482: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:39:51.531 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 70 (ObjNum 205 snap 58 seq_num 205) dirty exists 2024-09-21T09:39:51.536 INFO:tasks.rados.rados.0.smithi076.stdout:482: done (1 left) 2024-09-21T09:39:51.537 INFO:tasks.rados.rados.0.smithi076.stdout:484: done (0 left) 2024-09-21T09:39:51.537 INFO:tasks.rados.rados.0.smithi076.stdout:485: read oid 37 snap -1 2024-09-21T09:39:51.537 INFO:tasks.rados.rados.0.smithi076.stdout:485: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:39:51.537 INFO:tasks.rados.rados.0.smithi076.stdout:486: read oid 47 snap 48 2024-09-21T09:39:51.537 INFO:tasks.rados.rados.0.smithi076.stdout:486: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:51.537 INFO:tasks.rados.rados.0.smithi076.stdout:487: snap_remove snap 49 2024-09-21T09:39:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:51 smithi191 ceph-mon[33301]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:39:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:51 smithi191 ceph-mon[33301]: pgmap v24: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T09:39:51.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:51 smithi076 ceph-mon[29313]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:39:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:51 smithi076 ceph-mon[29313]: pgmap v24: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T09:39:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:51 smithi076 ceph-mon[41325]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:39:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:51 smithi076 ceph-mon[41325]: pgmap v24: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 0 B/s wr, 3 op/s 2024-09-21T09:39:52.527 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 systemd[1]: Stopping Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:487: done (2 left) 2024-09-21T09:39:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:488: rollback oid 17 current snap is 63 2024-09-21T09:39:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 36 2024-09-21T09:39:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:489: write oid 19 current snap is 63 2024-09-21T09:39:52.537 INFO:tasks.rados.rados.0.smithi076.stdout:489: seq_num 213 ranges {779891=485921,1947841=685395,3383215=196676} 2024-09-21T09:39:52.544 INFO:tasks.rados.rados.0.smithi076.stdout:489: writing smithi07685715-19 from 779891 to 1265812 tid 1 2024-09-21T09:39:52.554 INFO:tasks.rados.rados.0.smithi076.stdout:489: writing smithi07685715-19 from 1947841 to 2633236 tid 2 2024-09-21T09:39:52.557 INFO:tasks.rados.rados.0.smithi076.stdout:489: writing smithi07685715-19 from 3383215 to 3579891 tid 3 2024-09-21T09:39:52.557 INFO:tasks.rados.rados.0.smithi076.stdout:490: delete oid 39 current snap is 63 2024-09-21T09:39:52.557 INFO:tasks.rados.rados.0.smithi076.stdout:485: expect (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:39:52.639 INFO:tasks.rados.rados.0.smithi076.stdout:486: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:52.751 INFO:tasks.rados.rados.0.smithi076.stdout:488: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:39:52.751 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 164 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:39:52.751 INFO:tasks.rados.rados.0.smithi076.stdout:489: finishing write tid 1 to smithi07685715-19 2024-09-21T09:39:52.751 INFO:tasks.rados.rados.0.smithi076.stdout:489: finishing write tid 2 to smithi07685715-19 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: finishing write tid 3 to smithi07685715-19 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: finishing write tid 4 to smithi07685715-19 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: finishing write tid 5 to smithi07685715-19 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: oid 19 updating version 0 to 137 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: oid 19 updating version 137 to 138 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: oid 19 version 138 is already newer than 136 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: oid 19 updating version 138 to 139 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 139 (ObjNum 213 snap 63 seq_num 213) dirty exists 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: left oid 19 (ObjNum 213 snap 63 seq_num 213) 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:485: done (4 left) 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:486: done (3 left) 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:488: done (2 left) 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:489: done (1 left) 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:490: done (0 left) 2024-09-21T09:39:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:491: write oid 18 current snap is 63 2024-09-21T09:39:52.753 INFO:tasks.rados.rados.0.smithi076.stdout:491: seq_num 215 ranges {413505=636419,1664805=689064,3038061=492071} 2024-09-21T09:39:52.757 INFO:tasks.rados.rados.0.smithi076.stdout:491: writing smithi07685715-18 from 413505 to 1049924 tid 1 2024-09-21T09:39:52.764 INFO:tasks.rados.rados.0.smithi076.stdout:491: writing smithi07685715-18 from 1664805 to 2353869 tid 2 2024-09-21T09:39:52.770 INFO:tasks.rados.rados.0.smithi076.stdout:491: writing smithi07685715-18 from 3038061 to 3530132 tid 3 2024-09-21T09:39:52.770 INFO:tasks.rados.rados.0.smithi076.stdout:492: read oid 21 snap 62 2024-09-21T09:39:52.771 INFO:tasks.rados.rados.0.smithi076.stdout:492: expect (ObjNum 201 snap 57 seq_num 201) 2024-09-21T09:39:52.771 INFO:tasks.rados.rados.0.smithi076.stdout:493: write oid 29 current snap is 63 2024-09-21T09:39:52.771 INFO:tasks.rados.rados.0.smithi076.stdout:491: finishing write tid 1 to smithi07685715-18 2024-09-21T09:39:52.771 INFO:tasks.rados.rados.0.smithi076.stdout:491: finishing write tid 2 to smithi07685715-18 2024-09-21T09:39:52.771 INFO:tasks.rados.rados.0.smithi076.stdout:493: seq_num 216 ranges {430312=711668,1723287=549075,2781837=703343} 2024-09-21T09:39:52.777 INFO:tasks.rados.rados.0.smithi076.stdout:493: writing smithi07685715-29 from 430312 to 1141980 tid 1 2024-09-21T09:39:52.783 INFO:tasks.rados.rados.0.smithi076.stdout:493: writing smithi07685715-29 from 1723287 to 2272362 tid 2 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:493: writing smithi07685715-29 from 2781837 to 3485180 tid 3 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:494: read oid 28 snap -1 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:494: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:495: rollback oid 42 current snap is 63 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 60 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:491: finishing write tid 3 to smithi07685715-18 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:491: finishing write tid 4 to smithi07685715-18 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:491: finishing write tid 5 to smithi07685715-18 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:491: oid 18 updating version 0 to 78 2024-09-21T09:39:52.791 INFO:tasks.rados.rados.0.smithi076.stdout:491: oid 18 version 78 is already newer than 77 2024-09-21T09:39:52.792 INFO:tasks.rados.rados.0.smithi076.stdout:491: oid 18 updating version 78 to 79 2024-09-21T09:39:52.792 INFO:tasks.rados.rados.0.smithi076.stdout:491: oid 18 version 79 is already newer than 76 2024-09-21T09:39:52.792 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 79 (ObjNum 215 snap 63 seq_num 215) dirty exists 2024-09-21T09:39:52.792 INFO:tasks.rados.rados.0.smithi076.stdout:491: left oid 18 (ObjNum 215 snap 63 seq_num 215) 2024-09-21T09:39:52.792 INFO:tasks.rados.rados.0.smithi076.stdout:492: expect (ObjNum 201 snap 57 seq_num 201) 2024-09-21T09:39:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-mon[33301]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.656Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.657Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.657Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[66029]: ts=2024-09-21T09:39:52.657Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:39:52.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:52 smithi191 podman[77686]: 2024-09-21 09:39:52.666828387 +0000 UTC m=+0.108799262 container died 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:52.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:52 smithi076 ceph-mon[29313]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:39:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:52 smithi076 ceph-mon[41325]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:493: finishing write tid 1 to smithi07685715-29 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:493: finishing write tid 2 to smithi07685715-29 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:495: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:495: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:495: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 82 (ObjNum 205 snap 58 seq_num 205) dirty exists 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:493: finishing write tid 3 to smithi07685715-29 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:493: finishing write tid 4 to smithi07685715-29 2024-09-21T09:39:52.939 INFO:tasks.rados.rados.0.smithi076.stdout:491: done (4 left) 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:492: done (3 left) 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:495: done (2 left) 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:496: delete oid 26 current snap is 63 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:493: finishing write tid 5 to smithi07685715-29 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:493: oid 29 updating version 0 to 101 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:493: oid 29 version 101 is already newer than 100 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:493: oid 29 updating version 101 to 103 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:493: oid 29 version 103 is already newer than 102 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 103 (ObjNum 216 snap 63 seq_num 216) dirty exists 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:493: left oid 29 (ObjNum 216 snap 63 seq_num 216) 2024-09-21T09:39:52.940 INFO:tasks.rados.rados.0.smithi076.stdout:494: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:39:53.065 INFO:tasks.rados.rados.0.smithi076.stdout:493: done (2 left) 2024-09-21T09:39:53.065 INFO:tasks.rados.rados.0.smithi076.stdout:494: done (1 left) 2024-09-21T09:39:53.065 INFO:tasks.rados.rados.0.smithi076.stdout:496: done (0 left) 2024-09-21T09:39:53.065 INFO:tasks.rados.rados.0.smithi076.stdout:497: write oid 5 current snap is 63 2024-09-21T09:39:53.065 INFO:tasks.rados.rados.0.smithi076.stdout:497: seq_num 218 ranges {463926=462166,1723898=436391,2603977=725887} 2024-09-21T09:39:53.070 INFO:tasks.rados.rados.0.smithi076.stdout:497: writing smithi07685715-5 from 463926 to 926092 tid 1 2024-09-21T09:39:53.076 INFO:tasks.rados.rados.0.smithi076.stdout:497: writing smithi07685715-5 from 1723898 to 2160289 tid 2 2024-09-21T09:39:53.083 INFO:tasks.rados.rados.0.smithi076.stdout:497: writing smithi07685715-5 from 2603977 to 3329864 tid 3 2024-09-21T09:39:53.083 INFO:tasks.rados.rados.0.smithi076.stdout:498: rollback oid 41 current snap is 63 2024-09-21T09:39:53.083 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 48 2024-09-21T09:39:53.083 INFO:tasks.rados.rados.0.smithi076.stdout:497: finishing write tid 1 to smithi07685715-5 2024-09-21T09:39:53.083 INFO:tasks.rados.rados.0.smithi076.stdout:497: finishing write tid 2 to smithi07685715-5 2024-09-21T09:39:53.084 INFO:tasks.rados.rados.0.smithi076.stdout:499: write oid 9 current snap is 63 2024-09-21T09:39:53.084 INFO:tasks.rados.rados.0.smithi076.stdout:499: seq_num 219 ranges {480733=537415,1782380=696402,3031400=537159} 2024-09-21T09:39:53.089 INFO:tasks.rados.rados.0.smithi076.stdout:499: writing smithi07685715-9 from 480733 to 1018148 tid 1 2024-09-21T09:39:53.097 INFO:tasks.rados.rados.0.smithi076.stdout:499: writing smithi07685715-9 from 1782380 to 2478782 tid 2 2024-09-21T09:39:53.103 INFO:tasks.rados.rados.0.smithi076.stdout:499: writing smithi07685715-9 from 3031400 to 3568559 tid 3 2024-09-21T09:39:53.103 INFO:tasks.rados.rados.0.smithi076.stdout:500: read oid 43 snap -1 2024-09-21T09:39:53.103 INFO:tasks.rados.rados.0.smithi076.stdout:500: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:39:53.103 INFO:tasks.rados.rados.0.smithi076.stdout:501: write oid 38 current snap is 63 2024-09-21T09:39:53.104 INFO:tasks.rados.rados.0.smithi076.stdout:498: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:39:53.104 INFO:tasks.rados.rados.0.smithi076.stdout:501: seq_num 220 ranges {497540=612664,1840862=440060,3058823=464784} 2024-09-21T09:39:53.109 INFO:tasks.rados.rados.0.smithi076.stdout:501: writing smithi07685715-38 from 497540 to 1110204 tid 1 2024-09-21T09:39:53.115 INFO:tasks.rados.rados.0.smithi076.stdout:501: writing smithi07685715-38 from 1840862 to 2280922 tid 2 2024-09-21T09:39:53.122 INFO:tasks.rados.rados.0.smithi076.stdout:501: writing smithi07685715-38 from 3058823 to 3523607 tid 3 2024-09-21T09:39:53.123 INFO:tasks.rados.rados.0.smithi076.stdout:502: snap_remove snap 47 2024-09-21T09:39:53.325 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 podman[77686]: 2024-09-21 09:39:53.071867508 +0000 UTC m=+0.513838378 container cleanup 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:53.325 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 bash[77686]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:39:53.517 INFO:tasks.rados.rados.0.smithi076.stdout:502: done (5 left) 2024-09-21T09:39:53.518 INFO:tasks.rados.rados.0.smithi076.stdout:503: write oid 2 current snap is 63 2024-09-21T09:39:53.518 INFO:tasks.rados.rados.0.smithi076.stdout:503: seq_num 221 ranges {514347=404266,1499344=700071,2686246=676056} 2024-09-21T09:39:53.523 INFO:tasks.rados.rados.0.smithi076.stdout:503: writing smithi07685715-2 from 514347 to 918613 tid 1 2024-09-21T09:39:53.530 INFO:tasks.rados.rados.0.smithi076.stdout:503: writing smithi07685715-2 from 1499344 to 2199415 tid 2 2024-09-21T09:39:53.537 INFO:tasks.rados.rados.0.smithi076.stdout:503: writing smithi07685715-2 from 2686246 to 3362302 tid 3 2024-09-21T09:39:53.537 INFO:tasks.rados.rados.0.smithi076.stdout:504: snap_create 2024-09-21T09:39:53.537 INFO:tasks.rados.rados.0.smithi076.stdout:498: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:39:53.537 INFO:tasks.rados.rados.0.smithi076.stdout:498: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:39:53.537 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 247 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: finishing write tid 3 to smithi07685715-5 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: finishing write tid 4 to smithi07685715-5 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: finishing write tid 5 to smithi07685715-5 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: oid 5 updating version 0 to 257 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: oid 5 version 257 is already newer than 256 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: oid 5 version 257 is already newer than 255 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: oid 5 updating version 257 to 258 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 258 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:497: left oid 5 (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:499: finishing write tid 1 to smithi07685715-9 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:499: finishing write tid 2 to smithi07685715-9 2024-09-21T09:39:53.538 INFO:tasks.rados.rados.0.smithi076.stdout:499: finishing write tid 3 to smithi07685715-9 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: finishing write tid 4 to smithi07685715-9 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: finishing write tid 5 to smithi07685715-9 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: oid 9 updating version 0 to 209 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: oid 9 updating version 209 to 210 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: oid 9 updating version 210 to 211 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: oid 9 version 211 is already newer than 208 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 211 (ObjNum 219 snap 63 seq_num 219) dirty exists 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:499: left oid 9 (ObjNum 219 snap 63 seq_num 219) 2024-09-21T09:39:53.539 INFO:tasks.rados.rados.0.smithi076.stdout:500: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:39:53.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-mon[33301]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:39:53.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-mon[33301]: pgmap v27: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:53.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:53.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-mon[33301]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:39:53.612 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 podman[77704]: 2024-09-21 09:39:53.32418216 +0000 UTC m=+0.656637892 container remove 8726c547d40aacf2c699cccad04cbb32c8933cfaa6a694bfcfc72494ac3b2bfc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:53.612 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:39:53.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 systemd[1]: Stopped Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:53.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.784s CPU time. 2024-09-21T09:39:53.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 systemd[1]: Starting Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:53.682 INFO:tasks.rados.rados.0.smithi076.stdout:501: finishing write tid 1 to smithi07685715-38 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: finishing write tid 2 to smithi07685715-38 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: finishing write tid 3 to smithi07685715-38 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: finishing write tid 4 to smithi07685715-38 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: finishing write tid 5 to smithi07685715-38 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: oid 38 updating version 0 to 234 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: oid 38 version 234 is already newer than 233 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: oid 38 version 234 is already newer than 231 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: oid 38 version 234 is already newer than 232 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 234 (ObjNum 220 snap 63 seq_num 220) dirty exists 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:501: left oid 38 (ObjNum 220 snap 63 seq_num 220) 2024-09-21T09:39:53.683 INFO:tasks.rados.rados.0.smithi076.stdout:503: finishing write tid 1 to smithi07685715-2 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: finishing write tid 2 to smithi07685715-2 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: finishing write tid 3 to smithi07685715-2 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: finishing write tid 4 to smithi07685715-2 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: finishing write tid 5 to smithi07685715-2 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: oid 2 updating version 0 to 61 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: oid 2 version 61 is already newer than 60 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: oid 2 updating version 61 to 63 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: oid 2 version 63 is already newer than 62 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 63 (ObjNum 221 snap 63 seq_num 221) dirty exists 2024-09-21T09:39:53.684 INFO:tasks.rados.rados.0.smithi076.stdout:503: left oid 2 (ObjNum 221 snap 63 seq_num 221) 2024-09-21T09:39:53.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[29313]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[29313]: pgmap v27: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[29313]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[41325]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[41325]: pgmap v27: 89 pgs: 89 active+clean; 184 MiB data, 599 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:39:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:53 smithi076 ceph-mon[41325]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:39:53.971 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 podman[77802]: 2024-09-21 09:39:53.618815943 +0000 UTC m=+0.017089591 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:39:53.971 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 podman[77802]: 2024-09-21 09:39:53.717161146 +0000 UTC m=+0.115434784 container create 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:53.971 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 podman[77802]: 2024-09-21 09:39:53.970125963 +0000 UTC m=+0.368399607 container init 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 podman[77802]: 2024-09-21 09:39:53.974894957 +0000 UTC m=+0.373168596 container start 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:53.998Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:53.998Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:53.998Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi191 (none))" 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:53.998Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:53.998Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.000Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.000Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:39:54.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.001Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.001Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.002Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.002Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.793µs 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.002Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.004Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.011Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.012Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.012Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.893µs wal_replay_duration=9.728959ms wbl_replay_duration=133ns total_replay_duration=9.763431ms 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.014Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.014Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.014Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.035Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.537836ms db_storage=692ns remote_storage=1.162µs web_handler=303ns query_engine=621ns scrape=312.17µs scrape_sd=128.218µs notify=11.07µs notify_sd=19.495µs rules=20.680834ms tracing=5.91µs 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.035Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:39:54.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:39:54.035Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:39:54.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 bash[77802]: 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc 2024-09-21T09:39:54.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:39:54 smithi191 systemd[1]: Started Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:497: done (6 left) 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:498: done (5 left) 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:499: done (4 left) 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:500: done (3 left) 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:501: done (2 left) 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:503: done (1 left) 2024-09-21T09:39:54.543 INFO:tasks.rados.rados.0.smithi076.stdout:504: done (0 left) 2024-09-21T09:39:54.544 INFO:tasks.rados.rados.0.smithi076.stdout:505: write oid 32 current snap is 64 2024-09-21T09:39:54.544 INFO:tasks.rados.rados.0.smithi076.stdout:505: seq_num 222 ranges {531154=479515,1557826=443729,2713669=487328,3679295=51859} 2024-09-21T09:39:54.548 INFO:tasks.rados.rados.0.smithi076.stdout:505: writing smithi07685715-32 from 531154 to 1010669 tid 1 2024-09-21T09:39:54.554 INFO:tasks.rados.rados.0.smithi076.stdout:505: writing smithi07685715-32 from 1557826 to 2001555 tid 2 2024-09-21T09:39:54.560 INFO:tasks.rados.rados.0.smithi076.stdout:505: writing smithi07685715-32 from 2713669 to 3200997 tid 3 2024-09-21T09:39:54.564 INFO:tasks.rados.rados.0.smithi076.stdout:505: writing smithi07685715-32 from 3679295 to 3731154 tid 4 2024-09-21T09:39:54.564 INFO:tasks.rados.rados.0.smithi076.stdout:506: read oid 47 snap -1 2024-09-21T09:39:54.564 INFO:tasks.rados.rados.0.smithi076.stdout:506: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:54.564 INFO:tasks.rados.rados.0.smithi076.stdout:507: write oid 21 current snap is 64 2024-09-21T09:39:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:505: finishing write tid 1 to smithi07685715-32 2024-09-21T09:39:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:505: finishing write tid 2 to smithi07685715-32 2024-09-21T09:39:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:505: finishing write tid 3 to smithi07685715-32 2024-09-21T09:39:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:507: seq_num 223 ranges {547961=554764,1616308=703740,2741092=698600} 2024-09-21T09:39:54.570 INFO:tasks.rados.rados.0.smithi076.stdout:507: writing smithi07685715-21 from 547961 to 1102725 tid 1 2024-09-21T09:39:54.577 INFO:tasks.rados.rados.0.smithi076.stdout:507: writing smithi07685715-21 from 1616308 to 2320048 tid 2 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:507: writing smithi07685715-21 from 2741092 to 3439692 tid 3 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: finishing write tid 4 to smithi07685715-32 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: finishing write tid 5 to smithi07685715-32 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: finishing write tid 6 to smithi07685715-32 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: oid 32 updating version 0 to 144 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: oid 32 updating version 144 to 145 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: oid 32 updating version 145 to 146 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: oid 32 updating version 146 to 147 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: oid 32 version 147 is already newer than 143 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 147 (ObjNum 222 snap 64 seq_num 222) dirty exists 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:505: left oid 32 (ObjNum 222 snap 64 seq_num 222) 2024-09-21T09:39:54.584 INFO:tasks.rados.rados.0.smithi076.stdout:506: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:39:54.702 INFO:tasks.rados.rados.0.smithi076.stdout:507: finishing write tid 1 to smithi07685715-21 2024-09-21T09:39:54.702 INFO:tasks.rados.rados.0.smithi076.stdout:507: finishing write tid 2 to smithi07685715-21 2024-09-21T09:39:54.702 INFO:tasks.rados.rados.0.smithi076.stdout:507: finishing write tid 3 to smithi07685715-21 2024-09-21T09:39:54.702 INFO:tasks.rados.rados.0.smithi076.stdout:507: finishing write tid 4 to smithi07685715-21 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: finishing write tid 5 to smithi07685715-21 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: oid 21 updating version 0 to 274 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: oid 21 updating version 274 to 275 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: oid 21 version 275 is already newer than 272 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: oid 21 version 275 is already newer than 273 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 275 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: left oid 21 (ObjNum 223 snap 64 seq_num 223) 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:505: done (2 left) 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:506: done (1 left) 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:507: done (0 left) 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:508: rollback oid 11 current snap is 64 2024-09-21T09:39:54.703 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 61 2024-09-21T09:39:54.704 INFO:tasks.rados.rados.0.smithi076.stdout:509: snap_create 2024-09-21T09:39:54.704 INFO:tasks.rados.rados.0.smithi076.stdout:508: finishing rollback tid 0 to smithi07685715-11 2024-09-21T09:39:54.705 INFO:tasks.rados.rados.0.smithi076.stdout:508: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:39:54.705 INFO:tasks.rados.rados.0.smithi076.stdout:508: finishing rollback tid 2 to smithi07685715-11 2024-09-21T09:39:54.705 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 279 (ObjNum 183 snap 48 seq_num 183) dirty exists 2024-09-21T09:39:55.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:55.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:39:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:39:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:39:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:39:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:55 smithi191 ceph-mon[33301]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:39:55.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[29313]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:39:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:55.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:39:55.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:39:55.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:55 smithi076 ceph-mon[41325]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:39:55.542 INFO:tasks.rados.rados.0.smithi076.stdout:508: done (1 left) 2024-09-21T09:39:55.543 INFO:tasks.rados.rados.0.smithi076.stdout:509: done (0 left) 2024-09-21T09:39:55.543 INFO:tasks.rados.rados.0.smithi076.stdout:510: delete oid 45 current snap is 65 2024-09-21T09:39:55.545 INFO:tasks.rados.rados.0.smithi076.stdout:510: done (0 left) 2024-09-21T09:39:55.545 INFO:tasks.rados.rados.0.smithi076.stdout:511: snap_create 2024-09-21T09:39:56.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: pgmap v30: 89 pgs: 89 active+clean; 188 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[29313]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: pgmap v30: 89 pgs: 89 active+clean; 188 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-21T09:39:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:56 smithi076 ceph-mon[41325]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: pgmap v30: 89 pgs: 89 active+clean; 188 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:39:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:56 smithi191 ceph-mon[33301]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:39:56.545 INFO:tasks.rados.rados.0.smithi076.stdout:511: done (0 left) 2024-09-21T09:39:56.545 INFO:tasks.rados.rados.0.smithi076.stdout:512: read oid 2 snap -1 2024-09-21T09:39:56.545 INFO:tasks.rados.rados.0.smithi076.stdout:512: expect (ObjNum 221 snap 63 seq_num 221) 2024-09-21T09:39:56.545 INFO:tasks.rados.rados.0.smithi076.stdout:513: rollback oid 25 current snap is 66 2024-09-21T09:39:56.545 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 64 2024-09-21T09:39:56.545 INFO:tasks.rados.rados.0.smithi076.stdout:514: write oid 50 current snap is 66 2024-09-21T09:39:56.546 INFO:tasks.rados.rados.0.smithi076.stdout:514: seq_num 225 ranges {581575=705262,1733272=707409,3195938=437497} 2024-09-21T09:39:56.552 INFO:tasks.rados.rados.0.smithi076.stdout:514: writing smithi07685715-50 from 581575 to 1286837 tid 1 2024-09-21T09:39:56.558 INFO:tasks.rados.rados.0.smithi076.stdout:514: writing smithi07685715-50 from 1733272 to 2440681 tid 2 2024-09-21T09:39:56.565 INFO:tasks.rados.rados.0.smithi076.stdout:514: writing smithi07685715-50 from 3195938 to 3633435 tid 3 2024-09-21T09:39:56.565 INFO:tasks.rados.rados.0.smithi076.stdout:515: read oid 34 snap -1 2024-09-21T09:39:56.565 INFO:tasks.rados.rados.0.smithi076.stdout:515: expect (ObjNum 187 snap 49 seq_num 187) 2024-09-21T09:39:56.565 INFO:tasks.rados.rados.0.smithi076.stdout:516: read oid 1 snap 43 2024-09-21T09:39:56.565 INFO:tasks.rados.rados.0.smithi076.stdout:516: expect deleted 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:517: read oid 39 snap -1 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:517: expect deleted 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:518: write oid 27 current snap is 66 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:513: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:513: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:513: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 128 (ObjNum 194 snap 55 seq_num 194) dirty exists 2024-09-21T09:39:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:512: expect (ObjNum 221 snap 63 seq_num 221) 2024-09-21T09:39:56.684 INFO:tasks.rados.rados.0.smithi076.stdout:514: finishing write tid 1 to smithi07685715-50 2024-09-21T09:39:56.684 INFO:tasks.rados.rados.0.smithi076.stdout:514: finishing write tid 2 to smithi07685715-50 2024-09-21T09:39:56.684 INFO:tasks.rados.rados.0.smithi076.stdout:518: seq_num 226 ranges {598382=780511,2075401=451067,3107008=648769} 2024-09-21T09:39:56.691 INFO:tasks.rados.rados.0.smithi076.stdout:518: writing smithi07685715-27 from 598382 to 1378893 tid 1 2024-09-21T09:39:56.698 INFO:tasks.rados.rados.0.smithi076.stdout:518: writing smithi07685715-27 from 2075401 to 2526468 tid 2 2024-09-21T09:39:56.704 INFO:tasks.rados.rados.0.smithi076.stdout:518: writing smithi07685715-27 from 3107008 to 3755777 tid 3 2024-09-21T09:39:56.705 INFO:tasks.rados.rados.0.smithi076.stdout:512: done (6 left) 2024-09-21T09:39:56.705 INFO:tasks.rados.rados.0.smithi076.stdout:513: done (5 left) 2024-09-21T09:39:56.705 INFO:tasks.rados.rados.0.smithi076.stdout:519: write oid 20 current snap is 66 2024-09-21T09:39:56.705 INFO:tasks.rados.rados.0.smithi076.stdout:519: seq_num 227 ranges {615189=455760,1733883=711078,3134431=460041} 2024-09-21T09:39:56.710 INFO:tasks.rados.rados.0.smithi076.stdout:519: writing smithi07685715-20 from 615189 to 1070949 tid 1 2024-09-21T09:39:56.718 INFO:tasks.rados.rados.0.smithi076.stdout:519: writing smithi07685715-20 from 1733883 to 2444961 tid 2 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:519: writing smithi07685715-20 from 3134431 to 3594472 tid 3 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:520: read oid 18 snap 59 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:520: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:521: rollback oid 30 current snap is 66 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 40 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:514: finishing write tid 3 to smithi07685715-50 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:514: finishing write tid 4 to smithi07685715-50 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:514: finishing write tid 5 to smithi07685715-50 2024-09-21T09:39:56.725 INFO:tasks.rados.rados.0.smithi076.stdout:514: oid 50 updating version 0 to 146 2024-09-21T09:39:56.726 INFO:tasks.rados.rados.0.smithi076.stdout:514: oid 50 updating version 146 to 148 2024-09-21T09:39:56.726 INFO:tasks.rados.rados.0.smithi076.stdout:514: oid 50 version 148 is already newer than 147 2024-09-21T09:39:56.726 INFO:tasks.rados.rados.0.smithi076.stdout:514: oid 50 updating version 148 to 149 2024-09-21T09:39:56.726 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 149 (ObjNum 225 snap 66 seq_num 225) dirty exists 2024-09-21T09:39:56.726 INFO:tasks.rados.rados.0.smithi076.stdout:514: left oid 50 (ObjNum 225 snap 66 seq_num 225) 2024-09-21T09:39:56.726 INFO:tasks.rados.rados.0.smithi076.stdout:515: expect (ObjNum 187 snap 49 seq_num 187) 2024-09-21T09:39:56.831 INFO:tasks.rados.rados.0.smithi076.stdout:518: finishing write tid 1 to smithi07685715-27 2024-09-21T09:39:56.831 INFO:tasks.rados.rados.0.smithi076.stdout:518: finishing write tid 2 to smithi07685715-27 2024-09-21T09:39:56.831 INFO:tasks.rados.rados.0.smithi076.stdout:518: finishing write tid 3 to smithi07685715-27 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: finishing write tid 4 to smithi07685715-27 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: finishing write tid 5 to smithi07685715-27 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: oid 27 updating version 0 to 255 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: oid 27 updating version 255 to 258 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: oid 27 version 258 is already newer than 257 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: oid 27 version 258 is already newer than 256 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 258 (ObjNum 226 snap 66 seq_num 226) dirty exists 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: left oid 27 (ObjNum 226 snap 66 seq_num 226) 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:519: finishing write tid 1 to smithi07685715-20 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:519: finishing write tid 2 to smithi07685715-20 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:514: done (7 left) 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:515: done (6 left) 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:516: done (5 left) 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:517: done (4 left) 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:518: done (3 left) 2024-09-21T09:39:56.832 INFO:tasks.rados.rados.0.smithi076.stdout:522: snap_create 2024-09-21T09:39:56.833 INFO:tasks.rados.rados.0.smithi076.stdout:520: expect (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: finishing write tid 3 to smithi07685715-20 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: finishing write tid 4 to smithi07685715-20 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: finishing write tid 5 to smithi07685715-20 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: oid 20 updating version 0 to 106 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: oid 20 updating version 106 to 107 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: oid 20 updating version 107 to 109 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: oid 20 version 109 is already newer than 108 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 109 (ObjNum 227 snap 66 seq_num 227) dirty exists 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:519: left oid 20 (ObjNum 227 snap 66 seq_num 227) 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:521: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:521: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:39:56.904 INFO:tasks.rados.rados.0.smithi076.stdout:521: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:39:56.905 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 407 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:39:57.160 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:57 smithi076 ceph-mon[29313]: Upgrade: Updating mgr.y 2024-09-21T09:39:57.160 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:57 smithi076 ceph-mon[29313]: Deploying daemon mgr.y on smithi076 2024-09-21T09:39:57.160 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:57 smithi076 ceph-mon[29313]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:39:57.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:57 smithi076 ceph-mon[41325]: Upgrade: Updating mgr.y 2024-09-21T09:39:57.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:57 smithi076 ceph-mon[41325]: Deploying daemon mgr.y on smithi076 2024-09-21T09:39:57.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:57 smithi076 ceph-mon[41325]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:39:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:57 smithi191 ceph-mon[33301]: Upgrade: Updating mgr.y 2024-09-21T09:39:57.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:57 smithi191 ceph-mon[33301]: Deploying daemon mgr.y on smithi076 2024-09-21T09:39:57.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:57 smithi191 ceph-mon[33301]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:39:57.446 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:57 smithi076 systemd[1]: Stopping Ceph mgr.y for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:57.547 INFO:tasks.rados.rados.0.smithi076.stdout:519: done (3 left) 2024-09-21T09:39:57.547 INFO:tasks.rados.rados.0.smithi076.stdout:520: done (2 left) 2024-09-21T09:39:57.548 INFO:tasks.rados.rados.0.smithi076.stdout:521: done (1 left) 2024-09-21T09:39:57.548 INFO:tasks.rados.rados.0.smithi076.stdout:522: done (0 left) 2024-09-21T09:39:57.548 INFO:tasks.rados.rados.0.smithi076.stdout:523: delete oid 22 current snap is 67 2024-09-21T09:39:57.552 INFO:tasks.rados.rados.0.smithi076.stdout:523: done (0 left) 2024-09-21T09:39:57.552 INFO:tasks.rados.rados.0.smithi076.stdout:524: snap_create 2024-09-21T09:39:57.778 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:57 smithi076 podman[89041]: 2024-09-21 09:39:57.444359238 +0000 UTC m=+0.141150222 container died 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T09:39:58.109 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:57 smithi076 podman[89041]: 2024-09-21 09:39:57.777285578 +0000 UTC m=+0.474076555 container cleanup 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8) 2024-09-21T09:39:58.109 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:57 smithi076 bash[89041]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y 2024-09-21T09:39:58.109 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 podman[89058]: 2024-09-21 09:39:58.012493559 +0000 UTC m=+0.566123652 container remove 3f03f8aa752c0bf0322c04472c35e124e1c7f9e3f33947812c6c6a89ced85462 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:39:58.109 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:39:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:58 smithi191 ceph-mon[33301]: pgmap v33: 89 pgs: 89 active+clean; 188 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-21T09:39:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:58 smithi191 ceph-mon[33301]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:39:58.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:58 smithi076 ceph-mon[29313]: pgmap v33: 89 pgs: 89 active+clean; 188 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-21T09:39:58.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:58 smithi076 ceph-mon[29313]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:39:58.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service: Failed with result 'exit-code'. 2024-09-21T09:39:58.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 systemd[1]: Stopped Ceph mgr.y for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:58.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service: Consumed 42.993s CPU time. 2024-09-21T09:39:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:58 smithi076 ceph-mon[41325]: pgmap v33: 89 pgs: 89 active+clean; 188 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 3.3 MiB/s wr, 30 op/s 2024-09-21T09:39:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:58 smithi076 ceph-mon[41325]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:39:58.521 INFO:tasks.rados.rados.0.smithi076.stdout:524: done (0 left) 2024-09-21T09:39:58.521 INFO:tasks.rados.rados.0.smithi076.stdout:525: snap_remove snap 67 2024-09-21T09:39:58.813 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 systemd[1]: Starting Ceph mgr.y for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:39:58.813 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 podman[89189]: 2024-09-21 09:39:58.561995871 +0000 UTC m=+0.015897658 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:39:58.813 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 podman[89189]: 2024-09-21 09:39:58.68904373 +0000 UTC m=+0.142945496 container create 151afa33d0eeb7e634a87f1659f7562e17d0cc4fbb37a4fd13e75dc33c14dea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:39:59.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 podman[89189]: 2024-09-21 09:39:58.916519099 +0000 UTC m=+0.370420874 container init 151afa33d0eeb7e634a87f1659f7562e17d0cc4fbb37a4fd13e75dc33c14dea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T09:39:59.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 podman[89189]: 2024-09-21 09:39:58.92156987 +0000 UTC m=+0.375471647 container start 151afa33d0eeb7e634a87f1659f7562e17d0cc4fbb37a4fd13e75dc33c14dea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T09:39:59.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:58 smithi076 ceph-mgr[89227]: -- 172.21.15.76:0/1994473289 <== mon.2 v2:172.21.15.76:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562929f5f0e0 con 0x562929f3d000 2024-09-21T09:39:59.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:59 smithi076 bash[89189]: 151afa33d0eeb7e634a87f1659f7562e17d0cc4fbb37a4fd13e75dc33c14dea8 2024-09-21T09:39:59.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:39:59 smithi076 systemd[1]: Started Ceph mgr.y for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:39:59.534 INFO:tasks.rados.rados.0.smithi076.stdout:525: done (0 left) 2024-09-21T09:39:59.534 INFO:tasks.rados.rados.0.smithi076.stdout:526: write oid 45 current snap is 68 2024-09-21T09:39:59.534 INFO:tasks.rados.rados.0.smithi076.stdout:526: seq_num 229 ranges {648803=722611,1967200=714747,3421983=426820} 2024-09-21T09:39:59.541 INFO:tasks.rados.rados.0.smithi076.stdout:526: writing smithi07685715-45 from 648803 to 1371414 tid 1 2024-09-21T09:39:59.548 INFO:tasks.rados.rados.0.smithi076.stdout:526: writing smithi07685715-45 from 1967200 to 2681947 tid 2 2024-09-21T09:39:59.555 INFO:tasks.rados.rados.0.smithi076.stdout:526: writing smithi07685715-45 from 3421983 to 3848803 tid 3 2024-09-21T09:39:59.555 INFO:tasks.rados.rados.0.smithi076.stdout:527: snap_remove snap 62 2024-09-21T09:39:59.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:59 smithi191 ceph-mon[33301]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:39:59.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:59 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:59.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:59 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:59.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:59 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:59.806 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:39:59 smithi191 ceph-mon[33301]: pgmap v36: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:39:59.864 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[29313]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[29313]: pgmap v36: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[41325]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:39:59.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:39:59 smithi076 ceph-mon[41325]: pgmap v36: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:40:00.541 INFO:tasks.rados.rados.0.smithi076.stdout:527: done (1 left) 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:528: rollback oid 12 current snap is 68 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 66 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: finishing write tid 1 to smithi07685715-45 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: finishing write tid 2 to smithi07685715-45 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: finishing write tid 3 to smithi07685715-45 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: finishing write tid 4 to smithi07685715-45 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: finishing write tid 5 to smithi07685715-45 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: oid 45 updating version 0 to 81 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: oid 45 updating version 81 to 82 2024-09-21T09:40:00.542 INFO:tasks.rados.rados.0.smithi076.stdout:526: oid 45 version 82 is already newer than 80 2024-09-21T09:40:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:526: oid 45 updating version 82 to 83 2024-09-21T09:40:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 83 (ObjNum 229 snap 68 seq_num 229) dirty exists 2024-09-21T09:40:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:526: left oid 45 (ObjNum 229 snap 68 seq_num 229) 2024-09-21T09:40:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:526: done (1 left) 2024-09-21T09:40:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:529: snap_create 2024-09-21T09:40:00.548 INFO:tasks.rados.rados.0.smithi076.stdout:528: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:40:00.549 INFO:tasks.rados.rados.0.smithi076.stdout:528: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:40:00.549 INFO:tasks.rados.rados.0.smithi076.stdout:528: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:40:00.549 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 219 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-09-21T09:40:00.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:00 smithi191 ceph-mon[33301]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:40:00.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:00 smithi191 ceph-mon[33301]: overall HEALTH_OK 2024-09-21T09:40:00.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:00 smithi076 ceph-mon[29313]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:40:00.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:00 smithi076 ceph-mon[29313]: overall HEALTH_OK 2024-09-21T09:40:00.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:00 smithi076 ceph-mon[41325]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:40:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:00 smithi076 ceph-mon[41325]: overall HEALTH_OK 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:528: done (1 left) 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:529: done (0 left) 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:530: rollback oid 7 current snap is 69 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 59 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:531: read oid 4 snap -1 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:531: expect (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:532: write oid 34 current snap is 69 2024-09-21T09:40:01.548 INFO:tasks.rados.rados.0.smithi076.stdout:532: seq_num 230 ranges {665610=797860,1909329=458405,2816700=410210} 2024-09-21T09:40:01.556 INFO:tasks.rados.rados.0.smithi076.stdout:532: writing smithi07685715-34 from 665610 to 1463470 tid 1 2024-09-21T09:40:01.562 INFO:tasks.rados.rados.0.smithi076.stdout:532: writing smithi07685715-34 from 1909329 to 2367734 tid 2 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:532: writing smithi07685715-34 from 2816700 to 3226910 tid 3 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:533: snap_create 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:530: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:530: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:530: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 242 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-09-21T09:40:01.568 INFO:tasks.rados.rados.0.smithi076.stdout:531: expect (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:40:01.707 INFO:tasks.rados.rados.0.smithi076.stdout:532: finishing write tid 1 to smithi07685715-34 2024-09-21T09:40:01.707 INFO:tasks.rados.rados.0.smithi076.stdout:532: finishing write tid 2 to smithi07685715-34 2024-09-21T09:40:01.707 INFO:tasks.rados.rados.0.smithi076.stdout:532: finishing write tid 3 to smithi07685715-34 2024-09-21T09:40:01.707 INFO:tasks.rados.rados.0.smithi076.stdout:532: finishing write tid 4 to smithi07685715-34 2024-09-21T09:40:01.707 INFO:tasks.rados.rados.0.smithi076.stdout:532: finishing write tid 5 to smithi07685715-34 2024-09-21T09:40:01.708 INFO:tasks.rados.rados.0.smithi076.stdout:532: oid 34 updating version 0 to 76 2024-09-21T09:40:01.708 INFO:tasks.rados.rados.0.smithi076.stdout:532: oid 34 version 76 is already newer than 75 2024-09-21T09:40:01.708 INFO:tasks.rados.rados.0.smithi076.stdout:532: oid 34 updating version 76 to 77 2024-09-21T09:40:01.708 INFO:tasks.rados.rados.0.smithi076.stdout:532: oid 34 version 77 is already newer than 74 2024-09-21T09:40:01.708 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 77 (ObjNum 230 snap 69 seq_num 230) dirty exists 2024-09-21T09:40:01.708 INFO:tasks.rados.rados.0.smithi076.stdout:532: left oid 34 (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:40:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:01 smithi191 ceph-mon[33301]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:40:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:01 smithi191 ceph-mon[33301]: pgmap v39: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:40:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:01 smithi076 ceph-mon[29313]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:40:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:01 smithi076 ceph-mon[29313]: pgmap v39: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:40:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:01 smithi076 ceph-mon[41325]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:40:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:01 smithi076 ceph-mon[41325]: pgmap v39: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:40:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:530: done (3 left) 2024-09-21T09:40:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:531: done (2 left) 2024-09-21T09:40:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:532: done (1 left) 2024-09-21T09:40:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:533: done (0 left) 2024-09-21T09:40:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:534: delete oid 49 current snap is 70 2024-09-21T09:40:02.562 INFO:tasks.rados.rados.0.smithi076.stdout:534: done (0 left) 2024-09-21T09:40:02.562 INFO:tasks.rados.rados.0.smithi076.stdout:535: snap_remove snap 60 2024-09-21T09:40:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:02 smithi191 ceph-mon[33301]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:40:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:02 smithi076 ceph-mon[29313]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:40:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:02 smithi076 ceph-mon[41325]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:40:03.426 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:40:03] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-09-21T09:40:03.557 INFO:tasks.rados.rados.0.smithi076.stdout:535: done (0 left) 2024-09-21T09:40:03.558 INFO:tasks.rados.rados.0.smithi076.stdout:536: write oid 45 current snap is 70 2024-09-21T09:40:03.558 INFO:tasks.rados.rados.0.smithi076.stdout:536: seq_num 232 ranges {699224=548358,2026293=462074,3271546=432754} 2024-09-21T09:40:03.565 INFO:tasks.rados.rados.0.smithi076.stdout:536: writing smithi07685715-45 from 699224 to 1247582 tid 1 2024-09-21T09:40:03.572 INFO:tasks.rados.rados.0.smithi076.stdout:536: writing smithi07685715-45 from 2026293 to 2488367 tid 2 2024-09-21T09:40:03.579 INFO:tasks.rados.rados.0.smithi076.stdout:536: writing smithi07685715-45 from 3271546 to 3704300 tid 3 2024-09-21T09:40:03.579 INFO:tasks.rados.rados.0.smithi076.stdout:537: delete oid 10 current snap is 70 2024-09-21T09:40:03.579 INFO:tasks.rados.rados.0.smithi076.stdout:536: finishing write tid 1 to smithi07685715-45 2024-09-21T09:40:03.580 INFO:tasks.rados.rados.0.smithi076.stdout:536: finishing write tid 2 to smithi07685715-45 2024-09-21T09:40:03.580 INFO:tasks.rados.rados.0.smithi076.stdout:537: done (1 left) 2024-09-21T09:40:03.581 INFO:tasks.rados.rados.0.smithi076.stdout:538: write oid 48 current snap is 70 2024-09-21T09:40:03.581 INFO:tasks.rados.rados.0.smithi076.stdout:538: seq_num 234 ranges {732838=698856,2026904=465743,3210039=571651} 2024-09-21T09:40:03.588 INFO:tasks.rados.rados.0.smithi076.stdout:538: writing smithi07685715-48 from 732838 to 1431694 tid 1 2024-09-21T09:40:03.595 INFO:tasks.rados.rados.0.smithi076.stdout:538: writing smithi07685715-48 from 2026904 to 2492647 tid 2 2024-09-21T09:40:03.603 INFO:tasks.rados.rados.0.smithi076.stdout:538: writing smithi07685715-48 from 3210039 to 3781690 tid 3 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: finishing write tid 3 to smithi07685715-45 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: finishing write tid 4 to smithi07685715-45 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: finishing write tid 5 to smithi07685715-45 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: oid 45 updating version 0 to 92 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: oid 45 version 92 is already newer than 90 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: oid 45 version 92 is already newer than 91 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: oid 45 version 92 is already newer than 89 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 92 (ObjNum 232 snap 70 seq_num 232) dirty exists 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: left oid 45 (ObjNum 232 snap 70 seq_num 232) 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:538: finishing write tid 1 to smithi07685715-48 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:538: finishing write tid 2 to smithi07685715-48 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:536: done (1 left) 2024-09-21T09:40:03.604 INFO:tasks.rados.rados.0.smithi076.stdout:539: snap_create 2024-09-21T09:40:03.607 INFO:tasks.rados.rados.0.smithi076.stdout:538: finishing write tid 3 to smithi07685715-48 2024-09-21T09:40:03.607 INFO:tasks.rados.rados.0.smithi076.stdout:538: finishing write tid 4 to smithi07685715-48 2024-09-21T09:40:03.607 INFO:tasks.rados.rados.0.smithi076.stdout:538: finishing write tid 5 to smithi07685715-48 2024-09-21T09:40:03.608 INFO:tasks.rados.rados.0.smithi076.stdout:538: oid 48 updating version 0 to 423 2024-09-21T09:40:03.608 INFO:tasks.rados.rados.0.smithi076.stdout:538: oid 48 version 423 is already newer than 421 2024-09-21T09:40:03.608 INFO:tasks.rados.rados.0.smithi076.stdout:538: oid 48 updating version 423 to 424 2024-09-21T09:40:03.608 INFO:tasks.rados.rados.0.smithi076.stdout:538: oid 48 version 424 is already newer than 422 2024-09-21T09:40:03.608 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 424 (ObjNum 234 snap 70 seq_num 234) dirty exists 2024-09-21T09:40:03.608 INFO:tasks.rados.rados.0.smithi076.stdout:538: left oid 48 (ObjNum 234 snap 70 seq_num 234) 2024-09-21T09:40:03.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[29313]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:40:03.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[29313]: pgmap v42: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:40:03.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[41325]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[41325]: pgmap v42: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:03 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-mon[33301]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:40:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-mon[33301]: pgmap v42: 89 pgs: 89 active+clean; 201 MiB data, 633 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:40:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:03 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:04] ENGINE Bus STARTING 2024-09-21T09:40:04.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: CherryPy Checker: 2024-09-21T09:40:04.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: The Application mounted at '' has an empty config. 2024-09-21T09:40:04.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:04.574 INFO:tasks.rados.rados.0.smithi076.stdout:538: done (1 left) 2024-09-21T09:40:04.574 INFO:tasks.rados.rados.0.smithi076.stdout:539: done (0 left) 2024-09-21T09:40:04.574 INFO:tasks.rados.rados.0.smithi076.stdout:540: read oid 42 snap -1 2024-09-21T09:40:04.574 INFO:tasks.rados.rados.0.smithi076.stdout:540: expect (ObjNum 205 snap 58 seq_num 205) 2024-09-21T09:40:04.574 INFO:tasks.rados.rados.0.smithi076.stdout:541: snap_create 2024-09-21T09:40:04.582 INFO:tasks.rados.rados.0.smithi076.stdout:540: expect (ObjNum 205 snap 58 seq_num 205) 2024-09-21T09:40:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:40:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: Standby manager daemon y restarted 2024-09-21T09:40:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: Standby manager daemon y started 2024-09-21T09:40:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:40:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:40:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:04 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:04.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:04] ENGINE Serving on http://:::9283 2024-09-21T09:40:04.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:04] ENGINE Bus STARTED 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: Standby manager daemon y restarted 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: Standby manager daemon y started 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: Standby manager daemon y restarted 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: Standby manager daemon y started 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:40:04.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:04 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:05.575 INFO:tasks.rados.rados.0.smithi076.stdout:540: done (1 left) 2024-09-21T09:40:05.575 INFO:tasks.rados.rados.0.smithi076.stdout:541: done (0 left) 2024-09-21T09:40:05.575 INFO:tasks.rados.rados.0.smithi076.stdout:542: delete oid 14 current snap is 72 2024-09-21T09:40:05.577 INFO:tasks.rados.rados.0.smithi076.stdout:542: done (0 left) 2024-09-21T09:40:05.578 INFO:tasks.rados.rados.0.smithi076.stdout:543: write oid 24 current snap is 72 2024-09-21T09:40:05.578 INFO:tasks.rados.rados.0.smithi076.stdout:543: seq_num 236 ranges {766452=565707,1860221=469412,2981238=594195} 2024-09-21T09:40:05.586 INFO:tasks.rados.rados.0.smithi076.stdout:543: writing smithi07685715-24 from 766452 to 1332159 tid 1 2024-09-21T09:40:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: writing smithi07685715-24 from 1860221 to 2329633 tid 2 2024-09-21T09:40:05.599 INFO:tasks.rados.rados.0.smithi076.stdout:543: writing smithi07685715-24 from 2981238 to 3575433 tid 3 2024-09-21T09:40:05.599 INFO:tasks.rados.rados.0.smithi076.stdout:544: read oid 16 snap 43 2024-09-21T09:40:05.599 INFO:tasks.rados.rados.0.smithi076.stdout:544: expect (ObjNum 117 snap 20 seq_num 117) 2024-09-21T09:40:05.599 INFO:tasks.rados.rados.0.smithi076.stdout:545: snap_remove snap 61 2024-09-21T09:40:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:05 smithi191 ceph-mon[33301]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:40:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:05 smithi191 ceph-mon[33301]: mgrmap e27: x(active, since 27s), standbys: y 2024-09-21T09:40:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:05 smithi191 ceph-mon[33301]: pgmap v45: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 208 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:05.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:05 smithi076 ceph-mon[41325]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:40:05.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:05 smithi076 ceph-mon[41325]: mgrmap e27: x(active, since 27s), standbys: y 2024-09-21T09:40:05.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:05 smithi076 ceph-mon[41325]: pgmap v45: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 208 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:05 smithi076 ceph-mon[29313]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:40:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:05 smithi076 ceph-mon[29313]: mgrmap e27: x(active, since 27s), standbys: y 2024-09-21T09:40:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:05 smithi076 ceph-mon[29313]: pgmap v45: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 208 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:40:06.590 INFO:tasks.rados.rados.0.smithi076.stdout:545: done (2 left) 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:546: snap_create 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: finishing write tid 1 to smithi07685715-24 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: finishing write tid 2 to smithi07685715-24 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: finishing write tid 3 to smithi07685715-24 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: finishing write tid 4 to smithi07685715-24 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: finishing write tid 5 to smithi07685715-24 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: oid 24 updating version 0 to 42 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: oid 24 version 42 is already newer than 40 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: oid 24 updating version 42 to 43 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: oid 24 version 43 is already newer than 41 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 43 (ObjNum 236 snap 72 seq_num 236) dirty exists 2024-09-21T09:40:06.591 INFO:tasks.rados.rados.0.smithi076.stdout:543: left oid 24 (ObjNum 236 snap 72 seq_num 236) 2024-09-21T09:40:06.592 INFO:tasks.rados.rados.0.smithi076.stdout:544: expect (ObjNum 117 snap 20 seq_num 117) 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi076.stdout:543: done (2 left) 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi076.stdout:544: done (1 left) 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi076.stdout:546: done (0 left) 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi076.stdout:547: write oid 42 current snap is 73 2024-09-21T09:40:06.731 INFO:tasks.rados.rados.0.smithi076.stdout:547: seq_num 237 ranges {783259=640956,1918703=613070,3008661=405467,3821939=161320} 2024-09-21T09:40:06.738 INFO:tasks.rados.rados.0.smithi076.stdout:547: writing smithi07685715-42 from 783259 to 1424215 tid 1 2024-09-21T09:40:06.744 INFO:tasks.rados.rados.0.smithi076.stdout:547: writing smithi07685715-42 from 1918703 to 2531773 tid 2 2024-09-21T09:40:06.749 INFO:tasks.rados.rados.0.smithi076.stdout:547: writing smithi07685715-42 from 3008661 to 3414128 tid 3 2024-09-21T09:40:06.752 INFO:tasks.rados.rados.0.smithi076.stdout:547: writing smithi07685715-42 from 3821939 to 3983259 tid 4 2024-09-21T09:40:06.753 INFO:tasks.rados.rados.0.smithi076.stdout:548: snap_remove snap 48 2024-09-21T09:40:06.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mon[33301]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:40:06.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:06.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:06.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: ignoring --setuser ceph since I am not root 2024-09-21T09:40:06.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: ignoring --setgroup ceph since I am not root 2024-09-21T09:40:06.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:06 smithi191 ceph-mgr[69643]: -- 172.21.15.191:0/276111116 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b088bfb0e0 con 0x55b088bd8800 2024-09-21T09:40:06.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:06] ENGINE Bus STOPPING 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[41325]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[29313]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:06.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' 2024-09-21T09:40:07.307 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:40:07.307 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:06 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:06] ENGINE Bus STOPPED 2024-09-21T09:40:07.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.305+0000 7fb21cdf2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.305+0000 7fb21cdf2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.305+0000 7fb21cdf2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.305+0000 7fb21cdf2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.305+0000 7fb21cdf2640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.308+0000 7fb2185e9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.308+0000 7fb2185e9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.308+0000 7fb2185e9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.308+0000 7fb2185e9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.587 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:07.308+0000 7fb2185e9640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:40:07.683 INFO:tasks.rados.rados.0.smithi076.stdout:548: done (1 left) 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:549: snap_create 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: finishing write tid 1 to smithi07685715-42 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: finishing write tid 2 to smithi07685715-42 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: finishing write tid 3 to smithi07685715-42 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: finishing write tid 4 to smithi07685715-42 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: finishing write tid 5 to smithi07685715-42 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: finishing write tid 6 to smithi07685715-42 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: oid 42 updating version 0 to 102 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: oid 42 version 102 is already newer than 101 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: oid 42 updating version 102 to 104 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: oid 42 version 104 is already newer than 103 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: oid 42 version 104 is already newer than 100 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 104 (ObjNum 237 snap 73 seq_num 237) dirty exists 2024-09-21T09:40:07.684 INFO:tasks.rados.rados.0.smithi076.stdout:547: left oid 42 (ObjNum 237 snap 73 seq_num 237) 2024-09-21T09:40:07.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:40:07.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:07.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:40:07.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:40:07.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: Activating manager daemon y 2024-09-21T09:40:07.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24619 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: mgrmap e28: y(active, starting, since 0.00684704s) 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:40:07.765 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: Manager daemon y is now available 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:40:07.766 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:07 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:40:07.872 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:07] ENGINE Bus STARTING 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:07] ENGINE Serving on http://:::9283 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:40:07] ENGINE Bus STARTED 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:40:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: Activating manager daemon y 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24619 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: mgrmap e28: y(active, starting, since 0.00684704s) 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:40:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: Manager daemon y is now available 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:40:07.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24619 172.21.15.191:0/2993456927' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: Activating manager daemon y 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24619 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: mgrmap e28: y(active, starting, since 0.00684704s) 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:40:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi076.jymebx"}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:40:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: Manager daemon y is now available 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:40:07.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:40:07.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:40:07.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:40:07.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:07 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:40:08.521 INFO:tasks.rados.rados.0.smithi076.stdout:547: done (1 left) 2024-09-21T09:40:08.521 INFO:tasks.rados.rados.0.smithi076.stdout:549: done (0 left) 2024-09-21T09:40:08.521 INFO:tasks.rados.rados.0.smithi076.stdout:550: snap_remove snap 69 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:08.381+0000 7fb1fab74640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:40:08] ENGINE Error in HTTPServer.serve 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: Traceback (most recent call last): 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self._connections.run(self.expiration_interval) 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self._run(expiration_interval) 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:40:08.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: s = self.context.wrap_socket( 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: return self.sslsocket_class._create( 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self.do_handshake() 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self._sslobj.do_handshake() 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:40:08.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:40:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:08 smithi191 ceph-mon[33301]: mgrmap e29: y(active, since 1.01177s) 2024-09-21T09:40:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:08 smithi191 ceph-mon[33301]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:40:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:08 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:40:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:08 smithi191 ceph-mon[33301]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 40 GiB/s rd, 12 GiB/s wr, 156.55k op/s; 204 MiB/s, 726 objects/s recovering 2024-09-21T09:40:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:08 smithi191 ceph-mon[33301]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:40:09.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[29313]: mgrmap e29: y(active, since 1.01177s) 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[29313]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[29313]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 40 GiB/s rd, 12 GiB/s wr, 156.55k op/s; 204 MiB/s, 726 objects/s recovering 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[29313]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[41325]: mgrmap e29: y(active, since 1.01177s) 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[41325]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[41325]: pgmap v4: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 40 GiB/s rd, 12 GiB/s wr, 156.55k op/s; 204 MiB/s, 726 objects/s recovering 2024-09-21T09:40:09.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:08 smithi076 ceph-mon[41325]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:40:09.523 INFO:tasks.rados.rados.0.smithi076.stdout:550: done (0 left) 2024-09-21T09:40:09.523 INFO:tasks.rados.rados.0.smithi076.stdout:551: rollback oid 39 current snap is 74 2024-09-21T09:40:09.523 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 68 2024-09-21T09:40:09.523 INFO:tasks.rados.rados.0.smithi076.stdout:552: read oid 9 snap -1 2024-09-21T09:40:09.523 INFO:tasks.rados.rados.0.smithi076.stdout:552: expect (ObjNum 219 snap 63 seq_num 219) 2024-09-21T09:40:09.523 INFO:tasks.rados.rados.0.smithi076.stdout:553: snap_create 2024-09-21T09:40:09.526 INFO:tasks.rados.rados.0.smithi076.stdout:551: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:40:09.526 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 253 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:40:09.530 INFO:tasks.rados.rados.0.smithi076.stdout:552: expect (ObjNum 219 snap 63 seq_num 219) 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: [21/Sep/2024:09:40:08] ENGINE Bus STARTING 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: [21/Sep/2024:09:40:08] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: [21/Sep/2024:09:40:08] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: [21/Sep/2024:09:40:08] ENGINE Bus STARTED 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: [21/Sep/2024:09:40:08] ENGINE Error in HTTPServer.serve 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout: Traceback (most recent call last): 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:40:10.013 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._run(expiration_interval) 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: s = self.context.wrap_socket( 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: return self.sslsocket_class._create( 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: self.do_handshake() 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._sslobj.do_handshake() 2024-09-21T09:40:10.014 INFO:journalctl@ceph.mon.b.smithi191.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:40:10.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: pgmap v6: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 341 MiB/s rd, 102 MiB/s wr, 1.30k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-09-21T09:40:10.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: mgrmap e30: y(active, since 2s) 2024-09-21T09:40:10.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:09 smithi191 ceph-mon[33301]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:40:10.038 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: [21/Sep/2024:09:40:08] ENGINE Bus STARTING 2024-09-21T09:40:10.038 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: [21/Sep/2024:09:40:08] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:40:10.038 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: [21/Sep/2024:09:40:08] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:40:10.038 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: [21/Sep/2024:09:40:08] ENGINE Bus STARTED 2024-09-21T09:40:10.038 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: [21/Sep/2024:09:40:08] ENGINE Error in HTTPServer.serve 2024-09-21T09:40:10.038 INFO:journalctl@ceph.mon.a.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:40:10.039 INFO:journalctl@ceph.mon.a.smithi076.stdout: self.do_handshake() 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.a.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: pgmap v6: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 341 MiB/s rd, 102 MiB/s wr, 1.30k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: mgrmap e30: y(active, since 2s) 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[29313]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: [21/Sep/2024:09:40:08] ENGINE Bus STARTING 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: [21/Sep/2024:09:40:08] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: [21/Sep/2024:09:40:08] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: [21/Sep/2024:09:40:08] ENGINE Bus STARTED 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: [21/Sep/2024:09:40:08] ENGINE Error in HTTPServer.serve 2024-09-21T09:40:10.040 INFO:journalctl@ceph.mon.c.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: self.do_handshake() 2024-09-21T09:40:10.041 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:40:10.042 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:40:10.042 INFO:journalctl@ceph.mon.c.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:40:10.042 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: pgmap v6: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 341 MiB/s rd, 102 MiB/s wr, 1.30k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-09-21T09:40:10.042 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: mgrmap e30: y(active, since 2s) 2024-09-21T09:40:10.042 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:09 smithi076 ceph-mon[41325]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:40:10.266 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:40:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:551: done (2 left) 2024-09-21T09:40:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:552: done (1 left) 2024-09-21T09:40:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:553: done (0 left) 2024-09-21T09:40:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:554: write oid 40 current snap is 75 2024-09-21T09:40:10.528 INFO:tasks.rados.rados.0.smithi076.stdout:554: seq_num 238 ranges {400066=716205,1860832=139234} 2024-09-21T09:40:10.534 INFO:tasks.rados.rados.0.smithi076.stdout:554: writing smithi07685715-40 from 400066 to 1116271 tid 1 2024-09-21T09:40:10.540 INFO:tasks.rados.rados.0.smithi076.stdout:554: writing smithi07685715-40 from 1860832 to 2000066 tid 2 2024-09-21T09:40:10.540 INFO:tasks.rados.rados.0.smithi076.stdout:555: read oid 4 snap 74 2024-09-21T09:40:10.540 INFO:tasks.rados.rados.0.smithi076.stdout:555: expect (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:40:10.540 INFO:tasks.rados.rados.0.smithi076.stdout:556: write oid 24 current snap is 75 2024-09-21T09:40:10.540 INFO:tasks.rados.rados.0.smithi076.stdout:554: finishing write tid 1 to smithi07685715-40 2024-09-21T09:40:10.540 INFO:tasks.rados.rados.0.smithi076.stdout:556: seq_num 239 ranges {416873=791454,1919314=97559} 2024-09-21T09:40:10.546 INFO:tasks.rados.rados.0.smithi076.stdout:556: writing smithi07685715-24 from 416873 to 1208327 tid 1 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:556: writing smithi07685715-24 from 1919314 to 2016873 tid 2 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:557: rollback oid 23 current snap is 75 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 66 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:554: finishing write tid 2 to smithi07685715-40 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:554: finishing write tid 3 to smithi07685715-40 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:554: finishing write tid 4 to smithi07685715-40 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:554: oid 40 updating version 0 to 293 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:554: oid 40 version 293 is already newer than 292 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:554: oid 40 version 293 is already newer than 291 2024-09-21T09:40:10.552 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 293 (ObjNum 238 snap 75 seq_num 238) dirty exists 2024-09-21T09:40:10.553 INFO:tasks.rados.rados.0.smithi076.stdout:554: left oid 40 (ObjNum 238 snap 75 seq_num 238) 2024-09-21T09:40:10.553 INFO:tasks.rados.rados.0.smithi076.stdout:554: done (3 left) 2024-09-21T09:40:10.553 INFO:tasks.rados.rados.0.smithi076.stdout:558: read oid 10 snap -1 2024-09-21T09:40:10.553 INFO:tasks.rados.rados.0.smithi076.stdout:558: expect deleted 2024-09-21T09:40:10.553 INFO:tasks.rados.rados.0.smithi076.stdout:559: write oid 20 current snap is 75 2024-09-21T09:40:10.553 INFO:tasks.rados.rados.0.smithi076.stdout:559: seq_num 240 ranges {433680=466703,1577796=455884} 2024-09-21T09:40:10.557 INFO:tasks.rados.rados.0.smithi076.stdout:559: writing smithi07685715-20 from 433680 to 900383 tid 1 2024-09-21T09:40:10.563 INFO:tasks.rados.rados.0.smithi076.stdout:559: writing smithi07685715-20 from 1577796 to 2033680 tid 2 2024-09-21T09:40:10.563 INFO:tasks.rados.rados.0.smithi076.stdout:560: delete oid 11 current snap is 75 2024-09-21T09:40:10.564 INFO:tasks.rados.rados.0.smithi076.stdout:556: finishing write tid 1 to smithi07685715-24 2024-09-21T09:40:10.564 INFO:tasks.rados.rados.0.smithi076.stdout:556: finishing write tid 2 to smithi07685715-24 2024-09-21T09:40:10.564 INFO:tasks.rados.rados.0.smithi076.stdout:555: expect (ObjNum 192 snap 51 seq_num 192) 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:556: finishing write tid 3 to smithi07685715-24 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:557: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 0 (ObjNum 129 snap 0 seq_num 864131728) dirty dne 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:556: finishing write tid 4 to smithi07685715-24 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:556: oid 24 updating version 0 to 47 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:556: oid 24 version 47 is already newer than 46 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:556: oid 24 version 47 is already newer than 45 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 47 (ObjNum 239 snap 75 seq_num 239) dirty exists 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:556: left oid 24 (ObjNum 239 snap 75 seq_num 239) 2024-09-21T09:40:10.713 INFO:tasks.rados.rados.0.smithi076.stdout:559: finishing write tid 1 to smithi07685715-20 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:555: done (5 left) 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:556: done (4 left) 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:557: done (3 left) 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:558: done (2 left) 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:560: done (1 left) 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:561: write oid 27 current snap is 75 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: finishing write tid 2 to smithi07685715-20 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: finishing write tid 3 to smithi07685715-20 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: finishing write tid 4 to smithi07685715-20 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: oid 20 updating version 0 to 115 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: oid 20 version 115 is already newer than 113 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: oid 20 version 115 is already newer than 114 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 115 (ObjNum 240 snap 75 seq_num 240) dirty exists 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:559: left oid 20 (ObjNum 240 snap 75 seq_num 240) 2024-09-21T09:40:10.714 INFO:tasks.rados.rados.0.smithi076.stdout:561: seq_num 242 ranges {467294=617201,1578407=488887} 2024-09-21T09:40:10.718 INFO:tasks.rados.rados.0.smithi076.stdout:561: writing smithi07685715-27 from 467294 to 1084495 tid 1 2024-09-21T09:40:10.724 INFO:tasks.rados.rados.0.smithi076.stdout:561: writing smithi07685715-27 from 1578407 to 2067294 tid 2 2024-09-21T09:40:10.724 INFO:tasks.rados.rados.0.smithi076.stdout:559: done (1 left) 2024-09-21T09:40:10.724 INFO:tasks.rados.rados.0.smithi076.stdout:562: rollback oid 21 current snap is 75 2024-09-21T09:40:10.724 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 73 2024-09-21T09:40:10.724 INFO:tasks.rados.rados.0.smithi076.stdout:561: finishing write tid 1 to smithi07685715-27 2024-09-21T09:40:10.724 INFO:tasks.rados.rados.0.smithi076.stdout:563: delete oid 44 current snap is 75 2024-09-21T09:40:10.725 INFO:tasks.rados.rados.0.smithi076.stdout:563: done (2 left) 2024-09-21T09:40:10.725 INFO:tasks.rados.rados.0.smithi076.stdout:564: delete oid 10 current snap is 75 2024-09-21T09:40:10.727 INFO:tasks.rados.rados.0.smithi076.stdout:562: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:40:10.727 INFO:tasks.rados.rados.0.smithi076.stdout:562: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:40:10.727 INFO:tasks.rados.rados.0.smithi076.stdout:561: finishing write tid 2 to smithi07685715-27 2024-09-21T09:40:10.727 INFO:tasks.rados.rados.0.smithi076.stdout:562: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 328 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:561: finishing write tid 3 to smithi07685715-27 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:562: done (2 left) 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:564: done (1 left) 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:565: delete oid 10 current snap is 75 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:561: finishing write tid 4 to smithi07685715-27 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:561: oid 27 updating version 0 to 295 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:561: oid 27 updating version 295 to 297 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:561: oid 27 version 297 is already newer than 296 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 297 (ObjNum 242 snap 75 seq_num 242) dirty exists 2024-09-21T09:40:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:561: left oid 27 (ObjNum 242 snap 75 seq_num 242) 2024-09-21T09:40:10.729 INFO:tasks.rados.rados.0.smithi076.stdout:561: done (1 left) 2024-09-21T09:40:10.729 INFO:tasks.rados.rados.0.smithi076.stdout:565: done (0 left) 2024-09-21T09:40:10.729 INFO:tasks.rados.rados.0.smithi076.stdout:566: write oid 4 current snap is 75 2024-09-21T09:40:10.729 INFO:tasks.rados.rados.0.smithi076.stdout:566: seq_num 246 ranges {534522=634550,1812335=322187} 2024-09-21T09:40:10.734 INFO:tasks.rados.rados.0.smithi076.stdout:566: writing smithi07685715-4 from 534522 to 1169072 tid 1 2024-09-21T09:40:10.740 INFO:tasks.rados.rados.0.smithi076.stdout:566: writing smithi07685715-4 from 1812335 to 2134522 tid 2 2024-09-21T09:40:10.740 INFO:tasks.rados.rados.0.smithi076.stdout:567: read oid 31 snap -1 2024-09-21T09:40:10.740 INFO:tasks.rados.rados.0.smithi076.stdout:567: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:40:10.740 INFO:tasks.rados.rados.0.smithi076.stdout:568: snap_create 2024-09-21T09:40:10.740 INFO:tasks.rados.rados.0.smithi076.stdout:566: finishing write tid 1 to smithi07685715-4 2024-09-21T09:40:10.742 INFO:tasks.rados.rados.0.smithi076.stdout:566: finishing write tid 2 to smithi07685715-4 2024-09-21T09:40:10.742 INFO:tasks.rados.rados.0.smithi076.stdout:567: expect (ObjNum 138 snap 30 seq_num 138) 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:566: finishing write tid 3 to smithi07685715-4 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:566: finishing write tid 4 to smithi07685715-4 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:566: oid 4 updating version 0 to 160 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:566: oid 4 version 160 is already newer than 159 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:566: oid 4 updating version 160 to 161 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 161 (ObjNum 246 snap 75 seq_num 246) dirty exists 2024-09-21T09:40:10.830 INFO:tasks.rados.rados.0.smithi076.stdout:566: left oid 4 (ObjNum 246 snap 75 seq_num 246) 2024-09-21T09:40:10.876 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:40:10.876 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (3m) 8s ago 4m 26.6M - 0.25.0 c8568f914cd2 d930ef8d31ba 2024-09-21T09:40:10.876 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (4m) 7s ago 4m 88.6M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (3m) 8s ago 3m 16.1M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (3m) 7s ago 3m 15.5M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (53s) 7s ago 8m 507M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 8ab387e15f88 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (12s) 8s ago 10m 66.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (10m) 8s ago 10m 53.2M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (8m) 7s ago 8m 43.5M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (8m) 8s ago 8m 40.5M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (4m) 8s ago 4m 13.9M - 1.5.0 0da6a335fe13 6dbd51c32fb5 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (4m) 7s ago 4m 16.2M - 1.5.0 0da6a335fe13 9722666fd438 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (8m) 8s ago 8m 76.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (7m) 8s ago 7m 69.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (7m) 8s ago 7m 74.9M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (7m) 8s ago 7m 65.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:40:10.877 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (6m) 7s ago 6m 66.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:40:10.878 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (6m) 7s ago 6m 82.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:40:10.878 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (5m) 7s ago 5m 58.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:40:10.878 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (5m) 7s ago 5m 55.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:40:10.878 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (17s) 7s ago 4m 34.4M - 2.43.0 a07b618ecd1d 178927022a13 2024-09-21T09:40:11.221 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:40:11.221 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:40:11.221 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:40:11.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:11.221 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:40:11.222 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:40:11.524 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [ 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr" 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "", 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:40:11.525 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:40:11.697 INFO:tasks.rados.rados.0.smithi076.stdout:566: done (2 left) 2024-09-21T09:40:11.697 INFO:tasks.rados.rados.0.smithi076.stdout:567: done (1 left) 2024-09-21T09:40:11.697 INFO:tasks.rados.rados.0.smithi076.stdout:568: done (0 left) 2024-09-21T09:40:11.697 INFO:tasks.rados.rados.0.smithi076.stdout:569: write oid 36 current snap is 76 2024-09-21T09:40:11.698 INFO:tasks.rados.rados.0.smithi076.stdout:569: seq_num 247 ranges {551329=709799,1870817=280512} 2024-09-21T09:40:11.704 INFO:tasks.rados.rados.0.smithi076.stdout:569: writing smithi07685715-36 from 551329 to 1261128 tid 1 2024-09-21T09:40:11.709 INFO:tasks.rados.rados.0.smithi076.stdout:569: writing smithi07685715-36 from 1870817 to 2151329 tid 2 2024-09-21T09:40:11.709 INFO:tasks.rados.rados.0.smithi076.stdout:570: rollback oid 12 current snap is 76 2024-09-21T09:40:11.709 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 66 2024-09-21T09:40:11.709 INFO:tasks.rados.rados.0.smithi076.stdout:571: write oid 18 current snap is 76 2024-09-21T09:40:11.709 INFO:tasks.rados.rados.0.smithi076.stdout:571: seq_num 248 ranges {568136=785048,1929299=238837} 2024-09-21T09:40:11.716 INFO:tasks.rados.rados.0.smithi076.stdout:571: writing smithi07685715-18 from 568136 to 1353184 tid 1 2024-09-21T09:40:11.722 INFO:tasks.rados.rados.0.smithi076.stdout:571: writing smithi07685715-18 from 1929299 to 2168136 tid 2 2024-09-21T09:40:11.722 INFO:tasks.rados.rados.0.smithi076.stdout:572: rollback oid 21 current snap is 76 2024-09-21T09:40:11.722 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 66 2024-09-21T09:40:11.722 INFO:tasks.rados.rados.0.smithi076.stdout:569: finishing write tid 1 to smithi07685715-36 2024-09-21T09:40:11.722 INFO:tasks.rados.rados.0.smithi076.stdout:569: finishing write tid 2 to smithi07685715-36 2024-09-21T09:40:11.722 INFO:tasks.rados.rados.0.smithi076.stdout:569: finishing write tid 3 to smithi07685715-36 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:569: finishing write tid 4 to smithi07685715-36 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:569: oid 36 updating version 0 to 165 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:569: oid 36 version 165 is already newer than 164 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:569: oid 36 version 165 is already newer than 163 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 165 (ObjNum 247 snap 76 seq_num 247) dirty exists 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:569: left oid 36 (ObjNum 247 snap 76 seq_num 247) 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:570: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:570: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:40:11.723 INFO:tasks.rados.rados.0.smithi076.stdout:570: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:40:11.724 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 245 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-09-21T09:40:11.724 INFO:tasks.rados.rados.0.smithi076.stdout:569: done (3 left) 2024-09-21T09:40:11.724 INFO:tasks.rados.rados.0.smithi076.stdout:570: done (2 left) 2024-09-21T09:40:11.724 INFO:tasks.rados.rados.0.smithi076.stdout:573: snap_remove snap 72 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: pgmap v9: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 113 MiB/s rd, 34 MiB/s wr, 431 op/s; 578 KiB/s, 2 objects/s recovering 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: mgrmap e31: y(active, since 4s) 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: from='client.24896 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3554485379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:11.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:11.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:40:11.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: pgmap v9: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 113 MiB/s rd, 34 MiB/s wr, 431 op/s; 578 KiB/s, 2 objects/s recovering 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: mgrmap e31: y(active, since 4s) 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: from='client.24896 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3554485379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: from='client.14991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: pgmap v9: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 217 MiB data, 663 MiB used, 715 GiB / 715 GiB avail; 113 MiB/s rd, 34 MiB/s wr, 431 op/s; 578 KiB/s, 2 objects/s recovering 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: mgrmap e31: y(active, since 4s) 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: from='client.24896 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3554485379' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:11 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:12.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:40:11] ENGINE Bus STARTING 2024-09-21T09:40:12.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: CherryPy Checker: 2024-09-21T09:40:12.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: The Application mounted at '' has an empty config. 2024-09-21T09:40:12.112 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:11 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: 2024-09-21T09:40:12.113 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:40:12] ENGINE Serving on http://:::9283 2024-09-21T09:40:12.113 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: [21/Sep/2024:09:40:12] ENGINE Bus STARTED 2024-09-21T09:40:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:573: done (2 left) 2024-09-21T09:40:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:574: read oid 15 snap -1 2024-09-21T09:40:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:574: expect deleted 2024-09-21T09:40:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:575: snap_remove snap 64 2024-09-21T09:40:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: Standby manager daemon x started 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:12 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: Standby manager daemon x started 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='client.24908 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: Standby manager daemon x started 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/2055880075' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:12 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:575: done (3 left) 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:576: snap_create 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: finishing write tid 1 to smithi07685715-18 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: finishing write tid 2 to smithi07685715-18 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: finishing write tid 3 to smithi07685715-18 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:572: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:572: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: finishing write tid 4 to smithi07685715-18 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: oid 18 updating version 0 to 114 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: oid 18 version 114 is already newer than 113 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:571: oid 18 version 114 is already newer than 112 2024-09-21T09:40:13.529 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 114 (ObjNum 248 snap 76 seq_num 248) dirty exists 2024-09-21T09:40:13.530 INFO:tasks.rados.rados.0.smithi076.stdout:571: left oid 18 (ObjNum 248 snap 76 seq_num 248) 2024-09-21T09:40:13.530 INFO:tasks.rados.rados.0.smithi076.stdout:572: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:40:13.530 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 332 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-09-21T09:40:13.612 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[69620]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:40:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:40:13.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: pgmap v11: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 219 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 698 KiB/s wr, 10 op/s 2024-09-21T09:40:13.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-21T09:40:13.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[41325]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: pgmap v11: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 219 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 698 KiB/s wr, 10 op/s 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:13.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:13 smithi076 ceph-mon[29313]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:40:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: pgmap v11: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 219 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 8.8 MiB/s rd, 698 KiB/s wr, 10 op/s 2024-09-21T09:40:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-21T09:40:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:40:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:40:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:13 smithi191 ceph-mon[33301]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:40:14.527 INFO:tasks.rados.rados.0.smithi076.stdout:571: done (3 left) 2024-09-21T09:40:14.527 INFO:tasks.rados.rados.0.smithi076.stdout:572: done (2 left) 2024-09-21T09:40:14.527 INFO:tasks.rados.rados.0.smithi076.stdout:574: done (1 left) 2024-09-21T09:40:14.527 INFO:tasks.rados.rados.0.smithi076.stdout:576: done (0 left) 2024-09-21T09:40:14.527 INFO:tasks.rados.rados.0.smithi076.stdout:577: snap_create 2024-09-21T09:40:15.533 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[29313]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:40:15.533 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[29313]: pgmap v15: 89 pgs: 89 active+clean; 220 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-09-21T09:40:15.539 INFO:tasks.rados.rados.0.smithi076.stdout:577: done (0 left) 2024-09-21T09:40:15.539 INFO:tasks.rados.rados.0.smithi076.stdout:578: snap_remove snap 65 2024-09-21T09:40:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:15 smithi191 ceph-mon[33301]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:40:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:15 smithi191 ceph-mon[33301]: pgmap v15: 89 pgs: 89 active+clean; 220 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-09-21T09:40:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:15 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:15 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:15 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:15 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:15.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:15.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:15.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[41325]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[41325]: pgmap v15: 89 pgs: 89 active+clean; 220 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:15 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:16.536 INFO:tasks.rados.rados.0.smithi076.stdout:578: done (0 left) 2024-09-21T09:40:16.537 INFO:tasks.rados.rados.0.smithi076.stdout:579: delete oid 39 current snap is 78 2024-09-21T09:40:16.541 INFO:tasks.rados.rados.0.smithi076.stdout:579: done (0 left) 2024-09-21T09:40:16.541 INFO:tasks.rados.rados.0.smithi076.stdout:580: delete oid 19 current snap is 78 2024-09-21T09:40:16.549 INFO:tasks.rados.rados.0.smithi076.stdout:580: done (0 left) 2024-09-21T09:40:16.550 INFO:tasks.rados.rados.0.smithi076.stdout:581: read oid 6 snap -1 2024-09-21T09:40:16.550 INFO:tasks.rados.rados.0.smithi076.stdout:581: expect deleted 2024-09-21T09:40:16.550 INFO:tasks.rados.rados.0.smithi076.stdout:582: write oid 45 current snap is 78 2024-09-21T09:40:16.550 INFO:tasks.rados.rados.0.smithi076.stdout:582: seq_num 251 ranges {618557=727148,2104745=113812} 2024-09-21T09:40:16.557 INFO:tasks.rados.rados.0.smithi076.stdout:582: writing smithi07685715-45 from 618557 to 1345705 tid 1 2024-09-21T09:40:16.562 INFO:tasks.rados.rados.0.smithi076.stdout:582: writing smithi07685715-45 from 2104745 to 2218557 tid 2 2024-09-21T09:40:16.562 INFO:tasks.rados.rados.0.smithi076.stdout:583: snap_create 2024-09-21T09:40:16.562 INFO:tasks.rados.rados.0.smithi076.stdout:582: finishing write tid 1 to smithi07685715-45 2024-09-21T09:40:16.563 INFO:tasks.rados.rados.0.smithi076.stdout:582: finishing write tid 2 to smithi07685715-45 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:582: finishing write tid 3 to smithi07685715-45 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:582: finishing write tid 4 to smithi07685715-45 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:582: oid 45 updating version 0 to 104 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:582: oid 45 version 104 is already newer than 102 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:582: oid 45 version 104 is already newer than 103 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 104 (ObjNum 251 snap 78 seq_num 251) dirty exists 2024-09-21T09:40:16.564 INFO:tasks.rados.rados.0.smithi076.stdout:582: left oid 45 (ObjNum 251 snap 78 seq_num 251) 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:40:16.815 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.816 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:16 smithi191 ceph-mon[33301]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:40:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[29313]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:40:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:16.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:16 smithi076 ceph-mon[41325]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:40:17.558 INFO:tasks.rados.rados.0.smithi076.stdout:581: done (2 left) 2024-09-21T09:40:17.558 INFO:tasks.rados.rados.0.smithi076.stdout:582: done (1 left) 2024-09-21T09:40:17.558 INFO:tasks.rados.rados.0.smithi076.stdout:583: done (0 left) 2024-09-21T09:40:17.559 INFO:tasks.rados.rados.0.smithi076.stdout:584: write oid 2 current snap is 79 2024-09-21T09:40:17.559 INFO:tasks.rados.rados.0.smithi076.stdout:584: seq_num 252 ranges {635364=402397,1763227=472137} 2024-09-21T09:40:17.564 INFO:tasks.rados.rados.0.smithi076.stdout:584: writing smithi07685715-2 from 635364 to 1037761 tid 1 2024-09-21T09:40:17.570 INFO:tasks.rados.rados.0.smithi076.stdout:584: writing smithi07685715-2 from 1763227 to 2235364 tid 2 2024-09-21T09:40:17.571 INFO:tasks.rados.rados.0.smithi076.stdout:585: write oid 17 current snap is 79 2024-09-21T09:40:17.571 INFO:tasks.rados.rados.0.smithi076.stdout:585: seq_num 253 ranges {652171=477646,1821709=430462} 2024-09-21T09:40:17.576 INFO:tasks.rados.rados.0.smithi076.stdout:585: writing smithi07685715-17 from 652171 to 1129817 tid 1 2024-09-21T09:40:17.583 INFO:tasks.rados.rados.0.smithi076.stdout:585: writing smithi07685715-17 from 1821709 to 2252171 tid 2 2024-09-21T09:40:17.583 INFO:tasks.rados.rados.0.smithi076.stdout:586: write oid 30 current snap is 79 2024-09-21T09:40:17.583 INFO:tasks.rados.rados.0.smithi076.stdout:584: finishing write tid 1 to smithi07685715-2 2024-09-21T09:40:17.583 INFO:tasks.rados.rados.0.smithi076.stdout:586: seq_num 254 ranges {668978=552895,1763838=505140} 2024-09-21T09:40:17.590 INFO:tasks.rados.rados.0.smithi076.stdout:586: writing smithi07685715-30 from 668978 to 1221873 tid 1 2024-09-21T09:40:17.596 INFO:tasks.rados.rados.0.smithi076.stdout:586: writing smithi07685715-30 from 1763838 to 2268978 tid 2 2024-09-21T09:40:17.596 INFO:tasks.rados.rados.0.smithi076.stdout:587: delete oid 23 current snap is 79 2024-09-21T09:40:17.596 INFO:tasks.rados.rados.0.smithi076.stdout:584: finishing write tid 2 to smithi07685715-2 2024-09-21T09:40:17.596 INFO:tasks.rados.rados.0.smithi076.stdout:584: finishing write tid 3 to smithi07685715-2 2024-09-21T09:40:17.596 INFO:tasks.rados.rados.0.smithi076.stdout:584: finishing write tid 4 to smithi07685715-2 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:584: oid 2 updating version 0 to 65 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:584: oid 2 updating version 65 to 66 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:584: oid 2 updating version 66 to 67 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 67 (ObjNum 252 snap 79 seq_num 252) dirty exists 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:584: left oid 2 (ObjNum 252 snap 79 seq_num 252) 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: finishing write tid 1 to smithi07685715-17 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: finishing write tid 2 to smithi07685715-17 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: finishing write tid 3 to smithi07685715-17 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: finishing write tid 4 to smithi07685715-17 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: oid 17 updating version 0 to 257 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: oid 17 updating version 257 to 259 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:585: oid 17 version 259 is already newer than 258 2024-09-21T09:40:17.597 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 259 (ObjNum 253 snap 79 seq_num 253) dirty exists 2024-09-21T09:40:17.598 INFO:tasks.rados.rados.0.smithi076.stdout:585: left oid 17 (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:17.598 INFO:tasks.rados.rados.0.smithi076.stdout:586: finishing write tid 1 to smithi07685715-30 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: finishing write tid 2 to smithi07685715-30 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: finishing write tid 3 to smithi07685715-30 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: finishing write tid 4 to smithi07685715-30 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: oid 30 updating version 0 to 456 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: oid 30 updating version 456 to 457 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: oid 30 version 457 is already newer than 455 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 457 (ObjNum 254 snap 79 seq_num 254) dirty exists 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: left oid 30 (ObjNum 254 snap 79 seq_num 254) 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:584: done (3 left) 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:585: done (2 left) 2024-09-21T09:40:17.600 INFO:tasks.rados.rados.0.smithi076.stdout:586: done (1 left) 2024-09-21T09:40:17.601 INFO:tasks.rados.rados.0.smithi076.stdout:587: done (0 left) 2024-09-21T09:40:17.601 INFO:tasks.rados.rados.0.smithi076.stdout:588: read oid 36 snap 43 2024-09-21T09:40:17.601 INFO:tasks.rados.rados.0.smithi076.stdout:588: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:40:17.601 INFO:tasks.rados.rados.0.smithi076.stdout:589: snap_remove snap 70 2024-09-21T09:40:17.851 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:17 smithi191 ceph-mon[33301]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:40:17.851 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:17 smithi191 ceph-mon[33301]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:40:17.851 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:17 smithi191 ceph-mon[33301]: pgmap v18: 89 pgs: 89 active+clean; 220 MiB data, 671 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:40:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:17 smithi076 ceph-mon[29313]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:40:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:17 smithi076 ceph-mon[29313]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:40:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:17 smithi076 ceph-mon[29313]: pgmap v18: 89 pgs: 89 active+clean; 220 MiB data, 671 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:40:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:17 smithi076 ceph-mon[41325]: Reconfiguring daemon prometheus.a on smithi191 2024-09-21T09:40:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:17 smithi076 ceph-mon[41325]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:40:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:17 smithi076 ceph-mon[41325]: pgmap v18: 89 pgs: 89 active+clean; 220 MiB data, 671 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:40:18.525 INFO:tasks.rados.rados.0.smithi076.stdout:589: done (1 left) 2024-09-21T09:40:18.525 INFO:tasks.rados.rados.0.smithi076.stdout:590: rollback oid 16 current snap is 79 2024-09-21T09:40:18.525 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 73 2024-09-21T09:40:18.525 INFO:tasks.rados.rados.0.smithi076.stdout:588: expect (ObjNum 134 snap 28 seq_num 134) 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:590: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 214 (ObjNum 129 snap 0 seq_num 864296224) dirty dne 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:588: done (1 left) 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:590: done (0 left) 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:591: rollback oid 21 current snap is 79 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 59 2024-09-21T09:40:18.598 INFO:tasks.rados.rados.0.smithi076.stdout:592: delete oid 39 current snap is 79 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:591: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:591: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:591: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 353 (ObjNum 201 snap 57 seq_num 201) dirty exists 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:591: done (1 left) 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:592: done (0 left) 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:593: read oid 40 snap 71 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:593: expect deleted 2024-09-21T09:40:18.601 INFO:tasks.rados.rados.0.smithi076.stdout:594: write oid 31 current snap is 79 2024-09-21T09:40:18.602 INFO:tasks.rados.rados.0.smithi076.stdout:594: seq_num 257 ranges {719399=778642,1939284=380115} 2024-09-21T09:40:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:594: writing smithi07685715-31 from 719399 to 1498041 tid 1 2024-09-21T09:40:18.614 INFO:tasks.rados.rados.0.smithi076.stdout:594: writing smithi07685715-31 from 1939284 to 2319399 tid 2 2024-09-21T09:40:18.614 INFO:tasks.rados.rados.0.smithi076.stdout:595: snap_create 2024-09-21T09:40:18.615 INFO:tasks.rados.rados.0.smithi076.stdout:594: finishing write tid 1 to smithi07685715-31 2024-09-21T09:40:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:594: finishing write tid 2 to smithi07685715-31 2024-09-21T09:40:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:594: finishing write tid 3 to smithi07685715-31 2024-09-21T09:40:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:594: finishing write tid 4 to smithi07685715-31 2024-09-21T09:40:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:594: oid 31 updating version 0 to 171 2024-09-21T09:40:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:594: oid 31 version 171 is already newer than 167 2024-09-21T09:40:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:594: oid 31 version 171 is already newer than 170 2024-09-21T09:40:18.618 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 171 (ObjNum 257 snap 79 seq_num 257) dirty exists 2024-09-21T09:40:18.618 INFO:tasks.rados.rados.0.smithi076.stdout:594: left oid 31 (ObjNum 257 snap 79 seq_num 257) 2024-09-21T09:40:18.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:18 smithi191 ceph-mon[33301]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:40:18.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:18 smithi191 ceph-mon[33301]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:40:18.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:18 smithi076 ceph-mon[29313]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:40:18.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:18 smithi076 ceph-mon[29313]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:40:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:18 smithi076 ceph-mon[41325]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:40:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:18 smithi076 ceph-mon[41325]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:40:19.559 INFO:tasks.rados.rados.0.smithi076.stdout:593: done (2 left) 2024-09-21T09:40:19.559 INFO:tasks.rados.rados.0.smithi076.stdout:594: done (1 left) 2024-09-21T09:40:19.559 INFO:tasks.rados.rados.0.smithi076.stdout:595: done (0 left) 2024-09-21T09:40:19.559 INFO:tasks.rados.rados.0.smithi076.stdout:596: snap_create 2024-09-21T09:40:19.820 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-mon[33301]: pgmap v21: 89 pgs: 89 active+clean; 224 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 900 KiB/s wr, 7 op/s 2024-09-21T09:40:19.821 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 systemd[1]: Stopping Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:19.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:19 smithi076 ceph-mon[29313]: pgmap v21: 89 pgs: 89 active+clean; 224 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 900 KiB/s wr, 7 op/s 2024-09-21T09:40:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:19 smithi076 ceph-mon[41325]: pgmap v21: 89 pgs: 89 active+clean; 224 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 900 KiB/s wr, 7 op/s 2024-09-21T09:40:20.112 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.818Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:40:20.112 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[77817]: ts=2024-09-21T09:40:19.819Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:40:20.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:19 smithi191 podman[79877]: 2024-09-21 09:40:19.829066117 +0000 UTC m=+0.109874494 container died 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:20.453 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 podman[79877]: 2024-09-21 09:40:20.233818142 +0000 UTC m=+0.514626518 container cleanup 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:20.453 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 bash[79877]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:40:20.563 INFO:tasks.rados.rados.0.smithi076.stdout:596: done (0 left) 2024-09-21T09:40:20.563 INFO:tasks.rados.rados.0.smithi076.stdout:597: read oid 13 snap -1 2024-09-21T09:40:20.563 INFO:tasks.rados.rados.0.smithi076.stdout:597: expect (ObjNum 168 snap 44 seq_num 168) 2024-09-21T09:40:20.563 INFO:tasks.rados.rados.0.smithi076.stdout:598: snap_remove snap 35 2024-09-21T09:40:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:20 smithi191 ceph-mon[33301]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:40:20.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 podman[79894]: 2024-09-21 09:40:20.452420979 +0000 UTC m=+0.625532948 container remove 178927022a138746b06ebf10a204e6a07586c4c1b1b45e716200e1dcde412ccc (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:20.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:40:20.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 systemd[1]: Stopped Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:20.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.100s CPU time. 2024-09-21T09:40:20.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 systemd[1]: Starting Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:20.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:20 smithi076 ceph-mon[29313]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:40:20.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:20 smithi076 ceph-mon[41325]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:40:21.174 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 podman[79992]: 2024-09-21 09:40:20.772179698 +0000 UTC m=+0.016933653 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:40:21.174 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:20 smithi191 podman[79992]: 2024-09-21 09:40:20.886870814 +0000 UTC m=+0.131624768 container create 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:21.565 INFO:tasks.rados.rados.0.smithi076.stdout:597: expect (ObjNum 168 snap 44 seq_num 168) 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 podman[79992]: 2024-09-21 09:40:21.173243549 +0000 UTC m=+0.417997497 container init 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 podman[79992]: 2024-09-21 09:40:21.177816589 +0000 UTC m=+0.422570545 container start 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.201Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.201Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.201Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi191 (none))" 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.201Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.201Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.203Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:40:21.577 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.203Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.204Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.204Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.206Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.206Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=16.845µs 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.206Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.208Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.214Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.217Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.217Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.217Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=117.64µs wal_replay_duration=10.84252ms wbl_replay_duration=85ns total_replay_duration=11.041935ms 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.219Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.219Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.219Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:40:21.578 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.238Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.25346ms db_storage=800ns remote_storage=7.743µs web_handler=625ns query_engine=614ns scrape=393.686µs scrape_sd=130.324µs notify=12.771µs notify_sd=20.28µs rules=18.338445ms tracing=11.723µs 2024-09-21T09:40:21.579 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.238Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:40:21.579 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:40:21.238Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:40:21.579 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 bash[79992]: 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd 2024-09-21T09:40:21.579 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:40:21 smithi191 systemd[1]: Started Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:21.690 INFO:tasks.rados.rados.0.smithi076.stdout:597: done (1 left) 2024-09-21T09:40:21.690 INFO:tasks.rados.rados.0.smithi076.stdout:598: done (0 left) 2024-09-21T09:40:21.690 INFO:tasks.rados.rados.0.smithi076.stdout:599: write oid 30 current snap is 81 2024-09-21T09:40:21.690 INFO:tasks.rados.rados.0.smithi076.stdout:599: seq_num 258 ranges {736206=453891,1881413=454793} 2024-09-21T09:40:21.696 INFO:tasks.rados.rados.0.smithi076.stdout:599: writing smithi07685715-30 from 736206 to 1190097 tid 1 2024-09-21T09:40:21.703 INFO:tasks.rados.rados.0.smithi076.stdout:599: writing smithi07685715-30 from 1881413 to 2336206 tid 2 2024-09-21T09:40:21.703 INFO:tasks.rados.rados.0.smithi076.stdout:600: read oid 23 snap 40 2024-09-21T09:40:21.703 INFO:tasks.rados.rados.0.smithi076.stdout:600: expect deleted 2024-09-21T09:40:21.703 INFO:tasks.rados.rados.0.smithi076.stdout:601: write oid 16 current snap is 81 2024-09-21T09:40:21.703 INFO:tasks.rados.rados.0.smithi076.stdout:599: finishing write tid 1 to smithi07685715-30 2024-09-21T09:40:21.703 INFO:tasks.rados.rados.0.smithi076.stdout:601: seq_num 259 ranges {753013=645493,2056248=296765} 2024-09-21T09:40:21.710 INFO:tasks.rados.rados.0.smithi076.stdout:601: writing smithi07685715-16 from 753013 to 1398506 tid 1 2024-09-21T09:40:21.715 INFO:tasks.rados.rados.0.smithi076.stdout:601: writing smithi07685715-16 from 2056248 to 2353013 tid 2 2024-09-21T09:40:21.715 INFO:tasks.rados.rados.0.smithi076.stdout:602: delete oid 41 current snap is 81 2024-09-21T09:40:21.715 INFO:tasks.rados.rados.0.smithi076.stdout:599: finishing write tid 2 to smithi07685715-30 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:599: finishing write tid 3 to smithi07685715-30 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:599: finishing write tid 4 to smithi07685715-30 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:599: oid 30 updating version 0 to 472 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:599: oid 30 updating version 472 to 473 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:599: oid 30 updating version 473 to 474 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 474 (ObjNum 258 snap 81 seq_num 258) dirty exists 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:599: left oid 30 (ObjNum 258 snap 81 seq_num 258) 2024-09-21T09:40:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:601: finishing write tid 1 to smithi07685715-16 2024-09-21T09:40:21.718 INFO:tasks.rados.rados.0.smithi076.stdout:601: finishing write tid 2 to smithi07685715-16 2024-09-21T09:40:21.718 INFO:tasks.rados.rados.0.smithi076.stdout:601: finishing write tid 3 to smithi07685715-16 2024-09-21T09:40:21.718 INFO:tasks.rados.rados.0.smithi076.stdout:599: done (3 left) 2024-09-21T09:40:21.718 INFO:tasks.rados.rados.0.smithi076.stdout:600: done (2 left) 2024-09-21T09:40:21.718 INFO:tasks.rados.rados.0.smithi076.stdout:602: done (1 left) 2024-09-21T09:40:21.718 INFO:tasks.rados.rados.0.smithi076.stdout:603: snap_remove snap 71 2024-09-21T09:40:21.831 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:40:21.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: pgmap v24: 89 pgs: 89 active+clean; 224 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 900 KiB/s wr, 7 op/s 2024-09-21T09:40:21.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:21.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:21.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:21.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:21.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:21 smithi191 ceph-mon[33301]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:40:21.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: pgmap v24: 89 pgs: 89 active+clean; 224 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 900 KiB/s wr, 7 op/s 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[29313]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: pgmap v24: 89 pgs: 89 active+clean; 224 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 900 KiB/s wr, 7 op/s 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:21.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:21 smithi076 ceph-mon[41325]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:40:22.586 INFO:tasks.rados.rados.0.smithi076.stdout:603: done (1 left) 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:604: snap_create 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:601: finishing write tid 4 to smithi07685715-16 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:601: oid 16 updating version 0 to 320 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:601: oid 16 updating version 320 to 321 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:601: oid 16 version 321 is already newer than 319 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 321 (ObjNum 259 snap 81 seq_num 259) dirty exists 2024-09-21T09:40:22.587 INFO:tasks.rados.rados.0.smithi076.stdout:601: left oid 16 (ObjNum 259 snap 81 seq_num 259) 2024-09-21T09:40:22.684 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:22 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:22.685 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:22 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:22.850 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:22 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:22.850 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:22 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:22.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:22 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:40:22.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:22 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:601: done (1 left) 2024-09-21T09:40:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:604: done (0 left) 2024-09-21T09:40:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:605: write oid 46 current snap is 82 2024-09-21T09:40:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:605: seq_num 261 ranges {786627=795991,2173212=213415} 2024-09-21T09:40:23.602 INFO:tasks.rados.rados.0.smithi076.stdout:605: writing smithi07685715-46 from 786627 to 1582618 tid 1 2024-09-21T09:40:23.607 INFO:tasks.rados.rados.0.smithi076.stdout:605: writing smithi07685715-46 from 2173212 to 2386627 tid 2 2024-09-21T09:40:23.607 INFO:tasks.rados.rados.0.smithi076.stdout:606: read oid 50 snap -1 2024-09-21T09:40:23.607 INFO:tasks.rados.rados.0.smithi076.stdout:606: expect (ObjNum 225 snap 66 seq_num 225) 2024-09-21T09:40:23.607 INFO:tasks.rados.rados.0.smithi076.stdout:607: delete oid 4 current snap is 82 2024-09-21T09:40:23.607 INFO:tasks.rados.rados.0.smithi076.stdout:605: finishing write tid 1 to smithi07685715-46 2024-09-21T09:40:23.609 INFO:tasks.rados.rados.0.smithi076.stdout:605: finishing write tid 2 to smithi07685715-46 2024-09-21T09:40:23.609 INFO:tasks.rados.rados.0.smithi076.stdout:605: finishing write tid 3 to smithi07685715-46 2024-09-21T09:40:23.609 INFO:tasks.rados.rados.0.smithi076.stdout:605: finishing write tid 4 to smithi07685715-46 2024-09-21T09:40:23.609 INFO:tasks.rados.rados.0.smithi076.stdout:605: oid 46 updating version 0 to 370 2024-09-21T09:40:23.609 INFO:tasks.rados.rados.0.smithi076.stdout:605: oid 46 version 370 is already newer than 369 2024-09-21T09:40:23.610 INFO:tasks.rados.rados.0.smithi076.stdout:605: oid 46 version 370 is already newer than 368 2024-09-21T09:40:23.610 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 370 (ObjNum 261 snap 82 seq_num 261) dirty exists 2024-09-21T09:40:23.610 INFO:tasks.rados.rados.0.smithi076.stdout:605: left oid 46 (ObjNum 261 snap 82 seq_num 261) 2024-09-21T09:40:23.623 INFO:tasks.rados.rados.0.smithi076.stdout:606: expect (ObjNum 225 snap 66 seq_num 225) 2024-09-21T09:40:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: pgmap v27: 89 pgs: 89 active+clean; 225 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 505 KiB/s wr, 4 op/s 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: Upgrade: Updating mgr.x 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:23 smithi191 ceph-mon[33301]: Deploying daemon mgr.x on smithi191 2024-09-21T09:40:23.864 INFO:tasks.rados.rados.0.smithi076.stdout:605: done (2 left) 2024-09-21T09:40:23.864 INFO:tasks.rados.rados.0.smithi076.stdout:606: done (1 left) 2024-09-21T09:40:23.864 INFO:tasks.rados.rados.0.smithi076.stdout:607: done (0 left) 2024-09-21T09:40:23.864 INFO:tasks.rados.rados.0.smithi076.stdout:608: write oid 29 current snap is 82 2024-09-21T09:40:23.864 INFO:tasks.rados.rados.0.smithi076.stdout:608: seq_num 263 ranges {420241=546489,1373823=544414} 2024-09-21T09:40:23.868 INFO:tasks.rados.rados.0.smithi076.stdout:608: writing smithi07685715-29 from 420241 to 966730 tid 1 2024-09-21T09:40:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:40:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: pgmap v27: 89 pgs: 89 active+clean; 225 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 505 KiB/s wr, 4 op/s 2024-09-21T09:40:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: Upgrade: Updating mgr.x 2024-09-21T09:40:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[29313]: Deploying daemon mgr.x on smithi191 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: pgmap v27: 89 pgs: 89 active+clean; 225 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 505 KiB/s wr, 4 op/s 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: Upgrade: Updating mgr.x 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:23 smithi076 ceph-mon[41325]: Deploying daemon mgr.x on smithi191 2024-09-21T09:40:23.874 INFO:tasks.rados.rados.0.smithi076.stdout:608: writing smithi07685715-29 from 1373823 to 1918237 tid 2 2024-09-21T09:40:23.874 INFO:tasks.rados.rados.0.smithi076.stdout:609: write oid 48 current snap is 82 2024-09-21T09:40:23.874 INFO:tasks.rados.rados.0.smithi076.stdout:608: finishing write tid 1 to smithi07685715-29 2024-09-21T09:40:23.874 INFO:tasks.rados.rados.0.smithi076.stdout:609: seq_num 264 ranges {437048=621738,1832305=404425} 2024-09-21T09:40:23.879 INFO:tasks.rados.rados.0.smithi076.stdout:609: writing smithi07685715-48 from 437048 to 1058786 tid 1 2024-09-21T09:40:23.886 INFO:tasks.rados.rados.0.smithi076.stdout:609: writing smithi07685715-48 from 1832305 to 2236730 tid 2 2024-09-21T09:40:23.886 INFO:tasks.rados.rados.0.smithi076.stdout:610: write oid 45 current snap is 82 2024-09-21T09:40:23.887 INFO:tasks.rados.rados.0.smithi076.stdout:608: finishing write tid 2 to smithi07685715-29 2024-09-21T09:40:23.887 INFO:tasks.rados.rados.0.smithi076.stdout:610: seq_num 265 ranges {453855=696987,1890787=548083} 2024-09-21T09:40:23.892 INFO:tasks.rados.rados.0.smithi076.stdout:610: writing smithi07685715-45 from 453855 to 1150842 tid 1 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:610: writing smithi07685715-45 from 1890787 to 2438870 tid 2 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:611: snap_create 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:608: finishing write tid 3 to smithi07685715-29 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:608: finishing write tid 4 to smithi07685715-29 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:608: oid 29 updating version 0 to 129 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:608: oid 29 version 129 is already newer than 127 2024-09-21T09:40:23.900 INFO:tasks.rados.rados.0.smithi076.stdout:608: oid 29 version 129 is already newer than 128 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 129 (ObjNum 263 snap 82 seq_num 263) dirty exists 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:608: left oid 29 (ObjNum 263 snap 82 seq_num 263) 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: finishing write tid 1 to smithi07685715-48 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: finishing write tid 2 to smithi07685715-48 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: finishing write tid 3 to smithi07685715-48 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: finishing write tid 4 to smithi07685715-48 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: oid 48 updating version 0 to 479 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: oid 48 updating version 479 to 480 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: oid 48 version 480 is already newer than 478 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 480 (ObjNum 264 snap 82 seq_num 264) dirty exists 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:609: left oid 48 (ObjNum 264 snap 82 seq_num 264) 2024-09-21T09:40:23.901 INFO:tasks.rados.rados.0.smithi076.stdout:610: finishing write tid 1 to smithi07685715-45 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:610: finishing write tid 2 to smithi07685715-45 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:610: finishing write tid 3 to smithi07685715-45 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:610: finishing write tid 4 to smithi07685715-45 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:610: oid 45 updating version 0 to 114 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:610: oid 45 version 114 is already newer than 113 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:610: oid 45 version 114 is already newer than 112 2024-09-21T09:40:23.902 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 114 (ObjNum 265 snap 82 seq_num 265) dirty exists 2024-09-21T09:40:23.903 INFO:tasks.rados.rados.0.smithi076.stdout:610: left oid 45 (ObjNum 265 snap 82 seq_num 265) 2024-09-21T09:40:24.596 INFO:tasks.rados.rados.0.smithi076.stdout:608: done (3 left) 2024-09-21T09:40:24.596 INFO:tasks.rados.rados.0.smithi076.stdout:609: done (2 left) 2024-09-21T09:40:24.596 INFO:tasks.rados.rados.0.smithi076.stdout:610: done (1 left) 2024-09-21T09:40:24.596 INFO:tasks.rados.rados.0.smithi076.stdout:611: done (0 left) 2024-09-21T09:40:24.596 INFO:tasks.rados.rados.0.smithi076.stdout:612: write oid 25 current snap is 83 2024-09-21T09:40:24.596 INFO:tasks.rados.rados.0.smithi076.stdout:612: seq_num 266 ranges {470662=772236,1832916=408094} 2024-09-21T09:40:24.602 INFO:tasks.rados.rados.0.smithi076.stdout:612: writing smithi07685715-25 from 470662 to 1242898 tid 1 2024-09-21T09:40:24.608 INFO:tasks.rados.rados.0.smithi076.stdout:612: writing smithi07685715-25 from 1832916 to 2241010 tid 2 2024-09-21T09:40:24.608 INFO:tasks.rados.rados.0.smithi076.stdout:613: snap_create 2024-09-21T09:40:24.608 INFO:tasks.rados.rados.0.smithi076.stdout:612: finishing write tid 1 to smithi07685715-25 2024-09-21T09:40:24.610 INFO:tasks.rados.rados.0.smithi076.stdout:612: finishing write tid 2 to smithi07685715-25 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:612: finishing write tid 3 to smithi07685715-25 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:612: finishing write tid 4 to smithi07685715-25 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:612: oid 25 updating version 0 to 148 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:612: oid 25 version 148 is already newer than 147 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:612: oid 25 version 148 is already newer than 146 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 148 (ObjNum 266 snap 83 seq_num 266) dirty exists 2024-09-21T09:40:24.611 INFO:tasks.rados.rados.0.smithi076.stdout:612: left oid 25 (ObjNum 266 snap 83 seq_num 266) 2024-09-21T09:40:24.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:24 smithi191 ceph-mon[33301]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:40:24.612 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:24 smithi191 systemd[1]: Stopping Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:24.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:24 smithi076 ceph-mon[29313]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:40:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:24 smithi076 ceph-mon[41325]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:40:24.941 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:24 smithi191 podman[80335]: 2024-09-21 09:40:24.641155801 +0000 UTC m=+0.144792354 container died 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:40:25.194 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:24 smithi191 podman[80335]: 2024-09-21 09:40:24.940672044 +0000 UTC m=+0.444308599 container cleanup 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:40:25.194 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:24 smithi191 bash[80335]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:40:25.553 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 podman[80352]: 2024-09-21 09:40:25.193137729 +0000 UTC m=+0.550907000 container remove 8ab387e15f887f99ad00c810b1b5edcb38651400d5f743245faf8f7ecd1f7626 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:40:25.554 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:25.554 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:40:25.554 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 systemd[1]: Stopped Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:25.554 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service: Consumed 18.134s CPU time. 2024-09-21T09:40:25.554 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 systemd[1]: Starting Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:25.607 INFO:tasks.rados.rados.0.smithi076.stdout:612: done (1 left) 2024-09-21T09:40:25.607 INFO:tasks.rados.rados.0.smithi076.stdout:613: done (0 left) 2024-09-21T09:40:25.607 INFO:tasks.rados.rados.0.smithi076.stdout:614: delete oid 25 current snap is 84 2024-09-21T09:40:25.608 INFO:tasks.rados.rados.0.smithi076.stdout:614: done (0 left) 2024-09-21T09:40:25.609 INFO:tasks.rados.rados.0.smithi076.stdout:615: snap_remove snap 77 2024-09-21T09:40:25.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:25 smithi191 ceph-mon[33301]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:40:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:25 smithi191 ceph-mon[33301]: pgmap v30: 89 pgs: 89 active+clean; 228 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 736 KiB/s wr, 6 op/s 2024-09-21T09:40:25.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 podman[80486]: 2024-09-21 09:40:25.666674641 +0000 UTC m=+0.015954312 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:40:25.863 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:25 smithi191 podman[80486]: 2024-09-21 09:40:25.79515979 +0000 UTC m=+0.144439450 container create 141a29e6e0055f07498cb1ef4e7c1be24573454ddd3742bd40886a11d7abb384 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:40:25.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:25 smithi076 ceph-mon[29313]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:40:25.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:25 smithi076 ceph-mon[29313]: pgmap v30: 89 pgs: 89 active+clean; 228 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 736 KiB/s wr, 6 op/s 2024-09-21T09:40:25.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:25 smithi076 ceph-mon[41325]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:40:25.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:25 smithi076 ceph-mon[41325]: pgmap v30: 89 pgs: 89 active+clean; 228 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 736 KiB/s wr, 6 op/s 2024-09-21T09:40:26.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:26 smithi191 podman[80486]: 2024-09-21 09:40:26.048550665 +0000 UTC m=+0.397830335 container init 141a29e6e0055f07498cb1ef4e7c1be24573454ddd3742bd40886a11d7abb384 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T09:40:26.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:26 smithi191 podman[80486]: 2024-09-21 09:40:26.053106488 +0000 UTC m=+0.402386149 container start 141a29e6e0055f07498cb1ef4e7c1be24573454ddd3742bd40886a11d7abb384 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T09:40:26.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:26 smithi191 ceph-mgr[80524]: -- 172.21.15.191:0/1335619746 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fa6ac8d0e0 con 0x55fa6ac6b400 2024-09-21T09:40:26.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:26 smithi191 bash[80486]: 141a29e6e0055f07498cb1ef4e7c1be24573454ddd3742bd40886a11d7abb384 2024-09-21T09:40:26.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:26 smithi191 systemd[1]: Started Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:26.606 INFO:tasks.rados.rados.0.smithi076.stdout:615: done (0 left) 2024-09-21T09:40:26.606 INFO:tasks.rados.rados.0.smithi076.stdout:616: snap_create 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[29313]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[41325]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:26 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:26.947 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:26 smithi191 ceph-mon[33301]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:40:26.947 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:26 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:26.947 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:26 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:26.947 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:26 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:27.607 INFO:tasks.rados.rados.0.smithi076.stdout:616: done (0 left) 2024-09-21T09:40:27.608 INFO:tasks.rados.rados.0.smithi076.stdout:617: snap_create 2024-09-21T09:40:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:27 smithi076 ceph-mon[29313]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:40:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:27 smithi076 ceph-mon[29313]: pgmap v33: 89 pgs: 89 active+clean; 228 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 230 KiB/s wr, 2 op/s 2024-09-21T09:40:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:27 smithi076 ceph-mon[29313]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:40:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:27 smithi076 ceph-mon[41325]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:40:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:27 smithi076 ceph-mon[41325]: pgmap v33: 89 pgs: 89 active+clean; 228 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 230 KiB/s wr, 2 op/s 2024-09-21T09:40:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:27 smithi076 ceph-mon[41325]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:40:27.909 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:27 smithi191 ceph-mon[33301]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:40:27.910 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:27 smithi191 ceph-mon[33301]: pgmap v33: 89 pgs: 89 active+clean; 228 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 230 KiB/s wr, 2 op/s 2024-09-21T09:40:27.910 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:27 smithi191 ceph-mon[33301]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:40:28.527 INFO:tasks.rados.rados.0.smithi076.stdout:617: done (0 left) 2024-09-21T09:40:28.527 INFO:tasks.rados.rados.0.smithi076.stdout:618: write oid 9 current snap is 86 2024-09-21T09:40:28.527 INFO:tasks.rados.rados.0.smithi076.stdout:618: seq_num 268 ranges {504276=639087,1666233=695410} 2024-09-21T09:40:28.533 INFO:tasks.rados.rados.0.smithi076.stdout:618: writing smithi07685715-9 from 504276 to 1143363 tid 1 2024-09-21T09:40:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:618: writing smithi07685715-9 from 1666233 to 2361643 tid 2 2024-09-21T09:40:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:619: write oid 15 current snap is 86 2024-09-21T09:40:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:619: seq_num 269 ranges {521083=714336,1724715=555421} 2024-09-21T09:40:28.547 INFO:tasks.rados.rados.0.smithi076.stdout:619: writing smithi07685715-15 from 521083 to 1235419 tid 1 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:619: writing smithi07685715-15 from 1724715 to 2280136 tid 2 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:620: read oid 50 snap -1 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:620: expect (ObjNum 225 snap 66 seq_num 225) 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:621: read oid 47 snap 63 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:621: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:622: read oid 34 snap -1 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:622: expect (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:623: write oid 13 current snap is 86 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:618: finishing write tid 1 to smithi07685715-9 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:618: finishing write tid 2 to smithi07685715-9 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:618: finishing write tid 3 to smithi07685715-9 2024-09-21T09:40:28.554 INFO:tasks.rados.rados.0.smithi076.stdout:618: finishing write tid 4 to smithi07685715-9 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:618: oid 9 updating version 0 to 262 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:618: oid 9 version 262 is already newer than 261 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:618: oid 9 updating version 262 to 263 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 263 (ObjNum 268 snap 86 seq_num 268) dirty exists 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:618: left oid 9 (ObjNum 268 snap 86 seq_num 268) 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:619: finishing write tid 1 to smithi07685715-15 2024-09-21T09:40:28.555 INFO:tasks.rados.rados.0.smithi076.stdout:623: seq_num 270 ranges {537890=789585,1783197=699079} 2024-09-21T09:40:28.560 INFO:tasks.rados.rados.0.smithi076.stdout:623: writing smithi07685715-13 from 537890 to 1327475 tid 1 2024-09-21T09:40:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:623: writing smithi07685715-13 from 1783197 to 2482276 tid 2 2024-09-21T09:40:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:618: done (5 left) 2024-09-21T09:40:28.568 INFO:tasks.rados.rados.0.smithi076.stdout:624: read oid 12 snap 73 2024-09-21T09:40:28.568 INFO:tasks.rados.rados.0.smithi076.stdout:624: expect (ObjNum 167 snap 43 seq_num 167) 2024-09-21T09:40:28.568 INFO:tasks.rados.rados.0.smithi076.stdout:625: snap_remove snap 36 2024-09-21T09:40:29.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:40:28] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.43.0" 2024-09-21T09:40:29.538 INFO:tasks.rados.rados.0.smithi076.stdout:625: done (6 left) 2024-09-21T09:40:29.538 INFO:tasks.rados.rados.0.smithi076.stdout:626: snap_remove snap 74 2024-09-21T09:40:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:29 smithi191 ceph-mon[33301]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:40:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:29 smithi191 ceph-mon[33301]: pgmap v36: 89 pgs: 89 active+clean; 234 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:40:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:29 smithi076 ceph-mon[29313]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:40:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:29 smithi076 ceph-mon[29313]: pgmap v36: 89 pgs: 89 active+clean; 234 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:40:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:29 smithi076 ceph-mon[41325]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:40:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:29 smithi076 ceph-mon[41325]: pgmap v36: 89 pgs: 89 active+clean; 234 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:40:30.537 INFO:tasks.rados.rados.0.smithi076.stdout:626: done (6 left) 2024-09-21T09:40:30.538 INFO:tasks.rados.rados.0.smithi076.stdout:627: rollback oid 45 current snap is 86 2024-09-21T09:40:30.538 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 83 2024-09-21T09:40:30.538 INFO:tasks.rados.rados.0.smithi076.stdout:628: snap_remove snap 68 2024-09-21T09:40:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:30 smithi191 ceph-mon[33301]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:40:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:30 smithi076 ceph-mon[29313]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:40:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:30 smithi076 ceph-mon[41325]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:40:31.539 INFO:tasks.rados.rados.0.smithi076.stdout:628: done (7 left) 2024-09-21T09:40:31.539 INFO:tasks.rados.rados.0.smithi076.stdout:629: read oid 17 snap -1 2024-09-21T09:40:31.539 INFO:tasks.rados.rados.0.smithi076.stdout:629: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:31.539 INFO:tasks.rados.rados.0.smithi076.stdout:630: write oid 22 current snap is 86 2024-09-21T09:40:31.539 INFO:tasks.rados.rados.0.smithi076.stdout:630: seq_num 271 ranges {554697=464834,1725326=559090} 2024-09-21T09:40:31.545 INFO:tasks.rados.rados.0.smithi076.stdout:630: writing smithi07685715-22 from 554697 to 1019531 tid 1 2024-09-21T09:40:31.553 INFO:tasks.rados.rados.0.smithi076.stdout:630: writing smithi07685715-22 from 1725326 to 2284416 tid 2 2024-09-21T09:40:31.553 INFO:tasks.rados.rados.0.smithi076.stdout:631: snap_remove snap 76 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: pgmap v39: 89 pgs: 89 active+clean; 234 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: Standby manager daemon x restarted 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: Standby manager daemon x started 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.832 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-mon[33301]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:40:31.833 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: [21/Sep/2024:09:40:31] ENGINE Bus STARTING 2024-09-21T09:40:31.833 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: CherryPy Checker: 2024-09-21T09:40:31.833 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: The Application mounted at '' has an empty config. 2024-09-21T09:40:31.833 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: 2024-09-21T09:40:31.833 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: [21/Sep/2024:09:40:31] ENGINE Serving on http://:::9283 2024-09-21T09:40:31.833 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:40:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: [21/Sep/2024:09:40:31] ENGINE Bus STARTED 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: pgmap v39: 89 pgs: 89 active+clean; 234 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: Standby manager daemon x restarted 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: Standby manager daemon x started 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[29313]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: pgmap v39: 89 pgs: 89 active+clean; 234 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: Standby manager daemon x restarted 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: Standby manager daemon x started 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:40:31.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/4236440874' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:40:31.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:31.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:31 smithi076 ceph-mon[41325]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:40:32.558 INFO:tasks.rados.rados.0.smithi076.stdout:619: finishing write tid 2 to smithi07685715-15 2024-09-21T09:40:32.558 INFO:tasks.rados.rados.0.smithi076.stdout:619: finishing write tid 3 to smithi07685715-15 2024-09-21T09:40:32.558 INFO:tasks.rados.rados.0.smithi076.stdout:619: finishing write tid 4 to smithi07685715-15 2024-09-21T09:40:32.558 INFO:tasks.rados.rados.0.smithi076.stdout:619: oid 15 updating version 0 to 119 2024-09-21T09:40:32.559 INFO:tasks.rados.rados.0.smithi076.stdout:619: oid 15 version 119 is already newer than 117 2024-09-21T09:40:32.559 INFO:tasks.rados.rados.0.smithi076.stdout:619: oid 15 version 119 is already newer than 118 2024-09-21T09:40:32.559 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 119 (ObjNum 269 snap 86 seq_num 269) dirty exists 2024-09-21T09:40:32.559 INFO:tasks.rados.rados.0.smithi076.stdout:619: left oid 15 (ObjNum 269 snap 86 seq_num 269) 2024-09-21T09:40:32.559 INFO:tasks.rados.rados.0.smithi076.stdout:622: expect (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:40:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:623: finishing write tid 1 to smithi07685715-13 2024-09-21T09:40:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:621: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: finishing write tid 2 to smithi07685715-13 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: finishing write tid 3 to smithi07685715-13 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: finishing write tid 4 to smithi07685715-13 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: oid 13 updating version 0 to 78 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: oid 13 updating version 78 to 79 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: oid 13 version 79 is already newer than 77 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 79 (ObjNum 270 snap 86 seq_num 270) dirty exists 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:623: left oid 13 (ObjNum 270 snap 86 seq_num 270) 2024-09-21T09:40:32.842 INFO:tasks.rados.rados.0.smithi076.stdout:624: expect (ObjNum 167 snap 43 seq_num 167) 2024-09-21T09:40:32.921 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:32 smithi191 ceph-mon[33301]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-21T09:40:32.921 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:32 smithi191 ceph-mon[33301]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:40:32.929 INFO:tasks.rados.rados.0.smithi076.stdout:620: expect (ObjNum 225 snap 66 seq_num 225) 2024-09-21T09:40:33.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:32 smithi076 ceph-mon[29313]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-21T09:40:33.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:32 smithi076 ceph-mon[29313]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:40:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:32 smithi076 ceph-mon[41325]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-21T09:40:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:32 smithi076 ceph-mon[41325]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:40:33.163 INFO:tasks.rados.rados.0.smithi076.stdout:627: finishing rollback tid 0 to smithi07685715-45 2024-09-21T09:40:33.163 INFO:tasks.rados.rados.0.smithi076.stdout:627: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:40:33.163 INFO:tasks.rados.rados.0.smithi076.stdout:627: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:40:33.163 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 127 (ObjNum 265 snap 82 seq_num 265) dirty exists 2024-09-21T09:40:33.163 INFO:tasks.rados.rados.0.smithi076.stdout:629: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:33.236 INFO:tasks.rados.rados.0.smithi076.stdout:630: finishing write tid 1 to smithi07685715-22 2024-09-21T09:40:33.236 INFO:tasks.rados.rados.0.smithi076.stdout:630: finishing write tid 2 to smithi07685715-22 2024-09-21T09:40:33.236 INFO:tasks.rados.rados.0.smithi076.stdout:630: finishing write tid 3 to smithi07685715-22 2024-09-21T09:40:33.236 INFO:tasks.rados.rados.0.smithi076.stdout:630: finishing write tid 4 to smithi07685715-22 2024-09-21T09:40:33.236 INFO:tasks.rados.rados.0.smithi076.stdout:630: oid 22 updating version 0 to 348 2024-09-21T09:40:33.236 INFO:tasks.rados.rados.0.smithi076.stdout:630: oid 22 version 348 is already newer than 347 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:630: oid 22 version 348 is already newer than 346 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 348 (ObjNum 271 snap 86 seq_num 271) dirty exists 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:630: left oid 22 (ObjNum 271 snap 86 seq_num 271) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:619: done (9 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:620: done (8 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:621: done (7 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:622: done (6 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:623: done (5 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:624: done (4 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:627: done (3 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:629: done (2 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:630: done (1 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:631: done (0 left) 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:632: write oid 21 current snap is 86 2024-09-21T09:40:33.237 INFO:tasks.rados.rados.0.smithi076.stdout:632: seq_num 272 ranges {571504=540083,1783808=702748} 2024-09-21T09:40:33.242 INFO:tasks.rados.rados.0.smithi076.stdout:632: writing smithi07685715-21 from 571504 to 1111587 tid 1 2024-09-21T09:40:33.250 INFO:tasks.rados.rados.0.smithi076.stdout:632: writing smithi07685715-21 from 1783808 to 2486556 tid 2 2024-09-21T09:40:33.250 INFO:tasks.rados.rados.0.smithi076.stdout:633: delete oid 45 current snap is 86 2024-09-21T09:40:33.250 INFO:tasks.rados.rados.0.smithi076.stdout:632: finishing write tid 1 to smithi07685715-21 2024-09-21T09:40:33.251 INFO:tasks.rados.rados.0.smithi076.stdout:633: done (1 left) 2024-09-21T09:40:33.251 INFO:tasks.rados.rados.0.smithi076.stdout:634: snap_create 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: finishing write tid 2 to smithi07685715-21 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: finishing write tid 3 to smithi07685715-21 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: finishing write tid 4 to smithi07685715-21 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: oid 21 updating version 0 to 401 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: oid 21 version 401 is already newer than 400 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: oid 21 updating version 401 to 402 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 402 (ObjNum 272 snap 86 seq_num 272) dirty exists 2024-09-21T09:40:33.254 INFO:tasks.rados.rados.0.smithi076.stdout:632: left oid 21 (ObjNum 272 snap 86 seq_num 272) 2024-09-21T09:40:33.876 INFO:tasks.rados.rados.0.smithi076.stdout:632: done (1 left) 2024-09-21T09:40:33.877 INFO:tasks.rados.rados.0.smithi076.stdout:634: done (0 left) 2024-09-21T09:40:33.877 INFO:tasks.rados.rados.0.smithi076.stdout:635: rollback oid 19 current snap is 87 2024-09-21T09:40:33.877 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 83 2024-09-21T09:40:33.877 INFO:tasks.rados.rados.0.smithi076.stdout:636: rollback oid 6 current snap is 87 2024-09-21T09:40:33.877 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 84 2024-09-21T09:40:33.877 INFO:tasks.rados.rados.0.smithi076.stdout:637: snap_remove snap 43 2024-09-21T09:40:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:33 smithi076 ceph-mon[29313]: pgmap v42: 89 pgs: 89 active+clean; 236 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1000 KiB/s wr, 11 op/s 2024-09-21T09:40:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:33 smithi076 ceph-mon[41325]: pgmap v42: 89 pgs: 89 active+clean; 236 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1000 KiB/s wr, 11 op/s 2024-09-21T09:40:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:33 smithi191 ceph-mon[33301]: pgmap v42: 89 pgs: 89 active+clean; 236 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1000 KiB/s wr, 11 op/s 2024-09-21T09:40:34.886 INFO:tasks.rados.rados.0.smithi076.stdout:637: done (2 left) 2024-09-21T09:40:34.886 INFO:tasks.rados.rados.0.smithi076.stdout:638: read oid 26 snap -1 2024-09-21T09:40:34.886 INFO:tasks.rados.rados.0.smithi076.stdout:638: expect deleted 2024-09-21T09:40:34.886 INFO:tasks.rados.rados.0.smithi076.stdout:639: rollback oid 14 current snap is 87 2024-09-21T09:40:34.886 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 82 2024-09-21T09:40:34.886 INFO:tasks.rados.rados.0.smithi076.stdout:640: snap_create 2024-09-21T09:40:34.887 INFO:tasks.rados.rados.0.smithi076.stdout:636: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:40:34.887 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 0 (ObjNum 129 snap 0 seq_num 864305136) dirty dne 2024-09-21T09:40:34.887 INFO:tasks.rados.rados.0.smithi076.stdout:635: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:40:34.887 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 165 (ObjNum 369 snap 0 seq_num 864449488) dirty dne 2024-09-21T09:40:34.891 INFO:tasks.rados.rados.0.smithi076.stdout:639: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:40:34.891 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 166 (ObjNum 289 snap 0 seq_num 1682288400) dirty dne 2024-09-21T09:40:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:34 smithi191 ceph-mon[33301]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:40:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:34 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:34 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[29313]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[41325]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:35.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:635: done (4 left) 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:636: done (3 left) 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:638: done (2 left) 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:639: done (1 left) 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:640: done (0 left) 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:641: read oid 18 snap -1 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:641: expect (ObjNum 248 snap 76 seq_num 248) 2024-09-21T09:40:35.892 INFO:tasks.rados.rados.0.smithi076.stdout:642: snap_remove snap 87 2024-09-21T09:40:36.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: pgmap v44: 89 pgs: 89 active+clean; 234 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:40:36.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:40:36.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: pgmap v44: 89 pgs: 89 active+clean; 234 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:40:36.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: pgmap v44: 89 pgs: 89 active+clean; 234 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:40:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:36.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:36.899 INFO:tasks.rados.rados.0.smithi076.stdout:642: done (1 left) 2024-09-21T09:40:36.900 INFO:tasks.rados.rados.0.smithi076.stdout:643: read oid 13 snap -1 2024-09-21T09:40:36.900 INFO:tasks.rados.rados.0.smithi076.stdout:643: expect (ObjNum 270 snap 86 seq_num 270) 2024-09-21T09:40:36.900 INFO:tasks.rados.rados.0.smithi076.stdout:644: delete oid 39 current snap is 88 2024-09-21T09:40:36.900 INFO:tasks.rados.rados.0.smithi076.stdout:641: expect (ObjNum 248 snap 76 seq_num 248) 2024-09-21T09:40:37.022 INFO:tasks.rados.rados.0.smithi076.stdout:643: expect (ObjNum 270 snap 86 seq_num 270) 2024-09-21T09:40:37.123 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:37 smithi076 systemd[1]: Stopping Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all mgr 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all crash 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all rgw 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:40:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all nfs 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[29313]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all mgr 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all crash 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all rgw 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all nfs 2024-09-21T09:40:37.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:40:37.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:36 smithi076 ceph-mon[41325]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:641: done (2 left) 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:643: done (1 left) 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:644: done (0 left) 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:645: read oid 40 snap -1 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:645: expect (ObjNum 238 snap 75 seq_num 238) 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:646: read oid 11 snap -1 2024-09-21T09:40:37.174 INFO:tasks.rados.rados.0.smithi076.stdout:646: expect deleted 2024-09-21T09:40:37.175 INFO:tasks.rados.rados.0.smithi076.stdout:647: snap_remove snap 83 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all mgr 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all crash 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all rgw 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all nfs 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:40:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:36 smithi191 ceph-mon[33301]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:40:37.622 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:37 smithi076 podman[97207]: 2024-09-21 09:40:37.305105201 +0000 UTC m=+0.133816343 container died 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:37.891 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:37 smithi076 podman[97207]: 2024-09-21 09:40:37.818107492 +0000 UTC m=+0.646818623 container cleanup 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:37.891 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:37 smithi076 bash[97207]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:647: done (2 left) 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:648: read oid 18 snap -1 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:648: expect (ObjNum 248 snap 76 seq_num 248) 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:649: read oid 25 snap -1 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:649: expect deleted 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:650: delete oid 14 current snap is 88 2024-09-21T09:40:37.905 INFO:tasks.rados.rados.0.smithi076.stdout:645: expect (ObjNum 238 snap 75 seq_num 238) 2024-09-21T09:40:37.974 INFO:tasks.rados.rados.0.smithi076.stdout:645: done (4 left) 2024-09-21T09:40:37.974 INFO:tasks.rados.rados.0.smithi076.stdout:646: done (3 left) 2024-09-21T09:40:37.974 INFO:tasks.rados.rados.0.smithi076.stdout:650: done (2 left) 2024-09-21T09:40:37.974 INFO:tasks.rados.rados.0.smithi076.stdout:651: snap_remove snap 57 2024-09-21T09:40:38.277 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 podman[97224]: 2024-09-21 09:40:38.061634173 +0000 UTC m=+0.760168348 container remove 6dbd51c32fb527481e646a4e73bd76a8354038bf3ffdf90640ee9cc6a9bed9f1 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:38.289 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:38.289 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[29313]: Upgrade: Updating node-exporter.a (1/2) 2024-09-21T09:40:38.289 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[29313]: Deploying daemon node-exporter.a on smithi076 2024-09-21T09:40:38.289 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[29313]: pgmap v47: 89 pgs: 89 active+clean; 234 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[29313]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[41325]: Upgrade: Updating node-exporter.a (1/2) 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[41325]: Deploying daemon node-exporter.a on smithi076 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[41325]: pgmap v47: 89 pgs: 89 active+clean; 234 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[41325]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:37 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:38.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:37 smithi191 ceph-mon[33301]: Upgrade: Updating node-exporter.a (1/2) 2024-09-21T09:40:38.291 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:37 smithi191 ceph-mon[33301]: Deploying daemon node-exporter.a on smithi076 2024-09-21T09:40:38.291 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:37 smithi191 ceph-mon[33301]: pgmap v47: 89 pgs: 89 active+clean; 234 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:40:38.291 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:37 smithi191 ceph-mon[33301]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:40:38.291 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:37 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:38.530 INFO:tasks.rados.rados.0.smithi076.stdout:651: done (2 left) 2024-09-21T09:40:38.531 INFO:tasks.rados.rados.0.smithi076.stdout:652: write oid 7 current snap is 88 2024-09-21T09:40:38.531 INFO:tasks.rados.rados.0.smithi076.stdout:652: seq_num 276 ranges {638732=557432,1617736=710086} 2024-09-21T09:40:38.537 INFO:tasks.rados.rados.0.smithi076.stdout:652: writing smithi07685715-7 from 638732 to 1196164 tid 1 2024-09-21T09:40:38.544 INFO:tasks.rados.rados.0.smithi076.stdout:652: writing smithi07685715-7 from 1617736 to 2327822 tid 2 2024-09-21T09:40:38.544 INFO:tasks.rados.rados.0.smithi076.stdout:653: delete oid 13 current snap is 88 2024-09-21T09:40:38.544 INFO:tasks.rados.rados.0.smithi076.stdout:648: expect (ObjNum 248 snap 76 seq_num 248) 2024-09-21T09:40:38.622 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T09:40:38.622 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 systemd[1]: Stopped Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:38.622 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 2.325s CPU time. 2024-09-21T09:40:38.623 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 systemd[1]: Starting Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: finishing write tid 1 to smithi07685715-7 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:648: done (3 left) 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:649: done (2 left) 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:653: done (1 left) 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:654: delete oid 6 current snap is 88 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: finishing write tid 2 to smithi07685715-7 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: finishing write tid 3 to smithi07685715-7 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: finishing write tid 4 to smithi07685715-7 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: oid 7 updating version 0 to 283 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: oid 7 version 283 is already newer than 281 2024-09-21T09:40:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:652: oid 7 version 283 is already newer than 282 2024-09-21T09:40:38.682 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 283 (ObjNum 276 snap 88 seq_num 276) dirty exists 2024-09-21T09:40:38.682 INFO:tasks.rados.rados.0.smithi076.stdout:652: left oid 7 (ObjNum 276 snap 88 seq_num 276) 2024-09-21T09:40:38.682 INFO:tasks.rados.rados.0.smithi076.stdout:652: done (1 left) 2024-09-21T09:40:38.682 INFO:tasks.rados.rados.0.smithi076.stdout:654: done (0 left) 2024-09-21T09:40:38.683 INFO:tasks.rados.rados.0.smithi076.stdout:655: read oid 4 snap -1 2024-09-21T09:40:38.683 INFO:tasks.rados.rados.0.smithi076.stdout:655: expect deleted 2024-09-21T09:40:38.683 INFO:tasks.rados.rados.0.smithi076.stdout:656: rollback oid 16 current snap is 88 2024-09-21T09:40:38.683 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 86 2024-09-21T09:40:38.683 INFO:tasks.rados.rados.0.smithi076.stdout:657: write oid 45 current snap is 88 2024-09-21T09:40:38.683 INFO:tasks.rados.rados.0.smithi076.stdout:657: seq_num 279 ranges {689153=783179,2076829=457413} 2024-09-21T09:40:38.690 INFO:tasks.rados.rados.0.smithi076.stdout:657: writing smithi07685715-45 from 689153 to 1472332 tid 1 2024-09-21T09:40:38.696 INFO:tasks.rados.rados.0.smithi076.stdout:657: writing smithi07685715-45 from 2076829 to 2534242 tid 2 2024-09-21T09:40:38.696 INFO:tasks.rados.rados.0.smithi076.stdout:658: delete oid 11 current snap is 88 2024-09-21T09:40:38.696 INFO:tasks.rados.rados.0.smithi076.stdout:656: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:40:38.696 INFO:tasks.rados.rados.0.smithi076.stdout:656: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:40:38.697 INFO:tasks.rados.rados.0.smithi076.stdout:656: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:40:38.697 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 370 (ObjNum 259 snap 81 seq_num 259) dirty exists 2024-09-21T09:40:38.697 INFO:tasks.rados.rados.0.smithi076.stdout:657: finishing write tid 1 to smithi07685715-45 2024-09-21T09:40:38.698 INFO:tasks.rados.rados.0.smithi076.stdout:655: done (3 left) 2024-09-21T09:40:38.698 INFO:tasks.rados.rados.0.smithi076.stdout:656: done (2 left) 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:658: done (1 left) 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:659: read oid 41 snap -1 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:659: expect deleted 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:660: read oid 8 snap 63 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:660: expect (ObjNum 207 snap 60 seq_num 207) 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:661: rollback oid 10 current snap is 88 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 78 2024-09-21T09:40:38.699 INFO:tasks.rados.rados.0.smithi076.stdout:662: snap_remove snap 58 2024-09-21T09:40:39.079 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:38 smithi076 ceph-mon[29313]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:40:39.079 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:38 smithi076 ceph-mon[29313]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:40:39.080 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:40:38] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.43.0" 2024-09-21T09:40:39.080 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:38 smithi076 bash[97355]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:40:39.080 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:38 smithi076 ceph-mon[41325]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:40:39.080 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:38 smithi076 ceph-mon[41325]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:40:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:38 smithi191 ceph-mon[33301]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:40:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:38 smithi191 ceph-mon[33301]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:40:39.372 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:39 smithi076 bash[97355]: Getting image source signatures 2024-09-21T09:40:39.372 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:39 smithi076 bash[97355]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:40:39.372 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:39 smithi076 bash[97355]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:40:39.372 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:39 smithi076 bash[97355]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:40:39.538 INFO:tasks.rados.rados.0.smithi076.stdout:662: done (4 left) 2024-09-21T09:40:39.538 INFO:tasks.rados.rados.0.smithi076.stdout:663: read oid 11 snap -1 2024-09-21T09:40:39.538 INFO:tasks.rados.rados.0.smithi076.stdout:663: expect deleted 2024-09-21T09:40:39.538 INFO:tasks.rados.rados.0.smithi076.stdout:664: snap_remove snap 75 2024-09-21T09:40:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:39 smithi191 ceph-mon[33301]: pgmap v51: 89 pgs: 89 active+clean; 229 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 319 KiB/s wr, 8 op/s 2024-09-21T09:40:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:39 smithi191 ceph-mon[33301]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:40:40.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:39 smithi076 ceph-mon[29313]: pgmap v51: 89 pgs: 89 active+clean; 229 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 319 KiB/s wr, 8 op/s 2024-09-21T09:40:40.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:39 smithi076 ceph-mon[29313]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:40:40.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:39 smithi076 ceph-mon[41325]: pgmap v51: 89 pgs: 89 active+clean; 229 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 319 KiB/s wr, 8 op/s 2024-09-21T09:40:40.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:39 smithi076 ceph-mon[41325]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:40:40.540 INFO:tasks.rados.rados.0.smithi076.stdout:664: done (5 left) 2024-09-21T09:40:40.540 INFO:tasks.rados.rados.0.smithi076.stdout:665: delete oid 16 current snap is 88 2024-09-21T09:40:40.540 INFO:tasks.rados.rados.0.smithi076.stdout:657: finishing write tid 2 to smithi07685715-45 2024-09-21T09:40:40.540 INFO:tasks.rados.rados.0.smithi076.stdout:657: finishing write tid 3 to smithi07685715-45 2024-09-21T09:40:40.540 INFO:tasks.rados.rados.0.smithi076.stdout:657: finishing write tid 4 to smithi07685715-45 2024-09-21T09:40:40.540 INFO:tasks.rados.rados.0.smithi076.stdout:657: oid 45 updating version 0 to 141 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:657: oid 45 version 141 is already newer than 140 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:657: oid 45 version 141 is already newer than 139 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 141 (ObjNum 279 snap 88 seq_num 279) dirty exists 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:657: left oid 45 (ObjNum 279 snap 88 seq_num 279) 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:661: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 0 (ObjNum 81 snap 0 seq_num 1682429920) dirty dne 2024-09-21T09:40:40.541 INFO:tasks.rados.rados.0.smithi076.stdout:660: expect (ObjNum 207 snap 60 seq_num 207) 2024-09-21T09:40:40.698 INFO:tasks.rados.rados.0.smithi076.stdout:657: done (5 left) 2024-09-21T09:40:40.698 INFO:tasks.rados.rados.0.smithi076.stdout:659: done (4 left) 2024-09-21T09:40:40.698 INFO:tasks.rados.rados.0.smithi076.stdout:660: done (3 left) 2024-09-21T09:40:40.698 INFO:tasks.rados.rados.0.smithi076.stdout:661: done (2 left) 2024-09-21T09:40:40.698 INFO:tasks.rados.rados.0.smithi076.stdout:663: done (1 left) 2024-09-21T09:40:40.698 INFO:tasks.rados.rados.0.smithi076.stdout:665: done (0 left) 2024-09-21T09:40:40.699 INFO:tasks.rados.rados.0.smithi076.stdout:666: rollback oid 37 current snap is 88 2024-09-21T09:40:40.699 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 66 2024-09-21T09:40:40.699 INFO:tasks.rados.rados.0.smithi076.stdout:667: rollback oid 21 current snap is 88 2024-09-21T09:40:40.699 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 66 2024-09-21T09:40:40.699 INFO:tasks.rados.rados.0.smithi076.stdout:668: write oid 40 current snap is 88 2024-09-21T09:40:40.699 INFO:tasks.rados.rados.0.smithi076.stdout:668: seq_num 282 ranges {739574=725279,1968628=721093} 2024-09-21T09:40:40.705 INFO:tasks.rados.rados.0.smithi076.stdout:668: writing smithi07685715-40 from 739574 to 1464853 tid 1 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:668: writing smithi07685715-40 from 1968628 to 2689721 tid 2 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:669: write oid 42 current snap is 88 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:666: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:666: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:666: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 155 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:667: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:667: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:40:40.713 INFO:tasks.rados.rados.0.smithi076.stdout:669: seq_num 283 ranges {756381=400528,1910757=464751} 2024-09-21T09:40:40.718 INFO:tasks.rados.rados.0.smithi076.stdout:669: writing smithi07685715-42 from 756381 to 1156909 tid 1 2024-09-21T09:40:40.725 INFO:tasks.rados.rados.0.smithi076.stdout:669: writing smithi07685715-42 from 1910757 to 2375508 tid 2 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:666: done (3 left) 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:670: snap_create 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:667: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 442 (ObjNum 223 snap 64 seq_num 223) dirty exists 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: finishing write tid 1 to smithi07685715-40 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: finishing write tid 2 to smithi07685715-40 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: finishing write tid 3 to smithi07685715-40 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: finishing write tid 4 to smithi07685715-40 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: oid 40 updating version 0 to 385 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: oid 40 version 385 is already newer than 383 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: oid 40 version 385 is already newer than 384 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 385 (ObjNum 282 snap 88 seq_num 282) dirty exists 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:668: left oid 40 (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:40:40.726 INFO:tasks.rados.rados.0.smithi076.stdout:669: finishing write tid 1 to smithi07685715-42 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: finishing write tid 2 to smithi07685715-42 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: finishing write tid 3 to smithi07685715-42 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: finishing write tid 4 to smithi07685715-42 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: oid 42 updating version 0 to 164 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: oid 42 updating version 164 to 166 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: oid 42 version 166 is already newer than 165 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 166 (ObjNum 283 snap 88 seq_num 283) dirty exists 2024-09-21T09:40:40.728 INFO:tasks.rados.rados.0.smithi076.stdout:669: left oid 42 (ObjNum 283 snap 88 seq_num 283) 2024-09-21T09:40:40.872 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:40 smithi076 bash[97355]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:40:40.873 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:40 smithi076 bash[97355]: Writing manifest to image destination 2024-09-21T09:40:41.276 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:40 smithi076 podman[97355]: 2024-09-21 09:40:40.932079402 +0000 UTC m=+2.317534840 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:40:41.276 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 podman[97355]: 2024-09-21 09:40:41.164136255 +0000 UTC m=+2.549591697 container create a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:41.551 INFO:tasks.rados.rados.0.smithi076.stdout:667: done (3 left) 2024-09-21T09:40:41.551 INFO:tasks.rados.rados.0.smithi076.stdout:668: done (2 left) 2024-09-21T09:40:41.552 INFO:tasks.rados.rados.0.smithi076.stdout:669: done (1 left) 2024-09-21T09:40:41.552 INFO:tasks.rados.rados.0.smithi076.stdout:670: done (0 left) 2024-09-21T09:40:41.552 INFO:tasks.rados.rados.0.smithi076.stdout:671: rollback oid 28 current snap is 89 2024-09-21T09:40:41.552 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 28 to 84 2024-09-21T09:40:41.552 INFO:tasks.rados.rados.0.smithi076.stdout:672: snap_remove snap 79 2024-09-21T09:40:41.837 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-mon[29313]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:40:41.837 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-mon[29313]: pgmap v54: 89 pgs: 89 active+clean; 229 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 321 KiB/s wr, 8 op/s 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 podman[97355]: 2024-09-21 09:40:41.407720503 +0000 UTC m=+2.793175949 container init a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 podman[97355]: 2024-09-21 09:40:41.413575223 +0000 UTC m=+2.799030659 container start a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.416Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.416Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:40:41.838 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:40:41.839 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:40:41.839 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:40:41.839 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:40:41.839 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:40:41.839 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:40:41.839 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:40:41.839 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-mon[41325]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:40:41.840 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-mon[41325]: pgmap v54: 89 pgs: 89 active+clean; 229 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 321 KiB/s wr, 8 op/s 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:40:41.840 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:40:41.841 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:40:41.842 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:40:41.843 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.417Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:40:41.843 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.418Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:40:41.843 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a[97430]: ts=2024-09-21T09:40:41.418Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:40:41.843 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 bash[97355]: a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b 2024-09-21T09:40:41.843 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 09:40:41 smithi076 systemd[1]: Started Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:41 smithi191 ceph-mon[33301]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:40:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:41 smithi191 ceph-mon[33301]: pgmap v54: 89 pgs: 89 active+clean; 229 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 321 KiB/s wr, 8 op/s 2024-09-21T09:40:42.397 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:40:42.397 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (4m) 30s ago 5m 26.6M - 0.25.0 c8568f914cd2 d930ef8d31ba 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (4m) 11s ago 4m 88.6M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (3m) 30s ago 3m 16.7M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (3m) 11s ago 3m 15.9M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (16s) 11s ago 9m 66.3M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (43s) 30s ago 10m 489M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (10m) 30s ago 10m 57.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (9m) 11s ago 9m 47.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (9m) 30s ago 9m 45.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 starting - - - - 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (4m) 11s ago 4m 15.4M - 1.5.0 0da6a335fe13 9722666fd438 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (8m) 30s ago 8m 78.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (8m) 30s ago 8m 70.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:40:42.398 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (7m) 30s ago 7m 78.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:40:42.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (7m) 30s ago 7m 67.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:40:42.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (7m) 11s ago 7m 71.4M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:40:42.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (6m) 11s ago 6m 84.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:40:42.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (6m) 11s ago 6m 62.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:40:42.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (5m) 11s ago 5m 57.4M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:40:42.399 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (21s) 11s ago 5m 34.5M - 2.43.0 a07b618ecd1d 2f21884f574e 2024-09-21T09:40:42.547 INFO:tasks.rados.rados.0.smithi076.stdout:672: done (1 left) 2024-09-21T09:40:42.547 INFO:tasks.rados.rados.0.smithi076.stdout:673: delete oid 29 current snap is 89 2024-09-21T09:40:42.547 INFO:tasks.rados.rados.0.smithi076.stdout:671: finishing rollback tid 0 to smithi07685715-28 2024-09-21T09:40:42.547 INFO:tasks.rados.rados.0.smithi076.stdout:671: finishing rollback tid 1 to smithi07685715-28 2024-09-21T09:40:42.547 INFO:tasks.rados.rados.0.smithi076.stdout:671: finishing rollback tid 2 to smithi07685715-28 2024-09-21T09:40:42.548 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 540 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-09-21T09:40:42.552 INFO:tasks.rados.rados.0.smithi076.stdout:671: done (1 left) 2024-09-21T09:40:42.552 INFO:tasks.rados.rados.0.smithi076.stdout:673: done (0 left) 2024-09-21T09:40:42.552 INFO:tasks.rados.rados.0.smithi076.stdout:674: read oid 5 snap -1 2024-09-21T09:40:42.552 INFO:tasks.rados.rados.0.smithi076.stdout:674: expect (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:40:42.553 INFO:tasks.rados.rados.0.smithi076.stdout:675: snap_remove snap 81 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:40:42.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:40:42.749 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:40:42.749 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:40:42.749 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:40:42.749 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:40:42.749 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:40:42.749 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[29313]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:40:42.749 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:42.749 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:42.749 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[29313]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:42.749 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[41325]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:40:42.750 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:42.750 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:42.750 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:42 smithi076 ceph-mon[41325]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:42 smithi191 ceph-mon[33301]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:40:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:42 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:42 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:42 smithi191 ceph-mon[33301]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [ 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr" 2024-09-21T09:40:43.032 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:40:43.033 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:40:43.033 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading node-exporter daemons", 2024-09-21T09:40:43.033 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:40:43.033 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:40:43.551 INFO:tasks.rados.rados.0.smithi076.stdout:674: expect (ObjNum 218 snap 63 seq_num 218) 2024-09-21T09:40:43.613 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:43 smithi191 systemd[1]: Stopping Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:43 smithi191 ceph-mon[33301]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:43 smithi191 ceph-mon[33301]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:43 smithi191 ceph-mon[33301]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:40:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:43 smithi191 ceph-mon[33301]: pgmap v57: 89 pgs: 89 active+clean; 232 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 824 KiB/s wr, 9 op/s 2024-09-21T09:40:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:43 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4155317718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:43 smithi191 ceph-mon[33301]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:40:43.741 INFO:tasks.rados.rados.0.smithi076.stdout:674: done (1 left) 2024-09-21T09:40:43.741 INFO:tasks.rados.rados.0.smithi076.stdout:675: done (0 left) 2024-09-21T09:40:43.741 INFO:tasks.rados.rados.0.smithi076.stdout:676: write oid 44 current snap is 89 2024-09-21T09:40:43.741 INFO:tasks.rados.rados.0.smithi076.stdout:676: seq_num 285 ranges {789995=551026,2027721=468420} 2024-09-21T09:40:43.748 INFO:tasks.rados.rados.0.smithi076.stdout:676: writing smithi07685715-44 from 789995 to 1341021 tid 1 2024-09-21T09:40:43.754 INFO:tasks.rados.rados.0.smithi076.stdout:676: writing smithi07685715-44 from 2027721 to 2496141 tid 2 2024-09-21T09:40:43.755 INFO:tasks.rados.rados.0.smithi076.stdout:677: snap_remove snap 73 2024-09-21T09:40:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[29313]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[29313]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[29313]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[29313]: pgmap v57: 89 pgs: 89 active+clean; 232 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 824 KiB/s wr, 9 op/s 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4155317718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[29313]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[41325]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[41325]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[41325]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[41325]: pgmap v57: 89 pgs: 89 active+clean; 232 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 824 KiB/s wr, 9 op/s 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4155317718' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:43 smithi076 ceph-mon[41325]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:40:44.112 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:43 smithi191 podman[84385]: 2024-09-21 09:40:43.798107299 +0000 UTC m=+0.113371804 container died 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:44.488 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 podman[84385]: 2024-09-21 09:40:44.218735889 +0000 UTC m=+0.534000377 container cleanup 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:44.488 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 bash[84385]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:677: done (1 left) 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:678: write oid 42 current snap is 89 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: finishing write tid 1 to smithi07685715-44 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: finishing write tid 2 to smithi07685715-44 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: finishing write tid 3 to smithi07685715-44 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: finishing write tid 4 to smithi07685715-44 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: oid 44 updating version 0 to 142 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: oid 44 version 142 is already newer than 141 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: oid 44 version 142 is already newer than 140 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 142 (ObjNum 285 snap 89 seq_num 285) dirty exists 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:676: left oid 44 (ObjNum 285 snap 89 seq_num 285) 2024-09-21T09:40:44.569 INFO:tasks.rados.rados.0.smithi076.stdout:678: seq_num 286 ranges {406802=626275,1686203=728431} 2024-09-21T09:40:44.574 INFO:tasks.rados.rados.0.smithi076.stdout:678: writing smithi07685715-42 from 406802 to 1033077 tid 1 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:678: writing smithi07685715-42 from 1686203 to 2414634 tid 2 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:676: done (1 left) 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:679: read oid 41 snap 78 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:679: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:680: read oid 49 snap 85 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:680: expect deleted 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:681: write oid 24 current snap is 89 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:678: finishing write tid 1 to smithi07685715-42 2024-09-21T09:40:44.584 INFO:tasks.rados.rados.0.smithi076.stdout:681: seq_num 287 ranges {423609=701524,1628332=472089,2648987=174622} 2024-09-21T09:40:44.590 INFO:tasks.rados.rados.0.smithi076.stdout:681: writing smithi07685715-24 from 423609 to 1125133 tid 1 2024-09-21T09:40:44.595 INFO:tasks.rados.rados.0.smithi076.stdout:681: writing smithi07685715-24 from 1628332 to 2100421 tid 2 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:681: writing smithi07685715-24 from 2648987 to 2823609 tid 3 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:682: read oid 43 snap 88 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:682: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:683: snap_create 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:678: finishing write tid 2 to smithi07685715-42 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:678: finishing write tid 3 to smithi07685715-42 2024-09-21T09:40:44.600 INFO:tasks.rados.rados.0.smithi076.stdout:678: finishing write tid 4 to smithi07685715-42 2024-09-21T09:40:44.601 INFO:tasks.rados.rados.0.smithi076.stdout:678: oid 42 updating version 0 to 172 2024-09-21T09:40:44.601 INFO:tasks.rados.rados.0.smithi076.stdout:678: oid 42 updating version 172 to 178 2024-09-21T09:40:44.601 INFO:tasks.rados.rados.0.smithi076.stdout:678: oid 42 version 178 is already newer than 177 2024-09-21T09:40:44.601 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 178 (ObjNum 286 snap 89 seq_num 286) dirty exists 2024-09-21T09:40:44.601 INFO:tasks.rados.rados.0.smithi076.stdout:678: left oid 42 (ObjNum 286 snap 89 seq_num 286) 2024-09-21T09:40:44.601 INFO:tasks.rados.rados.0.smithi076.stdout:679: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:40:44.686 INFO:tasks.rados.rados.0.smithi076.stdout:681: finishing write tid 1 to smithi07685715-24 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: finishing write tid 2 to smithi07685715-24 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: finishing write tid 3 to smithi07685715-24 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: finishing write tid 4 to smithi07685715-24 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: finishing write tid 5 to smithi07685715-24 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: oid 24 updating version 0 to 55 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: oid 24 updating version 55 to 57 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: oid 24 updating version 57 to 58 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: oid 24 version 58 is already newer than 56 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 58 (ObjNum 287 snap 89 seq_num 287) dirty exists 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:681: left oid 24 (ObjNum 287 snap 89 seq_num 287) 2024-09-21T09:40:44.687 INFO:tasks.rados.rados.0.smithi076.stdout:682: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:40:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:44 smithi191 ceph-mon[33301]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:44 smithi191 ceph-mon[33301]: Upgrade: Updating node-exporter.b (2/2) 2024-09-21T09:40:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:44 smithi191 ceph-mon[33301]: Deploying daemon node-exporter.b on smithi191 2024-09-21T09:40:44.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 podman[84402]: 2024-09-21 09:40:44.486917177 +0000 UTC m=+0.692616967 container remove 9722666fd4386cb9a698f95959c1677f2870258ec036d8942036e7528caaea57 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:44.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:40:44.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T09:40:44.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 systemd[1]: Stopped Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:44.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 2.349s CPU time. 2024-09-21T09:40:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:44 smithi076 ceph-mon[29313]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:44 smithi076 ceph-mon[29313]: Upgrade: Updating node-exporter.b (2/2) 2024-09-21T09:40:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:44 smithi076 ceph-mon[29313]: Deploying daemon node-exporter.b on smithi191 2024-09-21T09:40:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:44 smithi076 ceph-mon[41325]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:40:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:44 smithi076 ceph-mon[41325]: Upgrade: Updating node-exporter.b (2/2) 2024-09-21T09:40:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:44 smithi076 ceph-mon[41325]: Deploying daemon node-exporter.b on smithi191 2024-09-21T09:40:45.362 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:44 smithi191 systemd[1]: Starting Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:40:45.362 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:45 smithi191 bash[84537]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:40:45.575 INFO:tasks.rados.rados.0.smithi076.stdout:678: done (5 left) 2024-09-21T09:40:45.575 INFO:tasks.rados.rados.0.smithi076.stdout:679: done (4 left) 2024-09-21T09:40:45.575 INFO:tasks.rados.rados.0.smithi076.stdout:680: done (3 left) 2024-09-21T09:40:45.575 INFO:tasks.rados.rados.0.smithi076.stdout:681: done (2 left) 2024-09-21T09:40:45.575 INFO:tasks.rados.rados.0.smithi076.stdout:682: done (1 left) 2024-09-21T09:40:45.576 INFO:tasks.rados.rados.0.smithi076.stdout:683: done (0 left) 2024-09-21T09:40:45.576 INFO:tasks.rados.rados.0.smithi076.stdout:684: write oid 15 current snap is 90 2024-09-21T09:40:45.576 INFO:tasks.rados.rados.0.smithi076.stdout:684: seq_num 288 ranges {440416=776773,1686814=732100} 2024-09-21T09:40:45.582 INFO:tasks.rados.rados.0.smithi076.stdout:684: writing smithi07685715-15 from 440416 to 1217189 tid 1 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:684: writing smithi07685715-15 from 1686814 to 2418914 tid 2 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:685: rollback oid 5 current snap is 90 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 85 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:684: finishing write tid 1 to smithi07685715-15 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:686: read oid 36 snap -1 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:686: expect (ObjNum 247 snap 76 seq_num 247) 2024-09-21T09:40:45.589 INFO:tasks.rados.rados.0.smithi076.stdout:687: snap_remove snap 59 2024-09-21T09:40:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:45 smithi191 ceph-mon[33301]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:40:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:45 smithi191 ceph-mon[33301]: pgmap v60: 89 pgs: 89 active+clean; 235 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2024-09-21T09:40:45.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:45 smithi191 bash[84537]: Getting image source signatures 2024-09-21T09:40:45.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:45 smithi191 bash[84537]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:40:45.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:45 smithi191 bash[84537]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:40:45.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:45 smithi191 bash[84537]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:40:45.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:45 smithi076 ceph-mon[29313]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:40:45.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:45 smithi076 ceph-mon[29313]: pgmap v60: 89 pgs: 89 active+clean; 235 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2024-09-21T09:40:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:45 smithi076 ceph-mon[41325]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:40:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:45 smithi076 ceph-mon[41325]: pgmap v60: 89 pgs: 89 active+clean; 235 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.1 MiB/s wr, 15 op/s 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:687: done (3 left) 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:688: read oid 2 snap -1 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:688: expect (ObjNum 252 snap 79 seq_num 252) 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:689: snap_create 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:685: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:685: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:685: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 460 (ObjNum 218 snap 63 seq_num 218) dirty exists 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:684: finishing write tid 2 to smithi07685715-15 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:684: finishing write tid 3 to smithi07685715-15 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:684: finishing write tid 4 to smithi07685715-15 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:684: oid 15 updating version 0 to 155 2024-09-21T09:40:46.582 INFO:tasks.rados.rados.0.smithi076.stdout:684: oid 15 version 155 is already newer than 154 2024-09-21T09:40:46.583 INFO:tasks.rados.rados.0.smithi076.stdout:684: oid 15 version 155 is already newer than 153 2024-09-21T09:40:46.583 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 155 (ObjNum 288 snap 90 seq_num 288) dirty exists 2024-09-21T09:40:46.583 INFO:tasks.rados.rados.0.smithi076.stdout:684: left oid 15 (ObjNum 288 snap 90 seq_num 288) 2024-09-21T09:40:46.583 INFO:tasks.rados.rados.0.smithi076.stdout:686: expect (ObjNum 247 snap 76 seq_num 247) 2024-09-21T09:40:46.681 INFO:tasks.rados.rados.0.smithi076.stdout:688: expect (ObjNum 252 snap 79 seq_num 252) 2024-09-21T09:40:46.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:46 smithi191 ceph-mon[33301]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:40:46.862 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:46 smithi191 bash[84537]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:40:46.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:46 smithi076 ceph-mon[29313]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:40:46.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:46 smithi076 ceph-mon[41325]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:40:47.233 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:46 smithi191 bash[84537]: Writing manifest to image destination 2024-09-21T09:40:47.559 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 podman[84537]: 2024-09-21 09:40:47.198983517 +0000 UTC m=+2.193162443 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:40:47.559 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 podman[84537]: 2024-09-21 09:40:47.322540978 +0000 UTC m=+2.316719899 container create 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:684: done (4 left) 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:685: done (3 left) 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:686: done (2 left) 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:688: done (1 left) 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:689: done (0 left) 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:690: write oid 27 current snap is 91 2024-09-21T09:40:47.585 INFO:tasks.rados.rados.0.smithi076.stdout:690: seq_num 289 ranges {457223=568375,1461649=475758,2420186=437037} 2024-09-21T09:40:47.590 INFO:tasks.rados.rados.0.smithi076.stdout:690: writing smithi07685715-27 from 457223 to 1025598 tid 1 2024-09-21T09:40:47.595 INFO:tasks.rados.rados.0.smithi076.stdout:690: writing smithi07685715-27 from 1461649 to 1937407 tid 2 2024-09-21T09:40:47.601 INFO:tasks.rados.rados.0.smithi076.stdout:690: writing smithi07685715-27 from 2420186 to 2857223 tid 3 2024-09-21T09:40:47.601 INFO:tasks.rados.rados.0.smithi076.stdout:691: read oid 16 snap -1 2024-09-21T09:40:47.601 INFO:tasks.rados.rados.0.smithi076.stdout:691: expect deleted 2024-09-21T09:40:47.601 INFO:tasks.rados.rados.0.smithi076.stdout:692: snap_remove snap 66 2024-09-21T09:40:47.601 INFO:tasks.rados.rados.0.smithi076.stdout:690: finishing write tid 1 to smithi07685715-27 2024-09-21T09:40:47.601 INFO:tasks.rados.rados.0.smithi076.stdout:690: finishing write tid 2 to smithi07685715-27 2024-09-21T09:40:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-mon[33301]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:40:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-mon[33301]: pgmap v63: 89 pgs: 89 active+clean; 235 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 353 KiB/s wr, 5 op/s 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 podman[84537]: 2024-09-21 09:40:47.558117754 +0000 UTC m=+2.552296675 container init 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 podman[84537]: 2024-09-21 09:40:47.563923087 +0000 UTC m=+2.558102008 container start 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.567Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.567Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:40:47.863 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:40:47.864 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:40:47.865 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:40:47.866 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.568Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.569Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b[84612]: ts=2024-09-21T09:40:47.569Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 bash[84537]: 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 2024-09-21T09:40:47.867 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 09:40:47 smithi191 systemd[1]: Started Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:40:47.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:47 smithi076 ceph-mon[29313]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:40:47.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:47 smithi076 ceph-mon[29313]: pgmap v63: 89 pgs: 89 active+clean; 235 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 353 KiB/s wr, 5 op/s 2024-09-21T09:40:47.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:47 smithi076 ceph-mon[41325]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:40:47.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:47 smithi076 ceph-mon[41325]: pgmap v63: 89 pgs: 89 active+clean; 235 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 353 KiB/s wr, 5 op/s 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:692: done (2 left) 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:693: rollback oid 3 current snap is 91 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 86 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:690: finishing write tid 3 to smithi07685715-27 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:690: finishing write tid 4 to smithi07685715-27 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:690: finishing write tid 5 to smithi07685715-27 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:690: oid 27 updating version 0 to 407 2024-09-21T09:40:48.533 INFO:tasks.rados.rados.0.smithi076.stdout:690: oid 27 updating version 407 to 409 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:690: oid 27 version 409 is already newer than 408 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:690: oid 27 updating version 409 to 410 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 410 (ObjNum 289 snap 91 seq_num 289) dirty exists 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:690: left oid 27 (ObjNum 289 snap 91 seq_num 289) 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:690: done (2 left) 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:691: done (1 left) 2024-09-21T09:40:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:694: snap_create 2024-09-21T09:40:48.536 INFO:tasks.rados.rados.0.smithi076.stdout:693: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:40:48.536 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 53 (ObjNum 97 snap 0 seq_num 864316064) dirty dne 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.639 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:48 smithi191 ceph-mon[33301]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[29313]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:40:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:48.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-mon[41325]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:40:49.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:48 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:40:48] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.43.0" 2024-09-21T09:40:49.532 INFO:tasks.rados.rados.0.smithi076.stdout:693: done (1 left) 2024-09-21T09:40:49.532 INFO:tasks.rados.rados.0.smithi076.stdout:694: done (0 left) 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:695: read oid 17 snap 91 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:695: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:696: rollback oid 3 current snap is 92 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 82 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:697: rollback oid 39 current snap is 92 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 80 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:698: write oid 37 current snap is 92 2024-09-21T09:40:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:698: seq_num 290 ranges {474030=643624,1520131=735769} 2024-09-21T09:40:49.539 INFO:tasks.rados.rados.0.smithi076.stdout:698: writing smithi07685715-37 from 474030 to 1117654 tid 1 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:698: writing smithi07685715-37 from 1520131 to 2255900 tid 2 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:699: snap_create 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:697: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 253 (ObjNum 129 snap 0 seq_num 864255848) dirty dne 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:696: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 53 (ObjNum 129 snap 0 seq_num 1682320073) dirty dne 2024-09-21T09:40:49.546 INFO:tasks.rados.rados.0.smithi076.stdout:695: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:49.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:49 smithi191 ceph-mon[33301]: pgmap v66: 89 pgs: 89 active+clean; 227 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:40:49.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:49 smithi191 ceph-mon[33301]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:40:49.623 INFO:tasks.rados.rados.0.smithi076.stdout:698: finishing write tid 1 to smithi07685715-37 2024-09-21T09:40:49.623 INFO:tasks.rados.rados.0.smithi076.stdout:698: finishing write tid 2 to smithi07685715-37 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:698: finishing write tid 3 to smithi07685715-37 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:698: finishing write tid 4 to smithi07685715-37 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:698: oid 37 updating version 0 to 181 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:698: oid 37 updating version 181 to 183 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:698: oid 37 version 183 is already newer than 182 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 183 (ObjNum 290 snap 92 seq_num 290) dirty exists 2024-09-21T09:40:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:698: left oid 37 (ObjNum 290 snap 92 seq_num 290) 2024-09-21T09:40:49.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:49 smithi076 ceph-mon[41325]: pgmap v66: 89 pgs: 89 active+clean; 227 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:40:49.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:49 smithi076 ceph-mon[41325]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:40:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:49 smithi076 ceph-mon[29313]: pgmap v66: 89 pgs: 89 active+clean; 227 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:40:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:49 smithi076 ceph-mon[29313]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:695: done (4 left) 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:696: done (3 left) 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:697: done (2 left) 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:698: done (1 left) 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:699: done (0 left) 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:700: rollback oid 45 current snap is 93 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 78 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:701: read oid 9 snap -1 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:701: expect (ObjNum 268 snap 86 seq_num 268) 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:702: write oid 10 current snap is 93 2024-09-21T09:40:50.534 INFO:tasks.rados.rados.0.smithi076.stdout:702: seq_num 291 ranges {490837=718873,1862260=479427,2875032=15805} 2024-09-21T09:40:50.540 INFO:tasks.rados.rados.0.smithi076.stdout:702: writing smithi07685715-10 from 490837 to 1209710 tid 1 2024-09-21T09:40:50.547 INFO:tasks.rados.rados.0.smithi076.stdout:702: writing smithi07685715-10 from 1862260 to 2341687 tid 2 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:702: writing smithi07685715-10 from 2875032 to 2890837 tid 3 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:703: write oid 46 current snap is 93 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:700: finishing rollback tid 0 to smithi07685715-45 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:700: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:700: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 161 (ObjNum 251 snap 78 seq_num 251) dirty exists 2024-09-21T09:40:50.551 INFO:tasks.rados.rados.0.smithi076.stdout:701: expect (ObjNum 268 snap 86 seq_num 268) 2024-09-21T09:40:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:702: finishing write tid 1 to smithi07685715-10 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: finishing write tid 2 to smithi07685715-10 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: finishing write tid 3 to smithi07685715-10 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: finishing write tid 4 to smithi07685715-10 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: finishing write tid 5 to smithi07685715-10 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: oid 10 updating version 0 to 302 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: oid 10 version 302 is already newer than 300 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: oid 10 version 302 is already newer than 301 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: oid 10 version 302 is already newer than 299 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 302 (ObjNum 291 snap 93 seq_num 291) dirty exists 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:702: left oid 10 (ObjNum 291 snap 93 seq_num 291) 2024-09-21T09:40:50.729 INFO:tasks.rados.rados.0.smithi076.stdout:703: seq_num 292 ranges {507644=794122,1920742=623085} 2024-09-21T09:40:50.735 INFO:tasks.rados.rados.0.smithi076.stdout:703: writing smithi07685715-46 from 507644 to 1301766 tid 1 2024-09-21T09:40:50.742 INFO:tasks.rados.rados.0.smithi076.stdout:703: writing smithi07685715-46 from 1920742 to 2543827 tid 2 2024-09-21T09:40:50.743 INFO:tasks.rados.rados.0.smithi076.stdout:700: done (3 left) 2024-09-21T09:40:50.743 INFO:tasks.rados.rados.0.smithi076.stdout:701: done (2 left) 2024-09-21T09:40:50.743 INFO:tasks.rados.rados.0.smithi076.stdout:702: done (1 left) 2024-09-21T09:40:50.743 INFO:tasks.rados.rados.0.smithi076.stdout:704: write oid 9 current snap is 93 2024-09-21T09:40:50.743 INFO:tasks.rados.rados.0.smithi076.stdout:703: finishing write tid 1 to smithi07685715-46 2024-09-21T09:40:50.743 INFO:tasks.rados.rados.0.smithi076.stdout:704: seq_num 293 ranges {524451=469371,1579224=483096,2529878=394573} 2024-09-21T09:40:50.748 INFO:tasks.rados.rados.0.smithi076.stdout:704: writing smithi07685715-9 from 524451 to 993822 tid 1 2024-09-21T09:40:50.754 INFO:tasks.rados.rados.0.smithi076.stdout:704: writing smithi07685715-9 from 1579224 to 2062320 tid 2 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:704: writing smithi07685715-9 from 2529878 to 2924451 tid 3 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:705: write oid 7 current snap is 93 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:703: finishing write tid 2 to smithi07685715-46 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:703: finishing write tid 3 to smithi07685715-46 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:703: finishing write tid 4 to smithi07685715-46 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:703: oid 46 updating version 0 to 481 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:703: oid 46 updating version 481 to 483 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:703: oid 46 version 483 is already newer than 482 2024-09-21T09:40:50.760 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 483 (ObjNum 292 snap 93 seq_num 292) dirty exists 2024-09-21T09:40:50.761 INFO:tasks.rados.rados.0.smithi076.stdout:703: left oid 46 (ObjNum 292 snap 93 seq_num 292) 2024-09-21T09:40:50.761 INFO:tasks.rados.rados.0.smithi076.stdout:704: finishing write tid 1 to smithi07685715-9 2024-09-21T09:40:50.761 INFO:tasks.rados.rados.0.smithi076.stdout:705: seq_num 294 ranges {541258=544620,1637706=626754,2840948=100310} 2024-09-21T09:40:50.765 INFO:tasks.rados.rados.0.smithi076.stdout:705: writing smithi07685715-7 from 541258 to 1085878 tid 1 2024-09-21T09:40:50.772 INFO:tasks.rados.rados.0.smithi076.stdout:705: writing smithi07685715-7 from 1637706 to 2264460 tid 2 2024-09-21T09:40:50.776 INFO:tasks.rados.rados.0.smithi076.stdout:705: writing smithi07685715-7 from 2840948 to 2941258 tid 3 2024-09-21T09:40:50.776 INFO:tasks.rados.rados.0.smithi076.stdout:703: done (2 left) 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:706: read oid 17 snap -1 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:706: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:707: write oid 5 current snap is 93 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: finishing write tid 2 to smithi07685715-9 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: finishing write tid 3 to smithi07685715-9 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: finishing write tid 4 to smithi07685715-9 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: finishing write tid 5 to smithi07685715-9 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: oid 9 updating version 0 to 324 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: oid 9 version 324 is already newer than 321 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: oid 9 version 324 is already newer than 323 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: oid 9 version 324 is already newer than 322 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 324 (ObjNum 293 snap 93 seq_num 293) dirty exists 2024-09-21T09:40:50.777 INFO:tasks.rados.rados.0.smithi076.stdout:704: left oid 9 (ObjNum 293 snap 93 seq_num 293) 2024-09-21T09:40:50.778 INFO:tasks.rados.rados.0.smithi076.stdout:705: finishing write tid 1 to smithi07685715-7 2024-09-21T09:40:50.778 INFO:tasks.rados.rados.0.smithi076.stdout:705: finishing write tid 2 to smithi07685715-7 2024-09-21T09:40:50.778 INFO:tasks.rados.rados.0.smithi076.stdout:707: seq_num 295 ranges {558065=619869,1579835=486765,2468371=489694} 2024-09-21T09:40:50.782 INFO:tasks.rados.rados.0.smithi076.stdout:707: writing smithi07685715-5 from 558065 to 1177934 tid 1 2024-09-21T09:40:50.788 INFO:tasks.rados.rados.0.smithi076.stdout:707: writing smithi07685715-5 from 1579835 to 2066600 tid 2 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:707: writing smithi07685715-5 from 2468371 to 2958065 tid 3 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:704: done (3 left) 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:708: read oid 23 snap -1 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:708: expect deleted 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:709: write oid 14 current snap is 93 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:705: finishing write tid 3 to smithi07685715-7 2024-09-21T09:40:50.793 INFO:tasks.rados.rados.0.smithi076.stdout:709: seq_num 296 ranges {574872=695118,2038317=630423} 2024-09-21T09:40:50.799 INFO:tasks.rados.rados.0.smithi076.stdout:709: writing smithi07685715-14 from 574872 to 1269990 tid 1 2024-09-21T09:40:50.807 INFO:tasks.rados.rados.0.smithi076.stdout:709: writing smithi07685715-14 from 2038317 to 2668740 tid 2 2024-09-21T09:40:50.808 INFO:tasks.rados.rados.0.smithi076.stdout:710: write oid 31 current snap is 93 2024-09-21T09:40:50.808 INFO:tasks.rados.rados.0.smithi076.stdout:710: seq_num 297 ranges {591679=486720,1813152=490434} 2024-09-21T09:40:50.813 INFO:tasks.rados.rados.0.smithi076.stdout:710: writing smithi07685715-31 from 591679 to 1078399 tid 1 2024-09-21T09:40:50.820 INFO:tasks.rados.rados.0.smithi076.stdout:710: writing smithi07685715-31 from 1813152 to 2303586 tid 2 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:711: read oid 10 snap -1 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:711: expect (ObjNum 291 snap 93 seq_num 291) 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:712: delete oid 29 current snap is 93 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: finishing write tid 4 to smithi07685715-7 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: finishing write tid 5 to smithi07685715-7 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: oid 7 updating version 0 to 306 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: oid 7 version 306 is already newer than 305 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: oid 7 updating version 306 to 307 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: oid 7 version 307 is already newer than 304 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 307 (ObjNum 294 snap 93 seq_num 294) dirty exists 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:705: left oid 7 (ObjNum 294 snap 93 seq_num 294) 2024-09-21T09:40:50.821 INFO:tasks.rados.rados.0.smithi076.stdout:706: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: finishing write tid 1 to smithi07685715-5 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: finishing write tid 2 to smithi07685715-5 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: finishing write tid 3 to smithi07685715-5 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: finishing write tid 4 to smithi07685715-5 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: finishing write tid 5 to smithi07685715-5 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: oid 5 updating version 0 to 488 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: oid 5 version 488 is already newer than 487 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: oid 5 version 488 is already newer than 486 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:707: oid 5 version 488 is already newer than 485 2024-09-21T09:40:50.901 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 488 (ObjNum 295 snap 93 seq_num 295) dirty exists 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:707: left oid 5 (ObjNum 295 snap 93 seq_num 295) 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:709: finishing write tid 1 to smithi07685715-14 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:705: done (7 left) 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:706: done (6 left) 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:707: done (5 left) 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:708: done (4 left) 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:712: done (3 left) 2024-09-21T09:40:50.902 INFO:tasks.rados.rados.0.smithi076.stdout:713: snap_remove snap 40 2024-09-21T09:40:51.541 INFO:tasks.rados.rados.0.smithi076.stdout:713: done (3 left) 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:714: read oid 1 snap -1 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:714: expect deleted 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:715: read oid 20 snap -1 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:715: expect (ObjNum 240 snap 75 seq_num 240) 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:716: rollback oid 47 current snap is 93 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 91 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:709: finishing write tid 2 to smithi07685715-14 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:709: finishing write tid 3 to smithi07685715-14 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:709: finishing write tid 4 to smithi07685715-14 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:709: oid 14 updating version 0 to 309 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:709: oid 14 version 309 is already newer than 308 2024-09-21T09:40:51.542 INFO:tasks.rados.rados.0.smithi076.stdout:709: oid 14 updating version 309 to 310 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 310 (ObjNum 296 snap 93 seq_num 296) dirty exists 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:709: left oid 14 (ObjNum 296 snap 93 seq_num 296) 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: finishing write tid 1 to smithi07685715-31 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: finishing write tid 2 to smithi07685715-31 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: finishing write tid 3 to smithi07685715-31 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: finishing write tid 4 to smithi07685715-31 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: oid 31 updating version 0 to 243 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: oid 31 version 243 is already newer than 242 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: oid 31 updating version 243 to 244 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 244 (ObjNum 297 snap 93 seq_num 297) dirty exists 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:710: left oid 31 (ObjNum 297 snap 93 seq_num 297) 2024-09-21T09:40:51.543 INFO:tasks.rados.rados.0.smithi076.stdout:711: expect (ObjNum 291 snap 93 seq_num 291) 2024-09-21T09:40:51.638 INFO:tasks.rados.rados.0.smithi076.stdout:715: expect (ObjNum 240 snap 75 seq_num 240) 2024-09-21T09:40:51.730 INFO:tasks.rados.rados.0.smithi076.stdout:716: finishing rollback tid 0 to smithi07685715-47 2024-09-21T09:40:51.730 INFO:tasks.rados.rados.0.smithi076.stdout:716: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:40:51.730 INFO:tasks.rados.rados.0.smithi076.stdout:716: finishing rollback tid 2 to smithi07685715-47 2024-09-21T09:40:51.730 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 578 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:709: done (5 left) 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:710: done (4 left) 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:711: done (3 left) 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:714: done (2 left) 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:715: done (1 left) 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:716: done (0 left) 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:717: write oid 50 current snap is 93 2024-09-21T09:40:51.731 INFO:tasks.rados.rados.0.smithi076.stdout:717: seq_num 299 ranges {625293=637218,1813763=494103,2978063=47230} 2024-09-21T09:40:51.737 INFO:tasks.rados.rados.0.smithi076.stdout:717: writing smithi07685715-50 from 625293 to 1262511 tid 1 2024-09-21T09:40:51.743 INFO:tasks.rados.rados.0.smithi076.stdout:717: writing smithi07685715-50 from 1813763 to 2307866 tid 2 2024-09-21T09:40:51.747 INFO:tasks.rados.rados.0.smithi076.stdout:717: writing smithi07685715-50 from 2978063 to 3025293 tid 3 2024-09-21T09:40:51.747 INFO:tasks.rados.rados.0.smithi076.stdout:718: write oid 29 current snap is 93 2024-09-21T09:40:51.747 INFO:tasks.rados.rados.0.smithi076.stdout:718: seq_num 300 ranges {642100=712467,1872245=637761} 2024-09-21T09:40:51.754 INFO:tasks.rados.rados.0.smithi076.stdout:718: writing smithi07685715-29 from 642100 to 1354567 tid 1 2024-09-21T09:40:51.760 INFO:tasks.rados.rados.0.smithi076.stdout:718: writing smithi07685715-29 from 1872245 to 2510006 tid 2 2024-09-21T09:40:51.760 INFO:tasks.rados.rados.0.smithi076.stdout:719: write oid 31 current snap is 93 2024-09-21T09:40:51.760 INFO:tasks.rados.rados.0.smithi076.stdout:719: seq_num 301 ranges {658907=787716,1930727=497772,3032909=25998} 2024-09-21T09:40:51.768 INFO:tasks.rados.rados.0.smithi076.stdout:719: writing smithi07685715-31 from 658907 to 1446623 tid 1 2024-09-21T09:40:51.773 INFO:tasks.rados.rados.0.smithi076.stdout:719: writing smithi07685715-31 from 1930727 to 2428499 tid 2 2024-09-21T09:40:51.777 INFO:tasks.rados.rados.0.smithi076.stdout:719: writing smithi07685715-31 from 3032909 to 3058907 tid 3 2024-09-21T09:40:51.777 INFO:tasks.rados.rados.0.smithi076.stdout:720: write oid 15 current snap is 93 2024-09-21T09:40:51.777 INFO:tasks.rados.rados.0.smithi076.stdout:720: seq_num 302 ranges {675714=462965,1589209=641430,2943979=131735} 2024-09-21T09:40:51.783 INFO:tasks.rados.rados.0.smithi076.stdout:720: writing smithi07685715-15 from 675714 to 1138679 tid 1 2024-09-21T09:40:51.789 INFO:tasks.rados.rados.0.smithi076.stdout:720: writing smithi07685715-15 from 1589209 to 2230639 tid 2 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:720: writing smithi07685715-15 from 2943979 to 3075714 tid 3 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:721: snap_create 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:717: finishing write tid 1 to smithi07685715-50 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:717: finishing write tid 2 to smithi07685715-50 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:717: finishing write tid 3 to smithi07685715-50 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:717: finishing write tid 4 to smithi07685715-50 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:717: finishing write tid 5 to smithi07685715-50 2024-09-21T09:40:51.795 INFO:tasks.rados.rados.0.smithi076.stdout:717: oid 50 updating version 0 to 255 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:717: oid 50 updating version 255 to 258 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:717: oid 50 version 258 is already newer than 257 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:717: oid 50 version 258 is already newer than 256 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 258 (ObjNum 299 snap 93 seq_num 299) dirty exists 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:717: left oid 50 (ObjNum 299 snap 93 seq_num 299) 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: finishing write tid 1 to smithi07685715-29 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: finishing write tid 2 to smithi07685715-29 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: finishing write tid 3 to smithi07685715-29 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: finishing write tid 4 to smithi07685715-29 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: oid 29 updating version 0 to 188 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: oid 29 updating version 188 to 189 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: oid 29 version 189 is already newer than 187 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 189 (ObjNum 300 snap 93 seq_num 300) dirty exists 2024-09-21T09:40:51.796 INFO:tasks.rados.rados.0.smithi076.stdout:718: left oid 29 (ObjNum 300 snap 93 seq_num 300) 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: finishing write tid 1 to smithi07685715-31 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: finishing write tid 2 to smithi07685715-31 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: finishing write tid 3 to smithi07685715-31 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: finishing write tid 4 to smithi07685715-31 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: finishing write tid 5 to smithi07685715-31 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: oid 31 updating version 0 to 253 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: oid 31 version 253 is already newer than 251 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: oid 31 updating version 253 to 254 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: oid 31 version 254 is already newer than 252 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 254 (ObjNum 301 snap 93 seq_num 301) dirty exists 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:719: left oid 31 (ObjNum 301 snap 93 seq_num 301) 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:720: finishing write tid 1 to smithi07685715-15 2024-09-21T09:40:51.797 INFO:tasks.rados.rados.0.smithi076.stdout:720: finishing write tid 2 to smithi07685715-15 2024-09-21T09:40:51.798 INFO:tasks.rados.rados.0.smithi076.stdout:720: finishing write tid 3 to smithi07685715-15 2024-09-21T09:40:51.798 INFO:tasks.rados.rados.0.smithi076.stdout:720: finishing write tid 4 to smithi07685715-15 2024-09-21T09:40:51.798 INFO:tasks.rados.rados.0.smithi076.stdout:720: finishing write tid 5 to smithi07685715-15 2024-09-21T09:40:51.798 INFO:tasks.rados.rados.0.smithi076.stdout:720: oid 15 updating version 0 to 168 2024-09-21T09:40:51.798 INFO:tasks.rados.rados.0.smithi076.stdout:720: oid 15 version 168 is already newer than 167 2024-09-21T09:40:51.798 INFO:tasks.rados.rados.0.smithi076.stdout:720: oid 15 version 168 is already newer than 166 2024-09-21T09:40:51.799 INFO:tasks.rados.rados.0.smithi076.stdout:720: oid 15 version 168 is already newer than 165 2024-09-21T09:40:51.799 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 168 (ObjNum 302 snap 93 seq_num 302) dirty exists 2024-09-21T09:40:51.799 INFO:tasks.rados.rados.0.smithi076.stdout:720: left oid 15 (ObjNum 302 snap 93 seq_num 302) 2024-09-21T09:40:51.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:51 smithi191 ceph-mon[33301]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:40:51.814 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:51 smithi191 ceph-mon[33301]: pgmap v69: 89 pgs: 89 active+clean; 227 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:40:51.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:51 smithi076 ceph-mon[41325]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:40:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:51 smithi076 ceph-mon[41325]: pgmap v69: 89 pgs: 89 active+clean; 227 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:40:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:51 smithi076 ceph-mon[29313]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:40:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:51 smithi076 ceph-mon[29313]: pgmap v69: 89 pgs: 89 active+clean; 227 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:40:52.720 INFO:tasks.rados.rados.0.smithi076.stdout:717: done (4 left) 2024-09-21T09:40:52.720 INFO:tasks.rados.rados.0.smithi076.stdout:718: done (3 left) 2024-09-21T09:40:52.720 INFO:tasks.rados.rados.0.smithi076.stdout:719: done (2 left) 2024-09-21T09:40:52.720 INFO:tasks.rados.rados.0.smithi076.stdout:720: done (1 left) 2024-09-21T09:40:52.720 INFO:tasks.rados.rados.0.smithi076.stdout:721: done (0 left) 2024-09-21T09:40:52.720 INFO:tasks.rados.rados.0.smithi076.stdout:722: snap_remove snap 63 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:40:52.828 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:52.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:52 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:52.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:52 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:53.723 INFO:tasks.rados.rados.0.smithi076.stdout:722: done (0 left) 2024-09-21T09:40:53.724 INFO:tasks.rados.rados.0.smithi076.stdout:723: read oid 6 snap -1 2024-09-21T09:40:53.724 INFO:tasks.rados.rados.0.smithi076.stdout:723: expect deleted 2024-09-21T09:40:53.724 INFO:tasks.rados.rados.0.smithi076.stdout:724: delete oid 29 current snap is 94 2024-09-21T09:40:53.727 INFO:tasks.rados.rados.0.smithi076.stdout:723: done (1 left) 2024-09-21T09:40:53.727 INFO:tasks.rados.rados.0.smithi076.stdout:724: done (0 left) 2024-09-21T09:40:53.727 INFO:tasks.rados.rados.0.smithi076.stdout:725: write oid 32 current snap is 94 2024-09-21T09:40:53.727 INFO:tasks.rados.rados.0.smithi076.stdout:725: seq_num 304 ranges {709328=613463,1989820=645099} 2024-09-21T09:40:53.734 INFO:tasks.rados.rados.0.smithi076.stdout:725: writing smithi07685715-32 from 709328 to 1322791 tid 1 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:725: writing smithi07685715-32 from 1989820 to 2634919 tid 2 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:726: read oid 36 snap -1 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:726: expect (ObjNum 247 snap 76 seq_num 247) 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:727: rollback oid 30 current snap is 94 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 84 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:725: finishing write tid 1 to smithi07685715-32 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:728: write oid 31 current snap is 94 2024-09-21T09:40:53.743 INFO:tasks.rados.rados.0.smithi076.stdout:728: seq_num 305 ranges {726135=405065,1764655=788757,3026248=99887} 2024-09-21T09:40:53.749 INFO:tasks.rados.rados.0.smithi076.stdout:728: writing smithi07685715-31 from 726135 to 1131200 tid 1 2024-09-21T09:40:53.757 INFO:tasks.rados.rados.0.smithi076.stdout:728: writing smithi07685715-31 from 1764655 to 2553412 tid 2 2024-09-21T09:40:53.760 INFO:tasks.rados.rados.0.smithi076.stdout:728: writing smithi07685715-31 from 3026248 to 3126135 tid 3 2024-09-21T09:40:53.760 INFO:tasks.rados.rados.0.smithi076.stdout:729: write oid 33 current snap is 94 2024-09-21T09:40:53.760 INFO:tasks.rados.rados.0.smithi076.stdout:725: finishing write tid 2 to smithi07685715-32 2024-09-21T09:40:53.761 INFO:tasks.rados.rados.0.smithi076.stdout:729: seq_num 306 ranges {742942=480314,1823137=648768} 2024-09-21T09:40:53.767 INFO:tasks.rados.rados.0.smithi076.stdout:729: writing smithi07685715-33 from 742942 to 1223256 tid 1 2024-09-21T09:40:53.774 INFO:tasks.rados.rados.0.smithi076.stdout:729: writing smithi07685715-33 from 1823137 to 2471905 tid 2 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:730: delete oid 7 current snap is 94 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:725: finishing write tid 3 to smithi07685715-32 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:725: finishing write tid 4 to smithi07685715-32 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:725: oid 32 updating version 0 to 261 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:725: oid 32 updating version 261 to 262 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:725: oid 32 version 262 is already newer than 256 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 262 (ObjNum 304 snap 94 seq_num 304) dirty exists 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:725: left oid 32 (ObjNum 304 snap 94 seq_num 304) 2024-09-21T09:40:53.775 INFO:tasks.rados.rados.0.smithi076.stdout:726: expect (ObjNum 247 snap 76 seq_num 247) 2024-09-21T09:40:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:53 smithi191 ceph-mon[33301]: pgmap v71: 89 pgs: 89 active+clean; 219 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:40:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:53 smithi191 ceph-mon[33301]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:40:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:53 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:53 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:53.871 INFO:tasks.rados.rados.0.smithi076.stdout:727: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:40:53.871 INFO:tasks.rados.rados.0.smithi076.stdout:727: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:40:53.871 INFO:tasks.rados.rados.0.smithi076.stdout:727: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 600 (ObjNum 258 snap 81 seq_num 258) dirty exists 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:728: finishing write tid 1 to smithi07685715-31 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:728: finishing write tid 2 to smithi07685715-31 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:725: done (5 left) 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:726: done (4 left) 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:727: done (3 left) 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:730: done (2 left) 2024-09-21T09:40:53.872 INFO:tasks.rados.rados.0.smithi076.stdout:731: snap_remove snap 93 2024-09-21T09:40:54.004 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[29313]: pgmap v71: 89 pgs: 89 active+clean; 219 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:40:54.004 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[29313]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:40:54.005 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:54.005 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:54.005 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[41325]: pgmap v71: 89 pgs: 89 active+clean; 219 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2024-09-21T09:40:54.005 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[41325]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:40:54.005 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:54.005 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:53 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:54.736 INFO:tasks.rados.rados.0.smithi076.stdout:731: done (2 left) 2024-09-21T09:40:54.736 INFO:tasks.rados.rados.0.smithi076.stdout:732: write oid 21 current snap is 94 2024-09-21T09:40:54.736 INFO:tasks.rados.rados.0.smithi076.stdout:732: seq_num 308 ranges {776556=630812,1823748=652437,3108517=68039} 2024-09-21T09:40:54.743 INFO:tasks.rados.rados.0.smithi076.stdout:732: writing smithi07685715-21 from 776556 to 1407368 tid 1 2024-09-21T09:40:54.749 INFO:tasks.rados.rados.0.smithi076.stdout:732: writing smithi07685715-21 from 1823748 to 2476185 tid 2 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:732: writing smithi07685715-21 from 3108517 to 3176556 tid 3 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:733: delete oid 13 current snap is 94 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:728: finishing write tid 3 to smithi07685715-31 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:728: finishing write tid 4 to smithi07685715-31 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:728: finishing write tid 5 to smithi07685715-31 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:728: oid 31 updating version 0 to 266 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:728: oid 31 updating version 266 to 267 2024-09-21T09:40:54.754 INFO:tasks.rados.rados.0.smithi076.stdout:728: oid 31 updating version 267 to 269 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:728: oid 31 version 269 is already newer than 268 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 269 (ObjNum 305 snap 94 seq_num 305) dirty exists 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:728: left oid 31 (ObjNum 305 snap 94 seq_num 305) 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: finishing write tid 1 to smithi07685715-33 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: finishing write tid 2 to smithi07685715-33 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: finishing write tid 3 to smithi07685715-33 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: finishing write tid 4 to smithi07685715-33 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: oid 33 updating version 0 to 188 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: oid 33 updating version 188 to 190 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: oid 33 version 190 is already newer than 189 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 190 (ObjNum 306 snap 94 seq_num 306) dirty exists 2024-09-21T09:40:54.755 INFO:tasks.rados.rados.0.smithi076.stdout:729: left oid 33 (ObjNum 306 snap 94 seq_num 306) 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: finishing write tid 1 to smithi07685715-21 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: finishing write tid 2 to smithi07685715-21 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: finishing write tid 3 to smithi07685715-21 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: finishing write tid 4 to smithi07685715-21 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: finishing write tid 5 to smithi07685715-21 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: oid 21 updating version 0 to 508 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: oid 21 version 508 is already newer than 506 2024-09-21T09:40:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:732: oid 21 updating version 508 to 509 2024-09-21T09:40:54.757 INFO:tasks.rados.rados.0.smithi076.stdout:732: oid 21 version 509 is already newer than 507 2024-09-21T09:40:54.757 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 509 (ObjNum 308 snap 94 seq_num 308) dirty exists 2024-09-21T09:40:54.757 INFO:tasks.rados.rados.0.smithi076.stdout:732: left oid 21 (ObjNum 308 snap 94 seq_num 308) 2024-09-21T09:40:54.757 INFO:tasks.rados.rados.0.smithi076.stdout:728: done (3 left) 2024-09-21T09:40:54.757 INFO:tasks.rados.rados.0.smithi076.stdout:729: done (2 left) 2024-09-21T09:40:54.757 INFO:tasks.rados.rados.0.smithi076.stdout:732: done (1 left) 2024-09-21T09:40:54.758 INFO:tasks.rados.rados.0.smithi076.stdout:733: done (0 left) 2024-09-21T09:40:54.758 INFO:tasks.rados.rados.0.smithi076.stdout:734: write oid 46 current snap is 94 2024-09-21T09:40:54.758 INFO:tasks.rados.rados.0.smithi076.stdout:734: seq_num 310 ranges {410170=781310,1940712=656106,3163363=46807} 2024-09-21T09:40:54.763 INFO:tasks.rados.rados.0.smithi076.stdout:734: writing smithi07685715-46 from 410170 to 1191480 tid 1 2024-09-21T09:40:54.772 INFO:tasks.rados.rados.0.smithi076.stdout:734: writing smithi07685715-46 from 1940712 to 2596818 tid 2 2024-09-21T09:40:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:734: writing smithi07685715-46 from 3163363 to 3210170 tid 3 2024-09-21T09:40:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:735: write oid 30 current snap is 94 2024-09-21T09:40:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:734: finishing write tid 1 to smithi07685715-46 2024-09-21T09:40:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:735: seq_num 311 ranges {426977=456559,1482841=799764,2958080=268897} 2024-09-21T09:40:54.781 INFO:tasks.rados.rados.0.smithi076.stdout:735: writing smithi07685715-30 from 426977 to 883536 tid 1 2024-09-21T09:40:54.789 INFO:tasks.rados.rados.0.smithi076.stdout:735: writing smithi07685715-30 from 1482841 to 2282605 tid 2 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:735: writing smithi07685715-30 from 2958080 to 3226977 tid 3 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:736: rollback oid 6 current snap is 94 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 85 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:734: finishing write tid 2 to smithi07685715-46 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:734: finishing write tid 3 to smithi07685715-46 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:734: finishing write tid 4 to smithi07685715-46 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:734: finishing write tid 5 to smithi07685715-46 2024-09-21T09:40:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:734: oid 46 updating version 0 to 511 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:734: oid 46 version 511 is already newer than 510 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:734: oid 46 updating version 511 to 512 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:734: oid 46 updating version 512 to 513 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 513 (ObjNum 310 snap 94 seq_num 310) dirty exists 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:734: left oid 46 (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:735: finishing write tid 1 to smithi07685715-30 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:735: finishing write tid 2 to smithi07685715-30 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:734: done (2 left) 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:737: rollback oid 23 current snap is 94 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 90 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:738: read oid 17 snap 89 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:738: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:739: read oid 22 snap -1 2024-09-21T09:40:54.796 INFO:tasks.rados.rados.0.smithi076.stdout:739: expect (ObjNum 271 snap 86 seq_num 271) 2024-09-21T09:40:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:740: snap_create 2024-09-21T09:40:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:736: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:40:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 0 (ObjNum 129 snap 0 seq_num 864323840) dirty dne 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: finishing write tid 3 to smithi07685715-30 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: finishing write tid 4 to smithi07685715-30 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: finishing write tid 5 to smithi07685715-30 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: oid 30 updating version 0 to 603 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: oid 30 updating version 603 to 604 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: oid 30 updating version 604 to 606 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: oid 30 version 606 is already newer than 605 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 606 (ObjNum 311 snap 94 seq_num 311) dirty exists 2024-09-21T09:40:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:735: left oid 30 (ObjNum 311 snap 94 seq_num 311) 2024-09-21T09:40:54.799 INFO:tasks.rados.rados.0.smithi076.stdout:737: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:40:54.799 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 0 (ObjNum 129 snap 0 seq_num 864051600) dirty dne 2024-09-21T09:40:54.799 INFO:tasks.rados.rados.0.smithi076.stdout:738: expect (ObjNum 253 snap 79 seq_num 253) 2024-09-21T09:40:54.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:54 smithi191 ceph-mon[33301]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:40:54.875 INFO:tasks.rados.rados.0.smithi076.stdout:739: expect (ObjNum 271 snap 86 seq_num 271) 2024-09-21T09:40:55.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:54 smithi076 ceph-mon[29313]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:40:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:54 smithi076 ceph-mon[41325]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:40:55.736 INFO:tasks.rados.rados.0.smithi076.stdout:735: done (5 left) 2024-09-21T09:40:55.736 INFO:tasks.rados.rados.0.smithi076.stdout:736: done (4 left) 2024-09-21T09:40:55.737 INFO:tasks.rados.rados.0.smithi076.stdout:737: done (3 left) 2024-09-21T09:40:55.737 INFO:tasks.rados.rados.0.smithi076.stdout:738: done (2 left) 2024-09-21T09:40:55.737 INFO:tasks.rados.rados.0.smithi076.stdout:739: done (1 left) 2024-09-21T09:40:55.737 INFO:tasks.rados.rados.0.smithi076.stdout:740: done (0 left) 2024-09-21T09:40:55.737 INFO:tasks.rados.rados.0.smithi076.stdout:741: snap_create 2024-09-21T09:40:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:55 smithi191 ceph-mon[33301]: pgmap v74: 89 pgs: 89 active+clean; 195 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 27 op/s 2024-09-21T09:40:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:55 smithi191 ceph-mon[33301]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:40:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:55 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:55 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:55.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[29313]: pgmap v74: 89 pgs: 89 active+clean; 195 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 27 op/s 2024-09-21T09:40:55.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[29313]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:40:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[41325]: pgmap v74: 89 pgs: 89 active+clean; 195 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 27 op/s 2024-09-21T09:40:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[41325]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:40:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:55 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:56.738 INFO:tasks.rados.rados.0.smithi076.stdout:741: done (0 left) 2024-09-21T09:40:56.738 INFO:tasks.rados.rados.0.smithi076.stdout:742: rollback oid 17 current snap is 96 2024-09-21T09:40:56.739 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 95 2024-09-21T09:40:56.739 INFO:tasks.rados.rados.0.smithi076.stdout:743: write oid 42 current snap is 96 2024-09-21T09:40:56.739 INFO:tasks.rados.rados.0.smithi076.stdout:743: seq_num 312 ranges {443784=648161,1657676=659775,2818209=425575} 2024-09-21T09:40:56.744 INFO:tasks.rados.rados.0.smithi076.stdout:743: writing smithi07685715-42 from 443784 to 1091945 tid 1 2024-09-21T09:40:56.752 INFO:tasks.rados.rados.0.smithi076.stdout:743: writing smithi07685715-42 from 1657676 to 2317451 tid 2 2024-09-21T09:40:56.757 INFO:tasks.rados.rados.0.smithi076.stdout:743: writing smithi07685715-42 from 2818209 to 3243784 tid 3 2024-09-21T09:40:56.757 INFO:tasks.rados.rados.0.smithi076.stdout:744: rollback oid 47 current snap is 96 2024-09-21T09:40:56.757 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 86 2024-09-21T09:40:56.757 INFO:tasks.rados.rados.0.smithi076.stdout:742: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:40:56.757 INFO:tasks.rados.rados.0.smithi076.stdout:742: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:40:56.757 INFO:tasks.rados.rados.0.smithi076.stdout:742: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:40:56.758 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 326 (ObjNum 253 snap 79 seq_num 253) dirty exists 2024-09-21T09:40:56.758 INFO:tasks.rados.rados.0.smithi076.stdout:743: finishing write tid 1 to smithi07685715-42 2024-09-21T09:40:56.758 INFO:tasks.rados.rados.0.smithi076.stdout:743: finishing write tid 2 to smithi07685715-42 2024-09-21T09:40:56.758 INFO:tasks.rados.rados.0.smithi076.stdout:742: done (2 left) 2024-09-21T09:40:56.758 INFO:tasks.rados.rados.0.smithi076.stdout:745: write oid 4 current snap is 96 2024-09-21T09:40:56.758 INFO:tasks.rados.rados.0.smithi076.stdout:745: seq_num 313 ranges {460591=723410,1716158=403433,2729279=440140} 2024-09-21T09:40:56.763 INFO:tasks.rados.rados.0.smithi076.stdout:745: writing smithi07685715-4 from 460591 to 1184001 tid 1 2024-09-21T09:40:56.769 INFO:tasks.rados.rados.0.smithi076.stdout:745: writing smithi07685715-4 from 1716158 to 2119591 tid 2 2024-09-21T09:40:56.775 INFO:tasks.rados.rados.0.smithi076.stdout:745: writing smithi07685715-4 from 2729279 to 3169419 tid 3 2024-09-21T09:40:56.775 INFO:tasks.rados.rados.0.smithi076.stdout:746: write oid 36 current snap is 96 2024-09-21T09:40:56.775 INFO:tasks.rados.rados.0.smithi076.stdout:746: seq_num 314 ranges {477398=798659,1774640=663444,2873055=404343} 2024-09-21T09:40:56.782 INFO:tasks.rados.rados.0.smithi076.stdout:746: writing smithi07685715-36 from 477398 to 1276057 tid 1 2024-09-21T09:40:56.788 INFO:tasks.rados.rados.0.smithi076.stdout:746: writing smithi07685715-36 from 1774640 to 2438084 tid 2 2024-09-21T09:40:56.793 INFO:tasks.rados.rados.0.smithi076.stdout:746: writing smithi07685715-36 from 2873055 to 3277398 tid 3 2024-09-21T09:40:56.793 INFO:tasks.rados.rados.0.smithi076.stdout:747: read oid 38 snap -1 2024-09-21T09:40:56.793 INFO:tasks.rados.rados.0.smithi076.stdout:747: expect (ObjNum 220 snap 63 seq_num 220) 2024-09-21T09:40:56.793 INFO:tasks.rados.rados.0.smithi076.stdout:748: delete oid 31 current snap is 96 2024-09-21T09:40:56.793 INFO:tasks.rados.rados.0.smithi076.stdout:743: finishing write tid 3 to smithi07685715-42 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: finishing write tid 4 to smithi07685715-42 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:744: finishing rollback tid 0 to smithi07685715-47 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: finishing write tid 5 to smithi07685715-42 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: oid 42 updating version 0 to 198 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: oid 42 version 198 is already newer than 197 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: oid 42 updating version 198 to 199 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: oid 42 version 199 is already newer than 196 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 199 (ObjNum 312 snap 96 seq_num 312) dirty exists 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:743: left oid 42 (ObjNum 312 snap 96 seq_num 312) 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:744: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:744: finishing rollback tid 2 to smithi07685715-47 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 611 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-09-21T09:40:56.794 INFO:tasks.rados.rados.0.smithi076.stdout:745: finishing write tid 1 to smithi07685715-4 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: finishing write tid 2 to smithi07685715-4 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: finishing write tid 3 to smithi07685715-4 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: finishing write tid 4 to smithi07685715-4 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: finishing write tid 5 to smithi07685715-4 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: oid 4 updating version 0 to 265 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: oid 4 updating version 265 to 268 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: oid 4 version 268 is already newer than 267 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: oid 4 version 268 is already newer than 266 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 268 (ObjNum 313 snap 96 seq_num 313) dirty exists 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:745: left oid 4 (ObjNum 313 snap 96 seq_num 313) 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:746: finishing write tid 1 to smithi07685715-36 2024-09-21T09:40:56.795 INFO:tasks.rados.rados.0.smithi076.stdout:746: finishing write tid 2 to smithi07685715-36 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: finishing write tid 3 to smithi07685715-36 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: finishing write tid 4 to smithi07685715-36 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: finishing write tid 5 to smithi07685715-36 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: oid 36 updating version 0 to 271 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: oid 36 updating version 271 to 273 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: oid 36 version 273 is already newer than 272 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: oid 36 version 273 is already newer than 270 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 273 (ObjNum 314 snap 96 seq_num 314) dirty exists 2024-09-21T09:40:56.796 INFO:tasks.rados.rados.0.smithi076.stdout:746: left oid 36 (ObjNum 314 snap 96 seq_num 314) 2024-09-21T09:40:56.798 INFO:tasks.rados.rados.0.smithi076.stdout:747: expect (ObjNum 220 snap 63 seq_num 220) 2024-09-21T09:40:56.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:40:56.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:56.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:56.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:56.836 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.837 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:56 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:56.918 INFO:tasks.rados.rados.0.smithi076.stdout:743: done (5 left) 2024-09-21T09:40:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:744: done (4 left) 2024-09-21T09:40:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:745: done (3 left) 2024-09-21T09:40:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:746: done (2 left) 2024-09-21T09:40:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:747: done (1 left) 2024-09-21T09:40:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:748: done (0 left) 2024-09-21T09:40:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:749: snap_create 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:40:57.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:56 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:40:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:749: done (0 left) 2024-09-21T09:40:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:750: rollback oid 17 current snap is 97 2024-09-21T09:40:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 84 2024-09-21T09:40:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:751: snap_remove snap 85 2024-09-21T09:40:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:57 smithi191 ceph-mon[33301]: pgmap v77: 89 pgs: 89 active+clean; 195 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-21T09:40:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:57 smithi191 ceph-mon[33301]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:40:58.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:57 smithi076 ceph-mon[29313]: pgmap v77: 89 pgs: 89 active+clean; 195 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-21T09:40:58.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:57 smithi076 ceph-mon[29313]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:40:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:57 smithi076 ceph-mon[41325]: pgmap v77: 89 pgs: 89 active+clean; 195 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2024-09-21T09:40:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:57 smithi076 ceph-mon[41325]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:40:58.534 INFO:tasks.rados.rados.0.smithi076.stdout:751: done (1 left) 2024-09-21T09:40:58.534 INFO:tasks.rados.rados.0.smithi076.stdout:752: rollback oid 3 current snap is 97 2024-09-21T09:40:58.534 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 84 2024-09-21T09:40:58.535 INFO:tasks.rados.rados.0.smithi076.stdout:750: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:40:58.535 INFO:tasks.rados.rados.0.smithi076.stdout:750: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:40:58.535 INFO:tasks.rados.rados.0.smithi076.stdout:750: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:40:58.535 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 330 (ObjNum 253 snap 79 seq_num 253) dirty exists 2024-09-21T09:40:58.535 INFO:tasks.rados.rados.0.smithi076.stdout:750: done (1 left) 2024-09-21T09:40:58.535 INFO:tasks.rados.rados.0.smithi076.stdout:753: delete oid 13 current snap is 97 2024-09-21T09:40:58.537 INFO:tasks.rados.rados.0.smithi076.stdout:752: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:40:58.537 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 129 snap 0 seq_num 864394736) dirty dne 2024-09-21T09:40:58.542 INFO:tasks.rados.rados.0.smithi076.stdout:752: done (1 left) 2024-09-21T09:40:58.542 INFO:tasks.rados.rados.0.smithi076.stdout:753: done (0 left) 2024-09-21T09:40:58.542 INFO:tasks.rados.rados.0.smithi076.stdout:754: delete oid 17 current snap is 97 2024-09-21T09:40:58.544 INFO:tasks.rados.rados.0.smithi076.stdout:754: done (0 left) 2024-09-21T09:40:58.544 INFO:tasks.rados.rados.0.smithi076.stdout:755: write oid 27 current snap is 97 2024-09-21T09:40:58.544 INFO:tasks.rados.rados.0.smithi076.stdout:755: seq_num 318 ranges {544626=699655,1892215=554429,3150041=194585} 2024-09-21T09:40:58.550 INFO:tasks.rados.rados.0.smithi076.stdout:755: writing smithi07685715-27 from 544626 to 1244281 tid 1 2024-09-21T09:40:58.557 INFO:tasks.rados.rados.0.smithi076.stdout:755: writing smithi07685715-27 from 1892215 to 2446644 tid 2 2024-09-21T09:40:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:755: writing smithi07685715-27 from 3150041 to 3344626 tid 3 2024-09-21T09:40:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:756: delete oid 39 current snap is 97 2024-09-21T09:40:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:755: finishing write tid 1 to smithi07685715-27 2024-09-21T09:40:58.562 INFO:tasks.rados.rados.0.smithi076.stdout:755: finishing write tid 2 to smithi07685715-27 2024-09-21T09:40:58.563 INFO:tasks.rados.rados.0.smithi076.stdout:756: done (1 left) 2024-09-21T09:40:58.563 INFO:tasks.rados.rados.0.smithi076.stdout:757: snap_create 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: finishing write tid 3 to smithi07685715-27 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: finishing write tid 4 to smithi07685715-27 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: finishing write tid 5 to smithi07685715-27 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: oid 27 updating version 0 to 441 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: oid 27 version 441 is already newer than 440 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: oid 27 version 441 is already newer than 438 2024-09-21T09:40:58.564 INFO:tasks.rados.rados.0.smithi076.stdout:755: oid 27 version 441 is already newer than 439 2024-09-21T09:40:58.565 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 441 (ObjNum 318 snap 97 seq_num 318) dirty exists 2024-09-21T09:40:58.565 INFO:tasks.rados.rados.0.smithi076.stdout:755: left oid 27 (ObjNum 318 snap 97 seq_num 318) 2024-09-21T09:40:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:58 smithi191 ceph-mon[33301]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:40:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:58 smithi191 ceph-mon[33301]: Upgrade: Updating prometheus.a 2024-09-21T09:40:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:58 smithi191 ceph-mon[33301]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:40:59.122 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:40:58] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-09-21T09:40:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-mon[29313]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:40:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-mon[29313]: Upgrade: Updating prometheus.a 2024-09-21T09:40:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-mon[29313]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:40:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-mon[41325]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:40:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-mon[41325]: Upgrade: Updating prometheus.a 2024-09-21T09:40:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:58 smithi076 ceph-mon[41325]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:40:59.543 INFO:tasks.rados.rados.0.smithi076.stdout:755: done (1 left) 2024-09-21T09:40:59.543 INFO:tasks.rados.rados.0.smithi076.stdout:757: done (0 left) 2024-09-21T09:40:59.543 INFO:tasks.rados.rados.0.smithi076.stdout:758: delete oid 21 current snap is 98 2024-09-21T09:40:59.546 INFO:tasks.rados.rados.0.smithi076.stdout:758: done (0 left) 2024-09-21T09:40:59.546 INFO:tasks.rados.rados.0.smithi076.stdout:759: snap_remove snap 91 2024-09-21T09:41:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:59 smithi191 ceph-mon[33301]: Deploying daemon prometheus.a on smithi191 2024-09-21T09:41:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:59 smithi191 ceph-mon[33301]: pgmap v81: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 24 op/s 2024-09-21T09:41:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:40:59 smithi191 ceph-mon[33301]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:41:00.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:59 smithi076 ceph-mon[29313]: Deploying daemon prometheus.a on smithi191 2024-09-21T09:41:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:59 smithi076 ceph-mon[29313]: pgmap v81: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 24 op/s 2024-09-21T09:41:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:40:59 smithi076 ceph-mon[29313]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:41:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:59 smithi076 ceph-mon[41325]: Deploying daemon prometheus.a on smithi191 2024-09-21T09:41:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:59 smithi076 ceph-mon[41325]: pgmap v81: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 24 op/s 2024-09-21T09:41:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:40:59 smithi076 ceph-mon[41325]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:41:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:759: done (0 left) 2024-09-21T09:41:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:760: read oid 41 snap 80 2024-09-21T09:41:00.543 INFO:tasks.rados.rados.0.smithi076.stdout:760: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:41:00.544 INFO:tasks.rados.rados.0.smithi076.stdout:761: write oid 14 current snap is 98 2024-09-21T09:41:00.544 INFO:tasks.rados.rados.0.smithi076.stdout:761: seq_num 321 ranges {595047=641755,1667661=418109,2548663=763022} 2024-09-21T09:41:00.550 INFO:tasks.rados.rados.0.smithi076.stdout:761: writing smithi07685715-14 from 595047 to 1236802 tid 1 2024-09-21T09:41:00.555 INFO:tasks.rados.rados.0.smithi076.stdout:761: writing smithi07685715-14 from 1667661 to 2085770 tid 2 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:761: writing smithi07685715-14 from 2548663 to 3311685 tid 3 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:762: read oid 10 snap 94 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:762: expect (ObjNum 291 snap 93 seq_num 291) 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:763: read oid 6 snap -1 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:763: expect deleted 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:764: snap_create 2024-09-21T09:41:00.562 INFO:tasks.rados.rados.0.smithi076.stdout:760: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:41:00.650 INFO:tasks.rados.rados.0.smithi076.stdout:761: finishing write tid 1 to smithi07685715-14 2024-09-21T09:41:00.650 INFO:tasks.rados.rados.0.smithi076.stdout:761: finishing write tid 2 to smithi07685715-14 2024-09-21T09:41:00.650 INFO:tasks.rados.rados.0.smithi076.stdout:761: finishing write tid 3 to smithi07685715-14 2024-09-21T09:41:00.650 INFO:tasks.rados.rados.0.smithi076.stdout:761: finishing write tid 4 to smithi07685715-14 2024-09-21T09:41:00.650 INFO:tasks.rados.rados.0.smithi076.stdout:761: finishing write tid 5 to smithi07685715-14 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:761: oid 14 updating version 0 to 341 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:761: oid 14 updating version 341 to 343 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:761: oid 14 updating version 343 to 344 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:761: oid 14 version 344 is already newer than 342 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 344 (ObjNum 321 snap 98 seq_num 321) dirty exists 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:761: left oid 14 (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:41:00.651 INFO:tasks.rados.rados.0.smithi076.stdout:762: expect (ObjNum 291 snap 93 seq_num 291) 2024-09-21T09:41:01.549 INFO:tasks.rados.rados.0.smithi076.stdout:760: done (4 left) 2024-09-21T09:41:01.549 INFO:tasks.rados.rados.0.smithi076.stdout:761: done (3 left) 2024-09-21T09:41:01.549 INFO:tasks.rados.rados.0.smithi076.stdout:762: done (2 left) 2024-09-21T09:41:01.549 INFO:tasks.rados.rados.0.smithi076.stdout:763: done (1 left) 2024-09-21T09:41:01.549 INFO:tasks.rados.rados.0.smithi076.stdout:764: done (0 left) 2024-09-21T09:41:01.549 INFO:tasks.rados.rados.0.smithi076.stdout:765: snap_create 2024-09-21T09:41:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:01 smithi191 ceph-mon[33301]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:41:01.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:01 smithi191 ceph-mon[33301]: pgmap v84: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 24 op/s 2024-09-21T09:41:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:01 smithi076 ceph-mon[29313]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:41:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:01 smithi076 ceph-mon[29313]: pgmap v84: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 24 op/s 2024-09-21T09:41:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:01 smithi076 ceph-mon[41325]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:41:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:01 smithi076 ceph-mon[41325]: pgmap v84: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 24 op/s 2024-09-21T09:41:02.553 INFO:tasks.rados.rados.0.smithi076.stdout:765: done (0 left) 2024-09-21T09:41:02.554 INFO:tasks.rados.rados.0.smithi076.stdout:766: write oid 42 current snap is 100 2024-09-21T09:41:02.554 INFO:tasks.rados.rados.0.smithi076.stdout:766: seq_num 322 ranges {611854=717004,2126143=561767,3259733=152121} 2024-09-21T09:41:02.560 INFO:tasks.rados.rados.0.smithi076.stdout:766: writing smithi07685715-42 from 611854 to 1328858 tid 1 2024-09-21T09:41:02.568 INFO:tasks.rados.rados.0.smithi076.stdout:766: writing smithi07685715-42 from 2126143 to 2687910 tid 2 2024-09-21T09:41:02.572 INFO:tasks.rados.rados.0.smithi076.stdout:766: writing smithi07685715-42 from 3259733 to 3411854 tid 3 2024-09-21T09:41:02.572 INFO:tasks.rados.rados.0.smithi076.stdout:767: read oid 4 snap 89 2024-09-21T09:41:02.572 INFO:tasks.rados.rados.0.smithi076.stdout:767: expect deleted 2024-09-21T09:41:02.572 INFO:tasks.rados.rados.0.smithi076.stdout:768: rollback oid 35 current snap is 100 2024-09-21T09:41:02.573 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 97 2024-09-21T09:41:02.573 INFO:tasks.rados.rados.0.smithi076.stdout:766: finishing write tid 1 to smithi07685715-42 2024-09-21T09:41:02.573 INFO:tasks.rados.rados.0.smithi076.stdout:766: finishing write tid 2 to smithi07685715-42 2024-09-21T09:41:02.573 INFO:tasks.rados.rados.0.smithi076.stdout:769: snap_create 2024-09-21T09:41:02.573 INFO:tasks.rados.rados.0.smithi076.stdout:766: finishing write tid 3 to smithi07685715-42 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: finishing write tid 4 to smithi07685715-42 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: finishing write tid 5 to smithi07685715-42 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: oid 42 updating version 0 to 205 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: oid 42 updating version 205 to 206 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: oid 42 updating version 206 to 208 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: oid 42 version 208 is already newer than 207 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 208 (ObjNum 322 snap 100 seq_num 322) dirty exists 2024-09-21T09:41:02.574 INFO:tasks.rados.rados.0.smithi076.stdout:766: left oid 42 (ObjNum 322 snap 100 seq_num 322) 2024-09-21T09:41:02.575 INFO:tasks.rados.rados.0.smithi076.stdout:768: finishing rollback tid 0 to smithi07685715-35 2024-09-21T09:41:02.575 INFO:tasks.rados.rados.0.smithi076.stdout:768: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:41:02.575 INFO:tasks.rados.rados.0.smithi076.stdout:768: finishing rollback tid 2 to smithi07685715-35 2024-09-21T09:41:02.575 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 532 (ObjNum 196 snap 55 seq_num 196) dirty exists 2024-09-21T09:41:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:02 smithi191 ceph-mon[33301]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:41:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:02 smithi076 ceph-mon[29313]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:41:02.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:02 smithi076 ceph-mon[41325]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:41:03.562 INFO:tasks.rados.rados.0.smithi076.stdout:766: done (3 left) 2024-09-21T09:41:03.562 INFO:tasks.rados.rados.0.smithi076.stdout:767: done (2 left) 2024-09-21T09:41:03.562 INFO:tasks.rados.rados.0.smithi076.stdout:768: done (1 left) 2024-09-21T09:41:03.562 INFO:tasks.rados.rados.0.smithi076.stdout:769: done (0 left) 2024-09-21T09:41:03.563 INFO:tasks.rados.rados.0.smithi076.stdout:770: write oid 19 current snap is 101 2024-09-21T09:41:03.563 INFO:tasks.rados.rados.0.smithi076.stdout:770: seq_num 323 ranges {628661=792253,2184625=421778,3403509=25152} 2024-09-21T09:41:03.569 INFO:tasks.rados.rados.0.smithi076.stdout:770: writing smithi07685715-19 from 628661 to 1420914 tid 1 2024-09-21T09:41:03.576 INFO:tasks.rados.rados.0.smithi076.stdout:770: writing smithi07685715-19 from 2184625 to 2606403 tid 2 2024-09-21T09:41:03.581 INFO:tasks.rados.rados.0.smithi076.stdout:770: writing smithi07685715-19 from 3403509 to 3428661 tid 3 2024-09-21T09:41:03.581 INFO:tasks.rados.rados.0.smithi076.stdout:771: write oid 44 current snap is 101 2024-09-21T09:41:03.581 INFO:tasks.rados.rados.0.smithi076.stdout:771: seq_num 324 ranges {645468=467502,1726754=565436,2798226=596838} 2024-09-21T09:41:03.587 INFO:tasks.rados.rados.0.smithi076.stdout:771: writing smithi07685715-44 from 645468 to 1112970 tid 1 2024-09-21T09:41:03.593 INFO:tasks.rados.rados.0.smithi076.stdout:771: writing smithi07685715-44 from 1726754 to 2292190 tid 2 2024-09-21T09:41:03.600 INFO:tasks.rados.rados.0.smithi076.stdout:771: writing smithi07685715-44 from 2798226 to 3395064 tid 3 2024-09-21T09:41:03.600 INFO:tasks.rados.rados.0.smithi076.stdout:772: snap_remove snap 94 2024-09-21T09:41:03.600 INFO:tasks.rados.rados.0.smithi076.stdout:770: finishing write tid 1 to smithi07685715-19 2024-09-21T09:41:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:03 smithi191 ceph-mon[33301]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:41:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:03 smithi191 ceph-mon[33301]: pgmap v87: 89 pgs: 89 active+clean; 174 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 795 KiB/s wr, 5 op/s 2024-09-21T09:41:03.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:03 smithi076 ceph-mon[29313]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:41:03.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:03 smithi076 ceph-mon[29313]: pgmap v87: 89 pgs: 89 active+clean; 174 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 795 KiB/s wr, 5 op/s 2024-09-21T09:41:03.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:03 smithi076 ceph-mon[41325]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:41:03.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:03 smithi076 ceph-mon[41325]: pgmap v87: 89 pgs: 89 active+clean; 174 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 795 KiB/s wr, 5 op/s 2024-09-21T09:41:04.565 INFO:tasks.rados.rados.0.smithi076.stdout:772: done (2 left) 2024-09-21T09:41:04.565 INFO:tasks.rados.rados.0.smithi076.stdout:773: delete oid 41 current snap is 101 2024-09-21T09:41:04.565 INFO:tasks.rados.rados.0.smithi076.stdout:770: finishing write tid 2 to smithi07685715-19 2024-09-21T09:41:04.565 INFO:tasks.rados.rados.0.smithi076.stdout:770: finishing write tid 3 to smithi07685715-19 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: finishing write tid 4 to smithi07685715-19 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: finishing write tid 5 to smithi07685715-19 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: oid 19 updating version 0 to 287 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: oid 19 version 287 is already newer than 285 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: oid 19 version 287 is already newer than 286 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: oid 19 version 287 is already newer than 284 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 287 (ObjNum 323 snap 101 seq_num 323) dirty exists 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:770: left oid 19 (ObjNum 323 snap 101 seq_num 323) 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: finishing write tid 1 to smithi07685715-44 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: finishing write tid 2 to smithi07685715-44 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: finishing write tid 3 to smithi07685715-44 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: finishing write tid 4 to smithi07685715-44 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: finishing write tid 5 to smithi07685715-44 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: oid 44 updating version 0 to 147 2024-09-21T09:41:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:771: oid 44 updating version 147 to 148 2024-09-21T09:41:04.567 INFO:tasks.rados.rados.0.smithi076.stdout:771: oid 44 updating version 148 to 150 2024-09-21T09:41:04.567 INFO:tasks.rados.rados.0.smithi076.stdout:771: oid 44 version 150 is already newer than 149 2024-09-21T09:41:04.567 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 150 (ObjNum 324 snap 101 seq_num 324) dirty exists 2024-09-21T09:41:04.567 INFO:tasks.rados.rados.0.smithi076.stdout:771: left oid 44 (ObjNum 324 snap 101 seq_num 324) 2024-09-21T09:41:04.570 INFO:tasks.rados.rados.0.smithi076.stdout:770: done (2 left) 2024-09-21T09:41:04.570 INFO:tasks.rados.rados.0.smithi076.stdout:771: done (1 left) 2024-09-21T09:41:04.570 INFO:tasks.rados.rados.0.smithi076.stdout:773: done (0 left) 2024-09-21T09:41:04.570 INFO:tasks.rados.rados.0.smithi076.stdout:774: snap_remove snap 96 2024-09-21T09:41:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:04 smithi191 ceph-mon[33301]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:41:04.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:04 smithi076 ceph-mon[29313]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:41:04.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:04 smithi076 ceph-mon[41325]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:41:05.570 INFO:tasks.rados.rados.0.smithi076.stdout:774: done (0 left) 2024-09-21T09:41:05.571 INFO:tasks.rados.rados.0.smithi076.stdout:775: write oid 24 current snap is 101 2024-09-21T09:41:05.571 INFO:tasks.rados.rados.0.smithi076.stdout:775: seq_num 326 ranges {679082=618000,1843718=569105,2853072=626010} 2024-09-21T09:41:05.578 INFO:tasks.rados.rados.0.smithi076.stdout:775: writing smithi07685715-24 from 679082 to 1297082 tid 1 2024-09-21T09:41:05.584 INFO:tasks.rados.rados.0.smithi076.stdout:775: writing smithi07685715-24 from 1843718 to 2412823 tid 2 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:775: writing smithi07685715-24 from 2853072 to 3479082 tid 3 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:776: read oid 19 snap 90 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:776: expect deleted 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:777: read oid 3 snap 99 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:777: expect deleted 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:778: read oid 43 snap -1 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:778: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:779: read oid 46 snap -1 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:779: expect (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:780: rollback oid 4 current snap is 101 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 80 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:775: finishing write tid 1 to smithi07685715-24 2024-09-21T09:41:05.591 INFO:tasks.rados.rados.0.smithi076.stdout:775: finishing write tid 2 to smithi07685715-24 2024-09-21T09:41:05.592 INFO:tasks.rados.rados.0.smithi076.stdout:781: write oid 18 current snap is 101 2024-09-21T09:41:05.592 INFO:tasks.rados.rados.0.smithi076.stdout:781: seq_num 327 ranges {695889=409602,1618553=712763,2996848=499041} 2024-09-21T09:41:05.596 INFO:tasks.rados.rados.0.smithi076.stdout:781: writing smithi07685715-18 from 695889 to 1105491 tid 1 2024-09-21T09:41:05.604 INFO:tasks.rados.rados.0.smithi076.stdout:781: writing smithi07685715-18 from 1618553 to 2331316 tid 2 2024-09-21T09:41:05.610 INFO:tasks.rados.rados.0.smithi076.stdout:781: writing smithi07685715-18 from 2996848 to 3495889 tid 3 2024-09-21T09:41:05.610 INFO:tasks.rados.rados.0.smithi076.stdout:782: snap_create 2024-09-21T09:41:05.610 INFO:tasks.rados.rados.0.smithi076.stdout:780: finishing rollback tid 0 to smithi07685715-4 2024-09-21T09:41:05.610 INFO:tasks.rados.rados.0.smithi076.stdout:780: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:41:05.610 INFO:tasks.rados.rados.0.smithi076.stdout:780: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:41:05.610 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 287 (ObjNum 246 snap 75 seq_num 246) dirty exists 2024-09-21T09:41:05.611 INFO:tasks.rados.rados.0.smithi076.stdout:778: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: finishing write tid 3 to smithi07685715-24 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: finishing write tid 4 to smithi07685715-24 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: finishing write tid 5 to smithi07685715-24 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: oid 24 updating version 0 to 63 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: oid 24 updating version 63 to 65 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: oid 24 version 65 is already newer than 62 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:775: oid 24 version 65 is already newer than 64 2024-09-21T09:41:05.754 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 65 (ObjNum 326 snap 101 seq_num 326) dirty exists 2024-09-21T09:41:05.755 INFO:tasks.rados.rados.0.smithi076.stdout:775: left oid 24 (ObjNum 326 snap 101 seq_num 326) 2024-09-21T09:41:05.755 INFO:tasks.rados.rados.0.smithi076.stdout:779: expect (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:41:05.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:05 smithi191 ceph-mon[33301]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:41:05.807 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:05 smithi191 ceph-mon[33301]: pgmap v90: 89 pgs: 89 active+clean; 175 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:41:05.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:05 smithi076 ceph-mon[29313]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:41:05.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:05 smithi076 ceph-mon[29313]: pgmap v90: 89 pgs: 89 active+clean; 175 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:41:05.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:05 smithi076 ceph-mon[41325]: osdmap e265: 8 total, 8 up, 8 in 2024-09-21T09:41:05.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:05 smithi076 ceph-mon[41325]: pgmap v90: 89 pgs: 89 active+clean; 175 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-09-21T09:41:05.965 INFO:tasks.rados.rados.0.smithi076.stdout:781: finishing write tid 1 to smithi07685715-18 2024-09-21T09:41:05.965 INFO:tasks.rados.rados.0.smithi076.stdout:781: finishing write tid 2 to smithi07685715-18 2024-09-21T09:41:05.965 INFO:tasks.rados.rados.0.smithi076.stdout:781: finishing write tid 3 to smithi07685715-18 2024-09-21T09:41:05.965 INFO:tasks.rados.rados.0.smithi076.stdout:781: finishing write tid 4 to smithi07685715-18 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:781: finishing write tid 5 to smithi07685715-18 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:781: oid 18 updating version 0 to 215 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:781: oid 18 updating version 215 to 217 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:781: oid 18 version 217 is already newer than 214 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:781: oid 18 version 217 is already newer than 216 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 217 (ObjNum 327 snap 101 seq_num 327) dirty exists 2024-09-21T09:41:05.966 INFO:tasks.rados.rados.0.smithi076.stdout:781: left oid 18 (ObjNum 327 snap 101 seq_num 327) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:775: done (7 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:776: done (6 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:777: done (5 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:778: done (4 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:779: done (3 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:780: done (2 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:781: done (1 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:782: done (0 left) 2024-09-21T09:41:06.576 INFO:tasks.rados.rados.0.smithi076.stdout:783: snap_remove snap 78 2024-09-21T09:41:06.860 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:06 smithi191 ceph-mon[33301]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:41:06.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:06 smithi076 ceph-mon[29313]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:41:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:06 smithi076 ceph-mon[41325]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:41:07.582 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:07 smithi191 ceph-mon[33301]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:41:07.582 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:07 smithi191 ceph-mon[33301]: pgmap v93: 89 pgs: 89 active+clean; 175 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 354 KiB/s wr, 4 op/s 2024-09-21T09:41:07.583 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:07.583 INFO:tasks.rados.rados.0.smithi076.stdout:783: done (0 left) 2024-09-21T09:41:07.583 INFO:tasks.rados.rados.0.smithi076.stdout:784: read oid 47 snap 97 2024-09-21T09:41:07.584 INFO:tasks.rados.rados.0.smithi076.stdout:784: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:41:07.584 INFO:tasks.rados.rados.0.smithi076.stdout:785: read oid 28 snap 86 2024-09-21T09:41:07.584 INFO:tasks.rados.rados.0.smithi076.stdout:785: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:41:07.584 INFO:tasks.rados.rados.0.smithi076.stdout:786: snap_remove snap 88 2024-09-21T09:41:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:07 smithi076 ceph-mon[29313]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:41:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:07 smithi076 ceph-mon[29313]: pgmap v93: 89 pgs: 89 active+clean; 175 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 354 KiB/s wr, 4 op/s 2024-09-21T09:41:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:07 smithi076 ceph-mon[41325]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:41:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:07 smithi076 ceph-mon[41325]: pgmap v93: 89 pgs: 89 active+clean; 175 MiB data, 633 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 354 KiB/s wr, 4 op/s 2024-09-21T09:41:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:08.512 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 systemd[1]: Stopping Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:08.539 INFO:tasks.rados.rados.0.smithi076.stdout:786: done (2 left) 2024-09-21T09:41:08.540 INFO:tasks.rados.rados.0.smithi076.stdout:787: write oid 36 current snap is 102 2024-09-21T09:41:08.540 INFO:tasks.rados.rados.0.smithi076.stdout:787: seq_num 328 ranges {712696=484851,1960682=572774,3307918=204778} 2024-09-21T09:41:08.547 INFO:tasks.rados.rados.0.smithi076.stdout:787: writing smithi07685715-36 from 712696 to 1197547 tid 1 2024-09-21T09:41:08.554 INFO:tasks.rados.rados.0.smithi076.stdout:787: writing smithi07685715-36 from 1960682 to 2533456 tid 2 2024-09-21T09:41:08.560 INFO:tasks.rados.rados.0.smithi076.stdout:787: writing smithi07685715-36 from 3307918 to 3512696 tid 3 2024-09-21T09:41:08.560 INFO:tasks.rados.rados.0.smithi076.stdout:788: snap_create 2024-09-21T09:41:08.560 INFO:tasks.rados.rados.0.smithi076.stdout:784: expect (ObjNum 155 snap 38 seq_num 155) 2024-09-21T09:41:08.672 INFO:tasks.rados.rados.0.smithi076.stdout:785: expect (ObjNum 173 snap 46 seq_num 173) 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: finishing write tid 1 to smithi07685715-36 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: finishing write tid 2 to smithi07685715-36 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: finishing write tid 3 to smithi07685715-36 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: finishing write tid 4 to smithi07685715-36 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: finishing write tid 5 to smithi07685715-36 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: oid 36 updating version 0 to 302 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: oid 36 version 302 is already newer than 301 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: oid 36 version 302 is already newer than 300 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: oid 36 version 302 is already newer than 299 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 302 (ObjNum 328 snap 102 seq_num 328) dirty exists 2024-09-21T09:41:08.799 INFO:tasks.rados.rados.0.smithi076.stdout:787: left oid 36 (ObjNum 328 snap 102 seq_num 328) 2024-09-21T09:41:08.851 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-mon[33301]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:41:08.851 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-mon[33301]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:41:08.851 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:41:08.851 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:41:08.851 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:41:08.851 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:41:08.851 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.511Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.512Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.512Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[80007]: ts=2024-09-21T09:41:08.512Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 podman[88804]: 2024-09-21 09:41:08.521786409 +0000 UTC m=+0.117639403 container died 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 podman[88804]: 2024-09-21 09:41:08.850321894 +0000 UTC m=+0.446174889 container cleanup 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:08.852 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:08 smithi191 bash[88804]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:41:08.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:08 smithi076 ceph-mon[29313]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:41:08.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:08 smithi076 ceph-mon[29313]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:41:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:08 smithi076 ceph-mon[41325]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:41:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:08 smithi076 ceph-mon[41325]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:41:09.112 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 podman[88823]: 2024-09-21 09:41:09.052295509 +0000 UTC m=+0.528939986 container remove 2f21884f574e614a1fe3e25d4266afad528200d00cc8c50b18d3b3d0102044fd (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:09.539 INFO:tasks.rados.rados.0.smithi076.stdout:784: done (3 left) 2024-09-21T09:41:09.539 INFO:tasks.rados.rados.0.smithi076.stdout:785: done (2 left) 2024-09-21T09:41:09.539 INFO:tasks.rados.rados.0.smithi076.stdout:787: done (1 left) 2024-09-21T09:41:09.540 INFO:tasks.rados.rados.0.smithi076.stdout:788: done (0 left) 2024-09-21T09:41:09.540 INFO:tasks.rados.rados.0.smithi076.stdout:789: delete oid 3 current snap is 103 2024-09-21T09:41:09.540 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:41:09.541 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 systemd[1]: Stopped Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:09.541 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.178s CPU time. 2024-09-21T09:41:09.541 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 systemd[1]: Starting Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:09.541 INFO:tasks.rados.rados.0.smithi076.stdout:789: done (0 left) 2024-09-21T09:41:09.541 INFO:tasks.rados.rados.0.smithi076.stdout:790: delete oid 4 current snap is 103 2024-09-21T09:41:09.546 INFO:tasks.rados.rados.0.smithi076.stdout:790: done (0 left) 2024-09-21T09:41:09.546 INFO:tasks.rados.rados.0.smithi076.stdout:791: read oid 46 snap 82 2024-09-21T09:41:09.546 INFO:tasks.rados.rados.0.smithi076.stdout:791: expect (ObjNum 261 snap 82 seq_num 261) 2024-09-21T09:41:09.546 INFO:tasks.rados.rados.0.smithi076.stdout:792: delete oid 9 current snap is 103 2024-09-21T09:41:09.550 INFO:tasks.rados.rados.0.smithi076.stdout:792: done (1 left) 2024-09-21T09:41:09.550 INFO:tasks.rados.rados.0.smithi076.stdout:793: read oid 3 snap 101 2024-09-21T09:41:09.550 INFO:tasks.rados.rados.0.smithi076.stdout:793: expect deleted 2024-09-21T09:41:09.550 INFO:tasks.rados.rados.0.smithi076.stdout:794: read oid 39 snap -1 2024-09-21T09:41:09.550 INFO:tasks.rados.rados.0.smithi076.stdout:794: expect deleted 2024-09-21T09:41:09.550 INFO:tasks.rados.rados.0.smithi076.stdout:795: snap_remove snap 80 2024-09-21T09:41:09.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-mon[33301]: pgmap v96: 89 pgs: 89 active+clean; 183 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:41:09.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-mon[33301]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:41:09.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 podman[88954]: 2024-09-21 09:41:09.609911024 +0000 UTC m=+0.021358790 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:41:09.863 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 podman[88954]: 2024-09-21 09:41:09.737297809 +0000 UTC m=+0.148745571 container create eda26012a522f9e7d74d220f464fb8511d96b74cbb416a4edf8d36db89b0d435 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:09.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:09 smithi076 ceph-mon[29313]: pgmap v96: 89 pgs: 89 active+clean; 183 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:41:09.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:09 smithi076 ceph-mon[29313]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:41:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:09 smithi076 ceph-mon[41325]: pgmap v96: 89 pgs: 89 active+clean; 183 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:41:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:09 smithi076 ceph-mon[41325]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 podman[88954]: 2024-09-21 09:41:09.972742683 +0000 UTC m=+0.384190453 container init eda26012a522f9e7d74d220f464fb8511d96b74cbb416a4edf8d36db89b0d435 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 podman[88954]: 2024-09-21 09:41:09.977270057 +0000 UTC m=+0.388717822 container start eda26012a522f9e7d74d220f464fb8511d96b74cbb416a4edf8d36db89b0d435 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.997Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.997Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.997Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi191 (none))" 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.997Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:41:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.997Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.999Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:09 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:09.999Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.001Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.001Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.003Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.003Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.071µs 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.003Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.004Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.011Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.012Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.012Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.013Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-21T09:41:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.013Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.806µs wal_replay_duration=9.910981ms wbl_replay_duration=83ns total_replay_duration=9.953891ms 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.015Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.015Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.015Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.031Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.20606ms db_storage=893ns remote_storage=1.525µs web_handler=414ns query_engine=596ns scrape=395.585µs scrape_sd=103.487µs notify=24.639µs notify_sd=15.652µs rules=15.311747ms tracing=5.956µs 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.031Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T09:41:10.031Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 bash[88954]: eda26012a522f9e7d74d220f464fb8511d96b74cbb416a4edf8d36db89b0d435 2024-09-21T09:41:10.365 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 09:41:10 smithi191 systemd[1]: Started Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:10.542 INFO:tasks.rados.rados.0.smithi076.stdout:795: done (3 left) 2024-09-21T09:41:10.542 INFO:tasks.rados.rados.0.smithi076.stdout:796: rollback oid 26 current snap is 103 2024-09-21T09:41:10.542 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 86 2024-09-21T09:41:10.542 INFO:tasks.rados.rados.0.smithi076.stdout:797: write oid 2 current snap is 103 2024-09-21T09:41:10.542 INFO:tasks.rados.rados.0.smithi076.stdout:797: seq_num 332 ranges {779924=785847,2078257=580112,3301257=278667} 2024-09-21T09:41:10.551 INFO:tasks.rados.rados.0.smithi076.stdout:797: writing smithi07685715-2 from 779924 to 1565771 tid 1 2024-09-21T09:41:10.557 INFO:tasks.rados.rados.0.smithi076.stdout:797: writing smithi07685715-2 from 2078257 to 2658369 tid 2 2024-09-21T09:41:10.563 INFO:tasks.rados.rados.0.smithi076.stdout:797: writing smithi07685715-2 from 3301257 to 3579924 tid 3 2024-09-21T09:41:10.563 INFO:tasks.rados.rados.0.smithi076.stdout:798: snap_remove snap 97 2024-09-21T09:41:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:41:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:11 smithi191 ceph-mon[33301]: pgmap v99: 89 pgs: 89 active+clean; 183 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:41:11.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[29313]: pgmap v99: 89 pgs: 89 active+clean; 183 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:11.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:41:11.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:11 smithi076 ceph-mon[41325]: pgmap v99: 89 pgs: 89 active+clean; 183 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:41:11.547 INFO:tasks.rados.rados.0.smithi076.stdout:798: done (5 left) 2024-09-21T09:41:11.547 INFO:tasks.rados.rados.0.smithi076.stdout:799: write oid 29 current snap is 103 2024-09-21T09:41:11.547 INFO:tasks.rados.rados.0.smithi076.stdout:799: seq_num 333 ranges {796731=461096,1736739=723770,2928680=668051} 2024-09-21T09:41:11.554 INFO:tasks.rados.rados.0.smithi076.stdout:799: writing smithi07685715-29 from 796731 to 1257827 tid 1 2024-09-21T09:41:11.560 INFO:tasks.rados.rados.0.smithi076.stdout:799: writing smithi07685715-29 from 1736739 to 2460509 tid 2 2024-09-21T09:41:11.567 INFO:tasks.rados.rados.0.smithi076.stdout:799: writing smithi07685715-29 from 2928680 to 3596731 tid 3 2024-09-21T09:41:11.567 INFO:tasks.rados.rados.0.smithi076.stdout:800: read oid 19 snap 99 2024-09-21T09:41:11.567 INFO:tasks.rados.rados.0.smithi076.stdout:800: expect deleted 2024-09-21T09:41:11.567 INFO:tasks.rados.rados.0.smithi076.stdout:801: snap_remove snap 90 2024-09-21T09:41:12.553 INFO:tasks.rados.rados.0.smithi076.stdout:801: done (7 left) 2024-09-21T09:41:12.553 INFO:tasks.rados.rados.0.smithi076.stdout:802: write oid 30 current snap is 103 2024-09-21T09:41:12.553 INFO:tasks.rados.rados.0.smithi076.stdout:802: seq_num 334 ranges {413538=536345,1395221=583781,2556103=658617} 2024-09-21T09:41:12.559 INFO:tasks.rados.rados.0.smithi076.stdout:802: writing smithi07685715-30 from 413538 to 949883 tid 1 2024-09-21T09:41:12.565 INFO:tasks.rados.rados.0.smithi076.stdout:802: writing smithi07685715-30 from 1395221 to 1979002 tid 2 2024-09-21T09:41:12.572 INFO:tasks.rados.rados.0.smithi076.stdout:802: writing smithi07685715-30 from 2556103 to 3214720 tid 3 2024-09-21T09:41:12.572 INFO:tasks.rados.rados.0.smithi076.stdout:803: write oid 10 current snap is 103 2024-09-21T09:41:12.572 INFO:tasks.rados.rados.0.smithi076.stdout:803: seq_num 335 ranges {430345=727947,1570056=727439,2699879=469889} 2024-09-21T09:41:12.578 INFO:tasks.rados.rados.0.smithi076.stdout:803: writing smithi07685715-10 from 430345 to 1158292 tid 1 2024-09-21T09:41:12.585 INFO:tasks.rados.rados.0.smithi076.stdout:803: writing smithi07685715-10 from 1570056 to 2297495 tid 2 2024-09-21T09:41:12.590 INFO:tasks.rados.rados.0.smithi076.stdout:803: writing smithi07685715-10 from 2699879 to 3169768 tid 3 2024-09-21T09:41:12.590 INFO:tasks.rados.rados.0.smithi076.stdout:804: snap_create 2024-09-21T09:41:12.590 INFO:tasks.rados.rados.0.smithi076.stdout:791: expect (ObjNum 261 snap 82 seq_num 261) 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:796: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 0 (ObjNum 529 snap 0 seq_num 1283432160) dirty dne 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: finishing write tid 1 to smithi07685715-2 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: finishing write tid 2 to smithi07685715-2 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: finishing write tid 3 to smithi07685715-2 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: finishing write tid 4 to smithi07685715-2 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: finishing write tid 5 to smithi07685715-2 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: oid 2 updating version 0 to 140 2024-09-21T09:41:12.708 INFO:tasks.rados.rados.0.smithi076.stdout:797: oid 2 updating version 140 to 142 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:797: oid 2 version 142 is already newer than 139 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:797: oid 2 version 142 is already newer than 141 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 142 (ObjNum 332 snap 103 seq_num 332) dirty exists 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:797: left oid 2 (ObjNum 332 snap 103 seq_num 332) 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: finishing write tid 1 to smithi07685715-29 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: finishing write tid 2 to smithi07685715-29 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: finishing write tid 3 to smithi07685715-29 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: finishing write tid 4 to smithi07685715-29 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: finishing write tid 5 to smithi07685715-29 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: oid 29 updating version 0 to 217 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: oid 29 updating version 217 to 218 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: oid 29 updating version 218 to 219 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: oid 29 version 219 is already newer than 216 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 219 (ObjNum 333 snap 103 seq_num 333) dirty exists 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:799: left oid 29 (ObjNum 333 snap 103 seq_num 333) 2024-09-21T09:41:12.709 INFO:tasks.rados.rados.0.smithi076.stdout:802: finishing write tid 1 to smithi07685715-30 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: finishing write tid 2 to smithi07685715-30 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: finishing write tid 3 to smithi07685715-30 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: finishing write tid 4 to smithi07685715-30 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: finishing write tid 5 to smithi07685715-30 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: oid 30 updating version 0 to 653 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: oid 30 version 653 is already newer than 651 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: oid 30 version 653 is already newer than 652 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: oid 30 updating version 653 to 654 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 654 (ObjNum 334 snap 103 seq_num 334) dirty exists 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:802: left oid 30 (ObjNum 334 snap 103 seq_num 334) 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:803: finishing write tid 1 to smithi07685715-10 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:803: finishing write tid 2 to smithi07685715-10 2024-09-21T09:41:12.710 INFO:tasks.rados.rados.0.smithi076.stdout:803: finishing write tid 3 to smithi07685715-10 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: finishing write tid 4 to smithi07685715-10 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: finishing write tid 5 to smithi07685715-10 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: oid 10 updating version 0 to 370 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: oid 10 updating version 370 to 372 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: oid 10 updating version 372 to 373 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: oid 10 version 373 is already newer than 371 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 373 (ObjNum 335 snap 103 seq_num 335) dirty exists 2024-09-21T09:41:12.711 INFO:tasks.rados.rados.0.smithi076.stdout:803: left oid 10 (ObjNum 335 snap 103 seq_num 335) 2024-09-21T09:41:12.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:12 smithi191 ceph-mon[33301]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:41:12.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:12 smithi076 ceph-mon[29313]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:41:12.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:12 smithi076 ceph-mon[41325]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:41:13.330 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:41:13.558 INFO:tasks.rados.rados.0.smithi076.stdout:791: done (9 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:793: done (8 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:794: done (7 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:796: done (6 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:797: done (5 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:799: done (4 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:800: done (3 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:802: done (2 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:803: done (1 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:804: done (0 left) 2024-09-21T09:41:13.559 INFO:tasks.rados.rados.0.smithi076.stdout:805: delete oid 35 current snap is 104 2024-09-21T09:41:13.564 INFO:tasks.rados.rados.0.smithi076.stdout:805: done (0 left) 2024-09-21T09:41:13.565 INFO:tasks.rados.rados.0.smithi076.stdout:806: write oid 17 current snap is 104 2024-09-21T09:41:13.565 INFO:tasks.rados.rados.0.smithi076.stdout:806: seq_num 337 ranges {463959=478445,1570667=731108,3038372=492433} 2024-09-21T09:41:13.569 INFO:tasks.rados.rados.0.smithi076.stdout:806: writing smithi07685715-17 from 463959 to 942404 tid 1 2024-09-21T09:41:13.577 INFO:tasks.rados.rados.0.smithi076.stdout:806: writing smithi07685715-17 from 1570667 to 2301775 tid 2 2024-09-21T09:41:13.584 INFO:tasks.rados.rados.0.smithi076.stdout:806: writing smithi07685715-17 from 3038372 to 3530805 tid 3 2024-09-21T09:41:13.585 INFO:tasks.rados.rados.0.smithi076.stdout:807: read oid 50 snap -1 2024-09-21T09:41:13.585 INFO:tasks.rados.rados.0.smithi076.stdout:807: expect (ObjNum 299 snap 93 seq_num 299) 2024-09-21T09:41:13.585 INFO:tasks.rados.rados.0.smithi076.stdout:808: write oid 1 current snap is 104 2024-09-21T09:41:13.585 INFO:tasks.rados.rados.0.smithi076.stdout:806: finishing write tid 1 to smithi07685715-17 2024-09-21T09:41:13.585 INFO:tasks.rados.rados.0.smithi076.stdout:808: seq_num 338 ranges {480766=553694,1629149=591119,2782148=420058} 2024-09-21T09:41:13.590 INFO:tasks.rados.rados.0.smithi076.stdout:808: writing smithi07685715-1 from 480766 to 1034460 tid 1 2024-09-21T09:41:13.597 INFO:tasks.rados.rados.0.smithi076.stdout:808: writing smithi07685715-1 from 1629149 to 2220268 tid 2 2024-09-21T09:41:13.602 INFO:tasks.rados.rados.0.smithi076.stdout:808: writing smithi07685715-1 from 2782148 to 3202206 tid 3 2024-09-21T09:41:13.602 INFO:tasks.rados.rados.0.smithi076.stdout:809: snap_remove snap 82 2024-09-21T09:41:13.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:13 smithi076 ceph-mon[29313]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:41:13.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:13 smithi076 ceph-mon[29313]: pgmap v102: 89 pgs: 89 active+clean; 180 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-21T09:41:13.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:13 smithi076 ceph-mon[41325]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:41:13.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:13 smithi076 ceph-mon[41325]: pgmap v102: 89 pgs: 89 active+clean; 180 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-21T09:41:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:13 smithi191 ceph-mon[33301]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:41:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:13 smithi191 ceph-mon[33301]: pgmap v102: 89 pgs: 89 active+clean; 180 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.6 MiB/s wr, 6 op/s 2024-09-21T09:41:13.897 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:41:13.897 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (4m) 22s ago 5m 26.1M - 0.25.0 c8568f914cd2 d930ef8d31ba 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (5m) 21s ago 5m 88.6M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (4m) 22s ago 4m 17.2M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (4m) 21s ago 4m 16.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (48s) 21s ago 9m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (75s) 22s ago 11m 511M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (11m) 22s ago 11m 57.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (10m) 21s ago 10m 50.1M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (9m) 22s ago 9m 47.5M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (32s) 22s ago 5m 7872k - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:41:13.898 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (26s) 21s ago 5m 3213k - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (9m) 22s ago 9m 81.2M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (8m) 22s ago 8m 72.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (8m) 22s ago 8m 91.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (8m) 22s ago 8m 69.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (7m) 21s ago 7m 74.0M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (7m) 21s ago 7m 86.0M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (6m) 21s ago 6m 64.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (6m) 21s ago 6m 60.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:41:13.899 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 starting - - - - 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:41:14.233 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:41:14.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:14.234 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:41:14.234 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:41:14.234 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:41:14.234 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:41:14.234 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [ 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr" 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading prometheus daemons", 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:41:14.520 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:41:14.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:14 smithi191 ceph-mon[33301]: from='client.15057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:14 smithi191 ceph-mon[33301]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:41:14.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:14 smithi191 ceph-mon[33301]: from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.555 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:14 smithi191 ceph-mon[33301]: from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.565 INFO:tasks.rados.rados.0.smithi076.stdout:809: done (3 left) 2024-09-21T09:41:14.565 INFO:tasks.rados.rados.0.smithi076.stdout:810: read oid 14 snap -1 2024-09-21T09:41:14.565 INFO:tasks.rados.rados.0.smithi076.stdout:810: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:41:14.565 INFO:tasks.rados.rados.0.smithi076.stdout:811: snap_remove snap 84 2024-09-21T09:41:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:14 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/1817431754' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:14.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[29313]: from='client.15057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[29313]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[29313]: from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[29313]: from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/1817431754' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[41325]: from='client.15057 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[41325]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[41325]: from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[41325]: from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:14 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/1817431754' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:811: done (4 left) 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:812: rollback oid 7 current snap is 104 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 89 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: finishing write tid 2 to smithi07685715-17 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: finishing write tid 3 to smithi07685715-17 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: finishing write tid 4 to smithi07685715-17 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: finishing write tid 5 to smithi07685715-17 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: oid 17 updating version 0 to 377 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: oid 17 version 377 is already newer than 376 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: oid 17 version 377 is already newer than 374 2024-09-21T09:41:15.572 INFO:tasks.rados.rados.0.smithi076.stdout:806: oid 17 version 377 is already newer than 375 2024-09-21T09:41:15.573 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 377 (ObjNum 337 snap 104 seq_num 337) dirty exists 2024-09-21T09:41:15.573 INFO:tasks.rados.rados.0.smithi076.stdout:806: left oid 17 (ObjNum 337 snap 104 seq_num 337) 2024-09-21T09:41:15.573 INFO:tasks.rados.rados.0.smithi076.stdout:808: finishing write tid 1 to smithi07685715-1 2024-09-21T09:41:15.573 INFO:tasks.rados.rados.0.smithi076.stdout:807: expect (ObjNum 299 snap 93 seq_num 299) 2024-09-21T09:41:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:808: finishing write tid 2 to smithi07685715-1 2024-09-21T09:41:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:808: finishing write tid 3 to smithi07685715-1 2024-09-21T09:41:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:806: done (4 left) 2024-09-21T09:41:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:807: done (3 left) 2024-09-21T09:41:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:813: read oid 44 snap 92 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:813: expect (ObjNum 285 snap 89 seq_num 285) 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:814: rollback oid 22 current snap is 104 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 89 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: finishing write tid 4 to smithi07685715-1 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: finishing write tid 5 to smithi07685715-1 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: oid 1 updating version 0 to 580 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: oid 1 updating version 580 to 581 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: oid 1 updating version 581 to 582 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: oid 1 version 582 is already newer than 579 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 582 (ObjNum 338 snap 104 seq_num 338) dirty exists 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:808: left oid 1 (ObjNum 338 snap 104 seq_num 338) 2024-09-21T09:41:15.812 INFO:tasks.rados.rados.0.smithi076.stdout:810: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:41:15.821 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:15.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:41:15.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: pgmap v105: 89 pgs: 89 active+clean; 177 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:41:15.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.822 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:15 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:41:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: pgmap v105: 89 pgs: 89 active+clean; 177 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:41:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:41:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: pgmap v105: 89 pgs: 89 active+clean; 177 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.0 MiB/s wr, 9 op/s 2024-09-21T09:41:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:15 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:812: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:41:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:812: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:41:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 387 (ObjNum 276 snap 88 seq_num 276) dirty exists 2024-09-21T09:41:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:814: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:41:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:814: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:41:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:814: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:41:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 480 (ObjNum 271 snap 86 seq_num 271) dirty exists 2024-09-21T09:41:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:813: expect (ObjNum 285 snap 89 seq_num 285) 2024-09-21T09:41:16.053 INFO:tasks.rados.rados.0.smithi076.stdout:808: done (4 left) 2024-09-21T09:41:16.053 INFO:tasks.rados.rados.0.smithi076.stdout:810: done (3 left) 2024-09-21T09:41:16.053 INFO:tasks.rados.rados.0.smithi076.stdout:812: done (2 left) 2024-09-21T09:41:16.054 INFO:tasks.rados.rados.0.smithi076.stdout:813: done (1 left) 2024-09-21T09:41:16.054 INFO:tasks.rados.rados.0.smithi076.stdout:814: done (0 left) 2024-09-21T09:41:16.054 INFO:tasks.rados.rados.0.smithi076.stdout:815: rollback oid 43 current snap is 104 2024-09-21T09:41:16.054 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 102 2024-09-21T09:41:16.054 INFO:tasks.rados.rados.0.smithi076.stdout:816: snap_remove snap 92 2024-09-21T09:41:16.576 INFO:tasks.rados.rados.0.smithi076.stdout:816: done (1 left) 2024-09-21T09:41:16.577 INFO:tasks.rados.rados.0.smithi076.stdout:817: write oid 22 current snap is 104 2024-09-21T09:41:16.577 INFO:tasks.rados.rados.0.smithi076.stdout:817: seq_num 339 ranges {497573=628943,1687631=734777,3093218=604355} 2024-09-21T09:41:16.583 INFO:tasks.rados.rados.0.smithi076.stdout:817: writing smithi07685715-22 from 497573 to 1126516 tid 1 2024-09-21T09:41:16.591 INFO:tasks.rados.rados.0.smithi076.stdout:817: writing smithi07685715-22 from 1687631 to 2422408 tid 2 2024-09-21T09:41:16.598 INFO:tasks.rados.rados.0.smithi076.stdout:817: writing smithi07685715-22 from 3093218 to 3697573 tid 3 2024-09-21T09:41:16.598 INFO:tasks.rados.rados.0.smithi076.stdout:818: snap_remove snap 89 2024-09-21T09:41:16.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:16 smithi191 ceph-mon[33301]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:41:16.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:16 smithi076 ceph-mon[29313]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:41:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:16 smithi076 ceph-mon[41325]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:41:17.584 INFO:tasks.rados.rados.0.smithi076.stdout:818: done (2 left) 2024-09-21T09:41:17.584 INFO:tasks.rados.rados.0.smithi076.stdout:819: delete oid 49 current snap is 104 2024-09-21T09:41:17.584 INFO:tasks.rados.rados.0.smithi076.stdout:815: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:41:17.584 INFO:tasks.rados.rados.0.smithi076.stdout:815: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:815: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 119 (ObjNum 199 snap 57 seq_num 199) dirty exists 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: finishing write tid 1 to smithi07685715-22 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: finishing write tid 2 to smithi07685715-22 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: finishing write tid 3 to smithi07685715-22 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: finishing write tid 4 to smithi07685715-22 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: finishing write tid 5 to smithi07685715-22 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: oid 22 updating version 0 to 487 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: oid 22 version 487 is already newer than 486 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: oid 22 updating version 487 to 488 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: oid 22 version 488 is already newer than 485 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 488 (ObjNum 339 snap 104 seq_num 339) dirty exists 2024-09-21T09:41:17.585 INFO:tasks.rados.rados.0.smithi076.stdout:817: left oid 22 (ObjNum 339 snap 104 seq_num 339) 2024-09-21T09:41:17.593 INFO:tasks.rados.rados.0.smithi076.stdout:815: done (2 left) 2024-09-21T09:41:17.593 INFO:tasks.rados.rados.0.smithi076.stdout:817: done (1 left) 2024-09-21T09:41:17.594 INFO:tasks.rados.rados.0.smithi076.stdout:819: done (0 left) 2024-09-21T09:41:17.594 INFO:tasks.rados.rados.0.smithi076.stdout:820: read oid 39 snap 103 2024-09-21T09:41:17.594 INFO:tasks.rados.rados.0.smithi076.stdout:820: expect deleted 2024-09-21T09:41:17.594 INFO:tasks.rados.rados.0.smithi076.stdout:821: read oid 43 snap 95 2024-09-21T09:41:17.594 INFO:tasks.rados.rados.0.smithi076.stdout:821: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:41:17.594 INFO:tasks.rados.rados.0.smithi076.stdout:822: snap_create 2024-09-21T09:41:17.599 INFO:tasks.rados.rados.0.smithi076.stdout:821: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:41:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:17 smithi191 ceph-mon[33301]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:41:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:17 smithi191 ceph-mon[33301]: pgmap v108: 89 pgs: 89 active+clean; 177 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 453 KiB/s wr, 3 op/s 2024-09-21T09:41:17.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:17 smithi076 ceph-mon[29313]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:41:17.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:17 smithi076 ceph-mon[29313]: pgmap v108: 89 pgs: 89 active+clean; 177 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 453 KiB/s wr, 3 op/s 2024-09-21T09:41:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:17 smithi076 ceph-mon[41325]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:41:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:17 smithi076 ceph-mon[41325]: pgmap v108: 89 pgs: 89 active+clean; 177 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 453 KiB/s wr, 3 op/s 2024-09-21T09:41:18.540 INFO:tasks.rados.rados.0.smithi076.stdout:820: done (2 left) 2024-09-21T09:41:18.540 INFO:tasks.rados.rados.0.smithi076.stdout:821: done (1 left) 2024-09-21T09:41:18.540 INFO:tasks.rados.rados.0.smithi076.stdout:822: done (0 left) 2024-09-21T09:41:18.540 INFO:tasks.rados.rados.0.smithi076.stdout:823: write oid 35 current snap is 105 2024-09-21T09:41:18.540 INFO:tasks.rados.rados.0.smithi076.stdout:823: seq_num 341 ranges {531187=779441,2088242=738446,3431711=299476} 2024-09-21T09:41:18.547 INFO:tasks.rados.rados.0.smithi076.stdout:823: writing smithi07685715-35 from 531187 to 1310628 tid 1 2024-09-21T09:41:18.555 INFO:tasks.rados.rados.0.smithi076.stdout:823: writing smithi07685715-35 from 2088242 to 2826688 tid 2 2024-09-21T09:41:18.561 INFO:tasks.rados.rados.0.smithi076.stdout:823: writing smithi07685715-35 from 3431711 to 3731187 tid 3 2024-09-21T09:41:18.561 INFO:tasks.rados.rados.0.smithi076.stdout:824: write oid 48 current snap is 105 2024-09-21T09:41:18.561 INFO:tasks.rados.rados.0.smithi076.stdout:824: seq_num 342 ranges {547994=454690,1746724=482104,2659134=465146,3605976=142018} 2024-09-21T09:41:18.566 INFO:tasks.rados.rados.0.smithi076.stdout:824: writing smithi07685715-48 from 547994 to 1002684 tid 1 2024-09-21T09:41:18.573 INFO:tasks.rados.rados.0.smithi076.stdout:824: writing smithi07685715-48 from 1746724 to 2228828 tid 2 2024-09-21T09:41:18.578 INFO:tasks.rados.rados.0.smithi076.stdout:824: writing smithi07685715-48 from 2659134 to 3124280 tid 3 2024-09-21T09:41:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:824: writing smithi07685715-48 from 3605976 to 3747994 tid 4 2024-09-21T09:41:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:825: write oid 49 current snap is 105 2024-09-21T09:41:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:823: finishing write tid 1 to smithi07685715-35 2024-09-21T09:41:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:825: seq_num 343 ranges {564801=646292,1921559=742115,3202910=561891} 2024-09-21T09:41:18.588 INFO:tasks.rados.rados.0.smithi076.stdout:825: writing smithi07685715-49 from 564801 to 1211093 tid 1 2024-09-21T09:41:18.596 INFO:tasks.rados.rados.0.smithi076.stdout:825: writing smithi07685715-49 from 1921559 to 2663674 tid 2 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:825: writing smithi07685715-49 from 3202910 to 3764801 tid 3 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:826: read oid 32 snap -1 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:826: expect (ObjNum 304 snap 94 seq_num 304) 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:827: delete oid 27 current snap is 105 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: finishing write tid 2 to smithi07685715-35 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: finishing write tid 3 to smithi07685715-35 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: finishing write tid 4 to smithi07685715-35 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: finishing write tid 5 to smithi07685715-35 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: oid 35 updating version 0 to 617 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: oid 35 updating version 617 to 618 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: oid 35 version 618 is already newer than 615 2024-09-21T09:41:18.604 INFO:tasks.rados.rados.0.smithi076.stdout:823: oid 35 version 618 is already newer than 616 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 618 (ObjNum 341 snap 105 seq_num 341) dirty exists 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:823: left oid 35 (ObjNum 341 snap 105 seq_num 341) 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: finishing write tid 1 to smithi07685715-48 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: finishing write tid 2 to smithi07685715-48 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: finishing write tid 3 to smithi07685715-48 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: finishing write tid 4 to smithi07685715-48 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: finishing write tid 5 to smithi07685715-48 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: finishing write tid 6 to smithi07685715-48 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: oid 48 updating version 0 to 678 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: oid 48 updating version 678 to 680 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: oid 48 version 680 is already newer than 677 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: oid 48 version 680 is already newer than 679 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: oid 48 updating version 680 to 681 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 681 (ObjNum 342 snap 105 seq_num 342) dirty exists 2024-09-21T09:41:18.605 INFO:tasks.rados.rados.0.smithi076.stdout:824: left oid 48 (ObjNum 342 snap 105 seq_num 342) 2024-09-21T09:41:18.606 INFO:tasks.rados.rados.0.smithi076.stdout:825: finishing write tid 1 to smithi07685715-49 2024-09-21T09:41:18.606 INFO:tasks.rados.rados.0.smithi076.stdout:825: finishing write tid 2 to smithi07685715-49 2024-09-21T09:41:18.606 INFO:tasks.rados.rados.0.smithi076.stdout:825: finishing write tid 3 to smithi07685715-49 2024-09-21T09:41:18.608 INFO:tasks.rados.rados.0.smithi076.stdout:825: finishing write tid 4 to smithi07685715-49 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:825: finishing write tid 5 to smithi07685715-49 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:825: oid 49 updating version 0 to 684 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:825: oid 49 version 684 is already newer than 682 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:825: oid 49 updating version 684 to 685 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:825: oid 49 version 685 is already newer than 683 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 685 (ObjNum 343 snap 105 seq_num 343) dirty exists 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:825: left oid 49 (ObjNum 343 snap 105 seq_num 343) 2024-09-21T09:41:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:823: done (4 left) 2024-09-21T09:41:18.610 INFO:tasks.rados.rados.0.smithi076.stdout:824: done (3 left) 2024-09-21T09:41:18.610 INFO:tasks.rados.rados.0.smithi076.stdout:825: done (2 left) 2024-09-21T09:41:18.610 INFO:tasks.rados.rados.0.smithi076.stdout:827: done (1 left) 2024-09-21T09:41:18.610 INFO:tasks.rados.rados.0.smithi076.stdout:828: write oid 5 current snap is 105 2024-09-21T09:41:18.610 INFO:tasks.rados.rados.0.smithi076.stdout:828: seq_num 345 ranges {598415=796790,1922170=745784,3141403=415315} 2024-09-21T09:41:18.617 INFO:tasks.rados.rados.0.smithi076.stdout:828: writing smithi07685715-5 from 598415 to 1395205 tid 1 2024-09-21T09:41:18.624 INFO:tasks.rados.rados.0.smithi076.stdout:828: writing smithi07685715-5 from 1922170 to 2667954 tid 2 2024-09-21T09:41:18.629 INFO:tasks.rados.rados.0.smithi076.stdout:828: writing smithi07685715-5 from 3141403 to 3556718 tid 3 2024-09-21T09:41:18.629 INFO:tasks.rados.rados.0.smithi076.stdout:829: write oid 42 current snap is 105 2024-09-21T09:41:18.629 INFO:tasks.rados.rados.0.smithi076.stdout:829: seq_num 346 ranges {615222=472039,1580652=489442,2768826=626587} 2024-09-21T09:41:18.635 INFO:tasks.rados.rados.0.smithi076.stdout:829: writing smithi07685715-42 from 615222 to 1087261 tid 1 2024-09-21T09:41:18.640 INFO:tasks.rados.rados.0.smithi076.stdout:829: writing smithi07685715-42 from 1580652 to 2070094 tid 2 2024-09-21T09:41:18.648 INFO:tasks.rados.rados.0.smithi076.stdout:829: writing smithi07685715-42 from 2768826 to 3395413 tid 3 2024-09-21T09:41:18.648 INFO:tasks.rados.rados.0.smithi076.stdout:830: read oid 46 snap 104 2024-09-21T09:41:18.648 INFO:tasks.rados.rados.0.smithi076.stdout:830: expect (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:41:18.648 INFO:tasks.rados.rados.0.smithi076.stdout:831: rollback oid 38 current snap is 105 2024-09-21T09:41:18.648 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 99 2024-09-21T09:41:18.648 INFO:tasks.rados.rados.0.smithi076.stdout:826: expect (ObjNum 304 snap 94 seq_num 304) 2024-09-21T09:41:18.845 INFO:tasks.rados.rados.0.smithi076.stdout:828: finishing write tid 1 to smithi07685715-5 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: finishing write tid 2 to smithi07685715-5 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:826: done (4 left) 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:832: delete oid 22 current snap is 105 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: finishing write tid 3 to smithi07685715-5 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: finishing write tid 4 to smithi07685715-5 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: finishing write tid 5 to smithi07685715-5 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: oid 5 updating version 0 to 623 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: oid 5 version 623 is already newer than 621 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: oid 5 version 623 is already newer than 620 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: oid 5 version 623 is already newer than 622 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 623 (ObjNum 345 snap 105 seq_num 345) dirty exists 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:828: left oid 5 (ObjNum 345 snap 105 seq_num 345) 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:829: finishing write tid 1 to smithi07685715-42 2024-09-21T09:41:18.846 INFO:tasks.rados.rados.0.smithi076.stdout:829: finishing write tid 2 to smithi07685715-42 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:831: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:831: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:831: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 399 (ObjNum 220 snap 63 seq_num 220) dirty exists 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: finishing write tid 3 to smithi07685715-42 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: finishing write tid 4 to smithi07685715-42 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: finishing write tid 5 to smithi07685715-42 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: oid 42 updating version 0 to 255 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: oid 42 updating version 255 to 257 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: oid 42 updating version 257 to 258 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: oid 42 version 258 is already newer than 256 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 258 (ObjNum 346 snap 105 seq_num 346) dirty exists 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:829: left oid 42 (ObjNum 346 snap 105 seq_num 346) 2024-09-21T09:41:18.847 INFO:tasks.rados.rados.0.smithi076.stdout:830: expect (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:41:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:41:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:41:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:18 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:18.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:41:18.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:41:18.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:18 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:19.060 INFO:tasks.rados.rados.0.smithi076.stdout:828: done (4 left) 2024-09-21T09:41:19.060 INFO:tasks.rados.rados.0.smithi076.stdout:829: done (3 left) 2024-09-21T09:41:19.060 INFO:tasks.rados.rados.0.smithi076.stdout:830: done (2 left) 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi076.stdout:831: done (1 left) 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi076.stdout:832: done (0 left) 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi076.stdout:833: read oid 43 snap -1 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi076.stdout:833: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:41:19.061 INFO:tasks.rados.rados.0.smithi076.stdout:834: snap_create 2024-09-21T09:41:19.063 INFO:tasks.rados.rados.0.smithi076.stdout:833: expect (ObjNum 199 snap 57 seq_num 199) 2024-09-21T09:41:19.591 INFO:tasks.rados.rados.0.smithi076.stdout:833: done (1 left) 2024-09-21T09:41:19.591 INFO:tasks.rados.rados.0.smithi076.stdout:834: done (0 left) 2024-09-21T09:41:19.591 INFO:tasks.rados.rados.0.smithi076.stdout:835: rollback oid 18 current snap is 106 2024-09-21T09:41:19.591 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 18 to 104 2024-09-21T09:41:19.591 INFO:tasks.rados.rados.0.smithi076.stdout:836: snap_create 2024-09-21T09:41:19.594 INFO:tasks.rados.rados.0.smithi076.stdout:835: finishing rollback tid 0 to smithi07685715-18 2024-09-21T09:41:19.594 INFO:tasks.rados.rados.0.smithi076.stdout:835: finishing rollback tid 1 to smithi07685715-18 2024-09-21T09:41:19.594 INFO:tasks.rados.rados.0.smithi076.stdout:835: finishing rollback tid 2 to smithi07685715-18 2024-09-21T09:41:19.594 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 262 (ObjNum 327 snap 101 seq_num 327) dirty exists 2024-09-21T09:41:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:19 smithi191 ceph-mon[33301]: pgmap v111: 89 pgs: 89 active+clean; 171 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-21T09:41:19.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[41325]: pgmap v111: 89 pgs: 89 active+clean; 171 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:19.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:19 smithi076 ceph-mon[29313]: pgmap v111: 89 pgs: 89 active+clean; 171 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:835: done (1 left) 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:836: done (0 left) 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:837: rollback oid 38 current snap is 107 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 103 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:838: read oid 40 snap 104 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:838: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:41:20.591 INFO:tasks.rados.rados.0.smithi076.stdout:839: write oid 39 current snap is 107 2024-09-21T09:41:20.592 INFO:tasks.rados.rados.0.smithi076.stdout:839: seq_num 348 ranges {648836=622537,1981263=493111,3107319=741517} 2024-09-21T09:41:20.598 INFO:tasks.rados.rados.0.smithi076.stdout:839: writing smithi07685715-39 from 648836 to 1271373 tid 1 2024-09-21T09:41:20.605 INFO:tasks.rados.rados.0.smithi076.stdout:839: writing smithi07685715-39 from 1981263 to 2474374 tid 2 2024-09-21T09:41:20.612 INFO:tasks.rados.rados.0.smithi076.stdout:839: writing smithi07685715-39 from 3107319 to 3848836 tid 3 2024-09-21T09:41:20.612 INFO:tasks.rados.rados.0.smithi076.stdout:840: write oid 36 current snap is 107 2024-09-21T09:41:20.613 INFO:tasks.rados.rados.0.smithi076.stdout:840: seq_num 349 ranges {665643=697786,2039745=753122,3534742=330901} 2024-09-21T09:41:20.620 INFO:tasks.rados.rados.0.smithi076.stdout:840: writing smithi07685715-36 from 665643 to 1363429 tid 1 2024-09-21T09:41:20.627 INFO:tasks.rados.rados.0.smithi076.stdout:840: writing smithi07685715-36 from 2039745 to 2792867 tid 2 2024-09-21T09:41:20.632 INFO:tasks.rados.rados.0.smithi076.stdout:840: writing smithi07685715-36 from 3534742 to 3865643 tid 3 2024-09-21T09:41:20.633 INFO:tasks.rados.rados.0.smithi076.stdout:841: snap_remove snap 95 2024-09-21T09:41:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:20 smithi191 ceph-mon[33301]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:41:20.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:20 smithi076 ceph-mon[29313]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:41:20.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:20 smithi076 ceph-mon[41325]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:41:21.607 INFO:tasks.rados.rados.0.smithi076.stdout:841: done (4 left) 2024-09-21T09:41:21.607 INFO:tasks.rados.rados.0.smithi076.stdout:842: write oid 47 current snap is 107 2024-09-21T09:41:21.607 INFO:tasks.rados.rados.0.smithi076.stdout:842: seq_num 350 ranges {682450=489388,1814580=496780,2878518=788028} 2024-09-21T09:41:21.614 INFO:tasks.rados.rados.0.smithi076.stdout:842: writing smithi07685715-47 from 682450 to 1171838 tid 1 2024-09-21T09:41:21.620 INFO:tasks.rados.rados.0.smithi076.stdout:842: writing smithi07685715-47 from 1814580 to 2311360 tid 2 2024-09-21T09:41:21.628 INFO:tasks.rados.rados.0.smithi076.stdout:842: writing smithi07685715-47 from 2878518 to 3666546 tid 3 2024-09-21T09:41:21.628 INFO:tasks.rados.rados.0.smithi076.stdout:843: write oid 16 current snap is 107 2024-09-21T09:41:21.628 INFO:tasks.rados.rados.0.smithi076.stdout:837: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:41:21.628 INFO:tasks.rados.rados.0.smithi076.stdout:837: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:41:21.628 INFO:tasks.rados.rados.0.smithi076.stdout:837: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:41:21.629 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 403 (ObjNum 220 snap 63 seq_num 220) dirty exists 2024-09-21T09:41:21.629 INFO:tasks.rados.rados.0.smithi076.stdout:838: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:41:21.761 INFO:tasks.rados.rados.0.smithi076.stdout:839: finishing write tid 1 to smithi07685715-39 2024-09-21T09:41:21.761 INFO:tasks.rados.rados.0.smithi076.stdout:839: finishing write tid 2 to smithi07685715-39 2024-09-21T09:41:21.761 INFO:tasks.rados.rados.0.smithi076.stdout:843: seq_num 351 ranges {699257=564637,1873062=640438,3305941=593316} 2024-09-21T09:41:21.768 INFO:tasks.rados.rados.0.smithi076.stdout:843: writing smithi07685715-16 from 699257 to 1263894 tid 1 2024-09-21T09:41:21.775 INFO:tasks.rados.rados.0.smithi076.stdout:843: writing smithi07685715-16 from 1873062 to 2513500 tid 2 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:843: writing smithi07685715-16 from 3305941 to 3899257 tid 3 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:837: done (5 left) 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:838: done (4 left) 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:844: write oid 19 current snap is 107 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:839: finishing write tid 3 to smithi07685715-39 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:839: finishing write tid 4 to smithi07685715-39 2024-09-21T09:41:21.784 INFO:tasks.rados.rados.0.smithi076.stdout:839: finishing write tid 5 to smithi07685715-39 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:839: oid 39 updating version 0 to 626 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:839: oid 39 updating version 626 to 627 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:839: oid 39 version 627 is already newer than 624 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:839: oid 39 version 627 is already newer than 625 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 627 (ObjNum 348 snap 107 seq_num 348) dirty exists 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:839: left oid 39 (ObjNum 348 snap 107 seq_num 348) 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:840: finishing write tid 1 to smithi07685715-36 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:840: finishing write tid 2 to smithi07685715-36 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:840: finishing write tid 3 to smithi07685715-36 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:840: finishing write tid 4 to smithi07685715-36 2024-09-21T09:41:21.785 INFO:tasks.rados.rados.0.smithi076.stdout:844: seq_num 352 ranges {716064=639886,1815191=500449,2817011=526925} 2024-09-21T09:41:21.791 INFO:tasks.rados.rados.0.smithi076.stdout:844: writing smithi07685715-19 from 716064 to 1355950 tid 1 2024-09-21T09:41:21.797 INFO:tasks.rados.rados.0.smithi076.stdout:844: writing smithi07685715-19 from 1815191 to 2315640 tid 2 2024-09-21T09:41:21.804 INFO:tasks.rados.rados.0.smithi076.stdout:844: writing smithi07685715-19 from 2817011 to 3343936 tid 3 2024-09-21T09:41:21.804 INFO:tasks.rados.rados.0.smithi076.stdout:839: done (4 left) 2024-09-21T09:41:21.804 INFO:tasks.rados.rados.0.smithi076.stdout:845: snap_create 2024-09-21T09:41:21.804 INFO:tasks.rados.rados.0.smithi076.stdout:840: finishing write tid 5 to smithi07685715-36 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:840: oid 36 updating version 0 to 336 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:840: oid 36 updating version 336 to 337 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:840: oid 36 updating version 337 to 339 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:840: oid 36 version 339 is already newer than 338 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 339 (ObjNum 349 snap 107 seq_num 349) dirty exists 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:840: left oid 36 (ObjNum 349 snap 107 seq_num 349) 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: finishing write tid 1 to smithi07685715-47 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: finishing write tid 2 to smithi07685715-47 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: finishing write tid 3 to smithi07685715-47 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: finishing write tid 4 to smithi07685715-47 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: finishing write tid 5 to smithi07685715-47 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: oid 47 updating version 0 to 695 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: oid 47 version 695 is already newer than 694 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: oid 47 version 695 is already newer than 693 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:842: oid 47 updating version 695 to 696 2024-09-21T09:41:21.805 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 696 (ObjNum 350 snap 107 seq_num 350) dirty exists 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:842: left oid 47 (ObjNum 350 snap 107 seq_num 350) 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: finishing write tid 1 to smithi07685715-16 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: finishing write tid 2 to smithi07685715-16 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: finishing write tid 3 to smithi07685715-16 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: finishing write tid 4 to smithi07685715-16 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: finishing write tid 5 to smithi07685715-16 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: oid 16 updating version 0 to 501 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: oid 16 updating version 501 to 503 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: oid 16 version 503 is already newer than 502 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: oid 16 updating version 503 to 504 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 504 (ObjNum 351 snap 107 seq_num 351) dirty exists 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:843: left oid 16 (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:41:21.806 INFO:tasks.rados.rados.0.smithi076.stdout:844: finishing write tid 1 to smithi07685715-19 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: finishing write tid 2 to smithi07685715-19 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: finishing write tid 3 to smithi07685715-19 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: finishing write tid 4 to smithi07685715-19 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: finishing write tid 5 to smithi07685715-19 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: oid 19 updating version 0 to 325 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: oid 19 updating version 325 to 327 2024-09-21T09:41:21.807 INFO:tasks.rados.rados.0.smithi076.stdout:844: oid 19 updating version 327 to 328 2024-09-21T09:41:21.808 INFO:tasks.rados.rados.0.smithi076.stdout:844: oid 19 version 328 is already newer than 326 2024-09-21T09:41:21.808 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 328 (ObjNum 352 snap 107 seq_num 352) dirty exists 2024-09-21T09:41:21.808 INFO:tasks.rados.rados.0.smithi076.stdout:844: left oid 19 (ObjNum 352 snap 107 seq_num 352) 2024-09-21T09:41:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:21 smithi191 ceph-mon[33301]: Upgrade: Updating alertmanager.a 2024-09-21T09:41:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:21 smithi191 ceph-mon[33301]: Deploying daemon alertmanager.a on smithi076 2024-09-21T09:41:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:21 smithi191 ceph-mon[33301]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:41:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:21 smithi191 ceph-mon[33301]: pgmap v114: 89 pgs: 89 active+clean; 171 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-21T09:41:21.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[41325]: Upgrade: Updating alertmanager.a 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[41325]: Deploying daemon alertmanager.a on smithi076 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[41325]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[41325]: pgmap v114: 89 pgs: 89 active+clean; 171 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[29313]: Upgrade: Updating alertmanager.a 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[29313]: Deploying daemon alertmanager.a on smithi076 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[29313]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:41:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:21 smithi076 ceph-mon[29313]: pgmap v114: 89 pgs: 89 active+clean; 171 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 14 op/s 2024-09-21T09:41:22.611 INFO:tasks.rados.rados.0.smithi076.stdout:840: done (4 left) 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:842: done (3 left) 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:843: done (2 left) 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:844: done (1 left) 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:845: done (0 left) 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:846: rollback oid 4 current snap is 108 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 100 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:847: write oid 5 current snap is 108 2024-09-21T09:41:22.612 INFO:tasks.rados.rados.0.smithi076.stdout:847: seq_num 353 ranges {732871=715135,1873673=644107,3244434=688437} 2024-09-21T09:41:22.619 INFO:tasks.rados.rados.0.smithi076.stdout:847: writing smithi07685715-5 from 732871 to 1448006 tid 1 2024-09-21T09:41:22.625 INFO:tasks.rados.rados.0.smithi076.stdout:847: writing smithi07685715-5 from 1873673 to 2517780 tid 2 2024-09-21T09:41:22.633 INFO:tasks.rados.rados.0.smithi076.stdout:847: writing smithi07685715-5 from 3244434 to 3932871 tid 3 2024-09-21T09:41:22.633 INFO:tasks.rados.rados.0.smithi076.stdout:848: snap_create 2024-09-21T09:41:22.633 INFO:tasks.rados.rados.0.smithi076.stdout:846: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:41:22.633 INFO:tasks.rados.rados.0.smithi076.stdout:846: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:41:22.633 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 343 (ObjNum 313 snap 96 seq_num 313) dirty exists 2024-09-21T09:41:22.634 INFO:tasks.rados.rados.0.smithi076.stdout:847: finishing write tid 1 to smithi07685715-5 2024-09-21T09:41:22.634 INFO:tasks.rados.rados.0.smithi076.stdout:847: finishing write tid 2 to smithi07685715-5 2024-09-21T09:41:22.636 INFO:tasks.rados.rados.0.smithi076.stdout:847: finishing write tid 3 to smithi07685715-5 2024-09-21T09:41:22.636 INFO:tasks.rados.rados.0.smithi076.stdout:847: finishing write tid 4 to smithi07685715-5 2024-09-21T09:41:22.636 INFO:tasks.rados.rados.0.smithi076.stdout:847: finishing write tid 5 to smithi07685715-5 2024-09-21T09:41:22.636 INFO:tasks.rados.rados.0.smithi076.stdout:847: oid 5 updating version 0 to 636 2024-09-21T09:41:22.637 INFO:tasks.rados.rados.0.smithi076.stdout:847: oid 5 updating version 636 to 638 2024-09-21T09:41:22.637 INFO:tasks.rados.rados.0.smithi076.stdout:847: oid 5 version 638 is already newer than 635 2024-09-21T09:41:22.637 INFO:tasks.rados.rados.0.smithi076.stdout:847: oid 5 version 638 is already newer than 637 2024-09-21T09:41:22.637 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 638 (ObjNum 353 snap 108 seq_num 353) dirty exists 2024-09-21T09:41:22.637 INFO:tasks.rados.rados.0.smithi076.stdout:847: left oid 5 (ObjNum 353 snap 108 seq_num 353) 2024-09-21T09:41:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:22 smithi191 ceph-mon[33301]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:41:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:22 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:22 smithi076 ceph-mon[29313]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:41:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:22 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:22.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:22 smithi076 ceph-mon[41325]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:41:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:22 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:23.612 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:41:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:41:23] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:846: done (2 left) 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:847: done (1 left) 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:848: done (0 left) 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:849: read oid 12 snap -1 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:849: expect (ObjNum 167 snap 43 seq_num 167) 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:850: rollback oid 9 current snap is 109 2024-09-21T09:41:23.616 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 104 2024-09-21T09:41:23.617 INFO:tasks.rados.rados.0.smithi076.stdout:851: snap_remove snap 108 2024-09-21T09:41:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:23 smithi076 ceph-mon[29313]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:41:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:23 smithi076 ceph-mon[29313]: pgmap v117: 89 pgs: 89 active+clean; 183 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2024-09-21T09:41:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:23 smithi076 ceph-mon[41325]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:41:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:23 smithi076 ceph-mon[41325]: pgmap v117: 89 pgs: 89 active+clean; 183 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2024-09-21T09:41:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:23 smithi191 ceph-mon[33301]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:41:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:23 smithi191 ceph-mon[33301]: pgmap v117: 89 pgs: 89 active+clean; 183 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.5 MiB/s wr, 12 op/s 2024-09-21T09:41:24.622 INFO:tasks.rados.rados.0.smithi076.stdout:851: done (2 left) 2024-09-21T09:41:24.623 INFO:tasks.rados.rados.0.smithi076.stdout:852: write oid 25 current snap is 109 2024-09-21T09:41:24.623 INFO:tasks.rados.rados.0.smithi076.stdout:849: expect (ObjNum 167 snap 43 seq_num 167) 2024-09-21T09:41:24.711 INFO:tasks.rados.rados.0.smithi076.stdout:850: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:41:24.711 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 342 (ObjNum 129 snap 0 seq_num 864359088) dirty dne 2024-09-21T09:41:24.712 INFO:tasks.rados.rados.0.smithi076.stdout:852: seq_num 354 ranges {749678=790384,2332155=504118,3271857=549469} 2024-09-21T09:41:24.719 INFO:tasks.rados.rados.0.smithi076.stdout:852: writing smithi07685715-25 from 749678 to 1540062 tid 1 2024-09-21T09:41:24.726 INFO:tasks.rados.rados.0.smithi076.stdout:852: writing smithi07685715-25 from 2332155 to 2836273 tid 2 2024-09-21T09:41:24.730 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:24 smithi076 ceph-mon[29313]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:41:24.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:24 smithi076 ceph-mon[41325]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:41:24.732 INFO:tasks.rados.rados.0.smithi076.stdout:852: writing smithi07685715-25 from 3271857 to 3821326 tid 3 2024-09-21T09:41:24.732 INFO:tasks.rados.rados.0.smithi076.stdout:849: done (2 left) 2024-09-21T09:41:24.733 INFO:tasks.rados.rados.0.smithi076.stdout:850: done (1 left) 2024-09-21T09:41:24.733 INFO:tasks.rados.rados.0.smithi076.stdout:853: delete oid 23 current snap is 109 2024-09-21T09:41:24.733 INFO:tasks.rados.rados.0.smithi076.stdout:852: finishing write tid 1 to smithi07685715-25 2024-09-21T09:41:24.733 INFO:tasks.rados.rados.0.smithi076.stdout:852: finishing write tid 2 to smithi07685715-25 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:853: done (1 left) 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:854: read oid 5 snap -1 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:854: expect (ObjNum 353 snap 108 seq_num 353) 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:855: rollback oid 27 current snap is 109 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 107 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:856: snap_create 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:852: finishing write tid 3 to smithi07685715-25 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:852: finishing write tid 4 to smithi07685715-25 2024-09-21T09:41:24.735 INFO:tasks.rados.rados.0.smithi076.stdout:852: finishing write tid 5 to smithi07685715-25 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:852: oid 25 updating version 0 to 221 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:852: oid 25 version 221 is already newer than 219 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:852: oid 25 updating version 221 to 222 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:852: oid 25 version 222 is already newer than 220 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 222 (ObjNum 354 snap 109 seq_num 354) dirty exists 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:852: left oid 25 (ObjNum 354 snap 109 seq_num 354) 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:855: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:41:24.736 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 494 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:41:24.752 INFO:tasks.rados.rados.0.smithi076.stdout:854: expect (ObjNum 353 snap 108 seq_num 353) 2024-09-21T09:41:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:24 smithi191 ceph-mon[33301]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:852: done (3 left) 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:854: done (2 left) 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:855: done (1 left) 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:856: done (0 left) 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:857: read oid 25 snap -1 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:857: expect (ObjNum 354 snap 109 seq_num 354) 2024-09-21T09:41:25.631 INFO:tasks.rados.rados.0.smithi076.stdout:858: snap_remove snap 98 2024-09-21T09:41:25.927 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:25 smithi076 ceph-mon[29313]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:41:25.927 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:25 smithi076 ceph-mon[29313]: pgmap v120: 89 pgs: 89 active+clean; 195 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 4.7 MiB/s wr, 27 op/s 2024-09-21T09:41:25.927 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:25 smithi076 ceph-mon[41325]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:41:25.927 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:25 smithi076 ceph-mon[41325]: pgmap v120: 89 pgs: 89 active+clean; 195 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 4.7 MiB/s wr, 27 op/s 2024-09-21T09:41:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:25 smithi191 ceph-mon[33301]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:41:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:25 smithi191 ceph-mon[33301]: pgmap v120: 89 pgs: 89 active+clean; 195 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 4.7 MiB/s wr, 27 op/s 2024-09-21T09:41:26.640 INFO:tasks.rados.rados.0.smithi076.stdout:858: done (1 left) 2024-09-21T09:41:26.641 INFO:tasks.rados.rados.0.smithi076.stdout:859: write oid 12 current snap is 110 2024-09-21T09:41:26.641 INFO:tasks.rados.rados.0.smithi076.stdout:859: seq_num 356 ranges {783292=540882,1932766=507787,3210350=572013} 2024-09-21T09:41:26.646 INFO:tasks.rados.rados.0.smithi076.stdout:859: writing smithi07685715-12 from 783292 to 1324174 tid 1 2024-09-21T09:41:26.652 INFO:tasks.rados.rados.0.smithi076.stdout:859: writing smithi07685715-12 from 1932766 to 2440553 tid 2 2024-09-21T09:41:26.660 INFO:tasks.rados.rados.0.smithi076.stdout:859: writing smithi07685715-12 from 3210350 to 3782363 tid 3 2024-09-21T09:41:26.660 INFO:tasks.rados.rados.0.smithi076.stdout:860: delete oid 11 current snap is 110 2024-09-21T09:41:26.660 INFO:tasks.rados.rados.0.smithi076.stdout:857: expect (ObjNum 354 snap 109 seq_num 354) 2024-09-21T09:41:26.788 INFO:tasks.rados.rados.0.smithi076.stdout:859: finishing write tid 1 to smithi07685715-12 2024-09-21T09:41:26.788 INFO:tasks.rados.rados.0.smithi076.stdout:859: finishing write tid 2 to smithi07685715-12 2024-09-21T09:41:26.788 INFO:tasks.rados.rados.0.smithi076.stdout:859: finishing write tid 3 to smithi07685715-12 2024-09-21T09:41:26.788 INFO:tasks.rados.rados.0.smithi076.stdout:859: finishing write tid 4 to smithi07685715-12 2024-09-21T09:41:26.788 INFO:tasks.rados.rados.0.smithi076.stdout:859: finishing write tid 5 to smithi07685715-12 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:859: oid 12 updating version 0 to 365 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:859: oid 12 version 365 is already newer than 364 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:859: oid 12 version 365 is already newer than 363 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:859: oid 12 updating version 365 to 366 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 366 (ObjNum 356 snap 110 seq_num 356) dirty exists 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:859: left oid 12 (ObjNum 356 snap 110 seq_num 356) 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:857: done (2 left) 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:859: done (1 left) 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:860: done (0 left) 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:861: read oid 37 snap -1 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:861: expect (ObjNum 290 snap 92 seq_num 290) 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:862: read oid 31 snap -1 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:862: expect deleted 2024-09-21T09:41:26.789 INFO:tasks.rados.rados.0.smithi076.stdout:863: snap_remove snap 103 2024-09-21T09:41:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:26 smithi191 ceph-mon[33301]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:41:27.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:26 smithi076 ceph-mon[29313]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:41:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:26 smithi076 ceph-mon[41325]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:41:27.650 INFO:tasks.rados.rados.0.smithi076.stdout:863: done (2 left) 2024-09-21T09:41:27.650 INFO:tasks.rados.rados.0.smithi076.stdout:864: delete oid 36 current snap is 110 2024-09-21T09:41:27.650 INFO:tasks.rados.rados.0.smithi076.stdout:861: expect (ObjNum 290 snap 92 seq_num 290) 2024-09-21T09:41:27.783 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:27 smithi076 systemd[1]: Stopping Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:27.784 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:27 smithi076 ceph-mon[29313]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:41:27.784 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:27 smithi076 ceph-mon[29313]: pgmap v123: 89 pgs: 89 active+clean; 195 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-09-21T09:41:27.784 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:27 smithi076 ceph-mon[41325]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:41:27.784 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:27 smithi076 ceph-mon[41325]: pgmap v123: 89 pgs: 89 active+clean; 195 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:861: done (2 left) 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:862: done (1 left) 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:864: done (0 left) 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:865: read oid 11 snap 105 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:865: expect deleted 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:866: rollback oid 3 current snap is 110 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 105 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:867: rollback oid 41 current snap is 110 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 86 2024-09-21T09:41:27.787 INFO:tasks.rados.rados.0.smithi076.stdout:868: snap_remove snap 99 2024-09-21T09:41:28.099 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:27 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[77314]: ts=2024-09-21T09:41:27.782Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:41:28.099 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:27 smithi076 podman[101352]: 2024-09-21 09:41:27.833448653 +0000 UTC m=+0.177121921 container died d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:27 smithi191 ceph-mon[33301]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:41:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:27 smithi191 ceph-mon[33301]: pgmap v123: 89 pgs: 89 active+clean; 195 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.3 MiB/s wr, 14 op/s 2024-09-21T09:41:28.372 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 podman[101352]: 2024-09-21 09:41:28.306169701 +0000 UTC m=+0.649842969 container cleanup d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:28.372 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 bash[101352]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:867: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 0 (ObjNum 81 snap 0 seq_num 1682071259) dirty dne 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:866: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:865: done (3 left) 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:866: done (2 left) 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:867: done (1 left) 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:868: done (0 left) 2024-09-21T09:41:28.541 INFO:tasks.rados.rados.0.smithi076.stdout:869: snap_remove snap 86 2024-09-21T09:41:28.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:28 smithi076 ceph-mon[41325]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:41:28.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:28 smithi076 ceph-mon[41325]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:41:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:28 smithi076 ceph-mon[29313]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:41:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:28 smithi076 ceph-mon[29313]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:41:28.873 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 podman[101369]: 2024-09-21 09:41:28.557787235 +0000 UTC m=+0.723125930 container remove d930ef8d31ba929be6770d6c29a36d5e4e6d782812d6a6028dcfea570b4eb002 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:28.873 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 podman[101369]: 2024-09-21 09:41:28.758834204 +0000 UTC m=+0.924172863 volume remove d62ce2f7b62824ce0414f2030995d3cdd679c124a116ffa1633a4c8097377420 2024-09-21T09:41:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:28 smithi191 ceph-mon[33301]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:41:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:28 smithi191 ceph-mon[33301]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:41:29.372 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:41:29.372 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 systemd[1]: Stopped Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:29.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:28 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.317s CPU time. 2024-09-21T09:41:29.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 systemd[1]: Starting Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:29.545 INFO:tasks.rados.rados.0.smithi076.stdout:869: done (0 left) 2024-09-21T09:41:29.545 INFO:tasks.rados.rados.0.smithi076.stdout:870: read oid 26 snap -1 2024-09-21T09:41:29.546 INFO:tasks.rados.rados.0.smithi076.stdout:870: expect deleted 2024-09-21T09:41:29.546 INFO:tasks.rados.rados.0.smithi076.stdout:871: write oid 47 current snap is 110 2024-09-21T09:41:29.546 INFO:tasks.rados.rados.0.smithi076.stdout:871: seq_num 359 ranges {433713=482982,1424565=609148} 2024-09-21T09:41:29.551 INFO:tasks.rados.rados.0.smithi076.stdout:871: writing smithi07685715-47 from 433713 to 916695 tid 1 2024-09-21T09:41:29.558 INFO:tasks.rados.rados.0.smithi076.stdout:871: writing smithi07685715-47 from 1424565 to 2033713 tid 2 2024-09-21T09:41:29.558 INFO:tasks.rados.rados.0.smithi076.stdout:872: write oid 32 current snap is 110 2024-09-21T09:41:29.558 INFO:tasks.rados.rados.0.smithi076.stdout:872: seq_num 360 ranges {450520=558231,1483047=515125} 2024-09-21T09:41:29.563 INFO:tasks.rados.rados.0.smithi076.stdout:872: writing smithi07685715-32 from 450520 to 1008751 tid 1 2024-09-21T09:41:29.569 INFO:tasks.rados.rados.0.smithi076.stdout:872: writing smithi07685715-32 from 1483047 to 1998172 tid 2 2024-09-21T09:41:29.569 INFO:tasks.rados.rados.0.smithi076.stdout:873: read oid 31 snap 105 2024-09-21T09:41:29.569 INFO:tasks.rados.rados.0.smithi076.stdout:873: expect deleted 2024-09-21T09:41:29.569 INFO:tasks.rados.rados.0.smithi076.stdout:874: write oid 25 current snap is 110 2024-09-21T09:41:29.569 INFO:tasks.rados.rados.0.smithi076.stdout:871: finishing write tid 1 to smithi07685715-47 2024-09-21T09:41:29.569 INFO:tasks.rados.rados.0.smithi076.stdout:871: finishing write tid 2 to smithi07685715-47 2024-09-21T09:41:29.570 INFO:tasks.rados.rados.0.smithi076.stdout:874: seq_num 361 ranges {467327=633480,1825176=242151} 2024-09-21T09:41:29.575 INFO:tasks.rados.rados.0.smithi076.stdout:874: writing smithi07685715-25 from 467327 to 1100807 tid 1 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:874: writing smithi07685715-25 from 1825176 to 2067327 tid 2 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:870: done (4 left) 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:875: read oid 34 snap -1 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:875: expect (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:876: read oid 11 snap 104 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:876: expect deleted 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:877: write oid 44 current snap is 110 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:871: finishing write tid 3 to smithi07685715-47 2024-09-21T09:41:29.581 INFO:tasks.rados.rados.0.smithi076.stdout:871: finishing write tid 4 to smithi07685715-47 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:871: oid 47 updating version 0 to 725 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:871: oid 47 version 725 is already newer than 723 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:871: oid 47 version 725 is already newer than 724 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 725 (ObjNum 359 snap 110 seq_num 359) dirty exists 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:871: left oid 47 (ObjNum 359 snap 110 seq_num 359) 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:872: finishing write tid 1 to smithi07685715-32 2024-09-21T09:41:29.582 INFO:tasks.rados.rados.0.smithi076.stdout:877: seq_num 362 ranges {484134=708729,1883658=200476} 2024-09-21T09:41:29.587 INFO:tasks.rados.rados.0.smithi076.stdout:877: writing smithi07685715-44 from 484134 to 1192863 tid 1 2024-09-21T09:41:29.592 INFO:tasks.rados.rados.0.smithi076.stdout:877: writing smithi07685715-44 from 1883658 to 2084134 tid 2 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:871: done (6 left) 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:878: snap_create 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: finishing write tid 2 to smithi07685715-32 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: finishing write tid 3 to smithi07685715-32 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: finishing write tid 4 to smithi07685715-32 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: oid 32 updating version 0 to 337 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: oid 32 version 337 is already newer than 336 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: oid 32 updating version 337 to 338 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 338 (ObjNum 360 snap 110 seq_num 360) dirty exists 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:872: left oid 32 (ObjNum 360 snap 110 seq_num 360) 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:874: finishing write tid 1 to smithi07685715-25 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:874: finishing write tid 2 to smithi07685715-25 2024-09-21T09:41:29.593 INFO:tasks.rados.rados.0.smithi076.stdout:875: expect (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:41:29.648 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 podman[101501]: 2024-09-21 09:41:29.291820098 +0000 UTC m=+0.016047070 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-21T09:41:29.649 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 podman[101501]: 2024-09-21 09:41:29.420654799 +0000 UTC m=+0.144881753 volume create 87951f3be4ce3b79cb42b938ca3b56cec9f91e0d7d33deceb0f2f4b5211a4038 2024-09-21T09:41:29.649 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 podman[101501]: 2024-09-21 09:41:29.529474578 +0000 UTC m=+0.253701532 container create 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:874: finishing write tid 3 to smithi07685715-25 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:874: finishing write tid 4 to smithi07685715-25 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:874: oid 25 updating version 0 to 226 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:874: oid 25 updating version 226 to 227 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:874: oid 25 updating version 227 to 228 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 228 (ObjNum 361 snap 110 seq_num 361) dirty exists 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:874: left oid 25 (ObjNum 361 snap 110 seq_num 361) 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:877: finishing write tid 1 to smithi07685715-44 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:877: finishing write tid 2 to smithi07685715-44 2024-09-21T09:41:29.767 INFO:tasks.rados.rados.0.smithi076.stdout:877: finishing write tid 3 to smithi07685715-44 2024-09-21T09:41:29.768 INFO:tasks.rados.rados.0.smithi076.stdout:877: finishing write tid 4 to smithi07685715-44 2024-09-21T09:41:29.768 INFO:tasks.rados.rados.0.smithi076.stdout:877: oid 44 updating version 0 to 170 2024-09-21T09:41:29.768 INFO:tasks.rados.rados.0.smithi076.stdout:877: oid 44 updating version 170 to 171 2024-09-21T09:41:29.768 INFO:tasks.rados.rados.0.smithi076.stdout:877: oid 44 updating version 171 to 172 2024-09-21T09:41:29.768 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 172 (ObjNum 362 snap 110 seq_num 362) dirty exists 2024-09-21T09:41:29.768 INFO:tasks.rados.rados.0.smithi076.stdout:877: left oid 44 (ObjNum 362 snap 110 seq_num 362) 2024-09-21T09:41:29.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-mon[29313]: pgmap v126: 89 pgs: 89 active+clean; 201 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 846 KiB/s wr, 11 op/s 2024-09-21T09:41:29.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-mon[29313]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:41:29.901 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-mon[41325]: pgmap v126: 89 pgs: 89 active+clean; 201 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 846 KiB/s wr, 11 op/s 2024-09-21T09:41:29.901 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-mon[41325]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:41:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:29 smithi191 ceph-mon[33301]: pgmap v126: 89 pgs: 89 active+clean; 201 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 846 KiB/s wr, 11 op/s 2024-09-21T09:41:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:29 smithi191 ceph-mon[33301]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:41:30.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 podman[101501]: 2024-09-21 09:41:29.899618798 +0000 UTC m=+0.623845752 container init 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:30.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 podman[101501]: 2024-09-21 09:41:29.904268059 +0000 UTC m=+0.628495013 container start 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:41:30.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.914Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-21T09:41:30.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.914Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-21T09:41:30.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.915Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.76 port=9094 2024-09-21T09:41:30.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.916Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:41:30.374 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.938Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:41:30.374 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.938Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:41:30.374 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.940Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:41:30.374 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:29.940Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:41:30.374 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:29 smithi076 bash[101501]: 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe 2024-09-21T09:41:30.374 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:30 smithi076 systemd[1]: Started Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:872: done (6 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:873: done (5 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:874: done (4 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:875: done (3 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:876: done (2 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:877: done (1 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:878: done (0 left) 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:879: write oid 45 current snap is 111 2024-09-21T09:41:30.544 INFO:tasks.rados.rados.0.smithi076.stdout:879: seq_num 363 ranges {500941=783978,1942140=158801} 2024-09-21T09:41:30.550 INFO:tasks.rados.rados.0.smithi076.stdout:879: writing smithi07685715-45 from 500941 to 1284919 tid 1 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:879: writing smithi07685715-45 from 1942140 to 2100941 tid 2 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:880: rollback oid 30 current snap is 111 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 106 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:879: finishing write tid 1 to smithi07685715-45 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:881: read oid 50 snap -1 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:881: expect (ObjNum 299 snap 93 seq_num 299) 2024-09-21T09:41:30.556 INFO:tasks.rados.rados.0.smithi076.stdout:882: delete oid 22 current snap is 111 2024-09-21T09:41:30.557 INFO:tasks.rados.rados.0.smithi076.stdout:879: finishing write tid 2 to smithi07685715-45 2024-09-21T09:41:30.557 INFO:tasks.rados.rados.0.smithi076.stdout:879: finishing write tid 3 to smithi07685715-45 2024-09-21T09:41:30.557 INFO:tasks.rados.rados.0.smithi076.stdout:879: finishing write tid 4 to smithi07685715-45 2024-09-21T09:41:30.558 INFO:tasks.rados.rados.0.smithi076.stdout:879: oid 45 updating version 0 to 206 2024-09-21T09:41:30.558 INFO:tasks.rados.rados.0.smithi076.stdout:879: oid 45 version 206 is already newer than 205 2024-09-21T09:41:30.558 INFO:tasks.rados.rados.0.smithi076.stdout:879: oid 45 version 206 is already newer than 204 2024-09-21T09:41:30.558 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 206 (ObjNum 363 snap 111 seq_num 363) dirty exists 2024-09-21T09:41:30.558 INFO:tasks.rados.rados.0.smithi076.stdout:879: left oid 45 (ObjNum 363 snap 111 seq_num 363) 2024-09-21T09:41:30.559 INFO:tasks.rados.rados.0.smithi076.stdout:879: done (3 left) 2024-09-21T09:41:30.559 INFO:tasks.rados.rados.0.smithi076.stdout:882: done (2 left) 2024-09-21T09:41:30.559 INFO:tasks.rados.rados.0.smithi076.stdout:883: snap_create 2024-09-21T09:41:30.565 INFO:tasks.rados.rados.0.smithi076.stdout:880: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:41:30.567 INFO:tasks.rados.rados.0.smithi076.stdout:881: expect (ObjNum 299 snap 93 seq_num 299) 2024-09-21T09:41:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:880: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:41:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:880: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:41:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 729 (ObjNum 334 snap 103 seq_num 334) dirty exists 2024-09-21T09:41:31.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:31 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:31 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:31 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:31 smithi191 ceph-mon[33301]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:41:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:31 smithi191 ceph-mon[33301]: pgmap v129: 89 pgs: 89 active+clean; 201 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 846 KiB/s wr, 11 op/s 2024-09-21T09:41:31.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:31.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:31.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:31.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[41325]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:41:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[41325]: pgmap v129: 89 pgs: 89 active+clean; 201 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 846 KiB/s wr, 11 op/s 2024-09-21T09:41:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[29313]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:41:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-mon[29313]: pgmap v129: 89 pgs: 89 active+clean; 201 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 846 KiB/s wr, 11 op/s 2024-09-21T09:41:31.554 INFO:tasks.rados.rados.0.smithi076.stdout:880: done (2 left) 2024-09-21T09:41:31.554 INFO:tasks.rados.rados.0.smithi076.stdout:881: done (1 left) 2024-09-21T09:41:31.554 INFO:tasks.rados.rados.0.smithi076.stdout:883: done (0 left) 2024-09-21T09:41:31.554 INFO:tasks.rados.rados.0.smithi076.stdout:884: snap_create 2024-09-21T09:41:32.370 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:31 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:31.916Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00016774s 2024-09-21T09:41:32.553 INFO:tasks.rados.rados.0.smithi076.stdout:884: done (0 left) 2024-09-21T09:41:32.553 INFO:tasks.rados.rados.0.smithi076.stdout:885: snap_create 2024-09-21T09:41:32.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:32 smithi191 ceph-mon[33301]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:41:32.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:32 smithi076 ceph-mon[29313]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:41:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:32 smithi076 ceph-mon[41325]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:41:33.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:41:33] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:41:33.554 INFO:tasks.rados.rados.0.smithi076.stdout:885: done (0 left) 2024-09-21T09:41:33.554 INFO:tasks.rados.rados.0.smithi076.stdout:886: delete oid 27 current snap is 114 2024-09-21T09:41:33.557 INFO:tasks.rados.rados.0.smithi076.stdout:886: done (0 left) 2024-09-21T09:41:33.557 INFO:tasks.rados.rados.0.smithi076.stdout:887: read oid 34 snap -1 2024-09-21T09:41:33.557 INFO:tasks.rados.rados.0.smithi076.stdout:887: expect (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:41:33.557 INFO:tasks.rados.rados.0.smithi076.stdout:888: delete oid 17 current snap is 114 2024-09-21T09:41:33.562 INFO:tasks.rados.rados.0.smithi076.stdout:888: done (1 left) 2024-09-21T09:41:33.562 INFO:tasks.rados.rados.0.smithi076.stdout:889: snap_remove snap 110 2024-09-21T09:41:33.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-mon[29313]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:41:33.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-mon[29313]: pgmap v132: 89 pgs: 89 active+clean; 181 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 760 KiB/s wr, 6 op/s 2024-09-21T09:41:33.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-mon[29313]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:41:33.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-mon[41325]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:41:33.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-mon[41325]: pgmap v132: 89 pgs: 89 active+clean; 181 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 760 KiB/s wr, 6 op/s 2024-09-21T09:41:33.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:33 smithi076 ceph-mon[41325]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:41:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:33 smithi191 ceph-mon[33301]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:41:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:33 smithi191 ceph-mon[33301]: pgmap v132: 89 pgs: 89 active+clean; 181 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 2.5 MiB/s rd, 760 KiB/s wr, 6 op/s 2024-09-21T09:41:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:33 smithi191 ceph-mon[33301]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:41:34.575 INFO:tasks.rados.rados.0.smithi076.stdout:889: done (1 left) 2024-09-21T09:41:34.575 INFO:tasks.rados.rados.0.smithi076.stdout:890: read oid 47 snap 107 2024-09-21T09:41:34.575 INFO:tasks.rados.rados.0.smithi076.stdout:890: expect (ObjNum 350 snap 107 seq_num 350) 2024-09-21T09:41:34.575 INFO:tasks.rados.rados.0.smithi076.stdout:891: snap_create 2024-09-21T09:41:34.575 INFO:tasks.rados.rados.0.smithi076.stdout:887: expect (ObjNum 230 snap 69 seq_num 230) 2024-09-21T09:41:34.756 INFO:tasks.rados.rados.0.smithi076.stdout:890: expect (ObjNum 350 snap 107 seq_num 350) 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[29313]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:34 smithi076 ceph-mon[41325]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:41:34.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:34 smithi191 ceph-mon[33301]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:887: done (2 left) 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:890: done (1 left) 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:891: done (0 left) 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:892: read oid 5 snap -1 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:892: expect (ObjNum 353 snap 108 seq_num 353) 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:893: read oid 36 snap -1 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:893: expect deleted 2024-09-21T09:41:35.573 INFO:tasks.rados.rados.0.smithi076.stdout:894: snap_remove snap 111 2024-09-21T09:41:35.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:35 smithi076 ceph-mon[41325]: pgmap v135: 89 pgs: 89 active+clean; 168 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-09-21T09:41:35.852 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:35 smithi076 ceph-mon[41325]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:41:35.852 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:35 smithi076 ceph-mon[29313]: pgmap v135: 89 pgs: 89 active+clean; 168 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-09-21T09:41:35.852 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:35 smithi076 ceph-mon[29313]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:41:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:35 smithi191 ceph-mon[33301]: pgmap v135: 89 pgs: 89 active+clean; 168 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-09-21T09:41:35.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:35 smithi191 ceph-mon[33301]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:41:36.575 INFO:tasks.rados.rados.0.smithi076.stdout:894: done (2 left) 2024-09-21T09:41:36.575 INFO:tasks.rados.rados.0.smithi076.stdout:895: snap_create 2024-09-21T09:41:36.575 INFO:tasks.rados.rados.0.smithi076.stdout:892: expect (ObjNum 353 snap 108 seq_num 353) 2024-09-21T09:41:37.582 INFO:tasks.rados.rados.0.smithi076.stdout:892: done (2 left) 2024-09-21T09:41:37.582 INFO:tasks.rados.rados.0.smithi076.stdout:893: done (1 left) 2024-09-21T09:41:37.582 INFO:tasks.rados.rados.0.smithi076.stdout:895: done (0 left) 2024-09-21T09:41:37.582 INFO:tasks.rados.rados.0.smithi076.stdout:896: write oid 35 current snap is 116 2024-09-21T09:41:37.582 INFO:tasks.rados.rados.0.smithi076.stdout:896: seq_num 367 ranges {568169=401327,1376068=669790} 2024-09-21T09:41:37.587 INFO:tasks.rados.rados.0.smithi076.stdout:896: writing smithi07685715-35 from 568169 to 969496 tid 1 2024-09-21T09:41:37.593 INFO:tasks.rados.rados.0.smithi076.stdout:896: writing smithi07685715-35 from 1376068 to 2045858 tid 2 2024-09-21T09:41:37.593 INFO:tasks.rados.rados.0.smithi076.stdout:897: read oid 45 snap -1 2024-09-21T09:41:37.593 INFO:tasks.rados.rados.0.smithi076.stdout:897: expect (ObjNum 363 snap 111 seq_num 363) 2024-09-21T09:41:37.593 INFO:tasks.rados.rados.0.smithi076.stdout:898: snap_remove snap 102 2024-09-21T09:41:37.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:37 smithi191 ceph-mon[33301]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:41:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:37 smithi191 ceph-mon[33301]: pgmap v138: 89 pgs: 89 active+clean; 168 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 436 KiB/s wr, 4 op/s 2024-09-21T09:41:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:37 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:37.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:37 smithi076 ceph-mon[41325]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:41:37.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:37 smithi076 ceph-mon[41325]: pgmap v138: 89 pgs: 89 active+clean; 168 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 436 KiB/s wr, 4 op/s 2024-09-21T09:41:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:37 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:37 smithi076 ceph-mon[29313]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:41:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:37 smithi076 ceph-mon[29313]: pgmap v138: 89 pgs: 89 active+clean; 168 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 436 KiB/s wr, 4 op/s 2024-09-21T09:41:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:37 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:38.545 INFO:tasks.rados.rados.0.smithi076.stdout:898: done (2 left) 2024-09-21T09:41:38.545 INFO:tasks.rados.rados.0.smithi076.stdout:899: write oid 18 current snap is 116 2024-09-21T09:41:38.545 INFO:tasks.rados.rados.0.smithi076.stdout:896: finishing write tid 1 to smithi07685715-35 2024-09-21T09:41:38.545 INFO:tasks.rados.rados.0.smithi076.stdout:899: seq_num 368 ranges {584976=476576,1834550=350426} 2024-09-21T09:41:38.551 INFO:tasks.rados.rados.0.smithi076.stdout:899: writing smithi07685715-18 from 584976 to 1061552 tid 1 2024-09-21T09:41:38.558 INFO:tasks.rados.rados.0.smithi076.stdout:899: writing smithi07685715-18 from 1834550 to 2184976 tid 2 2024-09-21T09:41:38.558 INFO:tasks.rados.rados.0.smithi076.stdout:900: snap_remove snap 109 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:38 smithi191 ceph-mon[33301]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[29313]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:38.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:38 smithi076 ceph-mon[41325]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:41:39.547 INFO:tasks.rados.rados.0.smithi076.stdout:896: finishing write tid 2 to smithi07685715-35 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:896: finishing write tid 3 to smithi07685715-35 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:896: finishing write tid 4 to smithi07685715-35 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:896: oid 35 updating version 0 to 662 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:896: oid 35 updating version 662 to 663 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:896: oid 35 version 663 is already newer than 661 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 663 (ObjNum 367 snap 116 seq_num 367) dirty exists 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:896: left oid 35 (ObjNum 367 snap 116 seq_num 367) 2024-09-21T09:41:39.548 INFO:tasks.rados.rados.0.smithi076.stdout:897: expect (ObjNum 363 snap 111 seq_num 363) 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:899: finishing write tid 1 to smithi07685715-18 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:899: finishing write tid 2 to smithi07685715-18 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:899: finishing write tid 3 to smithi07685715-18 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:896: done (3 left) 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:897: done (2 left) 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:900: done (1 left) 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:901: rollback oid 6 current snap is 116 2024-09-21T09:41:39.660 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 115 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:899: finishing write tid 4 to smithi07685715-18 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:899: oid 18 updating version 0 to 289 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:899: oid 18 version 289 is already newer than 284 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:899: oid 18 updating version 289 to 290 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 290 (ObjNum 368 snap 116 seq_num 368) dirty exists 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:899: left oid 18 (ObjNum 368 snap 116 seq_num 368) 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:899: done (1 left) 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:902: read oid 32 snap -1 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:902: expect (ObjNum 360 snap 110 seq_num 360) 2024-09-21T09:41:39.661 INFO:tasks.rados.rados.0.smithi076.stdout:903: snap_create 2024-09-21T09:41:39.662 INFO:tasks.rados.rados.0.smithi076.stdout:901: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:41:39.662 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 0 (ObjNum 289 snap 0 seq_num 1472411680) dirty dne 2024-09-21T09:41:39.663 INFO:tasks.rados.rados.0.smithi076.stdout:902: expect (ObjNum 360 snap 110 seq_num 360) 2024-09-21T09:41:39.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-mon[29313]: mgrmap e34: y(active, since 91s), standbys: x 2024-09-21T09:41:39.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-mon[29313]: pgmap v141: 89 pgs: 89 active+clean; 169 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 262 KiB/s wr, 10 op/s 2024-09-21T09:41:39.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-mon[29313]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:41:39.923 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-mon[41325]: mgrmap e34: y(active, since 91s), standbys: x 2024-09-21T09:41:39.923 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-mon[41325]: pgmap v141: 89 pgs: 89 active+clean; 169 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 262 KiB/s wr, 10 op/s 2024-09-21T09:41:39.923 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-mon[41325]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:41:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:39 smithi191 ceph-mon[33301]: mgrmap e34: y(active, since 91s), standbys: x 2024-09-21T09:41:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:39 smithi191 ceph-mon[33301]: pgmap v141: 89 pgs: 89 active+clean; 169 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 262 KiB/s wr, 10 op/s 2024-09-21T09:41:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:39 smithi191 ceph-mon[33301]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:41:40.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 09:41:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T09:41:39.920Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003889946s 2024-09-21T09:41:40.664 INFO:tasks.rados.rados.0.smithi076.stdout:901: done (2 left) 2024-09-21T09:41:40.664 INFO:tasks.rados.rados.0.smithi076.stdout:902: done (1 left) 2024-09-21T09:41:40.665 INFO:tasks.rados.rados.0.smithi076.stdout:903: done (0 left) 2024-09-21T09:41:40.665 INFO:tasks.rados.rados.0.smithi076.stdout:904: write oid 12 current snap is 117 2024-09-21T09:41:40.665 INFO:tasks.rados.rados.0.smithi076.stdout:904: seq_num 369 ranges {601783=551825,1776679=425104} 2024-09-21T09:41:40.671 INFO:tasks.rados.rados.0.smithi076.stdout:904: writing smithi07685715-12 from 601783 to 1153608 tid 1 2024-09-21T09:41:40.677 INFO:tasks.rados.rados.0.smithi076.stdout:904: writing smithi07685715-12 from 1776679 to 2201783 tid 2 2024-09-21T09:41:40.677 INFO:tasks.rados.rados.0.smithi076.stdout:905: delete oid 36 current snap is 117 2024-09-21T09:41:40.677 INFO:tasks.rados.rados.0.smithi076.stdout:904: finishing write tid 1 to smithi07685715-12 2024-09-21T09:41:40.680 INFO:tasks.rados.rados.0.smithi076.stdout:904: finishing write tid 2 to smithi07685715-12 2024-09-21T09:41:40.680 INFO:tasks.rados.rados.0.smithi076.stdout:904: finishing write tid 3 to smithi07685715-12 2024-09-21T09:41:40.680 INFO:tasks.rados.rados.0.smithi076.stdout:904: finishing write tid 4 to smithi07685715-12 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:904: oid 12 updating version 0 to 384 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:904: oid 12 updating version 384 to 386 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:904: oid 12 version 386 is already newer than 385 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 386 (ObjNum 369 snap 117 seq_num 369) dirty exists 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:904: left oid 12 (ObjNum 369 snap 117 seq_num 369) 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:904: done (1 left) 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:905: done (0 left) 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:906: read oid 33 snap -1 2024-09-21T09:41:40.681 INFO:tasks.rados.rados.0.smithi076.stdout:906: expect (ObjNum 306 snap 94 seq_num 306) 2024-09-21T09:41:40.682 INFO:tasks.rados.rados.0.smithi076.stdout:907: delete oid 34 current snap is 117 2024-09-21T09:41:40.683 INFO:tasks.rados.rados.0.smithi076.stdout:907: done (1 left) 2024-09-21T09:41:40.683 INFO:tasks.rados.rados.0.smithi076.stdout:908: snap_create 2024-09-21T09:41:40.686 INFO:tasks.rados.rados.0.smithi076.stdout:906: expect (ObjNum 306 snap 94 seq_num 306) 2024-09-21T09:41:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:40 smithi191 ceph-mon[33301]: Upgrade: Updating grafana.a 2024-09-21T09:41:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:40 smithi191 ceph-mon[33301]: Deploying daemon grafana.a on smithi191 2024-09-21T09:41:41.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:40 smithi076 ceph-mon[29313]: Upgrade: Updating grafana.a 2024-09-21T09:41:41.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:40 smithi076 ceph-mon[29313]: Deploying daemon grafana.a on smithi191 2024-09-21T09:41:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:40 smithi076 ceph-mon[41325]: Upgrade: Updating grafana.a 2024-09-21T09:41:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:40 smithi076 ceph-mon[41325]: Deploying daemon grafana.a on smithi191 2024-09-21T09:41:41.675 INFO:tasks.rados.rados.0.smithi076.stdout:906: done (1 left) 2024-09-21T09:41:41.675 INFO:tasks.rados.rados.0.smithi076.stdout:908: done (0 left) 2024-09-21T09:41:41.675 INFO:tasks.rados.rados.0.smithi076.stdout:909: delete oid 23 current snap is 118 2024-09-21T09:41:41.678 INFO:tasks.rados.rados.0.smithi076.stdout:909: done (0 left) 2024-09-21T09:41:41.678 INFO:tasks.rados.rados.0.smithi076.stdout:910: delete oid 30 current snap is 118 2024-09-21T09:41:41.681 INFO:tasks.rados.rados.0.smithi076.stdout:910: done (0 left) 2024-09-21T09:41:41.681 INFO:tasks.rados.rados.0.smithi076.stdout:911: write oid 39 current snap is 118 2024-09-21T09:41:41.681 INFO:tasks.rados.rados.0.smithi076.stdout:911: seq_num 374 ranges {685818=644423,2069089=216729} 2024-09-21T09:41:41.689 INFO:tasks.rados.rados.0.smithi076.stdout:911: writing smithi07685715-39 from 685818 to 1330241 tid 1 2024-09-21T09:41:41.694 INFO:tasks.rados.rados.0.smithi076.stdout:911: writing smithi07685715-39 from 2069089 to 2285818 tid 2 2024-09-21T09:41:41.694 INFO:tasks.rados.rados.0.smithi076.stdout:912: snap_create 2024-09-21T09:41:41.694 INFO:tasks.rados.rados.0.smithi076.stdout:911: finishing write tid 1 to smithi07685715-39 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: finishing write tid 2 to smithi07685715-39 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: finishing write tid 3 to smithi07685715-39 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: finishing write tid 4 to smithi07685715-39 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: oid 39 updating version 0 to 673 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: oid 39 version 673 is already newer than 672 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: oid 39 version 673 is already newer than 671 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 673 (ObjNum 374 snap 118 seq_num 374) dirty exists 2024-09-21T09:41:41.697 INFO:tasks.rados.rados.0.smithi076.stdout:911: left oid 39 (ObjNum 374 snap 118 seq_num 374) 2024-09-21T09:41:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:41 smithi191 ceph-mon[33301]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:41:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:41 smithi191 ceph-mon[33301]: pgmap v144: 89 pgs: 89 active+clean; 169 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 262 KiB/s wr, 10 op/s 2024-09-21T09:41:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:41 smithi076 ceph-mon[29313]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:41:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:41 smithi076 ceph-mon[29313]: pgmap v144: 89 pgs: 89 active+clean; 169 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 262 KiB/s wr, 10 op/s 2024-09-21T09:41:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:41 smithi076 ceph-mon[41325]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:41:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:41 smithi076 ceph-mon[41325]: pgmap v144: 89 pgs: 89 active+clean; 169 MiB data, 619 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 262 KiB/s wr, 10 op/s 2024-09-21T09:41:42.676 INFO:tasks.rados.rados.0.smithi076.stdout:911: done (1 left) 2024-09-21T09:41:42.676 INFO:tasks.rados.rados.0.smithi076.stdout:912: done (0 left) 2024-09-21T09:41:42.676 INFO:tasks.rados.rados.0.smithi076.stdout:913: delete oid 43 current snap is 119 2024-09-21T09:41:42.678 INFO:tasks.rados.rados.0.smithi076.stdout:913: done (0 left) 2024-09-21T09:41:42.678 INFO:tasks.rados.rados.0.smithi076.stdout:914: delete oid 5 current snap is 119 2024-09-21T09:41:42.681 INFO:tasks.rados.rados.0.smithi076.stdout:914: done (0 left) 2024-09-21T09:41:42.681 INFO:tasks.rados.rados.0.smithi076.stdout:915: read oid 3 snap -1 2024-09-21T09:41:42.681 INFO:tasks.rados.rados.0.smithi076.stdout:915: expect deleted 2024-09-21T09:41:42.681 INFO:tasks.rados.rados.0.smithi076.stdout:916: rollback oid 37 current snap is 119 2024-09-21T09:41:42.681 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 115 2024-09-21T09:41:42.681 INFO:tasks.rados.rados.0.smithi076.stdout:917: snap_remove snap 104 2024-09-21T09:41:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:42 smithi191 ceph-mon[33301]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:41:43.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:42 smithi076 ceph-mon[29313]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:41:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:42 smithi076 ceph-mon[41325]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:41:43.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:41:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:41:43] "GET /metrics HTTP/1.1" 200 36134 "" "Prometheus/2.51.0" 2024-09-21T09:41:43.689 INFO:tasks.rados.rados.0.smithi076.stdout:916: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:41:43.689 INFO:tasks.rados.rados.0.smithi076.stdout:916: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:41:43.689 INFO:tasks.rados.rados.0.smithi076.stdout:916: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:41:43.689 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 239 (ObjNum 290 snap 92 seq_num 290) dirty exists 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:915: done (2 left) 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:916: done (1 left) 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:917: done (0 left) 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:918: rollback oid 35 current snap is 119 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 115 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:919: write oid 30 current snap is 119 2024-09-21T09:41:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:919: seq_num 377 ranges {736239=470170,1728182=571782} 2024-09-21T09:41:43.696 INFO:tasks.rados.rados.0.smithi076.stdout:919: writing smithi07685715-30 from 736239 to 1206409 tid 1 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:919: writing smithi07685715-30 from 1728182 to 2299964 tid 2 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:920: delete oid 26 current snap is 119 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:918: finishing rollback tid 0 to smithi07685715-35 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:918: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:918: finishing rollback tid 2 to smithi07685715-35 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 679 (ObjNum 341 snap 105 seq_num 341) dirty exists 2024-09-21T09:41:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:919: finishing write tid 1 to smithi07685715-30 2024-09-21T09:41:43.705 INFO:tasks.rados.rados.0.smithi076.stdout:918: done (2 left) 2024-09-21T09:41:43.705 INFO:tasks.rados.rados.0.smithi076.stdout:920: done (1 left) 2024-09-21T09:41:43.705 INFO:tasks.rados.rados.0.smithi076.stdout:921: rollback oid 26 current snap is 119 2024-09-21T09:41:43.705 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 105 2024-09-21T09:41:43.705 INFO:tasks.rados.rados.0.smithi076.stdout:922: read oid 38 snap -1 2024-09-21T09:41:43.706 INFO:tasks.rados.rados.0.smithi076.stdout:922: expect (ObjNum 220 snap 63 seq_num 220) 2024-09-21T09:41:43.706 INFO:tasks.rados.rados.0.smithi076.stdout:923: read oid 33 snap -1 2024-09-21T09:41:43.706 INFO:tasks.rados.rados.0.smithi076.stdout:923: expect (ObjNum 306 snap 94 seq_num 306) 2024-09-21T09:41:43.706 INFO:tasks.rados.rados.0.smithi076.stdout:924: rollback oid 7 current snap is 119 2024-09-21T09:41:43.706 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 118 2024-09-21T09:41:43.706 INFO:tasks.rados.rados.0.smithi076.stdout:925: snap_remove snap 112 2024-09-21T09:41:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:43 smithi191 ceph-mon[33301]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:41:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:43 smithi191 ceph-mon[33301]: pgmap v147: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 441 KiB/s wr, 5 op/s 2024-09-21T09:41:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:43 smithi076 ceph-mon[29313]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:41:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:43 smithi076 ceph-mon[29313]: pgmap v147: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 441 KiB/s wr, 5 op/s 2024-09-21T09:41:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:43 smithi076 ceph-mon[41325]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:41:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:43 smithi076 ceph-mon[41325]: pgmap v147: 89 pgs: 89 active+clean; 170 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 441 KiB/s wr, 5 op/s 2024-09-21T09:41:44.692 INFO:tasks.rados.rados.0.smithi076.stdout:919: finishing write tid 2 to smithi07685715-30 2024-09-21T09:41:44.692 INFO:tasks.rados.rados.0.smithi076.stdout:919: finishing write tid 3 to smithi07685715-30 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:921: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:919: finishing write tid 4 to smithi07685715-30 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:919: oid 30 updating version 0 to 749 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:919: oid 30 updating version 749 to 752 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:919: oid 30 updating version 752 to 753 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 753 (ObjNum 377 snap 119 seq_num 377) dirty exists 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:919: left oid 30 (ObjNum 377 snap 119 seq_num 377) 2024-09-21T09:41:44.693 INFO:tasks.rados.rados.0.smithi076.stdout:922: expect (ObjNum 220 snap 63 seq_num 220) 2024-09-21T09:41:44.830 INFO:tasks.rados.rados.0.smithi076.stdout:924: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:41:44.830 INFO:tasks.rados.rados.0.smithi076.stdout:924: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:41:44.830 INFO:tasks.rados.rados.0.smithi076.stdout:924: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:41:44.830 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 447 (ObjNum 276 snap 88 seq_num 276) dirty exists 2024-09-21T09:41:44.830 INFO:tasks.rados.rados.0.smithi076.stdout:923: expect (ObjNum 306 snap 94 seq_num 306) 2024-09-21T09:41:44.854 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:41:44.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:44 smithi076 ceph-mon[29313]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:41:44.959 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:44 smithi076 ceph-mon[41325]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:919: done (5 left) 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:921: done (4 left) 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:922: done (3 left) 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:923: done (2 left) 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:924: done (1 left) 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:925: done (0 left) 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:926: rollback oid 41 current snap is 119 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 100 2024-09-21T09:41:45.066 INFO:tasks.rados.rados.0.smithi076.stdout:927: snap_remove snap 115 2024-09-21T09:41:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:44 smithi191 ceph-mon[33301]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:41:45.431 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:41:45.431 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (16s) 11s ago 6m 17.4M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:41:45.431 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (6m) 11s ago 6m 87.3M - 9.4.7 954c08fa6188 d3e4c8ed868b 2024-09-21T09:41:45.431 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (4m) 11s ago 4m 17.7M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:41:45.431 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (4m) 11s ago 4m 16.1M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (79s) 11s ago 10m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (106s) 11s ago 11m 514M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (11m) 11s ago 11m 60.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (10m) 11s ago 10m 53.6M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (10m) 11s ago 10m 51.6M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (64s) 11s ago 5m 9709k - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (58s) 11s ago 5m 11.2M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (9m) 11s ago 9m 89.0M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (9m) 11s ago 9m 77.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (9m) 11s ago 9m 82.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (8m) 11s ago 8m 70.9M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (8m) 11s ago 8m 82.2M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:41:45.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (7m) 11s ago 7m 90.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:41:45.433 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (7m) 11s ago 7m 65.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:41:45.433 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (7m) 11s ago 7m 64.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:41:45.433 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (35s) 11s ago 6m 37.5M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:41:45.698 INFO:tasks.rados.rados.0.smithi076.stdout:926: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:41:45.698 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 0 (ObjNum 177 snap 0 seq_num 1682359683) dirty dne 2024-09-21T09:41:45.698 INFO:tasks.rados.rados.0.smithi076.stdout:926: done (1 left) 2024-09-21T09:41:45.698 INFO:tasks.rados.rados.0.smithi076.stdout:927: done (0 left) 2024-09-21T09:41:45.699 INFO:tasks.rados.rados.0.smithi076.stdout:928: read oid 14 snap -1 2024-09-21T09:41:45.699 INFO:tasks.rados.rados.0.smithi076.stdout:928: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:41:45.699 INFO:tasks.rados.rados.0.smithi076.stdout:929: write oid 35 current snap is 119 2024-09-21T09:41:45.699 INFO:tasks.rados.rados.0.smithi076.stdout:929: seq_num 379 ranges {769853=620668,1845146=524707} 2024-09-21T09:41:45.706 INFO:tasks.rados.rados.0.smithi076.stdout:929: writing smithi07685715-35 from 769853 to 1390521 tid 1 2024-09-21T09:41:45.711 INFO:tasks.rados.rados.0.smithi076.stdout:929: writing smithi07685715-35 from 1845146 to 2369853 tid 2 2024-09-21T09:41:45.711 INFO:tasks.rados.rados.0.smithi076.stdout:930: write oid 18 current snap is 119 2024-09-21T09:41:45.712 INFO:tasks.rados.rados.0.smithi076.stdout:930: seq_num 380 ranges {786660=695917,1903628=435462} 2024-09-21T09:41:45.719 INFO:tasks.rados.rados.0.smithi076.stdout:930: writing smithi07685715-18 from 786660 to 1482577 tid 1 2024-09-21T09:41:45.724 INFO:tasks.rados.rados.0.smithi076.stdout:930: writing smithi07685715-18 from 1903628 to 2339090 tid 2 2024-09-21T09:41:45.724 INFO:tasks.rados.rados.0.smithi076.stdout:931: read oid 31 snap -1 2024-09-21T09:41:45.724 INFO:tasks.rados.rados.0.smithi076.stdout:931: expect deleted 2024-09-21T09:41:45.724 INFO:tasks.rados.rados.0.smithi076.stdout:932: snap_remove snap 116 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:41:45.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:41:45.780 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:41:46.064 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:41:46.064 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:41:46.064 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [ 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr" 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "2/2 daemons upgraded", 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:41:46.065 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:41:46.065 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:45 smithi076 ceph-mon[29313]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:41:46.066 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:45 smithi076 ceph-mon[29313]: pgmap v150: 89 pgs: 89 active+clean; 169 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 651 KiB/s wr, 9 op/s 2024-09-21T09:41:46.066 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:45 smithi076 ceph-mon[29313]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:46.066 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:45 smithi076 ceph-mon[41325]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:41:46.066 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:45 smithi076 ceph-mon[41325]: pgmap v150: 89 pgs: 89 active+clean; 169 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 651 KiB/s wr, 9 op/s 2024-09-21T09:41:46.066 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:45 smithi076 ceph-mon[41325]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:45 smithi191 ceph-mon[33301]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:41:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:45 smithi191 ceph-mon[33301]: pgmap v150: 89 pgs: 89 active+clean; 169 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 651 KiB/s wr, 9 op/s 2024-09-21T09:41:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:45 smithi191 ceph-mon[33301]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:46.704 INFO:tasks.rados.rados.0.smithi076.stdout:932: done (4 left) 2024-09-21T09:41:46.704 INFO:tasks.rados.rados.0.smithi076.stdout:933: read oid 34 snap -1 2024-09-21T09:41:46.705 INFO:tasks.rados.rados.0.smithi076.stdout:933: expect deleted 2024-09-21T09:41:46.705 INFO:tasks.rados.rados.0.smithi076.stdout:934: snap_create 2024-09-21T09:41:46.705 INFO:tasks.rados.rados.0.smithi076.stdout:928: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:41:46.856 INFO:tasks.rados.rados.0.smithi076.stdout:929: finishing write tid 1 to smithi07685715-35 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: finishing write tid 2 to smithi07685715-35 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: finishing write tid 3 to smithi07685715-35 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: finishing write tid 4 to smithi07685715-35 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: oid 35 updating version 0 to 695 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: oid 35 version 695 is already newer than 694 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: oid 35 updating version 695 to 696 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 696 (ObjNum 379 snap 119 seq_num 379) dirty exists 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:929: left oid 35 (ObjNum 379 snap 119 seq_num 379) 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:930: finishing write tid 1 to smithi07685715-18 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:930: finishing write tid 2 to smithi07685715-18 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:930: finishing write tid 3 to smithi07685715-18 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:930: finishing write tid 4 to smithi07685715-18 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:930: oid 18 updating version 0 to 303 2024-09-21T09:41:46.857 INFO:tasks.rados.rados.0.smithi076.stdout:930: oid 18 version 303 is already newer than 302 2024-09-21T09:41:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:930: oid 18 updating version 303 to 304 2024-09-21T09:41:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 304 (ObjNum 380 snap 119 seq_num 380) dirty exists 2024-09-21T09:41:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:930: left oid 18 (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:41:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:46 smithi191 ceph-mon[33301]: from='client.15093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:46 smithi191 ceph-mon[33301]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:46 smithi191 ceph-mon[33301]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:41:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:46 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3778225444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:47.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[29313]: from='client.15093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[29313]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[29313]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/3778225444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[41325]: from='client.15093 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[41325]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[41325]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:41:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:46 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/3778225444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:41:47.706 INFO:tasks.rados.rados.0.smithi076.stdout:928: done (5 left) 2024-09-21T09:41:47.706 INFO:tasks.rados.rados.0.smithi076.stdout:929: done (4 left) 2024-09-21T09:41:47.706 INFO:tasks.rados.rados.0.smithi076.stdout:930: done (3 left) 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:931: done (2 left) 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:933: done (1 left) 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:934: done (0 left) 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:935: rollback oid 28 current snap is 120 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 28 to 117 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:936: read oid 29 snap -1 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:936: expect (ObjNum 333 snap 103 seq_num 333) 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:937: rollback oid 42 current snap is 120 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 119 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:938: read oid 46 snap -1 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:938: expect (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:939: write oid 1 current snap is 120 2024-09-21T09:41:47.707 INFO:tasks.rados.rados.0.smithi076.stdout:939: seq_num 381 ranges {403467=487519,1562110=579120} 2024-09-21T09:41:47.711 INFO:tasks.rados.rados.0.smithi076.stdout:939: writing smithi07685715-1 from 403467 to 890986 tid 1 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:939: writing smithi07685715-1 from 1562110 to 2141230 tid 2 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:940: read oid 30 snap -1 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:940: expect (ObjNum 377 snap 119 seq_num 377) 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:941: delete oid 48 current snap is 120 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:937: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:937: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:937: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 310 (ObjNum 346 snap 105 seq_num 346) dirty exists 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:935: finishing rollback tid 0 to smithi07685715-28 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:935: finishing rollback tid 1 to smithi07685715-28 2024-09-21T09:41:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:935: finishing rollback tid 2 to smithi07685715-28 2024-09-21T09:41:47.720 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 763 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-09-21T09:41:47.724 INFO:tasks.rados.rados.0.smithi076.stdout:936: expect (ObjNum 333 snap 103 seq_num 333) 2024-09-21T09:41:47.836 INFO:tasks.rados.rados.0.smithi076.stdout:938: expect (ObjNum 310 snap 94 seq_num 310) 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:939: finishing write tid 1 to smithi07685715-1 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:939: finishing write tid 2 to smithi07685715-1 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:935: done (6 left) 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:936: done (5 left) 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:937: done (4 left) 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:938: done (3 left) 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:941: done (2 left) 2024-09-21T09:41:48.047 INFO:tasks.rados.rados.0.smithi076.stdout:942: snap_remove snap 114 2024-09-21T09:41:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:47 smithi191 ceph-mon[33301]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:47 smithi191 ceph-mon[33301]: pgmap v152: 89 pgs: 89 active+clean; 169 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 648 KiB/s wr, 9 op/s 2024-09-21T09:41:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:47 smithi191 ceph-mon[33301]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:41:48.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:47 smithi076 ceph-mon[29313]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:47 smithi076 ceph-mon[29313]: pgmap v152: 89 pgs: 89 active+clean; 169 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 648 KiB/s wr, 9 op/s 2024-09-21T09:41:48.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:47 smithi076 ceph-mon[29313]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:41:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:47 smithi076 ceph-mon[41325]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:41:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:47 smithi076 ceph-mon[41325]: pgmap v152: 89 pgs: 89 active+clean; 169 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s rd, 648 KiB/s wr, 9 op/s 2024-09-21T09:41:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:47 smithi076 ceph-mon[41325]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:41:48.546 INFO:tasks.rados.rados.0.smithi076.stdout:942: done (2 left) 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:943: read oid 5 snap -1 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:943: expect deleted 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:944: delete oid 29 current snap is 120 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:939: finishing write tid 3 to smithi07685715-1 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:939: finishing write tid 4 to smithi07685715-1 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:939: oid 1 updating version 0 to 704 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:939: oid 1 updating version 704 to 706 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:939: oid 1 version 706 is already newer than 705 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 706 (ObjNum 381 snap 120 seq_num 381) dirty exists 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:939: left oid 1 (ObjNum 381 snap 120 seq_num 381) 2024-09-21T09:41:48.547 INFO:tasks.rados.rados.0.smithi076.stdout:940: expect (ObjNum 377 snap 119 seq_num 377) 2024-09-21T09:41:48.624 INFO:tasks.rados.rados.0.smithi076.stdout:939: done (3 left) 2024-09-21T09:41:48.624 INFO:tasks.rados.rados.0.smithi076.stdout:940: done (2 left) 2024-09-21T09:41:48.624 INFO:tasks.rados.rados.0.smithi076.stdout:943: done (1 left) 2024-09-21T09:41:48.624 INFO:tasks.rados.rados.0.smithi076.stdout:944: done (0 left) 2024-09-21T09:41:48.624 INFO:tasks.rados.rados.0.smithi076.stdout:945: write oid 9 current snap is 120 2024-09-21T09:41:48.624 INFO:tasks.rados.rados.0.smithi076.stdout:945: seq_num 384 ranges {453888=713266,1737556=442800} 2024-09-21T09:41:48.630 INFO:tasks.rados.rados.0.smithi076.stdout:945: writing smithi07685715-9 from 453888 to 1167154 tid 1 2024-09-21T09:41:48.635 INFO:tasks.rados.rados.0.smithi076.stdout:945: writing smithi07685715-9 from 1737556 to 2180356 tid 2 2024-09-21T09:41:48.636 INFO:tasks.rados.rados.0.smithi076.stdout:946: read oid 42 snap -1 2024-09-21T09:41:48.636 INFO:tasks.rados.rados.0.smithi076.stdout:946: expect (ObjNum 346 snap 105 seq_num 346) 2024-09-21T09:41:48.636 INFO:tasks.rados.rados.0.smithi076.stdout:947: snap_remove snap 100 2024-09-21T09:41:49.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:48 smithi191 ceph-mon[33301]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:41:49.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:48 smithi191 ceph-mon[33301]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:41:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:48 smithi076 ceph-mon[29313]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:41:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:48 smithi076 ceph-mon[29313]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:41:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:48 smithi076 ceph-mon[41325]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:41:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:48 smithi076 ceph-mon[41325]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:41:49.472 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:49 smithi191 systemd[1]: Stopping Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: finishing write tid 1 to smithi07685715-9 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: finishing write tid 2 to smithi07685715-9 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: finishing write tid 3 to smithi07685715-9 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: finishing write tid 4 to smithi07685715-9 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: oid 9 updating version 0 to 401 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: oid 9 version 401 is already newer than 400 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: oid 9 version 401 is already newer than 399 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 401 (ObjNum 384 snap 120 seq_num 384) dirty exists 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:945: left oid 9 (ObjNum 384 snap 120 seq_num 384) 2024-09-21T09:41:49.555 INFO:tasks.rados.rados.0.smithi076.stdout:946: expect (ObjNum 346 snap 105 seq_num 346) 2024-09-21T09:41:49.777 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:49 smithi191 ceph-mon[33301]: pgmap v156: 89 pgs: 89 active+clean; 171 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 811 KiB/s wr, 10 op/s 2024-09-21T09:41:49.778 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:49 smithi191 ceph-mon[33301]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:41:49.778 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:49 smithi191 podman[93349]: 2024-09-21 09:41:49.470499363 +0000 UTC m=+0.122818000 container died d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, version=8.5, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, name=ubi8, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.expose-services=) 2024-09-21T09:41:49.867 INFO:tasks.rados.rados.0.smithi076.stdout:945: done (2 left) 2024-09-21T09:41:49.867 INFO:tasks.rados.rados.0.smithi076.stdout:946: done (1 left) 2024-09-21T09:41:49.867 INFO:tasks.rados.rados.0.smithi076.stdout:947: done (0 left) 2024-09-21T09:41:49.867 INFO:tasks.rados.rados.0.smithi076.stdout:948: write oid 5 current snap is 120 2024-09-21T09:41:49.867 INFO:tasks.rados.rados.0.smithi076.stdout:948: seq_num 385 ranges {470695=788515,1679685=586458} 2024-09-21T09:41:49.873 INFO:tasks.rados.rados.0.smithi076.stdout:948: writing smithi07685715-5 from 470695 to 1259210 tid 1 2024-09-21T09:41:49.879 INFO:tasks.rados.rados.0.smithi076.stdout:948: writing smithi07685715-5 from 1679685 to 2266143 tid 2 2024-09-21T09:41:49.879 INFO:tasks.rados.rados.0.smithi076.stdout:949: write oid 38 current snap is 120 2024-09-21T09:41:49.879 INFO:tasks.rados.rados.0.smithi076.stdout:949: seq_num 386 ranges {487502=463764,1738167=446469} 2024-09-21T09:41:49.884 INFO:tasks.rados.rados.0.smithi076.stdout:949: writing smithi07685715-38 from 487502 to 951266 tid 1 2024-09-21T09:41:49.891 INFO:tasks.rados.rados.0.smithi076.stdout:949: writing smithi07685715-38 from 1738167 to 2184636 tid 2 2024-09-21T09:41:49.891 INFO:tasks.rados.rados.0.smithi076.stdout:950: rollback oid 24 current snap is 120 2024-09-21T09:41:49.891 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 24 to 117 2024-09-21T09:41:49.891 INFO:tasks.rados.rados.0.smithi076.stdout:948: finishing write tid 1 to smithi07685715-5 2024-09-21T09:41:49.891 INFO:tasks.rados.rados.0.smithi076.stdout:951: write oid 44 current snap is 120 2024-09-21T09:41:49.891 INFO:tasks.rados.rados.0.smithi076.stdout:948: finishing write tid 2 to smithi07685715-5 2024-09-21T09:41:49.892 INFO:tasks.rados.rados.0.smithi076.stdout:948: finishing write tid 3 to smithi07685715-5 2024-09-21T09:41:49.892 INFO:tasks.rados.rados.0.smithi076.stdout:948: finishing write tid 4 to smithi07685715-5 2024-09-21T09:41:49.892 INFO:tasks.rados.rados.0.smithi076.stdout:948: oid 5 updating version 0 to 720 2024-09-21T09:41:49.892 INFO:tasks.rados.rados.0.smithi076.stdout:948: oid 5 updating version 720 to 721 2024-09-21T09:41:49.893 INFO:tasks.rados.rados.0.smithi076.stdout:948: oid 5 version 721 is already newer than 719 2024-09-21T09:41:49.893 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 721 (ObjNum 385 snap 120 seq_num 385) dirty exists 2024-09-21T09:41:49.893 INFO:tasks.rados.rados.0.smithi076.stdout:948: left oid 5 (ObjNum 385 snap 120 seq_num 385) 2024-09-21T09:41:49.893 INFO:tasks.rados.rados.0.smithi076.stdout:949: finishing write tid 1 to smithi07685715-38 2024-09-21T09:41:49.894 INFO:tasks.rados.rados.0.smithi076.stdout:951: seq_num 387 ranges {504309=539013,1796649=590127} 2024-09-21T09:41:49.897 INFO:tasks.rados.rados.0.smithi076.stdout:951: writing smithi07685715-44 from 504309 to 1043322 tid 1 2024-09-21T09:41:49.904 INFO:tasks.rados.rados.0.smithi076.stdout:951: writing smithi07685715-44 from 1796649 to 2386776 tid 2 2024-09-21T09:41:49.904 INFO:tasks.rados.rados.0.smithi076.stdout:948: done (3 left) 2024-09-21T09:41:49.904 INFO:tasks.rados.rados.0.smithi076.stdout:952: snap_remove snap 106 2024-09-21T09:41:50.102 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:49 smithi191 podman[93349]: 2024-09-21 09:41:49.87460225 +0000 UTC m=+0.526920888 container cleanup d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, architecture=x86_64, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, vcs-type=git, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, maintainer=Paul Cuzner , release=239.1651231664, version=8.5, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T09:41:50.102 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:49 smithi191 bash[93349]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:41:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:49 smithi076 ceph-mon[29313]: pgmap v156: 89 pgs: 89 active+clean; 171 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 811 KiB/s wr, 10 op/s 2024-09-21T09:41:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:49 smithi076 ceph-mon[29313]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:41:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:49 smithi076 ceph-mon[41325]: pgmap v156: 89 pgs: 89 active+clean; 171 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 811 KiB/s wr, 10 op/s 2024-09-21T09:41:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:49 smithi076 ceph-mon[41325]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:41:50.362 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 podman[93365]: 2024-09-21 09:41:50.101046623 +0000 UTC m=+0.629496564 container remove d3e4c8ed868b62ffdce290ea710cbb376be781d4d06cccf479f42517f1e0222e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.openshift.expose-services=, com.redhat.component=ubi8-container, name=ubi8, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., description=Ceph Grafana Container, maintainer=Paul Cuzner , version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-21T09:41:50.362 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:41:50.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T09:41:50.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 systemd[1]: Stopped Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:50.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@grafana.a.service: Consumed 7.674s CPU time. 2024-09-21T09:41:50.726 INFO:tasks.rados.rados.0.smithi076.stdout:952: done (3 left) 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:953: snap_create 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:950: finishing rollback tid 0 to smithi07685715-24 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:950: finishing rollback tid 1 to smithi07685715-24 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:950: finishing rollback tid 2 to smithi07685715-24 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 97 (ObjNum 326 snap 101 seq_num 326) dirty exists 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:949: finishing write tid 2 to smithi07685715-38 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:949: finishing write tid 3 to smithi07685715-38 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:949: finishing write tid 4 to smithi07685715-38 2024-09-21T09:41:50.727 INFO:tasks.rados.rados.0.smithi076.stdout:949: oid 38 updating version 0 to 464 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:949: oid 38 version 464 is already newer than 463 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:949: oid 38 updating version 464 to 465 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 465 (ObjNum 386 snap 120 seq_num 386) dirty exists 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:949: left oid 38 (ObjNum 386 snap 120 seq_num 386) 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: finishing write tid 1 to smithi07685715-44 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: finishing write tid 2 to smithi07685715-44 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: finishing write tid 3 to smithi07685715-44 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: finishing write tid 4 to smithi07685715-44 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: oid 44 updating version 0 to 183 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: oid 44 updating version 183 to 185 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: oid 44 version 185 is already newer than 184 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 185 (ObjNum 387 snap 120 seq_num 387) dirty exists 2024-09-21T09:41:50.728 INFO:tasks.rados.rados.0.smithi076.stdout:951: left oid 44 (ObjNum 387 snap 120 seq_num 387) 2024-09-21T09:41:50.767 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 systemd[1]: Starting Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:41:50.767 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 podman[93499]: 2024-09-21 09:41:50.56650214 +0000 UTC m=+0.016801961 image pull 5aad1d7cf004c80a4acb170e1c80fb85781b022f00f08c6bcd22608882b4c748 quay.io/ceph/grafana:10.4.8 2024-09-21T09:41:50.767 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 podman[93499]: 2024-09-21 09:41:50.669250003 +0000 UTC m=+0.119549859 container create 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:41:51.141 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 podman[93499]: 2024-09-21 09:41:50.888288704 +0000 UTC m=+0.338588520 container init 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:41:51.141 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:50 smithi191 podman[93499]: 2024-09-21 09:41:50.893135022 +0000 UTC m=+0.343434834 container start 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T09:41:51.141 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 bash[93499]: 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 2024-09-21T09:41:51.141 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005030456Z level=info msg="Starting Grafana" version=10.4.8 commit=8c384b71bde142a25ddc01ed5a183ca572b2b7f7 branch=v10.4.x compiled=2024-09-21T09:41:51Z 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005300295Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005311278Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005315984Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005320413Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005325642Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005329595Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005333671Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005337912Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005342006Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005345537Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005349527Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:41:51.142 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005353281Z level=info msg=Target target=[all] 2024-09-21T09:41:51.143 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005360827Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:41:51.143 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005364644Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:41:51.143 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005368605Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:41:51.143 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005372522Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:41:51.143 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005376401Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=settings t=2024-09-21T09:41:51.005380757Z level=info msg="App mode production" 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=sqlstore t=2024-09-21T09:41:51.005745011Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=sqlstore t=2024-09-21T09:41:51.005765175Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.006626792Z level=info msg="Starting DB migrations" 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 systemd[1]: Started Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.022709443Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.028761274Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=6.050285ms 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.029235415Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.029379793Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=144.505µs 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.029776936Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.030528523Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=751.174µs 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.03266038Z level=info msg="Executing migration" id="update login field with orgid to allow for multiple service accounts with same name across orgs" 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.03291084Z level=info msg="Migration successfully executed" id="update login field with orgid to allow for multiple service accounts with same name across orgs" duration=252.513µs 2024-09-21T09:41:51.144 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.035728007Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.040866749Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.136344ms 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.041386992Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.046682589Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.293618ms 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.047238563Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.052380685Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.139541ms 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.052916091Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.053065864Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=149.815µs 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.055915347Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.056716216Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=800.292µs 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.058833302Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.059553617Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=721.131µs 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.061386309Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.066791867Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.404891ms 2024-09-21T09:41:51.145 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.067207615Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.07211278Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=4.904554ms 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.072492647Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.072532241Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=39.413µs 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.075429933Z level=info msg="Executing migration" id="add library_element folder uid" 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.080490488Z level=info msg="Migration successfully executed" id="add library_element folder uid" duration=5.059285ms 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.080888019Z level=info msg="Executing migration" id="populate library_element folder_uid" 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.081154252Z level=info msg="Migration successfully executed" id="populate library_element folder_uid" duration=266.264µs 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.083614641Z level=info msg="Executing migration" id="add index library_element org_id-folder_uid-name-kind" 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.084479421Z level=info msg="Migration successfully executed" id="add index library_element org_id-folder_uid-name-kind" duration=864.727µs 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.08491075Z level=info msg="Executing migration" id="permission kind migration" 2024-09-21T09:41:51.146 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.090028212Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.116542ms 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.090427477Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.095537864Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.109679ms 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.095921326Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.100935993Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.014016ms 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.101302178Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.102057235Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=754.848µs 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.10243133Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.103208153Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=776.751µs 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.105131534Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.105829542Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=697.755µs 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.107876968Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.107895737Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=19.039µs 2024-09-21T09:41:51.147 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.110465164Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.111149923Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=682.914µs 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.111500608Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.112171128Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=670.331µs 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.114184054Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.130780253Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=16.593673ms 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.131241847Z level=info msg="Executing migration" id="create correlation v2" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.131980394Z level=info msg="Migration successfully executed" id="create correlation v2" duration=732.321µs 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.133771783Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.134452891Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=680.94µs 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.134790132Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.135514166Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=723.827µs 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.137452979Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-21T09:41:51.148 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.138111023Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=657.762µs 2024-09-21T09:41:51.149 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.140128903Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-21T09:41:51.149 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.140330265Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=200.976µs 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.142744584Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.143350434Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=606.468µs 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.143720697Z level=info msg="Executing migration" id="add provisioning column" 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.148968587Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.247722ms 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.149355471Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.149501222Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=144.602µs 2024-09-21T09:41:51.407 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.152079284Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.152121189Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=63.66µs 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.152842173Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.158012596Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.169875ms 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.15838018Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.158588891Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=209.146µs 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.159176775Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.159292268Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=115.841µs 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.159993182Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.160964436Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=971.429µs 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.16131988Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.162038458Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=719.806µs 2024-09-21T09:41:51.408 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.16238374Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.163129098Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=745.353µs 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.163476647Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.163887359Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=411.074µs 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.164266819Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.164472399Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=205.532µs 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.165112656Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.165827087Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=714.301µs 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.166186041Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.166941731Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=755.24µs 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.167303649Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.168005909Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=701.02µs 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.168368354Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-21T09:41:51.409 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.169124867Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=756.226µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.169489885Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.170199469Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=711.076µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.17056615Z level=info msg="Executing migration" id="create anon_device table" 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.171154357Z level=info msg="Migration successfully executed" id="create anon_device table" duration=588.546µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.171516962Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.172229147Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=712.049µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.172610021Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.173310073Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=698.52µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.173670241Z level=info msg="Executing migration" id="create signing_key table" 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.174314692Z level=info msg="Migration successfully executed" id="create signing_key table" duration=644.176µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.174713429Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.175398918Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=685.14µs 2024-09-21T09:41:51.410 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.175753539Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.176422522Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=668.809µs 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.176760683Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.177179888Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=419.704µs 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.177599631Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.183061149Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.459837ms 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.183431279Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-21T09:41:51.411 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.185119352Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.688435ms 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.185674294Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.185685565Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=11.981µs 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.186471744Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.187198917Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=727.689µs 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.187557384Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.187566742Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=9.912µs 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.187959615Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.188800784Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=840.121µs 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.189147232Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.189981455Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=834.465µs 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.190355846Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-21T09:41:51.412 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.19111175Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=755.588µs 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.191514764Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.192176466Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=662.295µs 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.192516115Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.192922013Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=406.004µs 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.193349691Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.193389965Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=40.725µs 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.194197857Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.199424942Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.227067ms 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.199778784Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.205224518Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.445162ms 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.205613223Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.205838673Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=226.83µs 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=migrator t=2024-09-21T09:41:51.208361322Z level=info msg="migrations completed" performed=69 skipped=482 duration=185.757497ms 2024-09-21T09:41:51.413 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=sqlstore t=2024-09-21T09:41:51.209104039Z level=info msg="Created default organization" 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=secrets t=2024-09-21T09:41:51.211113044Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugin.store t=2024-09-21T09:41:51.225581405Z level=info msg="Loading plugins..." 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=local.finder t=2024-09-21T09:41:51.267497664Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugin.store t=2024-09-21T09:41:51.267517345Z level=info msg="Plugins loaded" count=55 duration=41.936532ms 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=query_data t=2024-09-21T09:41:51.271539194Z level=info msg="Query Service initialization" 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=live.push_http t=2024-09-21T09:41:51.275462261Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ngalert.migration t=2024-09-21T09:41:51.276743341Z level=info msg=Starting 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ngalert.state.manager t=2024-09-21T09:41:51.284652227Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=infra.usagestats.collector t=2024-09-21T09:41:51.28617936Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=provisioning.datasources t=2024-09-21T09:41:51.288744919Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=provisioning.datasources t=2024-09-21T09:41:51.289008835Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=provisioning.alerting t=2024-09-21T09:41:51.320798926Z level=info msg="starting to provision alerting" 2024-09-21T09:41:51.414 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=provisioning.alerting t=2024-09-21T09:41:51.320816193Z level=info msg="finished to provision alerting" 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ngalert.state.manager t=2024-09-21T09:41:51.320953628Z level=info msg="Warming state cache for startup" 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=grafanaStorageLogger t=2024-09-21T09:41:51.32138581Z level=info msg="Storage starting" 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:41:51.331420691Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=http.server t=2024-09-21T09:41:51.333716916Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=http.server t=2024-09-21T09:41:51.334065725Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ngalert.state.manager t=2024-09-21T09:41:51.345721063Z level=info msg="State cache has been initialized" states=0 duration=24.764043ms 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ngalert.scheduler t=2024-09-21T09:41:51.3458611Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ticker t=2024-09-21T09:41:51.345915106Z level=info msg=starting first_tick=2024-09-21T09:42:00Z 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=provisioning.dashboard t=2024-09-21T09:41:51.347659979Z level=info msg="starting to provision dashboards" 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=grafana.update.checker t=2024-09-21T09:41:51.405158854Z level=info msg="Update check succeeded" duration=84.194215ms 2024-09-21T09:41:51.415 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugins.update.checker t=2024-09-21T09:41:51.407112687Z level=info msg="Update check succeeded" duration=85.90836ms 2024-09-21T09:41:51.693 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=sqlstore.transactions t=2024-09-21T09:41:51.440301928Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:41:51.693 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=provisioning.dashboard t=2024-09-21T09:41:51.540758164Z level=info msg="finished to provision dashboards" 2024-09-21T09:41:51.693 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=grafana-apiserver t=2024-09-21T09:41:51.5506426Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-21T09:41:51.693 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=grafana-apiserver t=2024-09-21T09:41:51.551048707Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-21T09:41:51.725 INFO:tasks.rados.rados.0.smithi076.stdout:949: done (3 left) 2024-09-21T09:41:51.725 INFO:tasks.rados.rados.0.smithi076.stdout:950: done (2 left) 2024-09-21T09:41:51.725 INFO:tasks.rados.rados.0.smithi076.stdout:951: done (1 left) 2024-09-21T09:41:51.725 INFO:tasks.rados.rados.0.smithi076.stdout:953: done (0 left) 2024-09-21T09:41:51.725 INFO:tasks.rados.rados.0.smithi076.stdout:954: snap_remove snap 117 2024-09-21T09:41:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-mon[33301]: pgmap v158: 89 pgs: 89 active+clean; 171 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 811 KiB/s wr, 10 op/s 2024-09-21T09:41:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-mon[33301]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:41:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:51 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:52.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[29313]: pgmap v158: 89 pgs: 89 active+clean; 171 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 811 KiB/s wr, 10 op/s 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[29313]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[41325]: pgmap v158: 89 pgs: 89 active+clean; 171 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 811 KiB/s wr, 10 op/s 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[41325]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:51 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:41:52.732 INFO:tasks.rados.rados.0.smithi076.stdout:954: done (0 left) 2024-09-21T09:41:52.733 INFO:tasks.rados.rados.0.smithi076.stdout:955: delete oid 50 current snap is 121 2024-09-21T09:41:52.735 INFO:tasks.rados.rados.0.smithi076.stdout:955: done (0 left) 2024-09-21T09:41:52.735 INFO:tasks.rados.rados.0.smithi076.stdout:956: delete oid 37 current snap is 121 2024-09-21T09:41:52.739 INFO:tasks.rados.rados.0.smithi076.stdout:956: done (0 left) 2024-09-21T09:41:52.739 INFO:tasks.rados.rados.0.smithi076.stdout:957: write oid 39 current snap is 121 2024-09-21T09:41:52.739 INFO:tasks.rados.rados.0.smithi076.stdout:957: seq_num 390 ranges {554730=481113,1572095=737454} 2024-09-21T09:41:52.745 INFO:tasks.rados.rados.0.smithi076.stdout:957: writing smithi07685715-39 from 554730 to 1035843 tid 1 2024-09-21T09:41:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:957: writing smithi07685715-39 from 1572095 to 2309549 tid 2 2024-09-21T09:41:52.752 INFO:tasks.rados.rados.0.smithi076.stdout:958: rollback oid 32 current snap is 121 2024-09-21T09:41:52.754 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 120 2024-09-21T09:41:52.754 INFO:tasks.rados.rados.0.smithi076.stdout:957: finishing write tid 1 to smithi07685715-39 2024-09-21T09:41:52.754 INFO:tasks.rados.rados.0.smithi076.stdout:959: snap_remove snap 118 2024-09-21T09:41:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:52 smithi191 ceph-mon[33301]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:41:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:52 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:53.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:52 smithi076 ceph-mon[41325]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:41:53.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:52 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:52 smithi076 ceph-mon[29313]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:41:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:52 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:41:53.551 INFO:tasks.rados.rados.0.smithi076.stdout:959: done (2 left) 2024-09-21T09:41:53.552 INFO:tasks.rados.rados.0.smithi076.stdout:960: write oid 19 current snap is 121 2024-09-21T09:41:53.552 INFO:tasks.rados.rados.0.smithi076.stdout:960: seq_num 391 ranges {571537=556362,1630577=597465} 2024-09-21T09:41:53.558 INFO:tasks.rados.rados.0.smithi076.stdout:960: writing smithi07685715-19 from 571537 to 1127899 tid 1 2024-09-21T09:41:53.564 INFO:tasks.rados.rados.0.smithi076.stdout:960: writing smithi07685715-19 from 1630577 to 2228042 tid 2 2024-09-21T09:41:53.564 INFO:tasks.rados.rados.0.smithi076.stdout:961: snap_remove snap 119 2024-09-21T09:41:53.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:41:53] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.51.0" 2024-09-21T09:41:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:53 smithi191 ceph-mon[33301]: pgmap v161: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-21T09:41:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:53 smithi191 ceph-mon[33301]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:41:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:53 smithi191 ceph-mon[33301]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:41:54.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-mon[29313]: pgmap v161: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-21T09:41:54.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-mon[29313]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:41:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-mon[29313]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:41:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-mon[41325]: pgmap v161: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-21T09:41:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-mon[41325]: osdmap e313: 8 total, 8 up, 8 in 2024-09-21T09:41:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:53 smithi076 ceph-mon[41325]: osdmap e314: 8 total, 8 up, 8 in 2024-09-21T09:41:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:961: done (3 left) 2024-09-21T09:41:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:962: rollback oid 28 current snap is 121 2024-09-21T09:41:54.565 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 28 to 101 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:958: finishing rollback tid 0 to smithi07685715-32 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:958: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:958: finishing rollback tid 2 to smithi07685715-32 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 358 (ObjNum 360 snap 110 seq_num 360) dirty exists 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: finishing write tid 2 to smithi07685715-39 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: finishing write tid 3 to smithi07685715-39 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: finishing write tid 4 to smithi07685715-39 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: oid 39 updating version 0 to 739 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: oid 39 version 739 is already newer than 738 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: oid 39 version 739 is already newer than 737 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 739 (ObjNum 390 snap 121 seq_num 390) dirty exists 2024-09-21T09:41:54.566 INFO:tasks.rados.rados.0.smithi076.stdout:957: left oid 39 (ObjNum 390 snap 121 seq_num 390) 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: finishing write tid 1 to smithi07685715-19 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: finishing write tid 2 to smithi07685715-19 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: finishing write tid 3 to smithi07685715-19 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: finishing write tid 4 to smithi07685715-19 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: oid 19 updating version 0 to 363 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: oid 19 updating version 363 to 364 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: oid 19 version 364 is already newer than 362 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 364 (ObjNum 391 snap 121 seq_num 391) dirty exists 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: left oid 19 (ObjNum 391 snap 121 seq_num 391) 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:957: done (3 left) 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:958: done (2 left) 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:960: done (1 left) 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:963: write oid 17 current snap is 121 2024-09-21T09:41:54.567 INFO:tasks.rados.rados.0.smithi076.stdout:963: seq_num 392 ranges {588344=631611,1689059=741123} 2024-09-21T09:41:54.572 INFO:tasks.rados.rados.0.smithi076.stdout:963: writing smithi07685715-17 from 588344 to 1219955 tid 1 2024-09-21T09:41:54.579 INFO:tasks.rados.rados.0.smithi076.stdout:963: writing smithi07685715-17 from 1689059 to 2430182 tid 2 2024-09-21T09:41:54.579 INFO:tasks.rados.rados.0.smithi076.stdout:964: delete oid 5 current snap is 121 2024-09-21T09:41:54.579 INFO:tasks.rados.rados.0.smithi076.stdout:962: finishing rollback tid 0 to smithi07685715-28 2024-09-21T09:41:54.580 INFO:tasks.rados.rados.0.smithi076.stdout:962: finishing rollback tid 1 to smithi07685715-28 2024-09-21T09:41:54.580 INFO:tasks.rados.rados.0.smithi076.stdout:962: finishing rollback tid 2 to smithi07685715-28 2024-09-21T09:41:54.580 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 805 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-09-21T09:41:54.580 INFO:tasks.rados.rados.0.smithi076.stdout:963: finishing write tid 1 to smithi07685715-17 2024-09-21T09:41:54.581 INFO:tasks.rados.rados.0.smithi076.stdout:962: done (2 left) 2024-09-21T09:41:54.581 INFO:tasks.rados.rados.0.smithi076.stdout:964: done (1 left) 2024-09-21T09:41:54.581 INFO:tasks.rados.rados.0.smithi076.stdout:965: write oid 11 current snap is 121 2024-09-21T09:41:54.581 INFO:tasks.rados.rados.0.smithi076.stdout:965: seq_num 394 ranges {621958=782109,2089670=532288} 2024-09-21T09:41:54.588 INFO:tasks.rados.rados.0.smithi076.stdout:965: writing smithi07685715-11 from 621958 to 1404067 tid 1 2024-09-21T09:41:54.595 INFO:tasks.rados.rados.0.smithi076.stdout:965: writing smithi07685715-11 from 2089670 to 2621958 tid 2 2024-09-21T09:41:54.595 INFO:tasks.rados.rados.0.smithi076.stdout:966: snap_remove snap 120 2024-09-21T09:41:55.565 INFO:tasks.rados.rados.0.smithi076.stdout:966: done (2 left) 2024-09-21T09:41:55.565 INFO:tasks.rados.rados.0.smithi076.stdout:967: snap_create 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: finishing write tid 2 to smithi07685715-17 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: finishing write tid 3 to smithi07685715-17 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: finishing write tid 4 to smithi07685715-17 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: oid 17 updating version 0 to 482 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: oid 17 updating version 482 to 483 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: oid 17 updating version 483 to 484 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 484 (ObjNum 392 snap 121 seq_num 392) dirty exists 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: left oid 17 (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:965: finishing write tid 1 to smithi07685715-11 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:965: finishing write tid 2 to smithi07685715-11 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:965: finishing write tid 3 to smithi07685715-11 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:965: finishing write tid 4 to smithi07685715-11 2024-09-21T09:41:55.566 INFO:tasks.rados.rados.0.smithi076.stdout:965: oid 11 updating version 0 to 755 2024-09-21T09:41:55.567 INFO:tasks.rados.rados.0.smithi076.stdout:965: oid 11 version 755 is already newer than 754 2024-09-21T09:41:55.567 INFO:tasks.rados.rados.0.smithi076.stdout:965: oid 11 updating version 755 to 756 2024-09-21T09:41:55.567 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 756 (ObjNum 394 snap 121 seq_num 394) dirty exists 2024-09-21T09:41:55.567 INFO:tasks.rados.rados.0.smithi076.stdout:965: left oid 11 (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:41:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:55 smithi191 ceph-mon[33301]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:41:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:55 smithi191 ceph-mon[33301]: pgmap v165: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-09-21T09:41:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:55 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:55 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:55.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[29313]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[29313]: pgmap v165: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[41325]: osdmap e315: 8 total, 8 up, 8 in 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[41325]: pgmap v165: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:55 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:963: done (2 left) 2024-09-21T09:41:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:965: done (1 left) 2024-09-21T09:41:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:967: done (0 left) 2024-09-21T09:41:56.566 INFO:tasks.rados.rados.0.smithi076.stdout:968: snap_create 2024-09-21T09:41:56.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:56 smithi191 ceph-mon[33301]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:41:56.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:56 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:56.665 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:56 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:56.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:56 smithi076 ceph-mon[29313]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:41:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:56 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:56 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:56 smithi076 ceph-mon[41325]: osdmap e316: 8 total, 8 up, 8 in 2024-09-21T09:41:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:56 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:56 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:41:57.567 INFO:tasks.rados.rados.0.smithi076.stdout:968: done (0 left) 2024-09-21T09:41:57.567 INFO:tasks.rados.rados.0.smithi076.stdout:969: write oid 31 current snap is 123 2024-09-21T09:41:57.567 INFO:tasks.rados.rados.0.smithi076.stdout:969: seq_num 395 ranges {638765=457358,1748152=604803} 2024-09-21T09:41:57.573 INFO:tasks.rados.rados.0.smithi076.stdout:969: writing smithi07685715-31 from 638765 to 1096123 tid 1 2024-09-21T09:41:57.581 INFO:tasks.rados.rados.0.smithi076.stdout:969: writing smithi07685715-31 from 1748152 to 2352955 tid 2 2024-09-21T09:41:57.581 INFO:tasks.rados.rados.0.smithi076.stdout:970: write oid 7 current snap is 123 2024-09-21T09:41:57.581 INFO:tasks.rados.rados.0.smithi076.stdout:969: finishing write tid 1 to smithi07685715-31 2024-09-21T09:41:57.581 INFO:tasks.rados.rados.0.smithi076.stdout:970: seq_num 396 ranges {655572=648960,1922987=732585} 2024-09-21T09:41:57.587 INFO:tasks.rados.rados.0.smithi076.stdout:970: writing smithi07685715-7 from 655572 to 1304532 tid 1 2024-09-21T09:41:57.595 INFO:tasks.rados.rados.0.smithi076.stdout:970: writing smithi07685715-7 from 1922987 to 2655572 tid 2 2024-09-21T09:41:57.595 INFO:tasks.rados.rados.0.smithi076.stdout:971: write oid 45 current snap is 123 2024-09-21T09:41:57.595 INFO:tasks.rados.rados.0.smithi076.stdout:969: finishing write tid 2 to smithi07685715-31 2024-09-21T09:41:57.595 INFO:tasks.rados.rados.0.smithi076.stdout:969: finishing write tid 3 to smithi07685715-31 2024-09-21T09:41:57.595 INFO:tasks.rados.rados.0.smithi076.stdout:969: finishing write tid 4 to smithi07685715-31 2024-09-21T09:41:57.595 INFO:tasks.rados.rados.0.smithi076.stdout:969: oid 31 updating version 0 to 374 2024-09-21T09:41:57.596 INFO:tasks.rados.rados.0.smithi076.stdout:969: oid 31 updating version 374 to 375 2024-09-21T09:41:57.596 INFO:tasks.rados.rados.0.smithi076.stdout:969: oid 31 version 375 is already newer than 373 2024-09-21T09:41:57.596 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 375 (ObjNum 395 snap 123 seq_num 395) dirty exists 2024-09-21T09:41:57.596 INFO:tasks.rados.rados.0.smithi076.stdout:969: left oid 31 (ObjNum 395 snap 123 seq_num 395) 2024-09-21T09:41:57.596 INFO:tasks.rados.rados.0.smithi076.stdout:970: finishing write tid 1 to smithi07685715-7 2024-09-21T09:41:57.596 INFO:tasks.rados.rados.0.smithi076.stdout:971: seq_num 397 ranges {672379=724209,1865116=608472} 2024-09-21T09:41:57.603 INFO:tasks.rados.rados.0.smithi076.stdout:971: writing smithi07685715-45 from 672379 to 1396588 tid 1 2024-09-21T09:41:57.611 INFO:tasks.rados.rados.0.smithi076.stdout:971: writing smithi07685715-45 from 1865116 to 2473588 tid 2 2024-09-21T09:41:57.611 INFO:tasks.rados.rados.0.smithi076.stdout:969: done (2 left) 2024-09-21T09:41:57.611 INFO:tasks.rados.rados.0.smithi076.stdout:972: read oid 34 snap -1 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:972: expect deleted 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:973: write oid 36 current snap is 123 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: finishing write tid 2 to smithi07685715-7 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: finishing write tid 3 to smithi07685715-7 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: finishing write tid 4 to smithi07685715-7 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: oid 7 updating version 0 to 487 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: oid 7 updating version 487 to 488 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: oid 7 version 488 is already newer than 486 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 488 (ObjNum 396 snap 123 seq_num 396) dirty exists 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:970: left oid 7 (ObjNum 396 snap 123 seq_num 396) 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:971: finishing write tid 1 to smithi07685715-45 2024-09-21T09:41:57.612 INFO:tasks.rados.rados.0.smithi076.stdout:973: seq_num 398 ranges {689186=799458,1923598=752130} 2024-09-21T09:41:57.617 INFO:tasks.rados.rados.0.smithi076.stdout:973: writing smithi07685715-36 from 689186 to 1488644 tid 1 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:973: writing smithi07685715-36 from 1923598 to 2675728 tid 2 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:970: done (3 left) 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:974: read oid 11 snap -1 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:974: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:975: read oid 4 snap -1 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:975: expect (ObjNum 313 snap 96 seq_num 313) 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:976: rollback oid 8 current snap is 123 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 101 2024-09-21T09:41:57.624 INFO:tasks.rados.rados.0.smithi076.stdout:971: finishing write tid 2 to smithi07685715-45 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:977: delete oid 39 current snap is 123 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:971: finishing write tid 3 to smithi07685715-45 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:971: finishing write tid 4 to smithi07685715-45 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:971: oid 45 updating version 0 to 222 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:971: oid 45 version 222 is already newer than 220 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:971: oid 45 version 222 is already newer than 221 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 222 (ObjNum 397 snap 123 seq_num 397) dirty exists 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:971: left oid 45 (ObjNum 397 snap 123 seq_num 397) 2024-09-21T09:41:57.625 INFO:tasks.rados.rados.0.smithi076.stdout:973: finishing write tid 1 to smithi07685715-36 2024-09-21T09:41:57.630 INFO:tasks.rados.rados.0.smithi076.stdout:971: done (6 left) 2024-09-21T09:41:57.630 INFO:tasks.rados.rados.0.smithi076.stdout:972: done (5 left) 2024-09-21T09:41:57.630 INFO:tasks.rados.rados.0.smithi076.stdout:977: done (4 left) 2024-09-21T09:41:57.631 INFO:tasks.rados.rados.0.smithi076.stdout:978: read oid 26 snap -1 2024-09-21T09:41:57.631 INFO:tasks.rados.rados.0.smithi076.stdout:978: expect deleted 2024-09-21T09:41:57.631 INFO:tasks.rados.rados.0.smithi076.stdout:979: rollback oid 12 current snap is 123 2024-09-21T09:41:57.631 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 101 2024-09-21T09:41:57.631 INFO:tasks.rados.rados.0.smithi076.stdout:974: expect (ObjNum 394 snap 121 seq_num 394) 2024-09-21T09:41:57.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:57 smithi191 ceph-mon[33301]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:41:57.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:57 smithi191 ceph-mon[33301]: pgmap v168: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:41:57.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:57 smithi191 ceph-mon[33301]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:41:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:979: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:41:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:979: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:41:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:979: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:41:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 411 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-09-21T09:41:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:973: finishing write tid 2 to smithi07685715-36 2024-09-21T09:41:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:973: finishing write tid 3 to smithi07685715-36 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:973: finishing write tid 4 to smithi07685715-36 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:973: oid 36 updating version 0 to 385 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:973: oid 36 version 385 is already newer than 383 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:973: oid 36 version 385 is already newer than 384 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 385 (ObjNum 398 snap 123 seq_num 398) dirty exists 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:973: left oid 36 (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:41:57.717 INFO:tasks.rados.rados.0.smithi076.stdout:975: expect (ObjNum 313 snap 96 seq_num 313) 2024-09-21T09:41:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:976: finishing rollback tid 0 to smithi07685715-8 2024-09-21T09:41:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:976: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:976: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 811 (ObjNum 207 snap 60 seq_num 207) dirty exists 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:973: done (5 left) 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:974: done (4 left) 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:975: done (3 left) 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:976: done (2 left) 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:978: done (1 left) 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:979: done (0 left) 2024-09-21T09:41:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:980: snap_remove snap 113 2024-09-21T09:41:57.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:57 smithi076 ceph-mon[29313]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:41:57.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:57 smithi076 ceph-mon[29313]: pgmap v168: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:41:57.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:57 smithi076 ceph-mon[29313]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:41:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:57 smithi076 ceph-mon[41325]: osdmap e317: 8 total, 8 up, 8 in 2024-09-21T09:41:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:57 smithi076 ceph-mon[41325]: pgmap v168: 89 pgs: 89 active+clean; 173 MiB data, 651 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:41:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:57 smithi076 ceph-mon[41325]: osdmap e318: 8 total, 8 up, 8 in 2024-09-21T09:41:58.579 INFO:tasks.rados.rados.0.smithi076.stdout:980: done (0 left) 2024-09-21T09:41:58.579 INFO:tasks.rados.rados.0.smithi076.stdout:981: snap_remove snap 121 2024-09-21T09:41:59.586 INFO:tasks.rados.rados.0.smithi076.stdout:981: done (0 left) 2024-09-21T09:41:59.586 INFO:tasks.rados.rados.0.smithi076.stdout:982: write oid 15 current snap is 123 2024-09-21T09:41:59.586 INFO:tasks.rados.rados.0.smithi076.stdout:982: seq_num 400 ranges {722800=549956,2040562=682238} 2024-09-21T09:41:59.592 INFO:tasks.rados.rados.0.smithi076.stdout:982: writing smithi07685715-15 from 722800 to 1272756 tid 1 2024-09-21T09:41:59.601 INFO:tasks.rados.rados.0.smithi076.stdout:982: writing smithi07685715-15 from 2040562 to 2722800 tid 2 2024-09-21T09:41:59.601 INFO:tasks.rados.rados.0.smithi076.stdout:983: snap_remove snap 122 2024-09-21T09:41:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:59 smithi191 ceph-mon[33301]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:41:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:41:59 smithi191 ceph-mon[33301]: pgmap v171: 89 pgs: 89 active+clean; 185 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:41:59.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:59 smithi076 ceph-mon[29313]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:41:59.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:41:59 smithi076 ceph-mon[29313]: pgmap v171: 89 pgs: 89 active+clean; 185 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:41:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:59 smithi076 ceph-mon[41325]: osdmap e319: 8 total, 8 up, 8 in 2024-09-21T09:41:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:41:59 smithi076 ceph-mon[41325]: pgmap v171: 89 pgs: 89 active+clean; 185 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:42:00.587 INFO:tasks.rados.rados.0.smithi076.stdout:983: done (1 left) 2024-09-21T09:42:00.587 INFO:tasks.rados.rados.0.smithi076.stdout:984: snap_remove snap 101 2024-09-21T09:42:00.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:00 smithi191 ceph-mon[33301]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:42:00.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:00 smithi076 ceph-mon[29313]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:42:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:00 smithi076 ceph-mon[41325]: osdmap e320: 8 total, 8 up, 8 in 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:984: done (1 left) 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:985: delete oid 35 current snap is 123 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: finishing write tid 1 to smithi07685715-15 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: finishing write tid 2 to smithi07685715-15 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: finishing write tid 3 to smithi07685715-15 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: finishing write tid 4 to smithi07685715-15 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: oid 15 updating version 0 to 229 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: oid 15 updating version 229 to 230 2024-09-21T09:42:01.591 INFO:tasks.rados.rados.0.smithi076.stdout:982: oid 15 version 230 is already newer than 226 2024-09-21T09:42:01.592 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 230 (ObjNum 400 snap 123 seq_num 400) dirty exists 2024-09-21T09:42:01.592 INFO:tasks.rados.rados.0.smithi076.stdout:982: left oid 15 (ObjNum 400 snap 123 seq_num 400) 2024-09-21T09:42:01.598 INFO:tasks.rados.rados.0.smithi076.stdout:982: done (1 left) 2024-09-21T09:42:01.599 INFO:tasks.rados.rados.0.smithi076.stdout:985: done (0 left) 2024-09-21T09:42:01.599 INFO:tasks.rados.rados.0.smithi076.stdout:986: snap_remove snap 107 2024-09-21T09:42:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:01 smithi191 ceph-mon[33301]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:42:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:01 smithi191 ceph-mon[33301]: pgmap v174: 89 pgs: 89 active+clean; 185 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:42:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:01 smithi076 ceph-mon[29313]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:42:01.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:01 smithi076 ceph-mon[29313]: pgmap v174: 89 pgs: 89 active+clean; 185 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:42:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:01 smithi076 ceph-mon[41325]: osdmap e321: 8 total, 8 up, 8 in 2024-09-21T09:42:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:01 smithi076 ceph-mon[41325]: pgmap v174: 89 pgs: 89 active+clean; 185 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:42:02.605 INFO:tasks.rados.rados.0.smithi076.stdout:986: done (0 left) 2024-09-21T09:42:02.606 INFO:tasks.rados.rados.0.smithi076.stdout:987: snap_remove snap 105 2024-09-21T09:42:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:02 smithi191 ceph-mon[33301]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:42:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:02 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:02 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:02 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:02.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:02 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:02.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:02 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[29313]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:42:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:02.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[41325]: osdmap e322: 8 total, 8 up, 8 in 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:02 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:987: done (0 left) 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:988: rollback oid 30 current snap is 123 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:988: done (0 left) 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:989: read oid 17 snap -1 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:989: expect (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:990: rollback oid 21 current snap is 123 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:990: done (1 left) 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:991: read oid 22 snap -1 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:991: expect deleted 2024-09-21T09:42:03.553 INFO:tasks.rados.rados.0.smithi076.stdout:992: read oid 47 snap -1 2024-09-21T09:42:03.554 INFO:tasks.rados.rados.0.smithi076.stdout:992: expect (ObjNum 359 snap 110 seq_num 359) 2024-09-21T09:42:03.554 INFO:tasks.rados.rados.0.smithi076.stdout:993: rollback oid 27 current snap is 123 2024-09-21T09:42:03.554 INFO:tasks.rados.rados.0.smithi076.stdout:993: done (3 left) 2024-09-21T09:42:03.554 INFO:tasks.rados.rados.0.smithi076.stdout:994: write oid 32 current snap is 123 2024-09-21T09:42:03.554 INFO:tasks.rados.rados.0.smithi076.stdout:994: seq_num 402 ranges {756414=700454,2041173=715241} 2024-09-21T09:42:03.561 INFO:tasks.rados.rados.0.smithi076.stdout:994: writing smithi07685715-32 from 756414 to 1456868 tid 1 2024-09-21T09:42:03.570 INFO:tasks.rados.rados.0.smithi076.stdout:994: writing smithi07685715-32 from 2041173 to 2756414 tid 2 2024-09-21T09:42:03.570 INFO:tasks.rados.rados.0.smithi076.stdout:995: snap_create 2024-09-21T09:42:03.570 INFO:tasks.rados.rados.0.smithi076.stdout:992: expect (ObjNum 359 snap 110 seq_num 359) 2024-09-21T09:42:03.609 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:42:03] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.51.0" 2024-09-21T09:42:03.670 INFO:tasks.rados.rados.0.smithi076.stdout:989: expect (ObjNum 392 snap 121 seq_num 392) 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: finishing write tid 1 to smithi07685715-32 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: finishing write tid 2 to smithi07685715-32 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: finishing write tid 3 to smithi07685715-32 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: finishing write tid 4 to smithi07685715-32 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: oid 32 updating version 0 to 393 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: oid 32 updating version 393 to 394 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: oid 32 version 394 is already newer than 392 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 394 (ObjNum 402 snap 123 seq_num 402) dirty exists 2024-09-21T09:42:03.751 INFO:tasks.rados.rados.0.smithi076.stdout:994: left oid 32 (ObjNum 402 snap 123 seq_num 402) 2024-09-21T09:42:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:42:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:42:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:42:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: Upgrade: Finalizing container_image settings 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: pgmap v177: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 301 KiB/s wr, 1 op/s 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:42:03.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:42:03.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: Upgrade: Complete! 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:03 smithi191 ceph-mon[33301]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: Upgrade: Finalizing container_image settings 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: pgmap v177: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 301 KiB/s wr, 1 op/s 2024-09-21T09:42:03.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: osdmap e323: 8 total, 8 up, 8 in 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: Upgrade: Finalizing container_image settings 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:42:03.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: pgmap v177: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 301 KiB/s wr, 1 op/s 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:42:03.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:42:03.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:42:03.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-21T09:42:03.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: Upgrade: Complete! 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:03.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[41325]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: Upgrade: Complete! 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:03.884 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:03 smithi076 ceph-mon[29313]: osdmap e324: 8 total, 8 up, 8 in 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:989: done (4 left) 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:991: done (3 left) 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:992: done (2 left) 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:994: done (1 left) 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:995: done (0 left) 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:996: read oid 44 snap 123 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:996: expect (ObjNum 387 snap 120 seq_num 387) 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:997: rollback oid 49 current snap is 124 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 123 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:998: write oid 5 current snap is 124 2024-09-21T09:42:04.553 INFO:tasks.rados.rados.0.smithi076.stdout:998: seq_num 403 ranges {773221=492056,1816008=503126} 2024-09-21T09:42:04.559 INFO:tasks.rados.rados.0.smithi076.stdout:998: writing smithi07685715-5 from 773221 to 1265277 tid 1 2024-09-21T09:42:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:998: writing smithi07685715-5 from 1816008 to 2319134 tid 2 2024-09-21T09:42:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:999: read oid 13 snap -1 2024-09-21T09:42:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:999: expect deleted 2024-09-21T09:42:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:1000: write oid 34 current snap is 124 2024-09-21T09:42:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:997: finishing rollback tid 0 to smithi07685715-49 2024-09-21T09:42:04.566 INFO:tasks.rados.rados.0.smithi076.stdout:1000: seq_num 404 ranges {790028=567305,1874490=763137} 2024-09-21T09:42:04.573 INFO:tasks.rados.rados.0.smithi076.stdout:1000: writing smithi07685715-34 from 790028 to 1357333 tid 1 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:1000: writing smithi07685715-34 from 1874490 to 2637627 tid 2 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:1001: read oid 45 snap 123 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:1001: expect (ObjNum 397 snap 123 seq_num 397) 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:1002: snap_create 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:997: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:997: finishing rollback tid 2 to smithi07685715-49 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 857 (ObjNum 343 snap 105 seq_num 343) dirty exists 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:998: finishing write tid 1 to smithi07685715-5 2024-09-21T09:42:04.581 INFO:tasks.rados.rados.0.smithi076.stdout:996: expect (ObjNum 387 snap 120 seq_num 387) 2024-09-21T09:42:04.749 INFO:tasks.rados.rados.0.smithi076.stdout:998: finishing write tid 2 to smithi07685715-5 2024-09-21T09:42:04.749 INFO:tasks.rados.rados.0.smithi076.stdout:998: finishing write tid 3 to smithi07685715-5 2024-09-21T09:42:04.749 INFO:tasks.rados.rados.0.smithi076.stdout:998: finishing write tid 4 to smithi07685715-5 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:998: oid 5 updating version 0 to 794 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:998: oid 5 updating version 794 to 795 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:998: oid 5 updating version 795 to 796 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 796 (ObjNum 403 snap 124 seq_num 403) dirty exists 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:998: left oid 5 (ObjNum 403 snap 124 seq_num 403) 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: finishing write tid 1 to smithi07685715-34 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: finishing write tid 2 to smithi07685715-34 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: finishing write tid 3 to smithi07685715-34 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: finishing write tid 4 to smithi07685715-34 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: oid 34 updating version 0 to 188 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: oid 34 updating version 188 to 189 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: oid 34 updating version 189 to 190 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 190 (ObjNum 404 snap 124 seq_num 404) dirty exists 2024-09-21T09:42:04.750 INFO:tasks.rados.rados.0.smithi076.stdout:1000: left oid 34 (ObjNum 404 snap 124 seq_num 404) 2024-09-21T09:42:04.751 INFO:tasks.rados.rados.0.smithi076.stdout:1001: expect (ObjNum 397 snap 123 seq_num 397) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:996: done (6 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:997: done (5 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:998: done (4 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:999: done (3 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:1000: done (2 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:1001: done (1 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:1002: done (0 left) 2024-09-21T09:42:05.563 INFO:tasks.rados.rados.0.smithi076.stdout:1003: snap_create 2024-09-21T09:42:05.845 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:05 smithi076 ceph-mon[29313]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:42:05.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:05 smithi076 ceph-mon[29313]: pgmap v180: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 301 KiB/s wr, 1 op/s 2024-09-21T09:42:05.846 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:05 smithi076 ceph-mon[41325]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:42:05.846 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:05 smithi076 ceph-mon[41325]: pgmap v180: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 301 KiB/s wr, 1 op/s 2024-09-21T09:42:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:05 smithi191 ceph-mon[33301]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:42:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:05 smithi191 ceph-mon[33301]: pgmap v180: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 301 KiB/s wr, 1 op/s 2024-09-21T09:42:06.565 INFO:tasks.rados.rados.0.smithi076.stdout:1003: done (0 left) 2024-09-21T09:42:06.565 INFO:tasks.rados.rados.0.smithi076.stdout:1004: rollback oid 43 current snap is 126 2024-09-21T09:42:06.565 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 124 2024-09-21T09:42:06.565 INFO:tasks.rados.rados.0.smithi076.stdout:1005: rollback oid 11 current snap is 126 2024-09-21T09:42:06.565 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 125 2024-09-21T09:42:06.566 INFO:tasks.rados.rados.0.smithi076.stdout:1006: snap_create 2024-09-21T09:42:06.568 INFO:tasks.rados.rados.0.smithi076.stdout:1005: finishing rollback tid 0 to smithi07685715-11 2024-09-21T09:42:06.569 INFO:tasks.rados.rados.0.smithi076.stdout:1005: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:42:06.569 INFO:tasks.rados.rados.0.smithi076.stdout:1005: finishing rollback tid 2 to smithi07685715-11 2024-09-21T09:42:06.569 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 800 (ObjNum 394 snap 121 seq_num 394) dirty exists 2024-09-21T09:42:06.572 INFO:tasks.rados.rados.0.smithi076.stdout:1004: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:42:06.572 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:42:06.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:06 smithi191 ceph-mon[33301]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:42:06.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:06 smithi076 ceph-mon[29313]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:42:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:06 smithi076 ceph-mon[41325]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:42:07.572 INFO:tasks.rados.rados.0.smithi076.stdout:1004: done (2 left) 2024-09-21T09:42:07.573 INFO:tasks.rados.rados.0.smithi076.stdout:1005: done (1 left) 2024-09-21T09:42:07.573 INFO:tasks.rados.rados.0.smithi076.stdout:1006: done (0 left) 2024-09-21T09:42:07.573 INFO:tasks.rados.rados.0.smithi076.stdout:1007: delete oid 21 current snap is 127 2024-09-21T09:42:07.575 INFO:tasks.rados.rados.0.smithi076.stdout:1007: done (0 left) 2024-09-21T09:42:07.575 INFO:tasks.rados.rados.0.smithi076.stdout:1008: snap_remove snap 124 2024-09-21T09:42:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:07 smithi191 ceph-mon[33301]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:42:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:07 smithi191 ceph-mon[33301]: pgmap v183: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:07.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:07 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:42:07.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:07 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:07.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[29313]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:42:07.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[29313]: pgmap v183: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:42:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[41325]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:42:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[41325]: pgmap v183: 89 pgs: 89 active+clean; 145 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:42:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:07 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:08.553 INFO:tasks.rados.rados.0.smithi076.stdout:1008: done (0 left) 2024-09-21T09:42:08.554 INFO:tasks.rados.rados.0.smithi076.stdout:1009: read oid 31 snap -1 2024-09-21T09:42:08.554 INFO:tasks.rados.rados.0.smithi076.stdout:1009: expect (ObjNum 395 snap 123 seq_num 395) 2024-09-21T09:42:08.554 INFO:tasks.rados.rados.0.smithi076.stdout:1010: snap_create 2024-09-21T09:42:08.566 INFO:tasks.rados.rados.0.smithi076.stdout:1009: expect (ObjNum 395 snap 123 seq_num 395) 2024-09-21T09:42:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:08 smithi191 ceph-mon[33301]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:42:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:08 smithi191 ceph-mon[33301]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:42:08.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:08 smithi076 ceph-mon[29313]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:42:08.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:08 smithi076 ceph-mon[29313]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:42:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:08 smithi076 ceph-mon[41325]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:42:08.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:08 smithi076 ceph-mon[41325]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:42:09.558 INFO:tasks.rados.rados.0.smithi076.stdout:1009: done (1 left) 2024-09-21T09:42:09.558 INFO:tasks.rados.rados.0.smithi076.stdout:1010: done (0 left) 2024-09-21T09:42:09.558 INFO:tasks.rados.rados.0.smithi076.stdout:1011: read oid 29 snap 125 2024-09-21T09:42:09.558 INFO:tasks.rados.rados.0.smithi076.stdout:1011: expect deleted 2024-09-21T09:42:09.558 INFO:tasks.rados.rados.0.smithi076.stdout:1012: delete oid 9 current snap is 128 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1011: done (1 left) 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1012: done (0 left) 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1013: read oid 39 snap 127 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1013: expect deleted 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1014: read oid 16 snap -1 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1014: expect (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1015: rollback oid 14 current snap is 128 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 126 2024-09-21T09:42:09.562 INFO:tasks.rados.rados.0.smithi076.stdout:1016: delete oid 48 current snap is 128 2024-09-21T09:42:09.564 INFO:tasks.rados.rados.0.smithi076.stdout:1015: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:42:09.564 INFO:tasks.rados.rados.0.smithi076.stdout:1015: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:42:09.564 INFO:tasks.rados.rados.0.smithi076.stdout:1015: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:42:09.565 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 518 (ObjNum 321 snap 98 seq_num 321) dirty exists 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1013: done (3 left) 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1015: done (2 left) 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1016: done (1 left) 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1017: read oid 21 snap -1 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1017: expect deleted 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1018: write oid 20 current snap is 128 2024-09-21T09:42:09.567 INFO:tasks.rados.rados.0.smithi076.stdout:1018: seq_num 408 ranges {457256=468301,1592065=770475,2854581=2675} 2024-09-21T09:42:09.572 INFO:tasks.rados.rados.0.smithi076.stdout:1018: writing smithi07685715-20 from 457256 to 925557 tid 1 2024-09-21T09:42:09.580 INFO:tasks.rados.rados.0.smithi076.stdout:1018: writing smithi07685715-20 from 1592065 to 2362540 tid 2 2024-09-21T09:42:09.583 INFO:tasks.rados.rados.0.smithi076.stdout:1018: writing smithi07685715-20 from 2854581 to 2857256 tid 3 2024-09-21T09:42:09.583 INFO:tasks.rados.rados.0.smithi076.stdout:1019: rollback oid 17 current snap is 128 2024-09-21T09:42:09.583 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 126 2024-09-21T09:42:09.583 INFO:tasks.rados.rados.0.smithi076.stdout:1014: expect (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: finishing write tid 1 to smithi07685715-20 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: finishing write tid 2 to smithi07685715-20 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: finishing write tid 3 to smithi07685715-20 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: finishing write tid 4 to smithi07685715-20 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1019: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: finishing write tid 5 to smithi07685715-20 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: oid 20 updating version 0 to 289 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: oid 20 updating version 289 to 292 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: oid 20 version 292 is already newer than 291 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: oid 20 version 292 is already newer than 290 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 292 (ObjNum 408 snap 128 seq_num 408) dirty exists 2024-09-21T09:42:09.708 INFO:tasks.rados.rados.0.smithi076.stdout:1018: left oid 20 (ObjNum 408 snap 128 seq_num 408) 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1019: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1019: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 522 (ObjNum 392 snap 121 seq_num 392) dirty exists 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1014: done (3 left) 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1017: done (2 left) 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1018: done (1 left) 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1019: done (0 left) 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1020: read oid 8 snap 126 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1020: expect (ObjNum 207 snap 60 seq_num 207) 2024-09-21T09:42:09.709 INFO:tasks.rados.rados.0.smithi076.stdout:1021: delete oid 45 current snap is 128 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:1021: done (1 left) 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:1022: rollback oid 25 current snap is 128 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 127 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:1023: rollback oid 27 current snap is 128 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 123 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:1024: write oid 44 current snap is 128 2024-09-21T09:42:09.710 INFO:tasks.rados.rados.0.smithi076.stdout:1024: seq_num 410 ranges {490870=618799,1592676=774144,2793074=97796} 2024-09-21T09:42:09.715 INFO:tasks.rados.rados.0.smithi076.stdout:1024: writing smithi07685715-44 from 490870 to 1109669 tid 1 2024-09-21T09:42:09.722 INFO:tasks.rados.rados.0.smithi076.stdout:1024: writing smithi07685715-44 from 1592676 to 2366820 tid 2 2024-09-21T09:42:09.725 INFO:tasks.rados.rados.0.smithi076.stdout:1024: writing smithi07685715-44 from 2793074 to 2890870 tid 3 2024-09-21T09:42:09.725 INFO:tasks.rados.rados.0.smithi076.stdout:1025: read oid 14 snap -1 2024-09-21T09:42:09.726 INFO:tasks.rados.rados.0.smithi076.stdout:1025: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:42:09.726 INFO:tasks.rados.rados.0.smithi076.stdout:1026: snap_remove snap 125 2024-09-21T09:42:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:09 smithi191 ceph-mon[33301]: pgmap v186: 89 pgs: 89 active+clean; 93 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 914 KiB/s wr, 13 op/s 2024-09-21T09:42:09.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:09 smithi191 ceph-mon[33301]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:42:09.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:09 smithi076 ceph-mon[29313]: pgmap v186: 89 pgs: 89 active+clean; 93 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 914 KiB/s wr, 13 op/s 2024-09-21T09:42:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:09 smithi076 ceph-mon[29313]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:42:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:09 smithi076 ceph-mon[41325]: pgmap v186: 89 pgs: 89 active+clean; 93 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 914 KiB/s wr, 13 op/s 2024-09-21T09:42:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:09 smithi076 ceph-mon[41325]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:42:10.596 INFO:tasks.rados.rados.0.smithi076.stdout:1026: done (5 left) 2024-09-21T09:42:10.596 INFO:tasks.rados.rados.0.smithi076.stdout:1027: write oid 4 current snap is 128 2024-09-21T09:42:10.596 INFO:tasks.rados.rados.0.smithi076.stdout:1027: seq_num 411 ranges {507677=410401,1367511=517802,2420497=487180} 2024-09-21T09:42:10.601 INFO:tasks.rados.rados.0.smithi076.stdout:1027: writing smithi07685715-4 from 507677 to 918078 tid 1 2024-09-21T09:42:10.606 INFO:tasks.rados.rados.0.smithi076.stdout:1027: writing smithi07685715-4 from 1367511 to 1885313 tid 2 2024-09-21T09:42:10.612 INFO:tasks.rados.rados.0.smithi076.stdout:1027: writing smithi07685715-4 from 2420497 to 2907677 tid 3 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1028: read oid 16 snap -1 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1028: expect (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1029: delete oid 33 current snap is 128 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1023: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1022: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1022: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1022: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 234 (ObjNum 361 snap 110 seq_num 361) dirty exists 2024-09-21T09:42:10.613 INFO:tasks.rados.rados.0.smithi076.stdout:1020: expect (ObjNum 207 snap 60 seq_num 207) 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: finishing write tid 1 to smithi07685715-44 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: finishing write tid 2 to smithi07685715-44 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: finishing write tid 3 to smithi07685715-44 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: finishing write tid 4 to smithi07685715-44 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: finishing write tid 5 to smithi07685715-44 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: oid 44 updating version 0 to 206 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: oid 44 version 206 is already newer than 203 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: oid 44 version 206 is already newer than 205 2024-09-21T09:42:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1024: oid 44 version 206 is already newer than 204 2024-09-21T09:42:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 206 (ObjNum 410 snap 128 seq_num 410) dirty exists 2024-09-21T09:42:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:1024: left oid 44 (ObjNum 410 snap 128 seq_num 410) 2024-09-21T09:42:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:1025: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:42:10.917 INFO:tasks.rados.rados.0.smithi076.stdout:1027: finishing write tid 1 to smithi07685715-4 2024-09-21T09:42:10.917 INFO:tasks.rados.rados.0.smithi076.stdout:1027: finishing write tid 2 to smithi07685715-4 2024-09-21T09:42:10.917 INFO:tasks.rados.rados.0.smithi076.stdout:1020: done (7 left) 2024-09-21T09:42:10.917 INFO:tasks.rados.rados.0.smithi076.stdout:1022: done (6 left) 2024-09-21T09:42:10.917 INFO:tasks.rados.rados.0.smithi076.stdout:1023: done (5 left) 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1024: done (4 left) 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1025: done (3 left) 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1029: done (2 left) 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1030: read oid 9 snap 126 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1030: expect (ObjNum 384 snap 120 seq_num 384) 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1031: write oid 15 current snap is 128 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: finishing write tid 3 to smithi07685715-4 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: finishing write tid 4 to smithi07685715-4 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: finishing write tid 5 to smithi07685715-4 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: oid 4 updating version 0 to 404 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: oid 4 updating version 404 to 405 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: oid 4 version 405 is already newer than 403 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: oid 4 updating version 405 to 406 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 406 (ObjNum 411 snap 128 seq_num 411) dirty exists 2024-09-21T09:42:10.918 INFO:tasks.rados.rados.0.smithi076.stdout:1027: left oid 4 (ObjNum 411 snap 128 seq_num 411) 2024-09-21T09:42:10.919 INFO:tasks.rados.rados.0.smithi076.stdout:1028: expect (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:42:11.040 INFO:tasks.rados.rados.0.smithi076.stdout:1030: expect (ObjNum 384 snap 120 seq_num 384) 2024-09-21T09:42:11.118 INFO:tasks.rados.rados.0.smithi076.stdout:1031: seq_num 413 ranges {541291=560899,1884475=521471} 2024-09-21T09:42:11.123 INFO:tasks.rados.rados.0.smithi076.stdout:1031: writing smithi07685715-15 from 541291 to 1102190 tid 1 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1031: writing smithi07685715-15 from 1884475 to 2405946 tid 2 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1027: done (3 left) 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1028: done (2 left) 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1030: done (1 left) 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1032: read oid 21 snap -1 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1032: expect deleted 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:1033: rollback oid 31 current snap is 128 2024-09-21T09:42:11.131 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 127 2024-09-21T09:42:11.132 INFO:tasks.rados.rados.0.smithi076.stdout:1031: finishing write tid 1 to smithi07685715-15 2024-09-21T09:42:11.132 INFO:tasks.rados.rados.0.smithi076.stdout:1034: snap_create 2024-09-21T09:42:11.133 INFO:tasks.rados.rados.0.smithi076.stdout:1031: finishing write tid 2 to smithi07685715-15 2024-09-21T09:42:11.133 INFO:tasks.rados.rados.0.smithi076.stdout:1031: finishing write tid 3 to smithi07685715-15 2024-09-21T09:42:11.133 INFO:tasks.rados.rados.0.smithi076.stdout:1033: finishing rollback tid 0 to smithi07685715-31 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1033: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1031: finishing write tid 4 to smithi07685715-15 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1031: oid 15 updating version 0 to 252 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1031: oid 15 updating version 252 to 254 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1031: oid 15 version 254 is already newer than 253 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 254 (ObjNum 413 snap 128 seq_num 413) dirty exists 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1031: left oid 15 (ObjNum 413 snap 128 seq_num 413) 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:1033: finishing rollback tid 2 to smithi07685715-31 2024-09-21T09:42:11.134 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 398 (ObjNum 395 snap 123 seq_num 395) dirty exists 2024-09-21T09:42:11.599 INFO:tasks.rados.rados.0.smithi076.stdout:1031: done (3 left) 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1032: done (2 left) 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1033: done (1 left) 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1034: done (0 left) 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1035: read oid 1 snap 127 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1035: expect (ObjNum 381 snap 120 seq_num 381) 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1036: write oid 28 current snap is 129 2024-09-21T09:42:11.600 INFO:tasks.rados.rados.0.smithi076.stdout:1036: seq_num 414 ranges {558098=636148,1826604=665129} 2024-09-21T09:42:11.605 INFO:tasks.rados.rados.0.smithi076.stdout:1036: writing smithi07685715-28 from 558098 to 1194246 tid 1 2024-09-21T09:42:11.613 INFO:tasks.rados.rados.0.smithi076.stdout:1036: writing smithi07685715-28 from 1826604 to 2491733 tid 2 2024-09-21T09:42:11.613 INFO:tasks.rados.rados.0.smithi076.stdout:1037: write oid 26 current snap is 129 2024-09-21T09:42:11.613 INFO:tasks.rados.rados.0.smithi076.stdout:1037: seq_num 415 ranges {574905=711397,1885086=525140,2930189=44716} 2024-09-21T09:42:11.620 INFO:tasks.rados.rados.0.smithi076.stdout:1037: writing smithi07685715-26 from 574905 to 1286302 tid 1 2024-09-21T09:42:11.626 INFO:tasks.rados.rados.0.smithi076.stdout:1037: writing smithi07685715-26 from 1885086 to 2410226 tid 2 2024-09-21T09:42:11.630 INFO:tasks.rados.rados.0.smithi076.stdout:1037: writing smithi07685715-26 from 2930189 to 2974905 tid 3 2024-09-21T09:42:11.630 INFO:tasks.rados.rados.0.smithi076.stdout:1038: snap_create 2024-09-21T09:42:11.630 INFO:tasks.rados.rados.0.smithi076.stdout:1035: expect (ObjNum 381 snap 120 seq_num 381) 2024-09-21T09:42:11.738 INFO:tasks.rados.rados.0.smithi076.stdout:1036: finishing write tid 1 to smithi07685715-28 2024-09-21T09:42:11.738 INFO:tasks.rados.rados.0.smithi076.stdout:1036: finishing write tid 2 to smithi07685715-28 2024-09-21T09:42:11.738 INFO:tasks.rados.rados.0.smithi076.stdout:1036: finishing write tid 3 to smithi07685715-28 2024-09-21T09:42:11.738 INFO:tasks.rados.rados.0.smithi076.stdout:1036: finishing write tid 4 to smithi07685715-28 2024-09-21T09:42:11.738 INFO:tasks.rados.rados.0.smithi076.stdout:1036: oid 28 updating version 0 to 860 2024-09-21T09:42:11.738 INFO:tasks.rados.rados.0.smithi076.stdout:1036: oid 28 updating version 860 to 861 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1036: oid 28 updating version 861 to 862 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 862 (ObjNum 414 snap 129 seq_num 414) dirty exists 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1036: left oid 28 (ObjNum 414 snap 129 seq_num 414) 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: finishing write tid 1 to smithi07685715-26 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: finishing write tid 2 to smithi07685715-26 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: finishing write tid 3 to smithi07685715-26 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: finishing write tid 4 to smithi07685715-26 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: finishing write tid 5 to smithi07685715-26 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: oid 26 updating version 0 to 433 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: oid 26 version 433 is already newer than 432 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: oid 26 version 433 is already newer than 430 2024-09-21T09:42:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1037: oid 26 version 433 is already newer than 431 2024-09-21T09:42:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 433 (ObjNum 415 snap 129 seq_num 415) dirty exists 2024-09-21T09:42:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1037: left oid 26 (ObjNum 415 snap 129 seq_num 415) 2024-09-21T09:42:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:11 smithi191 ceph-mon[33301]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:42:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:11 smithi191 ceph-mon[33301]: pgmap v189: 89 pgs: 89 active+clean; 93 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 914 KiB/s wr, 13 op/s 2024-09-21T09:42:11.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:11 smithi076 ceph-mon[29313]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:42:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:11 smithi076 ceph-mon[29313]: pgmap v189: 89 pgs: 89 active+clean; 93 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 914 KiB/s wr, 13 op/s 2024-09-21T09:42:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:11 smithi076 ceph-mon[41325]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:42:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:11 smithi076 ceph-mon[41325]: pgmap v189: 89 pgs: 89 active+clean; 93 MiB data, 477 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 914 KiB/s wr, 13 op/s 2024-09-21T09:42:12.606 INFO:tasks.rados.rados.0.smithi076.stdout:1035: done (3 left) 2024-09-21T09:42:12.606 INFO:tasks.rados.rados.0.smithi076.stdout:1036: done (2 left) 2024-09-21T09:42:12.606 INFO:tasks.rados.rados.0.smithi076.stdout:1037: done (1 left) 2024-09-21T09:42:12.606 INFO:tasks.rados.rados.0.smithi076.stdout:1038: done (0 left) 2024-09-21T09:42:12.606 INFO:tasks.rados.rados.0.smithi076.stdout:1039: delete oid 26 current snap is 130 2024-09-21T09:42:12.610 INFO:tasks.rados.rados.0.smithi076.stdout:1039: done (0 left) 2024-09-21T09:42:12.610 INFO:tasks.rados.rados.0.smithi076.stdout:1040: write oid 47 current snap is 130 2024-09-21T09:42:12.610 INFO:tasks.rados.rados.0.smithi076.stdout:1040: seq_num 417 ranges {608519=461895,1602050=528809,2585035=423484} 2024-09-21T09:42:12.615 INFO:tasks.rados.rados.0.smithi076.stdout:1040: writing smithi07685715-47 from 608519 to 1070414 tid 1 2024-09-21T09:42:12.621 INFO:tasks.rados.rados.0.smithi076.stdout:1040: writing smithi07685715-47 from 1602050 to 2130859 tid 2 2024-09-21T09:42:12.627 INFO:tasks.rados.rados.0.smithi076.stdout:1040: writing smithi07685715-47 from 2585035 to 3008519 tid 3 2024-09-21T09:42:12.627 INFO:tasks.rados.rados.0.smithi076.stdout:1041: write oid 20 current snap is 130 2024-09-21T09:42:12.627 INFO:tasks.rados.rados.0.smithi076.stdout:1041: seq_num 418 ranges {625326=537144,1944179=672467} 2024-09-21T09:42:12.634 INFO:tasks.rados.rados.0.smithi076.stdout:1041: writing smithi07685715-20 from 625326 to 1162470 tid 1 2024-09-21T09:42:12.643 INFO:tasks.rados.rados.0.smithi076.stdout:1041: writing smithi07685715-20 from 1944179 to 2616646 tid 2 2024-09-21T09:42:12.643 INFO:tasks.rados.rados.0.smithi076.stdout:1042: snap_create 2024-09-21T09:42:12.643 INFO:tasks.rados.rados.0.smithi076.stdout:1040: finishing write tid 1 to smithi07685715-47 2024-09-21T09:42:12.643 INFO:tasks.rados.rados.0.smithi076.stdout:1040: finishing write tid 2 to smithi07685715-47 2024-09-21T09:42:12.643 INFO:tasks.rados.rados.0.smithi076.stdout:1040: finishing write tid 3 to smithi07685715-47 2024-09-21T09:42:12.643 INFO:tasks.rados.rados.0.smithi076.stdout:1040: finishing write tid 4 to smithi07685715-47 2024-09-21T09:42:12.644 INFO:tasks.rados.rados.0.smithi076.stdout:1040: finishing write tid 5 to smithi07685715-47 2024-09-21T09:42:12.644 INFO:tasks.rados.rados.0.smithi076.stdout:1040: oid 47 updating version 0 to 867 2024-09-21T09:42:12.644 INFO:tasks.rados.rados.0.smithi076.stdout:1040: oid 47 version 867 is already newer than 866 2024-09-21T09:42:12.644 INFO:tasks.rados.rados.0.smithi076.stdout:1040: oid 47 version 867 is already newer than 865 2024-09-21T09:42:12.644 INFO:tasks.rados.rados.0.smithi076.stdout:1040: oid 47 version 867 is already newer than 864 2024-09-21T09:42:12.645 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 867 (ObjNum 417 snap 130 seq_num 417) dirty exists 2024-09-21T09:42:12.645 INFO:tasks.rados.rados.0.smithi076.stdout:1040: left oid 47 (ObjNum 417 snap 130 seq_num 417) 2024-09-21T09:42:12.645 INFO:tasks.rados.rados.0.smithi076.stdout:1041: finishing write tid 1 to smithi07685715-20 2024-09-21T09:42:12.646 INFO:tasks.rados.rados.0.smithi076.stdout:1041: finishing write tid 2 to smithi07685715-20 2024-09-21T09:42:12.646 INFO:tasks.rados.rados.0.smithi076.stdout:1041: finishing write tid 3 to smithi07685715-20 2024-09-21T09:42:12.647 INFO:tasks.rados.rados.0.smithi076.stdout:1041: finishing write tid 4 to smithi07685715-20 2024-09-21T09:42:12.647 INFO:tasks.rados.rados.0.smithi076.stdout:1041: oid 20 updating version 0 to 296 2024-09-21T09:42:12.647 INFO:tasks.rados.rados.0.smithi076.stdout:1041: oid 20 updating version 296 to 298 2024-09-21T09:42:12.647 INFO:tasks.rados.rados.0.smithi076.stdout:1041: oid 20 version 298 is already newer than 297 2024-09-21T09:42:12.647 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 298 (ObjNum 418 snap 130 seq_num 418) dirty exists 2024-09-21T09:42:12.647 INFO:tasks.rados.rados.0.smithi076.stdout:1041: left oid 20 (ObjNum 418 snap 130 seq_num 418) 2024-09-21T09:42:12.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:12 smithi191 ceph-mon[33301]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:42:12.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:12 smithi076 ceph-mon[29313]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:42:12.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:12 smithi076 ceph-mon[41325]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:42:13.609 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:42:13] "GET /metrics HTTP/1.1" 200 36132 "" "Prometheus/2.51.0" 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1040: done (2 left) 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1041: done (1 left) 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1042: done (0 left) 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1043: read oid 36 snap 127 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1043: expect (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1044: write oid 8 current snap is 131 2024-09-21T09:42:13.616 INFO:tasks.rados.rados.0.smithi076.stdout:1044: seq_num 419 ranges {642133=728746,2119014=532478} 2024-09-21T09:42:13.623 INFO:tasks.rados.rados.0.smithi076.stdout:1044: writing smithi07685715-8 from 642133 to 1370879 tid 1 2024-09-21T09:42:13.631 INFO:tasks.rados.rados.0.smithi076.stdout:1044: writing smithi07685715-8 from 2119014 to 2651492 tid 2 2024-09-21T09:42:13.631 INFO:tasks.rados.rados.0.smithi076.stdout:1045: write oid 16 current snap is 131 2024-09-21T09:42:13.632 INFO:tasks.rados.rados.0.smithi076.stdout:1045: seq_num 420 ranges {658940=403995,1777496=676136,2950951=107989} 2024-09-21T09:42:13.637 INFO:tasks.rados.rados.0.smithi076.stdout:1045: writing smithi07685715-16 from 658940 to 1062935 tid 1 2024-09-21T09:42:13.644 INFO:tasks.rados.rados.0.smithi076.stdout:1045: writing smithi07685715-16 from 1777496 to 2453632 tid 2 2024-09-21T09:42:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:1045: writing smithi07685715-16 from 2950951 to 3058940 tid 3 2024-09-21T09:42:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:1046: rollback oid 9 current snap is 131 2024-09-21T09:42:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 129 2024-09-21T09:42:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:1043: expect (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: finishing write tid 1 to smithi07685715-8 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: finishing write tid 2 to smithi07685715-8 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: finishing write tid 3 to smithi07685715-8 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: finishing write tid 4 to smithi07685715-8 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: oid 8 updating version 0 to 869 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: oid 8 updating version 869 to 870 2024-09-21T09:42:13.731 INFO:tasks.rados.rados.0.smithi076.stdout:1044: oid 8 updating version 870 to 871 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 871 (ObjNum 419 snap 131 seq_num 419) dirty exists 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1044: left oid 8 (ObjNum 419 snap 131 seq_num 419) 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: finishing write tid 1 to smithi07685715-16 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: finishing write tid 2 to smithi07685715-16 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1046: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 427 (ObjNum 33 snap 0 seq_num 830379256) dirty dne 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: finishing write tid 3 to smithi07685715-16 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: finishing write tid 4 to smithi07685715-16 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: finishing write tid 5 to smithi07685715-16 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: oid 16 updating version 0 to 547 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: oid 16 updating version 547 to 549 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: oid 16 version 549 is already newer than 548 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: oid 16 version 549 is already newer than 546 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 549 (ObjNum 420 snap 131 seq_num 420) dirty exists 2024-09-21T09:42:13.732 INFO:tasks.rados.rados.0.smithi076.stdout:1045: left oid 16 (ObjNum 420 snap 131 seq_num 420) 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1043: done (3 left) 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1044: done (2 left) 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1045: done (1 left) 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1046: done (0 left) 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1047: read oid 37 snap -1 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1047: expect deleted 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1048: write oid 16 current snap is 131 2024-09-21T09:42:13.733 INFO:tasks.rados.rados.0.smithi076.stdout:1048: seq_num 421 ranges {675747=479244,1835978=536147} 2024-09-21T09:42:13.736 INFO:tasks.rados.rados.0.smithi076.stdout:1048: writing smithi07685715-16 from 675747 to 1154991 tid 1 2024-09-21T09:42:13.743 INFO:tasks.rados.rados.0.smithi076.stdout:1048: writing smithi07685715-16 from 1835978 to 2372125 tid 2 2024-09-21T09:42:13.743 INFO:tasks.rados.rados.0.smithi076.stdout:1049: snap_create 2024-09-21T09:42:13.743 INFO:tasks.rados.rados.0.smithi076.stdout:1048: finishing write tid 1 to smithi07685715-16 2024-09-21T09:42:13.746 INFO:tasks.rados.rados.0.smithi076.stdout:1048: finishing write tid 2 to smithi07685715-16 2024-09-21T09:42:13.746 INFO:tasks.rados.rados.0.smithi076.stdout:1048: finishing write tid 3 to smithi07685715-16 2024-09-21T09:42:13.746 INFO:tasks.rados.rados.0.smithi076.stdout:1048: finishing write tid 4 to smithi07685715-16 2024-09-21T09:42:13.746 INFO:tasks.rados.rados.0.smithi076.stdout:1048: oid 16 updating version 0 to 552 2024-09-21T09:42:13.746 INFO:tasks.rados.rados.0.smithi076.stdout:1048: oid 16 version 552 is already newer than 550 2024-09-21T09:42:13.746 INFO:tasks.rados.rados.0.smithi076.stdout:1048: oid 16 version 552 is already newer than 551 2024-09-21T09:42:13.747 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 552 (ObjNum 421 snap 131 seq_num 421) dirty exists 2024-09-21T09:42:13.747 INFO:tasks.rados.rados.0.smithi076.stdout:1048: left oid 16 (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:13 smithi191 ceph-mon[33301]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:42:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:13 smithi191 ceph-mon[33301]: pgmap v192: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-09-21T09:42:13.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:13 smithi076 ceph-mon[41325]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:42:13.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:13 smithi076 ceph-mon[41325]: pgmap v192: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-09-21T09:42:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:13 smithi076 ceph-mon[29313]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:42:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:13 smithi076 ceph-mon[29313]: pgmap v192: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-09-21T09:42:14.626 INFO:tasks.rados.rados.0.smithi076.stdout:1047: done (2 left) 2024-09-21T09:42:14.629 INFO:tasks.rados.rados.0.smithi076.stdout:1048: done (1 left) 2024-09-21T09:42:14.629 INFO:tasks.rados.rados.0.smithi076.stdout:1049: done (0 left) 2024-09-21T09:42:14.629 INFO:tasks.rados.rados.0.smithi076.stdout:1050: rollback oid 27 current snap is 132 2024-09-21T09:42:14.629 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 131 2024-09-21T09:42:14.629 INFO:tasks.rados.rados.0.smithi076.stdout:1051: write oid 39 current snap is 132 2024-09-21T09:42:14.629 INFO:tasks.rados.rados.0.smithi076.stdout:1051: seq_num 422 ranges {692554=554493,1778107=679805,2889444=203110} 2024-09-21T09:42:14.632 INFO:tasks.rados.rados.0.smithi076.stdout:1051: writing smithi07685715-39 from 692554 to 1247047 tid 1 2024-09-21T09:42:14.639 INFO:tasks.rados.rados.0.smithi076.stdout:1051: writing smithi07685715-39 from 1778107 to 2457912 tid 2 2024-09-21T09:42:14.643 INFO:tasks.rados.rados.0.smithi076.stdout:1051: writing smithi07685715-39 from 2889444 to 3092554 tid 3 2024-09-21T09:42:14.643 INFO:tasks.rados.rados.0.smithi076.stdout:1052: write oid 38 current snap is 132 2024-09-21T09:42:14.643 INFO:tasks.rados.rados.0.smithi076.stdout:1052: seq_num 423 ranges {709361=629742,1836589=539816,3033220=76141} 2024-09-21T09:42:14.650 INFO:tasks.rados.rados.0.smithi076.stdout:1052: writing smithi07685715-38 from 709361 to 1339103 tid 1 2024-09-21T09:42:14.656 INFO:tasks.rados.rados.0.smithi076.stdout:1052: writing smithi07685715-38 from 1836589 to 2376405 tid 2 2024-09-21T09:42:14.661 INFO:tasks.rados.rados.0.smithi076.stdout:1052: writing smithi07685715-38 from 3033220 to 3109361 tid 3 2024-09-21T09:42:14.661 INFO:tasks.rados.rados.0.smithi076.stdout:1053: write oid 32 current snap is 132 2024-09-21T09:42:14.661 INFO:tasks.rados.rados.0.smithi076.stdout:1050: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:42:14.661 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 0 (ObjNum 49 snap 0 seq_num 1) dirty dne 2024-09-21T09:42:14.661 INFO:tasks.rados.rados.0.smithi076.stdout:1053: seq_num 424 ranges {726168=704991,1895071=683474} 2024-09-21T09:42:14.668 INFO:tasks.rados.rados.0.smithi076.stdout:1053: writing smithi07685715-32 from 726168 to 1431159 tid 1 2024-09-21T09:42:14.676 INFO:tasks.rados.rados.0.smithi076.stdout:1053: writing smithi07685715-32 from 1895071 to 2578545 tid 2 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1050: done (3 left) 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1054: read oid 16 snap 126 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1054: expect (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1055: rollback oid 8 current snap is 132 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 130 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1051: finishing write tid 1 to smithi07685715-39 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1051: finishing write tid 2 to smithi07685715-39 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1051: finishing write tid 3 to smithi07685715-39 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1051: finishing write tid 4 to smithi07685715-39 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1051: finishing write tid 5 to smithi07685715-39 2024-09-21T09:42:14.677 INFO:tasks.rados.rados.0.smithi076.stdout:1051: oid 39 updating version 0 to 809 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1051: oid 39 version 809 is already newer than 806 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1051: oid 39 version 809 is already newer than 808 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1051: oid 39 version 809 is already newer than 807 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 809 (ObjNum 422 snap 132 seq_num 422) dirty exists 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1051: left oid 39 (ObjNum 422 snap 132 seq_num 422) 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1052: finishing write tid 1 to smithi07685715-38 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1052: finishing write tid 2 to smithi07685715-38 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1051: done (4 left) 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1056: snap_create 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1052: finishing write tid 3 to smithi07685715-38 2024-09-21T09:42:14.678 INFO:tasks.rados.rados.0.smithi076.stdout:1052: finishing write tid 4 to smithi07685715-38 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1052: finishing write tid 5 to smithi07685715-38 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1052: oid 38 updating version 0 to 531 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1052: oid 38 version 531 is already newer than 529 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1052: oid 38 version 531 is already newer than 528 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1052: oid 38 version 531 is already newer than 530 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 531 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1052: left oid 38 (ObjNum 423 snap 132 seq_num 423) 2024-09-21T09:42:14.679 INFO:tasks.rados.rados.0.smithi076.stdout:1053: finishing write tid 1 to smithi07685715-32 2024-09-21T09:42:14.680 INFO:tasks.rados.rados.0.smithi076.stdout:1053: finishing write tid 2 to smithi07685715-32 2024-09-21T09:42:14.680 INFO:tasks.rados.rados.0.smithi076.stdout:1053: finishing write tid 3 to smithi07685715-32 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1053: finishing write tid 4 to smithi07685715-32 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1053: oid 32 updating version 0 to 402 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1053: oid 32 version 402 is already newer than 401 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1053: oid 32 version 402 is already newer than 400 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 402 (ObjNum 424 snap 132 seq_num 424) dirty exists 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1053: left oid 32 (ObjNum 424 snap 132 seq_num 424) 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1055: finishing rollback tid 0 to smithi07685715-8 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1055: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1055: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:42:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 875 (ObjNum 207 snap 60 seq_num 207) dirty exists 2024-09-21T09:42:14.686 INFO:tasks.rados.rados.0.smithi076.stdout:1054: expect (ObjNum 351 snap 107 seq_num 351) 2024-09-21T09:42:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:14 smithi191 ceph-mon[33301]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:42:14.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:14 smithi076 ceph-mon[29313]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:42:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:14 smithi076 ceph-mon[41325]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi076.stdout:1052: done (4 left) 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi076.stdout:1053: done (3 left) 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi076.stdout:1054: done (2 left) 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi076.stdout:1055: done (1 left) 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi076.stdout:1056: done (0 left) 2024-09-21T09:42:15.625 INFO:tasks.rados.rados.0.smithi076.stdout:1057: snap_remove snap 126 2024-09-21T09:42:15.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:15 smithi076 ceph-mon[29313]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:42:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:15 smithi076 ceph-mon[29313]: pgmap v195: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-09-21T09:42:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:15 smithi076 ceph-mon[41325]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:42:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:15 smithi076 ceph-mon[41325]: pgmap v195: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-09-21T09:42:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:15 smithi191 ceph-mon[33301]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:42:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:15 smithi191 ceph-mon[33301]: pgmap v195: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-09-21T09:42:16.627 INFO:tasks.rados.rados.0.smithi076.stdout:1057: done (0 left) 2024-09-21T09:42:16.627 INFO:tasks.rados.rados.0.smithi076.stdout:1058: snap_create 2024-09-21T09:42:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:16 smithi191 ceph-mon[33301]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:42:17.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:16 smithi076 ceph-mon[29313]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:42:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:16 smithi076 ceph-mon[41325]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:42:17.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-21T09:42:17.405 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:17.628 INFO:tasks.rados.rados.0.smithi076.stdout:1058: done (0 left) 2024-09-21T09:42:17.628 INFO:tasks.rados.rados.0.smithi076.stdout:1059: snap_remove snap 133 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[41325]: from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[41325]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[41325]: pgmap v198: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[41325]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[29313]: from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[29313]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[29313]: pgmap v198: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:17.805 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:17 smithi076 ceph-mon[29313]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:42:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:17 smithi191 ceph-mon[33301]: from='client.25010 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:17 smithi191 ceph-mon[33301]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:42:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:17 smithi191 ceph-mon[33301]: pgmap v198: 89 pgs: 89 active+clean; 107 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:17 smithi191 ceph-mon[33301]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:42:18.554 INFO:tasks.rados.rados.0.smithi076.stdout:1059: done (0 left) 2024-09-21T09:42:18.554 INFO:tasks.rados.rados.0.smithi076.stdout:1060: delete oid 17 current snap is 134 2024-09-21T09:42:18.559 INFO:tasks.rados.rados.0.smithi076.stdout:1060: done (0 left) 2024-09-21T09:42:18.559 INFO:tasks.rados.rados.0.smithi076.stdout:1061: read oid 16 snap -1 2024-09-21T09:42:18.559 INFO:tasks.rados.rados.0.smithi076.stdout:1061: expect (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:18.560 INFO:tasks.rados.rados.0.smithi076.stdout:1062: read oid 42 snap -1 2024-09-21T09:42:18.560 INFO:tasks.rados.rados.0.smithi076.stdout:1062: expect (ObjNum 346 snap 105 seq_num 346) 2024-09-21T09:42:18.560 INFO:tasks.rados.rados.0.smithi076.stdout:1063: snap_remove snap 130 2024-09-21T09:42:19.145 INFO:teuthology.orchestra.run.smithi076.stdout:wait for minority of mons to upgrade 2024-09-21T09:42:19.560 INFO:tasks.rados.rados.0.smithi076.stdout:1063: done (2 left) 2024-09-21T09:42:19.560 INFO:tasks.rados.rados.0.smithi076.stdout:1064: snap_create 2024-09-21T09:42:19.560 INFO:tasks.rados.rados.0.smithi076.stdout:1062: expect (ObjNum 346 snap 105 seq_num 346) 2024-09-21T09:42:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:19 smithi191 ceph-mon[33301]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:42:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:19 smithi191 ceph-mon[33301]: pgmap v201: 89 pgs: 89 active+clean; 117 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:42:19.871 INFO:tasks.rados.rados.0.smithi076.stdout:1061: expect (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:19.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:19 smithi076 ceph-mon[41325]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:42:19.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:19 smithi076 ceph-mon[41325]: pgmap v201: 89 pgs: 89 active+clean; 117 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:42:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:19 smithi076 ceph-mon[29313]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:42:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:19 smithi076 ceph-mon[29313]: pgmap v201: 89 pgs: 89 active+clean; 117 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:42:19.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-21T09:42:20.100 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:20.560 INFO:tasks.rados.rados.0.smithi076.stdout:1061: done (2 left) 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1062: done (1 left) 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1064: done (0 left) 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1065: read oid 27 snap -1 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1065: expect deleted 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1066: read oid 41 snap -1 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1066: expect deleted 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1067: write oid 6 current snap is 135 2024-09-21T09:42:20.561 INFO:tasks.rados.rados.0.smithi076.stdout:1067: seq_num 426 ranges {759782=571842,2012035=687143} 2024-09-21T09:42:20.567 INFO:tasks.rados.rados.0.smithi076.stdout:1067: writing smithi07685715-6 from 759782 to 1331624 tid 1 2024-09-21T09:42:20.575 INFO:tasks.rados.rados.0.smithi076.stdout:1067: writing smithi07685715-6 from 2012035 to 2699178 tid 2 2024-09-21T09:42:20.575 INFO:tasks.rados.rados.0.smithi076.stdout:1068: delete oid 23 current snap is 135 2024-09-21T09:42:20.575 INFO:tasks.rados.rados.0.smithi076.stdout:1067: finishing write tid 1 to smithi07685715-6 2024-09-21T09:42:20.578 INFO:tasks.rados.rados.0.smithi076.stdout:1065: done (3 left) 2024-09-21T09:42:20.578 INFO:tasks.rados.rados.0.smithi076.stdout:1066: done (2 left) 2024-09-21T09:42:20.578 INFO:tasks.rados.rados.0.smithi076.stdout:1068: done (1 left) 2024-09-21T09:42:20.578 INFO:tasks.rados.rados.0.smithi076.stdout:1069: read oid 19 snap 134 2024-09-21T09:42:20.578 INFO:tasks.rados.rados.0.smithi076.stdout:1069: expect (ObjNum 391 snap 121 seq_num 391) 2024-09-21T09:42:20.579 INFO:tasks.rados.rados.0.smithi076.stdout:1070: read oid 9 snap -1 2024-09-21T09:42:20.579 INFO:tasks.rados.rados.0.smithi076.stdout:1070: expect deleted 2024-09-21T09:42:20.579 INFO:tasks.rados.rados.0.smithi076.stdout:1071: read oid 16 snap -1 2024-09-21T09:42:20.579 INFO:tasks.rados.rados.0.smithi076.stdout:1071: expect (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:20.579 INFO:tasks.rados.rados.0.smithi076.stdout:1072: write oid 8 current snap is 135 2024-09-21T09:42:20.579 INFO:tasks.rados.rados.0.smithi076.stdout:1072: seq_num 428 ranges {793396=722340,2128999=690812} 2024-09-21T09:42:20.587 INFO:tasks.rados.rados.0.smithi076.stdout:1072: writing smithi07685715-8 from 793396 to 1515736 tid 1 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1072: writing smithi07685715-8 from 2128999 to 2819811 tid 2 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1073: read oid 41 snap -1 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1073: expect deleted 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1074: rollback oid 33 current snap is 135 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 127 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1067: finishing write tid 2 to smithi07685715-6 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1067: finishing write tid 3 to smithi07685715-6 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1067: finishing write tid 4 to smithi07685715-6 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1067: oid 6 updating version 0 to 212 2024-09-21T09:42:20.595 INFO:tasks.rados.rados.0.smithi076.stdout:1067: oid 6 version 212 is already newer than 211 2024-09-21T09:42:20.596 INFO:tasks.rados.rados.0.smithi076.stdout:1067: oid 6 updating version 212 to 213 2024-09-21T09:42:20.596 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 213 (ObjNum 426 snap 135 seq_num 426) dirty exists 2024-09-21T09:42:20.596 INFO:tasks.rados.rados.0.smithi076.stdout:1067: left oid 6 (ObjNum 426 snap 135 seq_num 426) 2024-09-21T09:42:20.596 INFO:tasks.rados.rados.0.smithi076.stdout:1071: expect (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:20.780 INFO:tasks.rados.rados.0.smithi076.stdout:1069: expect (ObjNum 391 snap 121 seq_num 391) 2024-09-21T09:42:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:20 smithi191 ceph-mon[33301]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:42:20.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:20 smithi076 ceph-mon[29313]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:42:20.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:20 smithi076 ceph-mon[41325]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1072: finishing write tid 1 to smithi07685715-8 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1074: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1074: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 244 (ObjNum 306 snap 94 seq_num 306) dirty exists 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1072: finishing write tid 2 to smithi07685715-8 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1072: finishing write tid 3 to smithi07685715-8 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1072: finishing write tid 4 to smithi07685715-8 2024-09-21T09:42:20.935 INFO:tasks.rados.rados.0.smithi076.stdout:1072: oid 8 updating version 0 to 887 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1072: oid 8 version 887 is already newer than 886 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1072: oid 8 updating version 887 to 888 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 888 (ObjNum 428 snap 135 seq_num 428) dirty exists 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1072: left oid 8 (ObjNum 428 snap 135 seq_num 428) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1067: done (6 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1069: done (5 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1070: done (4 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1071: done (3 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1072: done (2 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1073: done (1 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1074: done (0 left) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1075: read oid 28 snap -1 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1075: expect (ObjNum 414 snap 129 seq_num 414) 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1076: rollback oid 11 current snap is 135 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 131 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1077: write oid 43 current snap is 135 2024-09-21T09:42:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1077: seq_num 429 ranges {410203=797589,1787481=434470,2681405=528798} 2024-09-21T09:42:20.941 INFO:tasks.rados.rados.0.smithi076.stdout:1077: writing smithi07685715-43 from 410203 to 1207792 tid 1 2024-09-21T09:42:20.946 INFO:tasks.rados.rados.0.smithi076.stdout:1077: writing smithi07685715-43 from 1787481 to 2221951 tid 2 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:1077: writing smithi07685715-43 from 2681405 to 3210203 tid 3 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:1078: snap_create 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:1076: finishing rollback tid 0 to smithi07685715-11 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:1076: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:1076: finishing rollback tid 2 to smithi07685715-11 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 813 (ObjNum 394 snap 121 seq_num 394) dirty exists 2024-09-21T09:42:20.952 INFO:tasks.rados.rados.0.smithi076.stdout:1075: expect (ObjNum 414 snap 129 seq_num 414) 2024-09-21T09:42:21.126 INFO:tasks.rados.rados.0.smithi076.stdout:1077: finishing write tid 1 to smithi07685715-43 2024-09-21T09:42:21.126 INFO:tasks.rados.rados.0.smithi076.stdout:1077: finishing write tid 2 to smithi07685715-43 2024-09-21T09:42:21.126 INFO:tasks.rados.rados.0.smithi076.stdout:1077: finishing write tid 3 to smithi07685715-43 2024-09-21T09:42:21.126 INFO:tasks.rados.rados.0.smithi076.stdout:1077: finishing write tid 4 to smithi07685715-43 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:1077: finishing write tid 5 to smithi07685715-43 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:1077: oid 43 updating version 0 to 193 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:1077: oid 43 updating version 193 to 194 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:1077: oid 43 version 194 is already newer than 192 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:1077: oid 43 version 194 is already newer than 191 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 194 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:42:21.127 INFO:tasks.rados.rados.0.smithi076.stdout:1077: left oid 43 (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:42:21.568 INFO:tasks.rados.rados.0.smithi076.stdout:1075: done (3 left) 2024-09-21T09:42:21.569 INFO:tasks.rados.rados.0.smithi076.stdout:1076: done (2 left) 2024-09-21T09:42:21.569 INFO:tasks.rados.rados.0.smithi076.stdout:1077: done (1 left) 2024-09-21T09:42:21.569 INFO:tasks.rados.rados.0.smithi076.stdout:1078: done (0 left) 2024-09-21T09:42:21.569 INFO:tasks.rados.rados.0.smithi076.stdout:1079: snap_create 2024-09-21T09:42:21.611 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:21 smithi076 ceph-mon[41325]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:42:21.611 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:21 smithi076 ceph-mon[41325]: pgmap v204: 89 pgs: 89 active+clean; 117 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:42:21.611 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:21 smithi076 ceph-mon[29313]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:42:21.611 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:21 smithi076 ceph-mon[29313]: pgmap v204: 89 pgs: 89 active+clean; 117 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:42:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:21 smithi191 ceph-mon[33301]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:42:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:21 smithi191 ceph-mon[33301]: pgmap v204: 89 pgs: 89 active+clean; 117 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:42:22.578 INFO:tasks.rados.rados.0.smithi076.stdout:1079: done (0 left) 2024-09-21T09:42:22.579 INFO:tasks.rados.rados.0.smithi076.stdout:1080: read oid 47 snap -1 2024-09-21T09:42:22.579 INFO:tasks.rados.rados.0.smithi076.stdout:1080: expect (ObjNum 417 snap 130 seq_num 417) 2024-09-21T09:42:22.579 INFO:tasks.rados.rados.0.smithi076.stdout:1081: write oid 26 current snap is 137 2024-09-21T09:42:22.579 INFO:tasks.rados.rados.0.smithi076.stdout:1081: seq_num 430 ranges {427010=472838,1329610=694481,2708828=518182} 2024-09-21T09:42:22.583 INFO:tasks.rados.rados.0.smithi076.stdout:1081: writing smithi07685715-26 from 427010 to 899848 tid 1 2024-09-21T09:42:22.590 INFO:tasks.rados.rados.0.smithi076.stdout:1081: writing smithi07685715-26 from 1329610 to 2024091 tid 2 2024-09-21T09:42:22.597 INFO:tasks.rados.rados.0.smithi076.stdout:1081: writing smithi07685715-26 from 2708828 to 3227010 tid 3 2024-09-21T09:42:22.597 INFO:tasks.rados.rados.0.smithi076.stdout:1082: write oid 49 current snap is 137 2024-09-21T09:42:22.597 INFO:tasks.rados.rados.0.smithi076.stdout:1082: seq_num 431 ranges {443817=548087,1788092=438139,3019898=223919} 2024-09-21T09:42:22.602 INFO:tasks.rados.rados.0.smithi076.stdout:1082: writing smithi07685715-49 from 443817 to 991904 tid 1 2024-09-21T09:42:22.609 INFO:tasks.rados.rados.0.smithi076.stdout:1082: writing smithi07685715-49 from 1788092 to 2226231 tid 2 2024-09-21T09:42:22.615 INFO:tasks.rados.rados.0.smithi076.stdout:1082: writing smithi07685715-49 from 3019898 to 3243817 tid 3 2024-09-21T09:42:22.615 INFO:tasks.rados.rados.0.smithi076.stdout:1081: finishing write tid 1 to smithi07685715-26 2024-09-21T09:42:22.615 INFO:tasks.rados.rados.0.smithi076.stdout:1083: write oid 2 current snap is 137 2024-09-21T09:42:22.615 INFO:tasks.rados.rados.0.smithi076.stdout:1080: expect (ObjNum 417 snap 130 seq_num 417) 2024-09-21T09:42:22.784 INFO:tasks.rados.rados.0.smithi076.stdout:1081: finishing write tid 2 to smithi07685715-26 2024-09-21T09:42:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1081: finishing write tid 3 to smithi07685715-26 2024-09-21T09:42:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1081: finishing write tid 4 to smithi07685715-26 2024-09-21T09:42:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1083: seq_num 432 ranges {460624=623336,1846574=698150,3163674=96950} 2024-09-21T09:42:22.790 INFO:tasks.rados.rados.0.smithi076.stdout:1083: writing smithi07685715-2 from 460624 to 1083960 tid 1 2024-09-21T09:42:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1083: writing smithi07685715-2 from 1846574 to 2544724 tid 2 2024-09-21T09:42:22.803 INFO:tasks.rados.rados.0.smithi076.stdout:1083: writing smithi07685715-2 from 3163674 to 3260624 tid 3 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1080: done (3 left) 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1084: read oid 37 snap -1 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1084: expect deleted 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1085: read oid 14 snap -1 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1085: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1086: rollback oid 7 current snap is 137 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 123 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1081: finishing write tid 5 to smithi07685715-26 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1081: oid 26 updating version 0 to 440 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1081: oid 26 updating version 440 to 442 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1081: oid 26 version 442 is already newer than 441 2024-09-21T09:42:22.804 INFO:tasks.rados.rados.0.smithi076.stdout:1081: oid 26 version 442 is already newer than 439 2024-09-21T09:42:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 442 (ObjNum 430 snap 137 seq_num 430) dirty exists 2024-09-21T09:42:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:1081: left oid 26 (ObjNum 430 snap 137 seq_num 430) 2024-09-21T09:42:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:1082: finishing write tid 1 to smithi07685715-49 2024-09-21T09:42:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:1082: finishing write tid 2 to smithi07685715-49 2024-09-21T09:42:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:1082: finishing write tid 3 to smithi07685715-49 2024-09-21T09:42:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:1081: done (5 left) 2024-09-21T09:42:22.806 INFO:tasks.rados.rados.0.smithi076.stdout:1087: write oid 21 current snap is 137 2024-09-21T09:42:22.806 INFO:tasks.rados.rados.0.smithi076.stdout:1082: finishing write tid 4 to smithi07685715-49 2024-09-21T09:42:22.806 INFO:tasks.rados.rados.0.smithi076.stdout:1082: finishing write tid 5 to smithi07685715-49 2024-09-21T09:42:22.806 INFO:tasks.rados.rados.0.smithi076.stdout:1082: oid 49 updating version 0 to 892 2024-09-21T09:42:22.806 INFO:tasks.rados.rados.0.smithi076.stdout:1082: oid 49 version 892 is already newer than 890 2024-09-21T09:42:22.806 INFO:tasks.rados.rados.0.smithi076.stdout:1082: oid 49 version 892 is already newer than 891 2024-09-21T09:42:22.807 INFO:tasks.rados.rados.0.smithi076.stdout:1082: oid 49 updating version 892 to 893 2024-09-21T09:42:22.807 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 893 (ObjNum 431 snap 137 seq_num 431) dirty exists 2024-09-21T09:42:22.807 INFO:tasks.rados.rados.0.smithi076.stdout:1082: left oid 49 (ObjNum 431 snap 137 seq_num 431) 2024-09-21T09:42:22.807 INFO:tasks.rados.rados.0.smithi076.stdout:1083: finishing write tid 1 to smithi07685715-2 2024-09-21T09:42:22.807 INFO:tasks.rados.rados.0.smithi076.stdout:1083: finishing write tid 2 to smithi07685715-2 2024-09-21T09:42:22.807 INFO:tasks.rados.rados.0.smithi076.stdout:1087: seq_num 433 ranges {477431=698585,1905056=441808,3074744=202687} 2024-09-21T09:42:22.809 INFO:tasks.rados.rados.0.smithi076.stdout:1087: writing smithi07685715-21 from 477431 to 1176016 tid 1 2024-09-21T09:42:22.816 INFO:tasks.rados.rados.0.smithi076.stdout:1087: writing smithi07685715-21 from 1905056 to 2346864 tid 2 2024-09-21T09:42:22.821 INFO:tasks.rados.rados.0.smithi076.stdout:1087: writing smithi07685715-21 from 3074744 to 3277431 tid 3 2024-09-21T09:42:22.822 INFO:tasks.rados.rados.0.smithi076.stdout:1082: done (5 left) 2024-09-21T09:42:22.822 INFO:tasks.rados.rados.0.smithi076.stdout:1088: snap_create 2024-09-21T09:42:22.822 INFO:tasks.rados.rados.0.smithi076.stdout:1083: finishing write tid 3 to smithi07685715-2 2024-09-21T09:42:22.822 INFO:tasks.rados.rados.0.smithi076.stdout:1083: finishing write tid 4 to smithi07685715-2 2024-09-21T09:42:22.822 INFO:tasks.rados.rados.0.smithi076.stdout:1083: finishing write tid 5 to smithi07685715-2 2024-09-21T09:42:22.822 INFO:tasks.rados.rados.0.smithi076.stdout:1083: oid 2 updating version 0 to 178 2024-09-21T09:42:22.823 INFO:tasks.rados.rados.0.smithi076.stdout:1083: oid 2 updating version 178 to 179 2024-09-21T09:42:22.823 INFO:tasks.rados.rados.0.smithi076.stdout:1083: oid 2 version 179 is already newer than 177 2024-09-21T09:42:22.823 INFO:tasks.rados.rados.0.smithi076.stdout:1083: oid 2 version 179 is already newer than 176 2024-09-21T09:42:22.823 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 179 (ObjNum 432 snap 137 seq_num 432) dirty exists 2024-09-21T09:42:22.823 INFO:tasks.rados.rados.0.smithi076.stdout:1083: left oid 2 (ObjNum 432 snap 137 seq_num 432) 2024-09-21T09:42:22.823 INFO:tasks.rados.rados.0.smithi076.stdout:1085: expect (ObjNum 321 snap 98 seq_num 321) 2024-09-21T09:42:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:22 smithi191 ceph-mon[33301]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:42:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:22 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:42:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:22 smithi076 ceph-mon[29313]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:42:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:22 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:42:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:22 smithi076 ceph-mon[41325]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:42:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:22 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1086: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1086: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1086: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 547 (ObjNum 396 snap 123 seq_num 396) dirty exists 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: finishing write tid 1 to smithi07685715-21 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: finishing write tid 2 to smithi07685715-21 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: finishing write tid 3 to smithi07685715-21 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: finishing write tid 4 to smithi07685715-21 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: finishing write tid 5 to smithi07685715-21 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: oid 21 updating version 0 to 814 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: oid 21 updating version 814 to 817 2024-09-21T09:42:22.977 INFO:tasks.rados.rados.0.smithi076.stdout:1087: oid 21 version 817 is already newer than 815 2024-09-21T09:42:22.978 INFO:tasks.rados.rados.0.smithi076.stdout:1087: oid 21 version 817 is already newer than 816 2024-09-21T09:42:22.978 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 817 (ObjNum 433 snap 137 seq_num 433) dirty exists 2024-09-21T09:42:22.978 INFO:tasks.rados.rados.0.smithi076.stdout:1087: left oid 21 (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:23.589 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:42:23] "GET /metrics HTTP/1.1" 200 36144 "" "Prometheus/2.51.0" 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1083: done (5 left) 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1084: done (4 left) 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1085: done (3 left) 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1086: done (2 left) 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1087: done (1 left) 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1088: done (0 left) 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1089: rollback oid 17 current snap is 138 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 137 2024-09-21T09:42:23.593 INFO:tasks.rados.rados.0.smithi076.stdout:1090: rollback oid 23 current snap is 138 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 128 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1091: read oid 24 snap 131 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1091: expect (ObjNum 326 snap 101 seq_num 326) 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1092: read oid 20 snap -1 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1092: expect (ObjNum 418 snap 130 seq_num 418) 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1093: read oid 21 snap -1 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1093: expect (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1094: read oid 34 snap -1 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1094: expect (ObjNum 404 snap 124 seq_num 404) 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1095: read oid 1 snap -1 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1095: expect (ObjNum 381 snap 120 seq_num 381) 2024-09-21T09:42:23.594 INFO:tasks.rados.rados.0.smithi076.stdout:1096: write oid 48 current snap is 138 2024-09-21T09:42:23.595 INFO:tasks.rados.rados.0.smithi076.stdout:1096: seq_num 434 ranges {494238=490187,1563538=701819,2818520=475718} 2024-09-21T09:42:23.598 INFO:tasks.rados.rados.0.smithi076.stdout:1096: writing smithi07685715-48 from 494238 to 984425 tid 1 2024-09-21T09:42:23.605 INFO:tasks.rados.rados.0.smithi076.stdout:1096: writing smithi07685715-48 from 1563538 to 2265357 tid 2 2024-09-21T09:42:23.611 INFO:tasks.rados.rados.0.smithi076.stdout:1096: writing smithi07685715-48 from 2818520 to 3294238 tid 3 2024-09-21T09:42:23.611 INFO:tasks.rados.rados.0.smithi076.stdout:1097: rollback oid 2 current snap is 138 2024-09-21T09:42:23.611 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 127 2024-09-21T09:42:23.611 INFO:tasks.rados.rados.0.smithi076.stdout:1089: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:42:23.611 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 539 (ObjNum 864275424 snap 21884 seq_num 304) dirty dne 2024-09-21T09:42:23.612 INFO:tasks.rados.rados.0.smithi076.stdout:1089: done (8 left) 2024-09-21T09:42:23.612 INFO:tasks.rados.rados.0.smithi076.stdout:1098: delete oid 42 current snap is 138 2024-09-21T09:42:23.612 INFO:tasks.rados.rados.0.smithi076.stdout:1090: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:42:23.612 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:42:23.612 INFO:tasks.rados.rados.0.smithi076.stdout:1094: expect (ObjNum 404 snap 124 seq_num 404) 2024-09-21T09:42:23.701 INFO:tasks.rados.rados.0.smithi076.stdout:1091: expect (ObjNum 326 snap 101 seq_num 326) 2024-09-21T09:42:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:23 smithi191 ceph-mon[33301]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:23 smithi191 ceph-mon[33301]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:42:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:23 smithi191 ceph-mon[33301]: pgmap v207: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:42:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-mon[29313]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-mon[29313]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:42:23.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-mon[29313]: pgmap v207: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:42:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-mon[41325]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-mon[41325]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:42:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:23 smithi076 ceph-mon[41325]: pgmap v207: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:42:23.905 INFO:tasks.rados.rados.0.smithi076.stdout:1092: expect (ObjNum 418 snap 130 seq_num 418) 2024-09-21T09:42:24.086 INFO:tasks.rados.rados.0.smithi076.stdout:1096: finishing write tid 1 to smithi07685715-48 2024-09-21T09:42:24.086 INFO:tasks.rados.rados.0.smithi076.stdout:1093: expect (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:24.106 INFO:teuthology.orchestra.run.smithi076.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:24.205 INFO:tasks.rados.rados.0.smithi076.stdout:1096: finishing write tid 2 to smithi07685715-48 2024-09-21T09:42:24.205 INFO:tasks.rados.rados.0.smithi076.stdout:1095: expect (ObjNum 381 snap 120 seq_num 381) 2024-09-21T09:42:24.320 INFO:tasks.rados.rados.0.smithi076.stdout:1097: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:42:24.320 INFO:tasks.rados.rados.0.smithi076.stdout:1097: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:42:24.320 INFO:tasks.rados.rados.0.smithi076.stdout:1097: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:42:24.320 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 183 (ObjNum 332 snap 103 seq_num 332) dirty exists 2024-09-21T09:42:24.320 INFO:tasks.rados.rados.0.smithi076.stdout:1096: finishing write tid 3 to smithi07685715-48 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: finishing write tid 4 to smithi07685715-48 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: finishing write tid 5 to smithi07685715-48 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: oid 48 updating version 0 to 895 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: oid 48 updating version 895 to 896 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: oid 48 updating version 896 to 897 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: oid 48 updating version 897 to 898 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 898 (ObjNum 434 snap 138 seq_num 434) dirty exists 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1096: left oid 48 (ObjNum 434 snap 138 seq_num 434) 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1090: done (8 left) 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1091: done (7 left) 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1092: done (6 left) 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1093: done (5 left) 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1094: done (4 left) 2024-09-21T09:42:24.321 INFO:tasks.rados.rados.0.smithi076.stdout:1095: done (3 left) 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:1096: done (2 left) 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:1097: done (1 left) 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:1098: done (0 left) 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:1099: rollback oid 7 current snap is 138 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 129 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:1100: snap_create 2024-09-21T09:42:24.322 INFO:tasks.rados.rados.0.smithi076.stdout:1099: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:42:24.323 INFO:tasks.rados.rados.0.smithi076.stdout:1099: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:42:24.323 INFO:tasks.rados.rados.0.smithi076.stdout:1099: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:42:24.323 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 552 (ObjNum 396 snap 123 seq_num 396) dirty exists 2024-09-21T09:42:24.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:24 smithi191 ceph-mon[33301]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:42:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:24 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:24 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:24 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:24 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:24 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:24.869 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[41325]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[29313]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:42:24.870 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:24.871 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:24.871 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:24.871 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:42:24.871 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:24 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:24.891 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:42:25.101 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:25.118 INFO:tasks.rados.rados.0.smithi076.stdout:1099: done (1 left) 2024-09-21T09:42:25.118 INFO:tasks.rados.rados.0.smithi076.stdout:1100: done (0 left) 2024-09-21T09:42:25.118 INFO:tasks.rados.rados.0.smithi076.stdout:1101: rollback oid 11 current snap is 139 2024-09-21T09:42:25.118 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 128 2024-09-21T09:42:25.118 INFO:tasks.rados.rados.0.smithi076.stdout:1102: snap_remove snap 137 2024-09-21T09:42:25.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:25 smithi191 ceph-mon[33301]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:25 smithi191 ceph-mon[33301]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:25 smithi191 ceph-mon[33301]: pgmap v209: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:42:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:25 smithi191 ceph-mon[33301]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[29313]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[29313]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[29313]: pgmap v209: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[29313]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[41325]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[41325]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[41325]: pgmap v209: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:42:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:25 smithi076 ceph-mon[41325]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:42:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1102: done (1 left) 2024-09-21T09:42:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1103: write oid 26 current snap is 139 2024-09-21T09:42:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1101: finishing rollback tid 0 to smithi07685715-11 2024-09-21T09:42:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1101: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:42:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1101: finishing rollback tid 2 to smithi07685715-11 2024-09-21T09:42:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 821 (ObjNum 394 snap 121 seq_num 394) dirty exists 2024-09-21T09:42:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1103: seq_num 436 ranges {527852=640685,1680502=589135,2757013=570839} 2024-09-21T09:42:26.125 INFO:tasks.rados.rados.0.smithi076.stdout:1103: writing smithi07685715-26 from 527852 to 1168537 tid 1 2024-09-21T09:42:26.132 INFO:tasks.rados.rados.0.smithi076.stdout:1103: writing smithi07685715-26 from 1680502 to 2269637 tid 2 2024-09-21T09:42:26.138 INFO:tasks.rados.rados.0.smithi076.stdout:1103: writing smithi07685715-26 from 2757013 to 3327852 tid 3 2024-09-21T09:42:26.138 INFO:tasks.rados.rados.0.smithi076.stdout:1101: done (1 left) 2024-09-21T09:42:26.138 INFO:tasks.rados.rados.0.smithi076.stdout:1104: read oid 30 snap -1 2024-09-21T09:42:26.138 INFO:tasks.rados.rados.0.smithi076.stdout:1104: expect (ObjNum 377 snap 119 seq_num 377) 2024-09-21T09:42:26.139 INFO:tasks.rados.rados.0.smithi076.stdout:1105: write oid 10 current snap is 139 2024-09-21T09:42:26.139 INFO:tasks.rados.rados.0.smithi076.stdout:1105: seq_num 437 ranges {544659=715934,1738984=449146,2784436=560223} 2024-09-21T09:42:26.145 INFO:tasks.rados.rados.0.smithi076.stdout:1105: writing smithi07685715-10 from 544659 to 1260593 tid 1 2024-09-21T09:42:26.150 INFO:tasks.rados.rados.0.smithi076.stdout:1105: writing smithi07685715-10 from 1738984 to 2188130 tid 2 2024-09-21T09:42:26.157 INFO:tasks.rados.rados.0.smithi076.stdout:1105: writing smithi07685715-10 from 2784436 to 3344659 tid 3 2024-09-21T09:42:26.157 INFO:tasks.rados.rados.0.smithi076.stdout:1106: write oid 5 current snap is 139 2024-09-21T09:42:26.157 INFO:tasks.rados.rados.0.smithi076.stdout:1106: seq_num 438 ranges {561466=791183,2081113=592804,3095506=265960} 2024-09-21T09:42:26.164 INFO:tasks.rados.rados.0.smithi076.stdout:1106: writing smithi07685715-5 from 561466 to 1352649 tid 1 2024-09-21T09:42:26.171 INFO:tasks.rados.rados.0.smithi076.stdout:1106: writing smithi07685715-5 from 2081113 to 2673917 tid 2 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1106: writing smithi07685715-5 from 3095506 to 3361466 tid 3 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1107: read oid 7 snap -1 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1107: expect (ObjNum 396 snap 123 seq_num 396) 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1108: rollback oid 24 current snap is 139 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 24 to 135 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1103: finishing write tid 1 to smithi07685715-26 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1103: finishing write tid 2 to smithi07685715-26 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1103: finishing write tid 3 to smithi07685715-26 2024-09-21T09:42:26.176 INFO:tasks.rados.rados.0.smithi076.stdout:1103: finishing write tid 4 to smithi07685715-26 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1103: finishing write tid 5 to smithi07685715-26 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1103: oid 26 updating version 0 to 445 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1103: oid 26 updating version 445 to 447 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1103: oid 26 version 447 is already newer than 446 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1103: oid 26 version 447 is already newer than 444 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 447 (ObjNum 436 snap 139 seq_num 436) dirty exists 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1103: left oid 26 (ObjNum 436 snap 139 seq_num 436) 2024-09-21T09:42:26.177 INFO:tasks.rados.rados.0.smithi076.stdout:1104: expect (ObjNum 377 snap 119 seq_num 377) 2024-09-21T09:42:26.252 INFO:tasks.rados.rados.0.smithi076.stdout:1105: finishing write tid 1 to smithi07685715-10 2024-09-21T09:42:26.252 INFO:tasks.rados.rados.0.smithi076.stdout:1105: finishing write tid 2 to smithi07685715-10 2024-09-21T09:42:26.252 INFO:tasks.rados.rados.0.smithi076.stdout:1105: finishing write tid 3 to smithi07685715-10 2024-09-21T09:42:26.252 INFO:tasks.rados.rados.0.smithi076.stdout:1105: finishing write tid 4 to smithi07685715-10 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: finishing write tid 5 to smithi07685715-10 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: oid 10 updating version 0 to 559 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: oid 10 version 559 is already newer than 558 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: oid 10 version 559 is already newer than 557 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: oid 10 version 559 is already newer than 556 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 559 (ObjNum 437 snap 139 seq_num 437) dirty exists 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: left oid 10 (ObjNum 437 snap 139 seq_num 437) 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1106: finishing write tid 1 to smithi07685715-5 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1103: done (5 left) 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1104: done (4 left) 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1105: done (3 left) 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1109: delete oid 33 current snap is 139 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1106: finishing write tid 2 to smithi07685715-5 2024-09-21T09:42:26.253 INFO:tasks.rados.rados.0.smithi076.stdout:1106: finishing write tid 3 to smithi07685715-5 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1108: finishing rollback tid 0 to smithi07685715-24 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1108: finishing rollback tid 1 to smithi07685715-24 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1108: finishing rollback tid 2 to smithi07685715-24 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 117 (ObjNum 326 snap 101 seq_num 326) dirty exists 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: finishing write tid 4 to smithi07685715-5 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: finishing write tid 5 to smithi07685715-5 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: oid 5 updating version 0 to 825 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: oid 5 updating version 825 to 828 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: oid 5 version 828 is already newer than 827 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: oid 5 version 828 is already newer than 826 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 828 (ObjNum 438 snap 139 seq_num 438) dirty exists 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1106: left oid 5 (ObjNum 438 snap 139 seq_num 438) 2024-09-21T09:42:26.254 INFO:tasks.rados.rados.0.smithi076.stdout:1107: expect (ObjNum 396 snap 123 seq_num 396) 2024-09-21T09:42:26.400 INFO:tasks.rados.rados.0.smithi076.stdout:1106: done (3 left) 2024-09-21T09:42:26.400 INFO:tasks.rados.rados.0.smithi076.stdout:1107: done (2 left) 2024-09-21T09:42:26.400 INFO:tasks.rados.rados.0.smithi076.stdout:1108: done (1 left) 2024-09-21T09:42:26.400 INFO:tasks.rados.rados.0.smithi076.stdout:1109: done (0 left) 2024-09-21T09:42:26.400 INFO:tasks.rados.rados.0.smithi076.stdout:1110: write oid 14 current snap is 139 2024-09-21T09:42:26.401 INFO:tasks.rados.rados.0.smithi076.stdout:1110: seq_num 440 ranges {595080=541681,1798077=596473,3150352=244728} 2024-09-21T09:42:26.406 INFO:tasks.rados.rados.0.smithi076.stdout:1110: writing smithi07685715-14 from 595080 to 1136761 tid 1 2024-09-21T09:42:26.413 INFO:tasks.rados.rados.0.smithi076.stdout:1110: writing smithi07685715-14 from 1798077 to 2394550 tid 2 2024-09-21T09:42:26.419 INFO:tasks.rados.rados.0.smithi076.stdout:1110: writing smithi07685715-14 from 3150352 to 3395080 tid 3 2024-09-21T09:42:26.419 INFO:tasks.rados.rados.0.smithi076.stdout:1111: write oid 31 current snap is 139 2024-09-21T09:42:26.419 INFO:tasks.rados.rados.0.smithi076.stdout:1110: finishing write tid 1 to smithi07685715-14 2024-09-21T09:42:26.419 INFO:tasks.rados.rados.0.smithi076.stdout:1110: finishing write tid 2 to smithi07685715-14 2024-09-21T09:42:26.419 INFO:tasks.rados.rados.0.smithi076.stdout:1111: seq_num 441 ranges {611887=733283,1972912=456484,2894128=517759} 2024-09-21T09:42:26.426 INFO:tasks.rados.rados.0.smithi076.stdout:1111: writing smithi07685715-31 from 611887 to 1345170 tid 1 2024-09-21T09:42:26.432 INFO:tasks.rados.rados.0.smithi076.stdout:1111: writing smithi07685715-31 from 1972912 to 2429396 tid 2 2024-09-21T09:42:26.437 INFO:tasks.rados.rados.0.smithi076.stdout:1111: writing smithi07685715-31 from 2894128 to 3411887 tid 3 2024-09-21T09:42:26.437 INFO:tasks.rados.rados.0.smithi076.stdout:1112: snap_create 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: finishing write tid 3 to smithi07685715-14 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: finishing write tid 4 to smithi07685715-14 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: finishing write tid 5 to smithi07685715-14 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: oid 14 updating version 0 to 562 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: oid 14 version 562 is already newer than 561 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: oid 14 updating version 562 to 563 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: oid 14 updating version 563 to 564 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 564 (ObjNum 440 snap 139 seq_num 440) dirty exists 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1110: left oid 14 (ObjNum 440 snap 139 seq_num 440) 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1111: finishing write tid 1 to smithi07685715-31 2024-09-21T09:42:26.438 INFO:tasks.rados.rados.0.smithi076.stdout:1111: finishing write tid 2 to smithi07685715-31 2024-09-21T09:42:26.439 INFO:tasks.rados.rados.0.smithi076.stdout:1111: finishing write tid 3 to smithi07685715-31 2024-09-21T09:42:26.439 INFO:tasks.rados.rados.0.smithi076.stdout:1111: finishing write tid 4 to smithi07685715-31 2024-09-21T09:42:26.439 INFO:tasks.rados.rados.0.smithi076.stdout:1111: finishing write tid 5 to smithi07685715-31 2024-09-21T09:42:26.439 INFO:tasks.rados.rados.0.smithi076.stdout:1111: oid 31 updating version 0 to 413 2024-09-21T09:42:26.440 INFO:tasks.rados.rados.0.smithi076.stdout:1111: oid 31 version 413 is already newer than 410 2024-09-21T09:42:26.440 INFO:tasks.rados.rados.0.smithi076.stdout:1111: oid 31 version 413 is already newer than 411 2024-09-21T09:42:26.440 INFO:tasks.rados.rados.0.smithi076.stdout:1111: oid 31 version 413 is already newer than 412 2024-09-21T09:42:26.440 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 413 (ObjNum 441 snap 139 seq_num 441) dirty exists 2024-09-21T09:42:26.440 INFO:tasks.rados.rados.0.smithi076.stdout:1111: left oid 31 (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:42:27.126 INFO:tasks.rados.rados.0.smithi076.stdout:1110: done (2 left) 2024-09-21T09:42:27.126 INFO:tasks.rados.rados.0.smithi076.stdout:1111: done (1 left) 2024-09-21T09:42:27.126 INFO:tasks.rados.rados.0.smithi076.stdout:1112: done (0 left) 2024-09-21T09:42:27.126 INFO:tasks.rados.rados.0.smithi076.stdout:1113: read oid 45 snap -1 2024-09-21T09:42:27.126 INFO:tasks.rados.rados.0.smithi076.stdout:1113: expect deleted 2024-09-21T09:42:27.126 INFO:tasks.rados.rados.0.smithi076.stdout:1114: snap_create 2024-09-21T09:42:27.288 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:42:27.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:27 smithi076 ceph-mon[29313]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:42:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:27 smithi076 ceph-mon[29313]: pgmap v212: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:42:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:27 smithi076 ceph-mon[41325]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:42:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:27 smithi076 ceph-mon[41325]: pgmap v212: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:42:27.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:27 smithi191 ceph-mon[33301]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:42:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:27 smithi191 ceph-mon[33301]: pgmap v212: 89 pgs: 89 active+clean; 132 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:42:27.863 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (58s) 54s ago 6m 17.4M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (37s) 32s ago 6m 45.6M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (5m) 54s ago 5m 17.7M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (5m) 32s ago 5m 16.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (2m) 32s ago 10m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (2m) 54s ago 12m 514M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (12m) 54s ago 12m 60.8M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 87efbba0b9f3 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (11m) 32s ago 11m 55.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (11m) 54s ago 11m 51.6M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e 2906e77f2aff 2024-09-21T09:42:27.864 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (106s) 54s ago 6m 9709k - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (100s) 32s ago 6m 10.7M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (10m) 54s ago 10m 89.0M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (10m) 54s ago 10m 77.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (9m) 54s ago 9m 82.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (9m) 54s ago 9m 70.9M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (8m) 32s ago 8m 84.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (8m) 32s ago 8m 91.9M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (8m) 32s ago 8m 67.0M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (7m) 32s ago 7m 66.2M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:42:27.865 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (78s) 32s ago 7m 39.3M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:1113: done (1 left) 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:1114: done (0 left) 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:1115: read oid 37 snap -1 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:1115: expect deleted 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:1116: rollback oid 38 current snap is 141 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 138 2024-09-21T09:42:28.127 INFO:tasks.rados.rados.0.smithi076.stdout:1117: write oid 1 current snap is 141 2024-09-21T09:42:28.128 INFO:tasks.rados.rados.0.smithi076.stdout:1117: seq_num 442 ranges {628694=408532,1515041=600142,2805198=552112} 2024-09-21T09:42:28.132 INFO:tasks.rados.rados.0.smithi076.stdout:1117: writing smithi07685715-1 from 628694 to 1037226 tid 1 2024-09-21T09:42:28.138 INFO:tasks.rados.rados.0.smithi076.stdout:1117: writing smithi07685715-1 from 1515041 to 2115183 tid 2 2024-09-21T09:42:28.145 INFO:tasks.rados.rados.0.smithi076.stdout:1117: writing smithi07685715-1 from 2805198 to 3357310 tid 3 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:1118: delete oid 7 current snap is 141 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:1116: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:1116: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:1116: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 568 (ObjNum 423 snap 132 seq_num 423) dirty exists 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:1117: finishing write tid 1 to smithi07685715-1 2024-09-21T09:42:28.146 INFO:tasks.rados.rados.0.smithi076.stdout:1117: finishing write tid 2 to smithi07685715-1 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:1115: done (3 left) 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:1116: done (2 left) 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:1118: done (1 left) 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:1119: rollback oid 48 current snap is 141 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 127 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:1120: write oid 5 current snap is 141 2024-09-21T09:42:28.148 INFO:tasks.rados.rados.0.smithi076.stdout:1120: seq_num 444 ranges {662308=559030,1632005=603811,2860044=602264} 2024-09-21T09:42:28.154 INFO:tasks.rados.rados.0.smithi076.stdout:1120: writing smithi07685715-5 from 662308 to 1221338 tid 1 2024-09-21T09:42:28.160 INFO:tasks.rados.rados.0.smithi076.stdout:1120: writing smithi07685715-5 from 1632005 to 2235816 tid 2 2024-09-21T09:42:28.167 INFO:tasks.rados.rados.0.smithi076.stdout:1120: writing smithi07685715-5 from 2860044 to 3462308 tid 3 2024-09-21T09:42:28.167 INFO:tasks.rados.rados.0.smithi076.stdout:1121: write oid 29 current snap is 141 2024-09-21T09:42:28.167 INFO:tasks.rados.rados.0.smithi076.stdout:1121: seq_num 445 ranges {679115=634279,2090487=463822,3003820=475295} 2024-09-21T09:42:28.174 INFO:tasks.rados.rados.0.smithi076.stdout:1121: writing smithi07685715-29 from 679115 to 1313394 tid 1 2024-09-21T09:42:28.180 INFO:tasks.rados.rados.0.smithi076.stdout:1121: writing smithi07685715-29 from 2090487 to 2554309 tid 2 2024-09-21T09:42:28.186 INFO:tasks.rados.rados.0.smithi076.stdout:1121: writing smithi07685715-29 from 3003820 to 3479115 tid 3 2024-09-21T09:42:28.186 INFO:tasks.rados.rados.0.smithi076.stdout:1122: read oid 15 snap -1 2024-09-21T09:42:28.186 INFO:tasks.rados.rados.0.smithi076.stdout:1122: expect (ObjNum 413 snap 128 seq_num 413) 2024-09-21T09:42:28.186 INFO:tasks.rados.rados.0.smithi076.stdout:1123: read oid 16 snap -1 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1123: expect (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1124: snap_create 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: finishing write tid 3 to smithi07685715-1 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: finishing write tid 4 to smithi07685715-1 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: finishing write tid 5 to smithi07685715-1 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: oid 1 updating version 0 to 831 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: oid 1 updating version 831 to 832 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: oid 1 version 832 is already newer than 830 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: oid 1 updating version 832 to 833 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 833 (ObjNum 442 snap 141 seq_num 442) dirty exists 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1117: left oid 1 (ObjNum 442 snap 141 seq_num 442) 2024-09-21T09:42:28.187 INFO:tasks.rados.rados.0.smithi076.stdout:1119: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1119: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 901 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: finishing write tid 1 to smithi07685715-5 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: finishing write tid 2 to smithi07685715-5 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: finishing write tid 3 to smithi07685715-5 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: finishing write tid 4 to smithi07685715-5 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: finishing write tid 5 to smithi07685715-5 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: oid 5 updating version 0 to 836 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: oid 5 updating version 836 to 837 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: oid 5 updating version 837 to 838 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: oid 5 version 838 is already newer than 835 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 838 (ObjNum 444 snap 141 seq_num 444) dirty exists 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1120: left oid 5 (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1121: finishing write tid 1 to smithi07685715-29 2024-09-21T09:42:28.188 INFO:tasks.rados.rados.0.smithi076.stdout:1121: finishing write tid 2 to smithi07685715-29 2024-09-21T09:42:28.189 INFO:tasks.rados.rados.0.smithi076.stdout:1121: finishing write tid 3 to smithi07685715-29 2024-09-21T09:42:28.189 INFO:tasks.rados.rados.0.smithi076.stdout:1121: finishing write tid 4 to smithi07685715-29 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:1121: finishing write tid 5 to smithi07685715-29 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:1121: oid 29 updating version 0 to 304 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:1121: oid 29 version 304 is already newer than 303 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:1121: oid 29 version 304 is already newer than 301 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:1121: oid 29 version 304 is already newer than 302 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 304 (ObjNum 445 snap 141 seq_num 445) dirty exists 2024-09-21T09:42:28.190 INFO:tasks.rados.rados.0.smithi076.stdout:1121: left oid 29 (ObjNum 445 snap 141 seq_num 445) 2024-09-21T09:42:28.193 INFO:tasks.rados.rados.0.smithi076.stdout:1122: expect (ObjNum 413 snap 128 seq_num 413) 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 3 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:42:28.217 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:42:28.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:42:28.218 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:42:28.218 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 13, 2024-09-21T09:42:28.218 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:42:28.218 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:42:28.218 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:42:28.368 INFO:tasks.rados.rados.0.smithi076.stdout:1123: expect (ObjNum 421 snap 131 seq_num 421) 2024-09-21T09:42:28.522 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[29313]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:42:28.522 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[29313]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.522 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[29313]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.522 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[29313]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.522 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[41325]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:42:28.522 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[41325]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.523 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[41325]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.523 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:28 smithi076 ceph-mon[41325]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "0/3 daemons upgraded", 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:42:28.523 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1117: done (6 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1119: done (5 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1120: done (4 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1121: done (3 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1122: done (2 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1123: done (1 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1124: done (0 left) 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1125: rollback oid 34 current snap is 142 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 129 2024-09-21T09:42:28.567 INFO:tasks.rados.rados.0.smithi076.stdout:1126: snap_create 2024-09-21T09:42:28.569 INFO:tasks.rados.rados.0.smithi076.stdout:1125: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:42:28.569 INFO:tasks.rados.rados.0.smithi076.stdout:1125: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:42:28.569 INFO:tasks.rados.rados.0.smithi076.stdout:1125: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:42:28.569 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 198 (ObjNum 404 snap 124 seq_num 404) dirty exists 2024-09-21T09:42:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:28 smithi191 ceph-mon[33301]: osdmap e347: 8 total, 8 up, 8 in 2024-09-21T09:42:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:28 smithi191 ceph-mon[33301]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:28 smithi191 ceph-mon[33301]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:28 smithi191 ceph-mon[33301]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='client.? 172.21.15.76:0/4097220357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: Upgrade: Setting container_image for all mgr 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:42:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[29313]: pgmap v216: 89 pgs: 89 active+clean; 148 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 35 op/s 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='client.? 172.21.15.76:0/4097220357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:29.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: Upgrade: Setting container_image for all mgr 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:42:29.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:29 smithi076 ceph-mon[41325]: pgmap v216: 89 pgs: 89 active+clean; 148 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 35 op/s 2024-09-21T09:42:29.561 INFO:tasks.rados.rados.0.smithi076.stdout:1125: done (1 left) 2024-09-21T09:42:29.561 INFO:tasks.rados.rados.0.smithi076.stdout:1126: done (0 left) 2024-09-21T09:42:29.561 INFO:tasks.rados.rados.0.smithi076.stdout:1127: snap_create 2024-09-21T09:42:29.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: osdmap e348: 8 total, 8 up, 8 in 2024-09-21T09:42:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/4097220357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:29.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:42:29.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:42:29.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:29.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:42:29.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all mgr 2024-09-21T09:42:29.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:29.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:42:29.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:42:29.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:42:29.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: from='client.15153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:29.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: osdmap e349: 8 total, 8 up, 8 in 2024-09-21T09:42:29.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:29 smithi191 ceph-mon[33301]: pgmap v216: 89 pgs: 89 active+clean; 148 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 35 op/s 2024-09-21T09:42:30.563 INFO:tasks.rados.rados.0.smithi076.stdout:1127: done (0 left) 2024-09-21T09:42:30.563 INFO:tasks.rados.rados.0.smithi076.stdout:1128: write oid 16 current snap is 144 2024-09-21T09:42:30.563 INFO:tasks.rados.rados.0.smithi076.stdout:1128: seq_num 446 ranges {695922=709528,2032616=607480,3198537=297385} 2024-09-21T09:42:30.570 INFO:tasks.rados.rados.0.smithi076.stdout:1128: writing smithi07685715-16 from 695922 to 1405450 tid 1 2024-09-21T09:42:30.577 INFO:tasks.rados.rados.0.smithi076.stdout:1128: writing smithi07685715-16 from 2032616 to 2640096 tid 2 2024-09-21T09:42:30.582 INFO:tasks.rados.rados.0.smithi076.stdout:1128: writing smithi07685715-16 from 3198537 to 3495922 tid 3 2024-09-21T09:42:30.582 INFO:tasks.rados.rados.0.smithi076.stdout:1129: read oid 3 snap -1 2024-09-21T09:42:30.582 INFO:tasks.rados.rados.0.smithi076.stdout:1129: expect deleted 2024-09-21T09:42:30.582 INFO:tasks.rados.rados.0.smithi076.stdout:1130: snap_remove snap 131 2024-09-21T09:42:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:42:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: Upgrade: Updating mon.a 2024-09-21T09:42:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:42:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:42:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:30.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:30 smithi191 ceph-mon[33301]: Deploying daemon mon.a on smithi076 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: Upgrade: Updating mon.a 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[41325]: Deploying daemon mon.a on smithi076 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: osdmap e350: 8 total, 8 up, 8 in 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: Upgrade: Updating mon.a 2024-09-21T09:42:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: from='mgr.24836 ' entity='mgr.y' 2024-09-21T09:42:30.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:42:30.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:42:30.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: from='mgr.24836 172.21.15.76:0/3410589300' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:42:30.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:30 smithi076 ceph-mon[29313]: Deploying daemon mon.a on smithi076 2024-09-21T09:42:31.575 INFO:tasks.rados.rados.0.smithi076.stdout:1128: finishing write tid 1 to smithi07685715-16 2024-09-21T09:42:31.575 INFO:tasks.rados.rados.0.smithi076.stdout:1128: finishing write tid 2 to smithi07685715-16 2024-09-21T09:42:31.575 INFO:tasks.rados.rados.0.smithi076.stdout:1128: finishing write tid 3 to smithi07685715-16 2024-09-21T09:42:31.575 INFO:tasks.rados.rados.0.smithi076.stdout:1128: finishing write tid 4 to smithi07685715-16 2024-09-21T09:42:31.575 INFO:tasks.rados.rados.0.smithi076.stdout:1128: finishing write tid 5 to smithi07685715-16 2024-09-21T09:42:31.575 INFO:tasks.rados.rados.0.smithi076.stdout:1128: oid 16 updating version 0 to 562 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1128: oid 16 version 562 is already newer than 560 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1128: oid 16 version 562 is already newer than 559 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1128: oid 16 version 562 is already newer than 561 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 562 (ObjNum 446 snap 144 seq_num 446) dirty exists 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1128: left oid 16 (ObjNum 446 snap 144 seq_num 446) 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1128: done (2 left) 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1129: done (1 left) 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1130: done (0 left) 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1131: read oid 31 snap -1 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1131: expect (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:42:31.576 INFO:tasks.rados.rados.0.smithi076.stdout:1132: delete oid 12 current snap is 144 2024-09-21T09:42:31.580 INFO:tasks.rados.rados.0.smithi076.stdout:1132: done (1 left) 2024-09-21T09:42:31.580 INFO:tasks.rados.rados.0.smithi076.stdout:1133: snap_create 2024-09-21T09:42:31.584 INFO:tasks.rados.rados.0.smithi076.stdout:1131: expect (ObjNum 441 snap 139 seq_num 441) 2024-09-21T09:42:31.823 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:31 smithi076 ceph-mon[41325]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:42:31.823 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:31 smithi076 ceph-mon[41325]: pgmap v219: 89 pgs: 89 active+clean; 148 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 35 op/s 2024-09-21T09:42:31.824 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:31 smithi076 systemd[1]: Stopping Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:42:31.824 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:31 smithi076 ceph-mon[29313]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:42:31.824 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:31 smithi076 ceph-mon[29313]: pgmap v219: 89 pgs: 89 active+clean; 148 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 35 op/s 2024-09-21T09:42:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:31 smithi191 ceph-mon[33301]: osdmap e351: 8 total, 8 up, 8 in 2024-09-21T09:42:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:31 smithi191 ceph-mon[33301]: pgmap v219: 89 pgs: 89 active+clean; 148 MiB data, 570 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 35 op/s 2024-09-21T09:42:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:31 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[29290]: 2024-09-21T09:42:31.820+0000 7f2e76b66640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:42:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:31 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[29290]: 2024-09-21T09:42:31.820+0000 7f2e76b66640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T09:42:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:31 smithi076 podman[106290]: 2024-09-21 09:42:31.933545482 +0000 UTC m=+0.219199204 container died 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T09:42:32.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:32 smithi076 podman[106290]: 2024-09-21 09:42:32.436503232 +0000 UTC m=+0.722157003 container cleanup 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:42:32.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:32 smithi076 bash[106290]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:42:33.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:32 smithi076 podman[106308]: 2024-09-21 09:42:32.738866529 +0000 UTC m=+0.802335314 container remove 87efbba0b9f3d6c1790df3c11d9a505339089f091096397f9713f137c76209e1 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:42:33.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:32 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:42:33.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:32 smithi076 systemd[1]: Stopped Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:42:33.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:32 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service: Consumed 14.976s CPU time. 2024-09-21T09:42:33.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:42:33] "GET /metrics HTTP/1.1" 200 36144 "" "Prometheus/2.51.0" 2024-09-21T09:42:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 systemd[1]: Starting Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:42:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 podman[106444]: 2024-09-21 09:42:33.218278656 +0000 UTC m=+0.015696185 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:42:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 podman[106444]: 2024-09-21 09:42:33.315253212 +0000 UTC m=+0.112670725 container create 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:42:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 podman[106444]: 2024-09-21 09:42:33.551114541 +0000 UTC m=+0.348532066 container init 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True) 2024-09-21T09:42:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 podman[106444]: 2024-09-21 09:42:33.556026807 +0000 UTC m=+0.353444320 container start 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:42:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: pidfile_write: ignore empty --pid-file 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: load: jerasure load: lrc 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Git sha 0 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: DB SUMMARY 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: DB Session ID: PQ0OKEB9Q1RQKVHSCVLP 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:42:33.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1518 Bytes 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 7333135 ; 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.env: 0x5608fe172120 2024-09-21T09:42:33.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.info_log: 0x5608febfb700 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.statistics: (nil) 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.use_fsync: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:42:33.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.db_log_dir: 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.wal_dir: 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.write_buffer_manager: 0x5608febfe1e0 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:42:33.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.unordered_write: 0 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.row_cache: None 2024-09-21T09:42:33.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.wal_filter: None 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.wal_compression: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:42:33.880 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_open_files: -1 2024-09-21T09:42:33.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Compression algorithms supported: 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kZSTD supported: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:42:33.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.merge_operator: 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_filter: None 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5608febfb320) 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout: index_type: 0 2024-09-21T09:42:33.883 INFO:journalctl@ceph.mon.a.smithi076.stdout: data_block_index_type: 0 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: index_shortening: 1 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: checksum: 4 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: no_block_cache: 0 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache: 0x5608febef610 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache_options: 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: capacity : 536870912 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: num_shard_bits : 4 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: strict_capacity_limit : 0 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_cache_compressed: (nil) 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: persistent_cache: (nil) 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_size: 4096 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_size_deviation: 10 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_restart_interval: 16 2024-09-21T09:42:33.884 INFO:journalctl@ceph.mon.a.smithi076.stdout: index_block_restart_interval: 1 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: metadata_block_size: 4096 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: partition_filters: 0 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: use_delta_encoding: 1 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: filter_policy: bloomfilter 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: whole_key_filtering: 1 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: verify_compression: 0 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: format_version: 5 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: enable_index_compression: 1 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: block_align: 0 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: max_auto_readahead_size: 262144 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: prepopulate_block_cache: 0 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:42:33.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression: NoCompression 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.num_levels: 7 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:42:33.886 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:42:33.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:42:33.888 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:42:33.889 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:42:33.890 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.ttl: 2592000 2024-09-21T09:42:33.891 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:42:33.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12779, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dcd6f1be-f33b-4984-a0ba-b429944fcff3 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911753582307, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-09-21T09:42:33.893 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-09-21T09:42:33.894 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911753605516, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 6826056, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12780, "largest_seqno": 17413, "table_properties": {"data_size": 6804273, "index_size": 15424, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 5317, "raw_key_size": 50207, "raw_average_key_size": 23, "raw_value_size": 6758773, "raw_average_value_size": 3191, "num_data_blocks": 668, "num_entries": 2118, "num_filter_entries": 2118, "num_deletions": 357, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911753, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dcd6f1be-f33b-4984-a0ba-b429944fcff3", "db_session_id": "PQ0OKEB9Q1RQKVHSCVLP", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:42:33.894 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911753605664, "job": 1, "event": "recovery_finished"} 2024-09-21T09:42:33.894 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-09-21T09:42:33.894 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:42:33.894 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:42:33.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5608fec1ee00 2024-09-21T09:42:33.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: DB pointer 0x5608fed20000 2024-09-21T09:42:33.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:42:33.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:42:33.895 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** DB Stats ** 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:42:33.896 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: L0 1/0 6.51 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 524.8 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: L6 1/0 7.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: Sum 2/0 14.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 524.8 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-21T09:42:33.897 INFO:journalctl@ceph.mon.a.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 524.8 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 524.8 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.898 INFO:journalctl@ceph.mon.a.smithi076.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: Cumulative compaction: 0.01 GB write, 250.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:42:33.899 INFO:journalctl@ceph.mon.a.smithi076.stdout: Interval compaction: 0.01 GB write, 250.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout: Block cache BinnedLRUCache@0x5608febef610#7 capacity: 512.00 MB usage: 22.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,6.03 KB,0.00115037%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: starting mon.a rank 0 at public addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] at bind addrs [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???) e3 preinit fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).mds e9 new map 2024-09-21T09:42:33.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).mds e9 print_map 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: e9 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: legacy client fscid: 1 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: Filesystem 'foo' (1) 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: fs_name foo 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: epoch 8 2024-09-21T09:42:33.901 INFO:journalctl@ceph.mon.a.smithi076.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: created 2024-09-21T09:36:39.879731+0000 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: modified 2024-09-21T09:36:51.753955+0000 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: tableserver 0 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: root 0 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: session_timeout 60 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: session_autoclose 300 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: max_file_size 1099511627776 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: max_xattr_size 65536 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: required_client_features {} 2024-09-21T09:42:33.902 INFO:journalctl@ceph.mon.a.smithi076.stdout: last_failure 0 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: last_failure_osd_epoch 0 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: max_mds 1 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: in 0 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: up {0=14631} 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: failed 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: damaged 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: stopped 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: data_pools [4] 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: metadata_pool 3 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: inline_data disabled 2024-09-21T09:42:33.903 INFO:journalctl@ceph.mon.a.smithi076.stdout: balancer 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: bal_rank_mask -1 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: standby_count_wanted 1 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: [mds.foo.smithi076.jymebx{0:14631} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: Standby daemons: 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout: [mds.foo.smithi191.dhrpzx{-1:24466} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).osd e352 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).osd e352 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).osd e352 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).osd e352 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 ceph-mon[106482]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 bash[106444]: 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 2024-09-21T09:42:33.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:33 smithi076 systemd[1]: Started Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:42:35.094 INFO:tasks.rados.rados.0.smithi076.stdout:1131: done (1 left) 2024-09-21T09:42:35.094 INFO:tasks.rados.rados.0.smithi076.stdout:1133: done (0 left) 2024-09-21T09:42:35.094 INFO:tasks.rados.rados.0.smithi076.stdout:1134: delete oid 24 current snap is 145 2024-09-21T09:42:35.098 INFO:tasks.rados.rados.0.smithi076.stdout:1134: done (0 left) 2024-09-21T09:42:35.098 INFO:tasks.rados.rados.0.smithi076.stdout:1135: read oid 43 snap 127 2024-09-21T09:42:35.099 INFO:tasks.rados.rados.0.smithi076.stdout:1135: expect deleted 2024-09-21T09:42:35.099 INFO:tasks.rados.rados.0.smithi076.stdout:1136: delete oid 39 current snap is 145 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi076.stdout:1135: done (1 left) 2024-09-21T09:42:35.101 INFO:tasks.rados.rados.0.smithi076.stdout:1136: done (0 left) 2024-09-21T09:42:35.102 INFO:tasks.rados.rados.0.smithi076.stdout:1137: snap_create 2024-09-21T09:42:35.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: ignoring --setuser ceph since I am not root 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: ignoring --setgroup ceph since I am not root 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mgr[80524]: -- 172.21.15.191:0/1417643524 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a47d7e90e0 con 0x55a47d7c7000 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: pgmap v221: 89 pgs: 89 active+clean; 155 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: pgmap v222: 89 pgs: 89 active+clean; 155 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.0 MiB/s wr, 10 op/s 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: mon.a calling monitor election 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: monmap epoch 3 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: last_changed 2024-09-21T09:31:20.913061+0000 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:42:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: min_mon_release 18 (reef) 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: election_strategy: 1 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: overall HEALTH_OK 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: from='mgr.24836 ' entity='' 2024-09-21T09:42:35.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:35 smithi191 ceph-mon[33301]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ignoring --setuser ceph since I am not root 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ignoring --setgroup ceph since I am not root 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mgr[89227]: -- 172.21.15.76:0/3850411449 <== mon.2 v2:172.21.15.76:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fb82f7b0e0 con 0x55fb82f59400 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: pgmap v221: 89 pgs: 89 active+clean; 155 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: pgmap v222: 89 pgs: 89 active+clean; 155 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.0 MiB/s wr, 10 op/s 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: mon.a calling monitor election 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:42:35.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: monmap epoch 3 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: last_changed 2024-09-21T09:31:20.913061+0000 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: min_mon_release 18 (reef) 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: election_strategy: 1 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: overall HEALTH_OK 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: from='mgr.24836 ' entity='' 2024-09-21T09:42:35.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[106482]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: pgmap v221: 89 pgs: 89 active+clean; 155 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: pgmap v222: 89 pgs: 89 active+clean; 155 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.0 MiB/s wr, 10 op/s 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: mon.a calling monitor election 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: monmap epoch 3 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: last_changed 2024-09-21T09:31:20.913061+0000 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: min_mon_release 18 (reef) 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: election_strategy: 1 2024-09-21T09:42:35.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: osdmap e352: 8 total, 8 up, 8 in 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: overall HEALTH_OK 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: from='mgr.24836 ' entity='' 2024-09-21T09:42:35.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:35 smithi076 ceph-mon[41325]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-21T09:42:36.099 INFO:tasks.rados.rados.0.smithi076.stdout:1137: done (0 left) 2024-09-21T09:42:36.099 INFO:tasks.rados.rados.0.smithi076.stdout:1138: read oid 6 snap -1 2024-09-21T09:42:36.099 INFO:tasks.rados.rados.0.smithi076.stdout:1138: expect (ObjNum 426 snap 135 seq_num 426) 2024-09-21T09:42:36.099 INFO:tasks.rados.rados.0.smithi076.stdout:1139: read oid 21 snap -1 2024-09-21T09:42:36.099 INFO:tasks.rados.rados.0.smithi076.stdout:1139: expect (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:36.099 INFO:tasks.rados.rados.0.smithi076.stdout:1140: snap_create 2024-09-21T09:42:36.115 INFO:tasks.rados.rados.0.smithi076.stdout:1139: expect (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:36.228 INFO:tasks.rados.rados.0.smithi076.stdout:1138: expect (ObjNum 426 snap 135 seq_num 426) 2024-09-21T09:42:36.357 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:36 smithi076 ceph-mon[41325]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:42:36.357 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:36 smithi076 ceph-mon[106482]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:42:36.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:36 smithi191 ceph-mon[33301]: osdmap e353: 8 total, 8 up, 8 in 2024-09-21T09:42:37.104 INFO:tasks.rados.rados.0.smithi076.stdout:1138: done (2 left) 2024-09-21T09:42:37.105 INFO:tasks.rados.rados.0.smithi076.stdout:1139: done (1 left) 2024-09-21T09:42:37.105 INFO:tasks.rados.rados.0.smithi076.stdout:1140: done (0 left) 2024-09-21T09:42:37.105 INFO:tasks.rados.rados.0.smithi076.stdout:1141: write oid 49 current snap is 147 2024-09-21T09:42:37.105 INFO:tasks.rados.rados.0.smithi076.stdout:1141: seq_num 450 ranges {763150=726877,1982897=614818,3024582=474994} 2024-09-21T09:42:37.112 INFO:tasks.rados.rados.0.smithi076.stdout:1141: writing smithi07685715-49 from 763150 to 1490027 tid 1 2024-09-21T09:42:37.119 INFO:tasks.rados.rados.0.smithi076.stdout:1141: writing smithi07685715-49 from 1982897 to 2597715 tid 2 2024-09-21T09:42:37.124 INFO:tasks.rados.rados.0.smithi076.stdout:1141: writing smithi07685715-49 from 3024582 to 3499576 tid 3 2024-09-21T09:42:37.124 INFO:tasks.rados.rados.0.smithi076.stdout:1142: snap_remove snap 144 2024-09-21T09:42:37.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:37 smithi191 ceph-mon[33301]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:42:37.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:37 smithi076 ceph-mon[106482]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:42:37.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:37 smithi076 ceph-mon[41325]: osdmap e354: 8 total, 8 up, 8 in 2024-09-21T09:42:38.119 INFO:tasks.rados.rados.0.smithi076.stdout:1142: done (1 left) 2024-09-21T09:42:38.120 INFO:tasks.rados.rados.0.smithi076.stdout:1143: read oid 1 snap -1 2024-09-21T09:42:38.120 INFO:tasks.rados.rados.0.smithi076.stdout:1143: expect (ObjNum 442 snap 141 seq_num 442) 2024-09-21T09:42:38.120 INFO:tasks.rados.rados.0.smithi076.stdout:1144: read oid 40 snap 145 2024-09-21T09:42:38.120 INFO:tasks.rados.rados.0.smithi076.stdout:1144: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:42:38.120 INFO:tasks.rados.rados.0.smithi076.stdout:1145: snap_remove snap 146 2024-09-21T09:42:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:38 smithi191 ceph-mon[33301]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:42:38.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:38 smithi076 ceph-mon[106482]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:42:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:38 smithi076 ceph-mon[41325]: osdmap e355: 8 total, 8 up, 8 in 2024-09-21T09:42:38.619 INFO:tasks.rados.rados.0.smithi076.stdout:1145: done (3 left) 2024-09-21T09:42:38.619 INFO:tasks.rados.rados.0.smithi076.stdout:1146: read oid 37 snap 136 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1146: expect deleted 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1147: read oid 4 snap -1 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1147: expect (ObjNum 411 snap 128 seq_num 411) 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1148: read oid 20 snap -1 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1148: expect (ObjNum 418 snap 130 seq_num 418) 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1149: write oid 35 current snap is 147 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1141: finishing write tid 1 to smithi07685715-49 2024-09-21T09:42:38.620 INFO:tasks.rados.rados.0.smithi076.stdout:1149: seq_num 451 ranges {779957=402126,1925026=758476,3335652=244305} 2024-09-21T09:42:38.626 INFO:tasks.rados.rados.0.smithi076.stdout:1149: writing smithi07685715-35 from 779957 to 1182083 tid 1 2024-09-21T09:42:38.635 INFO:tasks.rados.rados.0.smithi076.stdout:1149: writing smithi07685715-35 from 1925026 to 2683502 tid 2 2024-09-21T09:42:38.640 INFO:tasks.rados.rados.0.smithi076.stdout:1149: writing smithi07685715-35 from 3335652 to 3579957 tid 3 2024-09-21T09:42:38.640 INFO:tasks.rados.rados.0.smithi076.stdout:1150: read oid 18 snap -1 2024-09-21T09:42:38.640 INFO:tasks.rados.rados.0.smithi076.stdout:1150: expect (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:42:38.640 INFO:tasks.rados.rados.0.smithi076.stdout:1151: snap_remove snap 141 2024-09-21T09:42:38.640 INFO:tasks.rados.rados.0.smithi076.stdout:1141: finishing write tid 2 to smithi07685715-49 2024-09-21T09:42:39.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:39 smithi076 ceph-mon[106482]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:42:39.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:39 smithi076 ceph-mon[106482]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:42:39.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:39 smithi076 ceph-mon[41325]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:42:39.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:39 smithi076 ceph-mon[41325]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:42:39.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:39 smithi191 ceph-mon[33301]: osdmap e356: 8 total, 8 up, 8 in 2024-09-21T09:42:39.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:39 smithi191 ceph-mon[33301]: osdmap e357: 8 total, 8 up, 8 in 2024-09-21T09:42:39.620 INFO:tasks.rados.rados.0.smithi076.stdout:1151: done (8 left) 2024-09-21T09:42:39.620 INFO:tasks.rados.rados.0.smithi076.stdout:1152: read oid 43 snap -1 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1152: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1153: read oid 25 snap -1 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1153: expect (ObjNum 361 snap 110 seq_num 361) 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1154: delete oid 17 current snap is 147 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: finishing write tid 3 to smithi07685715-49 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: finishing write tid 4 to smithi07685715-49 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: finishing write tid 5 to smithi07685715-49 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: oid 49 updating version 0 to 910 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: oid 49 version 910 is already newer than 908 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: oid 49 version 910 is already newer than 909 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: oid 49 version 910 is already newer than 907 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 910 (ObjNum 450 snap 147 seq_num 450) dirty exists 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1141: left oid 49 (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:42:39.621 INFO:tasks.rados.rados.0.smithi076.stdout:1144: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:42:39.749 INFO:tasks.rados.rados.0.smithi076.stdout:1143: expect (ObjNum 442 snap 141 seq_num 442) 2024-09-21T09:42:39.951 INFO:tasks.rados.rados.0.smithi076.stdout:1148: expect (ObjNum 418 snap 130 seq_num 418) 2024-09-21T09:42:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:1147: expect (ObjNum 411 snap 128 seq_num 411) 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1149: finishing write tid 1 to smithi07685715-35 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1149: finishing write tid 2 to smithi07685715-35 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1141: done (10 left) 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1143: done (9 left) 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1144: done (8 left) 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1146: done (7 left) 2024-09-21T09:42:40.263 INFO:tasks.rados.rados.0.smithi076.stdout:1147: done (6 left) 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1148: done (5 left) 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1154: done (4 left) 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1155: snap_create 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: finishing write tid 3 to smithi07685715-35 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: finishing write tid 4 to smithi07685715-35 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: finishing write tid 5 to smithi07685715-35 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: oid 35 updating version 0 to 852 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: oid 35 version 852 is already newer than 850 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: oid 35 version 852 is already newer than 851 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: oid 35 version 852 is already newer than 849 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 852 (ObjNum 451 snap 147 seq_num 451) dirty exists 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1149: left oid 35 (ObjNum 451 snap 147 seq_num 451) 2024-09-21T09:42:40.264 INFO:tasks.rados.rados.0.smithi076.stdout:1150: expect (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:42:40.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: [21/Sep/2024:09:42:40] ENGINE Bus STARTING 2024-09-21T09:42:40.362 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: CherryPy Checker: 2024-09-21T09:42:40.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: The Application mounted at '' has an empty config. 2024-09-21T09:42:40.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: 2024-09-21T09:42:40.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: [21/Sep/2024:09:42:40] ENGINE Serving on http://:::9283 2024-09-21T09:42:40.363 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x[80501]: [21/Sep/2024:09:42:40] ENGINE Bus STARTED 2024-09-21T09:42:40.419 INFO:tasks.rados.rados.0.smithi076.stdout:1153: expect (ObjNum 361 snap 110 seq_num 361) 2024-09-21T09:42:40.510 INFO:tasks.rados.rados.0.smithi076.stdout:1152: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:42:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.597+0000 7f868bebe640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.597+0000 7f868bebe640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.597+0000 7f868bebe640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.597+0000 7f868bebe640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.597+0000 7f868bebe640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.598+0000 7f8689eba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.598+0000 7f8689eba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.598+0000 7f8689eba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.598+0000 7f8689eba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:40.598+0000 7f8689eba640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: Standby manager daemon x restarted 2024-09-21T09:42:40.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: Standby manager daemon x started 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: Active manager daemon y restarted 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: Activating manager daemon y 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: mgrmap e36: y(active, starting, since 0.00756679s), standbys: x 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:42:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi076.jymebx"} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:42:40.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: Manager daemon y is now available 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:42:40.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: Standby manager daemon x restarted 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: Standby manager daemon x started 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: Active manager daemon y restarted 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: Activating manager daemon y 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: mgrmap e36: y(active, starting, since 0.00756679s), standbys: x 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:42:40.878 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi076.jymebx"} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:42:40.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: Manager daemon y is now available 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:42:40.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:40 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: osdmap e358: 8 total, 8 up, 8 in 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: Standby manager daemon x restarted 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: Standby manager daemon x started 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.? 172.21.15.191:0/806686099' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: Active manager daemon y restarted 2024-09-21T09:42:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: Activating manager daemon y 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: mgrmap e36: y(active, starting, since 0.00756679s), standbys: x 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi076.jymebx"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi191.dhrpzx"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:42:41.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: Manager daemon y is now available 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:42:41.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:42:41.223 INFO:tasks.rados.rados.0.smithi076.stdout:1149: done (4 left) 2024-09-21T09:42:41.223 INFO:tasks.rados.rados.0.smithi076.stdout:1150: done (3 left) 2024-09-21T09:42:41.223 INFO:tasks.rados.rados.0.smithi076.stdout:1152: done (2 left) 2024-09-21T09:42:41.223 INFO:tasks.rados.rados.0.smithi076.stdout:1153: done (1 left) 2024-09-21T09:42:41.223 INFO:tasks.rados.rados.0.smithi076.stdout:1155: done (0 left) 2024-09-21T09:42:41.223 INFO:tasks.rados.rados.0.smithi076.stdout:1156: snap_remove snap 145 2024-09-21T09:42:41.475 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:41.210+0000 7f86bc22c640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:42:41.476 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:42:41] ENGINE Bus STARTING 2024-09-21T09:42:41.476 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: CherryPy Checker: 2024-09-21T09:42:41.476 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: The Application mounted at '' has an empty config. 2024-09-21T09:42:41.476 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:41.476 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:42:41] ENGINE Serving on http://:::9283 2024-09-21T09:42:41.476 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:41 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:09:42:41] ENGINE Bus STARTED 2024-09-21T09:42:42.232 INFO:tasks.rados.rados.0.smithi076.stdout:1156: done (0 left) 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1157: rollback oid 42 current snap is 148 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 138 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1158: read oid 44 snap -1 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1158: expect (ObjNum 410 snap 128 seq_num 410) 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1159: read oid 43 snap -1 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1159: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1160: write oid 22 current snap is 148 2024-09-21T09:42:42.233 INFO:tasks.rados.rados.0.smithi076.stdout:1160: seq_num 453 ranges {413571=552624,1641990=762145,2990498=425163} 2024-09-21T09:42:42.238 INFO:tasks.rados.rados.0.smithi076.stdout:1160: writing smithi07685715-22 from 413571 to 966195 tid 1 2024-09-21T09:42:42.246 INFO:tasks.rados.rados.0.smithi076.stdout:1160: writing smithi07685715-22 from 1641990 to 2404135 tid 2 2024-09-21T09:42:42.253 INFO:tasks.rados.rados.0.smithi076.stdout:1160: writing smithi07685715-22 from 2990498 to 3415661 tid 3 2024-09-21T09:42:42.253 INFO:tasks.rados.rados.0.smithi076.stdout:1161: read oid 21 snap -1 2024-09-21T09:42:42.253 INFO:tasks.rados.rados.0.smithi076.stdout:1161: expect (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:42.253 INFO:tasks.rados.rados.0.smithi076.stdout:1162: write oid 48 current snap is 148 2024-09-21T09:42:42.253 INFO:tasks.rados.rados.0.smithi076.stdout:1162: seq_num 454 ranges {430378=627873,1700472=622156,3017921=612457} 2024-09-21T09:42:42.259 INFO:tasks.rados.rados.0.smithi076.stdout:1162: writing smithi07685715-48 from 430378 to 1058251 tid 1 2024-09-21T09:42:42.265 INFO:tasks.rados.rados.0.smithi076.stdout:1162: writing smithi07685715-48 from 1700472 to 2322628 tid 2 2024-09-21T09:42:42.273 INFO:tasks.rados.rados.0.smithi076.stdout:1162: writing smithi07685715-48 from 3017921 to 3630378 tid 3 2024-09-21T09:42:42.273 INFO:tasks.rados.rados.0.smithi076.stdout:1163: snap_create 2024-09-21T09:42:42.273 INFO:tasks.rados.rados.0.smithi076.stdout:1157: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:42:42.273 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 350 (ObjNum 49 snap 0 seq_num 0) dirty dne 2024-09-21T09:42:42.273 INFO:tasks.rados.rados.0.smithi076.stdout:1159: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:42:42.375 INFO:tasks.rados.rados.0.smithi076.stdout:1160: finishing write tid 1 to smithi07685715-22 2024-09-21T09:42:42.375 INFO:tasks.rados.rados.0.smithi076.stdout:1158: expect (ObjNum 410 snap 128 seq_num 410) 2024-09-21T09:42:42.560 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[106482]: mgrmap e37: y(active, since 1.01075s), standbys: x 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[106482]: pgmap v3: 89 pgs: 89 active+clean; 161 MiB data, 595 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[106482]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[41325]: mgrmap e37: y(active, since 1.01075s), standbys: x 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[41325]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[41325]: pgmap v3: 89 pgs: 89 active+clean; 161 MiB data, 595 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:42.561 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:42 smithi076 ceph-mon[41325]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:42:42.568 INFO:tasks.rados.rados.0.smithi076.stdout:1160: finishing write tid 2 to smithi07685715-22 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: finishing write tid 3 to smithi07685715-22 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: finishing write tid 4 to smithi07685715-22 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: finishing write tid 5 to smithi07685715-22 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: oid 22 updating version 0 to 567 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: oid 22 updating version 567 to 570 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: oid 22 version 570 is already newer than 569 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: oid 22 version 570 is already newer than 568 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 570 (ObjNum 453 snap 148 seq_num 453) dirty exists 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1160: left oid 22 (ObjNum 453 snap 148 seq_num 453) 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1162: finishing write tid 1 to smithi07685715-48 2024-09-21T09:42:42.569 INFO:tasks.rados.rados.0.smithi076.stdout:1161: expect (ObjNum 433 snap 137 seq_num 433) 2024-09-21T09:42:42.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:42 smithi191 ceph-mon[33301]: mgrmap e37: y(active, since 1.01075s), standbys: x 2024-09-21T09:42:42.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:42 smithi191 ceph-mon[33301]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:42:42.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:42 smithi191 ceph-mon[33301]: pgmap v3: 89 pgs: 89 active+clean; 161 MiB data, 595 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:42.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:42 smithi191 ceph-mon[33301]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:42:42.678 INFO:tasks.rados.rados.0.smithi076.stdout:1162: finishing write tid 2 to smithi07685715-48 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: finishing write tid 3 to smithi07685715-48 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: finishing write tid 4 to smithi07685715-48 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: finishing write tid 5 to smithi07685715-48 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: oid 48 updating version 0 to 921 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: oid 48 updating version 921 to 922 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: oid 48 version 922 is already newer than 920 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: oid 48 version 922 is already newer than 919 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 922 (ObjNum 454 snap 148 seq_num 454) dirty exists 2024-09-21T09:42:42.679 INFO:tasks.rados.rados.0.smithi076.stdout:1162: left oid 48 (ObjNum 454 snap 148 seq_num 454) 2024-09-21T09:42:43.233 INFO:tasks.rados.rados.0.smithi076.stdout:1157: done (6 left) 2024-09-21T09:42:43.233 INFO:tasks.rados.rados.0.smithi076.stdout:1158: done (5 left) 2024-09-21T09:42:43.233 INFO:tasks.rados.rados.0.smithi076.stdout:1159: done (4 left) 2024-09-21T09:42:43.233 INFO:tasks.rados.rados.0.smithi076.stdout:1160: done (3 left) 2024-09-21T09:42:43.233 INFO:tasks.rados.rados.0.smithi076.stdout:1161: done (2 left) 2024-09-21T09:42:43.234 INFO:tasks.rados.rados.0.smithi076.stdout:1162: done (1 left) 2024-09-21T09:42:43.234 INFO:tasks.rados.rados.0.smithi076.stdout:1163: done (0 left) 2024-09-21T09:42:43.234 INFO:tasks.rados.rados.0.smithi076.stdout:1164: write oid 7 current snap is 149 2024-09-21T09:42:43.234 INFO:tasks.rados.rados.0.smithi076.stdout:1164: seq_num 455 ranges {447185=703122,1642601=765814,2928991=447707} 2024-09-21T09:42:43.240 INFO:tasks.rados.rados.0.smithi076.stdout:1164: writing smithi07685715-7 from 447185 to 1150307 tid 1 2024-09-21T09:42:43.247 INFO:tasks.rados.rados.0.smithi076.stdout:1164: writing smithi07685715-7 from 1642601 to 2408415 tid 2 2024-09-21T09:42:43.252 INFO:tasks.rados.rados.0.smithi076.stdout:1164: writing smithi07685715-7 from 2928991 to 3376698 tid 3 2024-09-21T09:42:43.252 INFO:tasks.rados.rados.0.smithi076.stdout:1165: write oid 19 current snap is 149 2024-09-21T09:42:43.253 INFO:tasks.rados.rados.0.smithi076.stdout:1164: finishing write tid 1 to smithi07685715-7 2024-09-21T09:42:43.253 INFO:tasks.rados.rados.0.smithi076.stdout:1164: finishing write tid 2 to smithi07685715-7 2024-09-21T09:42:43.253 INFO:tasks.rados.rados.0.smithi076.stdout:1165: seq_num 456 ranges {463992=778371,1701083=625825,2956414=658979} 2024-09-21T09:42:43.259 INFO:tasks.rados.rados.0.smithi076.stdout:1165: writing smithi07685715-19 from 463992 to 1242363 tid 1 2024-09-21T09:42:43.265 INFO:tasks.rados.rados.0.smithi076.stdout:1165: writing smithi07685715-19 from 1701083 to 2326908 tid 2 2024-09-21T09:42:43.272 INFO:tasks.rados.rados.0.smithi076.stdout:1165: writing smithi07685715-19 from 2956414 to 3615393 tid 3 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1166: snap_create 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: finishing write tid 3 to smithi07685715-7 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: finishing write tid 4 to smithi07685715-7 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: finishing write tid 5 to smithi07685715-7 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: oid 7 updating version 0 to 583 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: oid 7 version 583 is already newer than 582 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: oid 7 updating version 583 to 584 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: oid 7 updating version 584 to 585 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 585 (ObjNum 455 snap 149 seq_num 455) dirty exists 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1164: left oid 7 (ObjNum 455 snap 149 seq_num 455) 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1165: finishing write tid 1 to smithi07685715-19 2024-09-21T09:42:43.273 INFO:tasks.rados.rados.0.smithi076.stdout:1165: finishing write tid 2 to smithi07685715-19 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: finishing write tid 3 to smithi07685715-19 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: finishing write tid 4 to smithi07685715-19 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: finishing write tid 5 to smithi07685715-19 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: oid 19 updating version 0 to 419 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: oid 19 updating version 419 to 421 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: oid 19 version 421 is already newer than 420 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: oid 19 updating version 421 to 422 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 422 (ObjNum 456 snap 149 seq_num 456) dirty exists 2024-09-21T09:42:43.275 INFO:tasks.rados.rados.0.smithi076.stdout:1165: left oid 19 (ObjNum 456 snap 149 seq_num 456) 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[106482]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[106482]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[106482]: [21/Sep/2024:09:42:42] ENGINE Bus STARTING 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[106482]: [21/Sep/2024:09:42:43] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[41325]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[41325]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[41325]: [21/Sep/2024:09:42:42] ENGINE Bus STARTING 2024-09-21T09:42:43.440 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-mon[41325]: [21/Sep/2024:09:42:43] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:43.156+0000 7f86664b9640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:42:43] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: Traceback (most recent call last): 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self._connections.run(self.expiration_interval) 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self._run(expiration_interval) 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: s = self.context.wrap_socket( 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:43.441 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: return self.sslsocket_class._create( 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self.do_handshake() 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: self._sslobj.do_handshake() 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:42:43.442 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:42:43] "GET /metrics HTTP/1.1" 200 33385 "" "Prometheus/2.51.0" 2024-09-21T09:42:43.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:43 smithi191 ceph-mon[33301]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:42:43.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:43 smithi191 ceph-mon[33301]: mgrmap e38: y(active, since 2s), standbys: x 2024-09-21T09:42:43.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:43 smithi191 ceph-mon[33301]: [21/Sep/2024:09:42:42] ENGINE Bus STARTING 2024-09-21T09:42:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:43 smithi191 ceph-mon[33301]: [21/Sep/2024:09:42:43] ENGINE Serving on http://172.21.15.76:8765 2024-09-21T09:42:43.621 INFO:tasks.rados.rados.0.smithi076.stdout:1164: done (2 left) 2024-09-21T09:42:43.621 INFO:tasks.rados.rados.0.smithi076.stdout:1165: done (1 left) 2024-09-21T09:42:43.621 INFO:tasks.rados.rados.0.smithi076.stdout:1166: done (0 left) 2024-09-21T09:42:43.621 INFO:tasks.rados.rados.0.smithi076.stdout:1167: read oid 47 snap 134 2024-09-21T09:42:43.621 INFO:tasks.rados.rados.0.smithi076.stdout:1167: expect (ObjNum 417 snap 130 seq_num 417) 2024-09-21T09:42:43.621 INFO:tasks.rados.rados.0.smithi076.stdout:1168: snap_remove snap 135 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[106482]: [21/Sep/2024:09:42:43] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[106482]: [21/Sep/2024:09:42:43] ENGINE Bus STARTED 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[106482]: [21/Sep/2024:09:42:43] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:42:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: self.do_handshake() 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:42:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[106482]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[106482]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[41325]: [21/Sep/2024:09:42:43] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[41325]: [21/Sep/2024:09:42:43] ENGINE Bus STARTED 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[41325]: [21/Sep/2024:09:42:43] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout: Traceback (most recent call last): 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:44.375 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._run(expiration_interval) 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: s = self.context.wrap_socket( 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: return self.sslsocket_class._create( 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: self.do_handshake() 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout: self._sslobj.do_handshake() 2024-09-21T09:42:44.377 INFO:journalctl@ceph.mon.c.smithi076.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:44.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[41325]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:42:44.377 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:44 smithi076 ceph-mon[41325]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:42:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:44 smithi191 ceph-mon[33301]: [21/Sep/2024:09:42:43] ENGINE Serving on https://172.21.15.76:7150 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:44 smithi191 ceph-mon[33301]: [21/Sep/2024:09:42:43] ENGINE Bus STARTED 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:44 smithi191 ceph-mon[33301]: [21/Sep/2024:09:42:43] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: Traceback (most recent call last): 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._run(expiration_interval) 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: s = self.context.wrap_socket( 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: return self.sslsocket_class._create( 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: self.do_handshake() 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: self._sslobj.do_handshake() 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:44 smithi191 ceph-mon[33301]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:42:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:44 smithi191 ceph-mon[33301]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:42:44.631 INFO:tasks.rados.rados.0.smithi076.stdout:1168: done (1 left) 2024-09-21T09:42:44.631 INFO:tasks.rados.rados.0.smithi076.stdout:1169: snap_remove snap 149 2024-09-21T09:42:45.632 INFO:tasks.rados.rados.0.smithi076.stdout:1169: done (1 left) 2024-09-21T09:42:45.632 INFO:tasks.rados.rados.0.smithi076.stdout:1170: delete oid 21 current snap is 150 2024-09-21T09:42:45.632 INFO:tasks.rados.rados.0.smithi076.stdout:1167: expect (ObjNum 417 snap 130 seq_num 417) 2024-09-21T09:42:45.723 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: pgmap v9: 89 pgs: 89 active+clean; 161 MiB data, 595 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:45.723 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: pgmap v9: 89 pgs: 89 active+clean; 161 MiB data, 595 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.724 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:42:45.725 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:42:45.725 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.725 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.725 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.725 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:45 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1167: done (1 left) 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1170: done (0 left) 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1171: rollback oid 19 current snap is 150 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 143 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1172: read oid 5 snap -1 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1172: expect (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1173: write oid 1 current snap is 150 2024-09-21T09:42:45.799 INFO:tasks.rados.rados.0.smithi076.stdout:1173: seq_num 458 ranges {497606=645222,1934400=629494,3243966=453640} 2024-09-21T09:42:45.805 INFO:tasks.rados.rados.0.smithi076.stdout:1173: writing smithi07685715-1 from 497606 to 1142828 tid 1 2024-09-21T09:42:45.813 INFO:tasks.rados.rados.0.smithi076.stdout:1173: writing smithi07685715-1 from 1934400 to 2563894 tid 2 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:1173: writing smithi07685715-1 from 3243966 to 3697606 tid 3 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:1174: rollback oid 10 current snap is 150 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 148 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:1171: finishing rollback tid 0 to smithi07685715-19 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:1171: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:1171: finishing rollback tid 2 to smithi07685715-19 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 429 (ObjNum 391 snap 121 seq_num 391) dirty exists 2024-09-21T09:42:45.820 INFO:tasks.rados.rados.0.smithi076.stdout:1172: expect (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:42:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: pgmap v9: 89 pgs: 89 active+clean; 161 MiB data, 595 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:42:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: mgrmap e39: y(active, since 4s), standbys: x 2024-09-21T09:42:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:46.010 INFO:tasks.rados.rados.0.smithi076.stdout:1173: finishing write tid 1 to smithi07685715-1 2024-09-21T09:42:46.010 INFO:tasks.rados.rados.0.smithi076.stdout:1173: finishing write tid 2 to smithi07685715-1 2024-09-21T09:42:46.010 INFO:tasks.rados.rados.0.smithi076.stdout:1173: finishing write tid 3 to smithi07685715-1 2024-09-21T09:42:46.010 INFO:tasks.rados.rados.0.smithi076.stdout:1173: finishing write tid 4 to smithi07685715-1 2024-09-21T09:42:46.010 INFO:tasks.rados.rados.0.smithi076.stdout:1174: finishing rollback tid 0 to smithi07685715-10 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: finishing write tid 5 to smithi07685715-1 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: oid 1 updating version 0 to 867 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: oid 1 version 867 is already newer than 866 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: oid 1 updating version 867 to 868 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: oid 1 updating version 868 to 869 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 869 (ObjNum 458 snap 150 seq_num 458) dirty exists 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: left oid 1 (ObjNum 458 snap 150 seq_num 458) 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1174: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1171: done (3 left) 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1172: done (2 left) 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1173: done (1 left) 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1175: read oid 6 snap -1 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1175: expect (ObjNum 426 snap 135 seq_num 426) 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1176: write oid 14 current snap is 150 2024-09-21T09:42:46.011 INFO:tasks.rados.rados.0.smithi076.stdout:1174: finishing rollback tid 2 to smithi07685715-10 2024-09-21T09:42:46.012 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 597 (ObjNum 437 snap 139 seq_num 437) dirty exists 2024-09-21T09:42:46.012 INFO:tasks.rados.rados.0.smithi076.stdout:1176: seq_num 459 ranges {514413=720471,1876529=773152,3438683=275730} 2024-09-21T09:42:46.017 INFO:tasks.rados.rados.0.smithi076.stdout:1176: writing smithi07685715-14 from 514413 to 1234884 tid 1 2024-09-21T09:42:46.024 INFO:tasks.rados.rados.0.smithi076.stdout:1176: writing smithi07685715-14 from 1876529 to 2649681 tid 2 2024-09-21T09:42:46.031 INFO:tasks.rados.rados.0.smithi076.stdout:1176: writing smithi07685715-14 from 3438683 to 3714413 tid 3 2024-09-21T09:42:46.031 INFO:tasks.rados.rados.0.smithi076.stdout:1174: done (2 left) 2024-09-21T09:42:46.031 INFO:tasks.rados.rados.0.smithi076.stdout:1177: write oid 46 current snap is 150 2024-09-21T09:42:46.031 INFO:tasks.rados.rados.0.smithi076.stdout:1175: expect (ObjNum 426 snap 135 seq_num 426) 2024-09-21T09:42:46.136 INFO:tasks.rados.rados.0.smithi076.stdout:1176: finishing write tid 1 to smithi07685715-14 2024-09-21T09:42:46.136 INFO:tasks.rados.rados.0.smithi076.stdout:1176: finishing write tid 2 to smithi07685715-14 2024-09-21T09:42:46.136 INFO:tasks.rados.rados.0.smithi076.stdout:1177: seq_num 460 ranges {531220=795720,1935011=516810,3066106=420420} 2024-09-21T09:42:46.143 INFO:tasks.rados.rados.0.smithi076.stdout:1177: writing smithi07685715-46 from 531220 to 1326940 tid 1 2024-09-21T09:42:46.149 INFO:tasks.rados.rados.0.smithi076.stdout:1177: writing smithi07685715-46 from 1935011 to 2451821 tid 2 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1177: writing smithi07685715-46 from 3066106 to 3486526 tid 3 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1175: done (2 left) 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1178: write oid 18 current snap is 150 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1176: finishing write tid 3 to smithi07685715-14 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1176: finishing write tid 4 to smithi07685715-14 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1176: finishing write tid 5 to smithi07685715-14 2024-09-21T09:42:46.155 INFO:tasks.rados.rados.0.smithi076.stdout:1176: oid 14 updating version 0 to 601 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1176: oid 14 version 601 is already newer than 599 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1176: oid 14 updating version 601 to 602 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1176: oid 14 version 602 is already newer than 600 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 602 (ObjNum 459 snap 150 seq_num 459) dirty exists 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1176: left oid 14 (ObjNum 459 snap 150 seq_num 459) 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1177: finishing write tid 1 to smithi07685715-46 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1177: finishing write tid 2 to smithi07685715-46 2024-09-21T09:42:46.156 INFO:tasks.rados.rados.0.smithi076.stdout:1178: seq_num 461 ranges {548027=470969,1593493=776821,3093529=631692} 2024-09-21T09:42:46.160 INFO:tasks.rados.rados.0.smithi076.stdout:1178: writing smithi07685715-18 from 548027 to 1018996 tid 1 2024-09-21T09:42:46.168 INFO:tasks.rados.rados.0.smithi076.stdout:1178: writing smithi07685715-18 from 1593493 to 2370314 tid 2 2024-09-21T09:42:46.175 INFO:tasks.rados.rados.0.smithi076.stdout:1178: writing smithi07685715-18 from 3093529 to 3725221 tid 3 2024-09-21T09:42:46.175 INFO:tasks.rados.rados.0.smithi076.stdout:1176: done (2 left) 2024-09-21T09:42:46.175 INFO:tasks.rados.rados.0.smithi076.stdout:1179: rollback oid 8 current snap is 150 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 147 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: finishing write tid 3 to smithi07685715-46 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: finishing write tid 4 to smithi07685715-46 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: finishing write tid 5 to smithi07685715-46 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: oid 46 updating version 0 to 872 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: oid 46 updating version 872 to 873 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: oid 46 version 873 is already newer than 871 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: oid 46 updating version 873 to 874 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 874 (ObjNum 460 snap 150 seq_num 460) dirty exists 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: left oid 46 (ObjNum 460 snap 150 seq_num 460) 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1178: finishing write tid 1 to smithi07685715-18 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1178: finishing write tid 2 to smithi07685715-18 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1177: done (2 left) 2024-09-21T09:42:46.176 INFO:tasks.rados.rados.0.smithi076.stdout:1180: write oid 42 current snap is 150 2024-09-21T09:42:46.177 INFO:tasks.rados.rados.0.smithi076.stdout:1180: seq_num 462 ranges {564834=546218,1651975=520479,2720952=559317} 2024-09-21T09:42:46.181 INFO:tasks.rados.rados.0.smithi076.stdout:1180: writing smithi07685715-42 from 564834 to 1111052 tid 1 2024-09-21T09:42:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:1180: writing smithi07685715-42 from 1651975 to 2172454 tid 2 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1180: writing smithi07685715-42 from 2720952 to 3280269 tid 3 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1181: snap_create 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1179: finishing rollback tid 0 to smithi07685715-8 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1178: finishing write tid 3 to smithi07685715-18 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1179: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1178: finishing write tid 4 to smithi07685715-18 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1179: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 928 (ObjNum 428 snap 135 seq_num 428) dirty exists 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1178: finishing write tid 5 to smithi07685715-18 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1178: oid 18 updating version 0 to 360 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1178: oid 18 updating version 360 to 362 2024-09-21T09:42:46.194 INFO:tasks.rados.rados.0.smithi076.stdout:1178: oid 18 version 362 is already newer than 361 2024-09-21T09:42:46.195 INFO:tasks.rados.rados.0.smithi076.stdout:1178: oid 18 version 362 is already newer than 359 2024-09-21T09:42:46.195 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 362 (ObjNum 461 snap 150 seq_num 461) dirty exists 2024-09-21T09:42:46.195 INFO:tasks.rados.rados.0.smithi076.stdout:1178: left oid 18 (ObjNum 461 snap 150 seq_num 461) 2024-09-21T09:42:46.195 INFO:tasks.rados.rados.0.smithi076.stdout:1180: finishing write tid 1 to smithi07685715-42 2024-09-21T09:42:46.195 INFO:tasks.rados.rados.0.smithi076.stdout:1180: finishing write tid 2 to smithi07685715-42 2024-09-21T09:42:46.196 INFO:tasks.rados.rados.0.smithi076.stdout:1180: finishing write tid 3 to smithi07685715-42 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: finishing write tid 4 to smithi07685715-42 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: finishing write tid 5 to smithi07685715-42 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: oid 42 updating version 0 to 363 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: oid 42 updating version 363 to 366 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: oid 42 version 366 is already newer than 364 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: oid 42 version 366 is already newer than 365 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 366 (ObjNum 462 snap 150 seq_num 462) dirty exists 2024-09-21T09:42:46.197 INFO:tasks.rados.rados.0.smithi076.stdout:1180: left oid 42 (ObjNum 462 snap 150 seq_num 462) 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1178: done (3 left) 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1179: done (2 left) 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1180: done (1 left) 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1181: done (0 left) 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1182: rollback oid 13 current snap is 151 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 127 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1183: read oid 18 snap 138 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1183: expect (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1184: delete oid 24 current snap is 151 2024-09-21T09:42:46.696 INFO:tasks.rados.rados.0.smithi076.stdout:1182: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:42:46.697 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 0 (ObjNum 864428544 snap 21884 seq_num 304) dirty dne 2024-09-21T09:42:46.699 INFO:tasks.rados.rados.0.smithi076.stdout:1183: expect (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:42:46.852 INFO:tasks.rados.rados.0.smithi076.stdout:1182: done (2 left) 2024-09-21T09:42:46.852 INFO:tasks.rados.rados.0.smithi076.stdout:1183: done (1 left) 2024-09-21T09:42:46.852 INFO:tasks.rados.rados.0.smithi076.stdout:1184: done (0 left) 2024-09-21T09:42:46.852 INFO:tasks.rados.rados.0.smithi076.stdout:1185: snap_create 2024-09-21T09:42:46.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:46 smithi076 ceph-mon[41325]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:42:46.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:46 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:46.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:46 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:46.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:46 smithi076 ceph-mon[106482]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:42:46.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:46.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:47.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:46 smithi191 ceph-mon[33301]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:42:47.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:46 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:47.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:46 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:47.710 INFO:tasks.rados.rados.0.smithi076.stdout:1185: done (0 left) 2024-09-21T09:42:47.710 INFO:tasks.rados.rados.0.smithi076.stdout:1186: snap_remove snap 128 2024-09-21T09:42:48.028 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:47 smithi076 ceph-mon[106482]: pgmap v12: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 3.9 MiB/s wr, 46 op/s 2024-09-21T09:42:48.028 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:47 smithi076 ceph-mon[106482]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:42:48.028 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:47 smithi076 ceph-mon[41325]: pgmap v12: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 3.9 MiB/s wr, 46 op/s 2024-09-21T09:42:48.028 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:47 smithi076 ceph-mon[41325]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:42:48.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:47 smithi191 ceph-mon[33301]: pgmap v12: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 3.9 MiB/s wr, 46 op/s 2024-09-21T09:42:48.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:47 smithi191 ceph-mon[33301]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:42:48.622 INFO:tasks.rados.rados.0.smithi076.stdout:1186: done (0 left) 2024-09-21T09:42:48.622 INFO:tasks.rados.rados.0.smithi076.stdout:1187: snap_remove snap 134 2024-09-21T09:42:48.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[106482]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[106482]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[41325]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:48 smithi076 ceph-mon[41325]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:42:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:48 smithi191 ceph-mon[33301]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:42:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:48 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:48 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:48 smithi191 ceph-mon[33301]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:42:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:1187: done (0 left) 2024-09-21T09:42:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:1188: rollback oid 2 current snap is 152 2024-09-21T09:42:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 140 2024-09-21T09:42:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:1189: write oid 26 current snap is 152 2024-09-21T09:42:49.624 INFO:tasks.rados.rados.0.smithi076.stdout:1189: seq_num 464 ranges {598448=413069,1768939=524148,2775798=581861,3781488=16960} 2024-09-21T09:42:49.630 INFO:tasks.rados.rados.0.smithi076.stdout:1189: writing smithi07685715-26 from 598448 to 1011517 tid 1 2024-09-21T09:42:49.637 INFO:tasks.rados.rados.0.smithi076.stdout:1189: writing smithi07685715-26 from 1768939 to 2293087 tid 2 2024-09-21T09:42:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1189: writing smithi07685715-26 from 2775798 to 3357659 tid 3 2024-09-21T09:42:49.646 INFO:tasks.rados.rados.0.smithi076.stdout:1189: writing smithi07685715-26 from 3781488 to 3798448 tid 4 2024-09-21T09:42:49.646 INFO:tasks.rados.rados.0.smithi076.stdout:1190: rollback oid 44 current snap is 152 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 150 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1188: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1188: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1188: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 195 (ObjNum 332 snap 103 seq_num 332) dirty exists 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1189: finishing write tid 1 to smithi07685715-26 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1189: finishing write tid 2 to smithi07685715-26 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1188: done (2 left) 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1191: rollback oid 14 current snap is 152 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 150 2024-09-21T09:42:49.647 INFO:tasks.rados.rados.0.smithi076.stdout:1192: snap_remove snap 142 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: pgmap v15: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 3.9 MiB/s wr, 45 op/s 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:42:50.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[106482]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:42:50.111 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: pgmap v15: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 3.9 MiB/s wr, 45 op/s 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:50.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:42:50.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:42:50.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:42:50.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:49 smithi076 ceph-mon[41325]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: pgmap v15: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 3.9 MiB/s wr, 45 op/s 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.conf 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.conf 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:42:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-21T09:42:50.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:49 smithi191 ceph-mon[33301]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:42:50.626 INFO:tasks.rados.rados.0.smithi076.stdout:1192: done (3 left) 2024-09-21T09:42:50.627 INFO:tasks.rados.rados.0.smithi076.stdout:1193: write oid 29 current snap is 152 2024-09-21T09:42:50.627 INFO:tasks.rados.rados.0.smithi076.stdout:1193: seq_num 465 ranges {615255=488318,1827421=784159,3203221=612034} 2024-09-21T09:42:50.633 INFO:tasks.rados.rados.0.smithi076.stdout:1193: writing smithi07685715-29 from 615255 to 1103573 tid 1 2024-09-21T09:42:50.641 INFO:tasks.rados.rados.0.smithi076.stdout:1193: writing smithi07685715-29 from 1827421 to 2611580 tid 2 2024-09-21T09:42:50.648 INFO:tasks.rados.rados.0.smithi076.stdout:1193: writing smithi07685715-29 from 3203221 to 3815255 tid 3 2024-09-21T09:42:50.648 INFO:tasks.rados.rados.0.smithi076.stdout:1194: snap_remove snap 129 2024-09-21T09:42:51.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[106482]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[106482]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[106482]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[106482]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[106482]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[106482]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[41325]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[41325]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[41325]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[41325]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[41325]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:42:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:50 smithi076 ceph-mon[41325]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:42:51.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:50 smithi191 ceph-mon[33301]: Updating smithi191:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:50 smithi191 ceph-mon[33301]: Updating smithi076:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:50 smithi191 ceph-mon[33301]: Updating smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:42:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:50 smithi191 ceph-mon[33301]: Updating smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:42:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:50 smithi191 ceph-mon[33301]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:42:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:50 smithi191 ceph-mon[33301]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1194: done (4 left) 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1195: rollback oid 41 current snap is 152 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 150 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1196: delete oid 17 current snap is 152 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1189: finishing write tid 3 to smithi07685715-26 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1189: finishing write tid 4 to smithi07685715-26 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1190: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:42:51.630 INFO:tasks.rados.rados.0.smithi076.stdout:1189: finishing write tid 5 to smithi07685715-26 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1191: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: finishing write tid 6 to smithi07685715-26 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: oid 26 updating version 0 to 463 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: oid 26 version 463 is already newer than 461 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: oid 26 version 463 is already newer than 459 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: oid 26 version 463 is already newer than 462 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: oid 26 version 463 is already newer than 460 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 463 (ObjNum 464 snap 152 seq_num 464) dirty exists 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1189: left oid 26 (ObjNum 464 snap 152 seq_num 464) 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1190: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1190: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 217 (ObjNum 410 snap 128 seq_num 410) dirty exists 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1191: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:1191: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:42:51.631 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 624 (ObjNum 459 snap 150 seq_num 459) dirty exists 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: finishing write tid 1 to smithi07685715-29 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: finishing write tid 2 to smithi07685715-29 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: finishing write tid 3 to smithi07685715-29 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: finishing write tid 4 to smithi07685715-29 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: finishing write tid 5 to smithi07685715-29 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: oid 29 updating version 0 to 308 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: oid 29 updating version 308 to 310 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: oid 29 version 310 is already newer than 309 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: oid 29 updating version 310 to 311 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 311 (ObjNum 465 snap 152 seq_num 465) dirty exists 2024-09-21T09:42:51.632 INFO:tasks.rados.rados.0.smithi076.stdout:1193: left oid 29 (ObjNum 465 snap 152 seq_num 465) 2024-09-21T09:42:51.635 INFO:tasks.rados.rados.0.smithi076.stdout:1195: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:42:51.635 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:42:51.636 INFO:tasks.rados.rados.0.smithi076.stdout:1189: done (5 left) 2024-09-21T09:42:51.636 INFO:tasks.rados.rados.0.smithi076.stdout:1190: done (4 left) 2024-09-21T09:42:51.636 INFO:tasks.rados.rados.0.smithi076.stdout:1191: done (3 left) 2024-09-21T09:42:51.636 INFO:tasks.rados.rados.0.smithi076.stdout:1193: done (2 left) 2024-09-21T09:42:51.636 INFO:tasks.rados.rados.0.smithi076.stdout:1195: done (1 left) 2024-09-21T09:42:51.637 INFO:tasks.rados.rados.0.smithi076.stdout:1196: done (0 left) 2024-09-21T09:42:51.637 INFO:tasks.rados.rados.0.smithi076.stdout:1197: rollback oid 48 current snap is 152 2024-09-21T09:42:51.637 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 140 2024-09-21T09:42:51.637 INFO:tasks.rados.rados.0.smithi076.stdout:1198: snap_create 2024-09-21T09:42:51.639 INFO:tasks.rados.rados.0.smithi076.stdout:1197: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:42:51.639 INFO:tasks.rados.rados.0.smithi076.stdout:1197: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:42:51.640 INFO:tasks.rados.rados.0.smithi076.stdout:1197: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:42:51.640 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 954 (ObjNum 434 snap 138 seq_num 434) dirty exists 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: pgmap v18: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: Upgrade: Updating mon.c 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: Deploying daemon mon.c on smithi076 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[106482]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: pgmap v18: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: Upgrade: Updating mon.c 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:42:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:42:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: Deploying daemon mon.c on smithi076 2024-09-21T09:42:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:51 smithi076 ceph-mon[41325]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:42:52.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: pgmap v18: 89 pgs: 89 active+clean; 184 MiB data, 624 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: Upgrade: Updating mon.c 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: Deploying daemon mon.c on smithi076 2024-09-21T09:42:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:42:51 smithi191 ceph-mon[33301]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:42:52.629 INFO:tasks.rados.rados.0.smithi076.stdout:1197: done (1 left) 2024-09-21T09:42:52.630 INFO:tasks.rados.rados.0.smithi076.stdout:1198: done (0 left) 2024-09-21T09:42:52.630 INFO:tasks.rados.rados.0.smithi076.stdout:1199: delete oid 11 current snap is 153 2024-09-21T09:42:52.634 INFO:tasks.rados.rados.0.smithi076.stdout:1199: done (0 left) 2024-09-21T09:42:52.634 INFO:tasks.rados.rados.0.smithi076.stdout:1200: delete oid 37 current snap is 153 2024-09-21T09:42:52.636 INFO:tasks.rados.rados.0.smithi076.stdout:1200: done (0 left) 2024-09-21T09:42:52.636 INFO:tasks.rados.rados.0.smithi076.stdout:1201: read oid 46 snap -1 2024-09-21T09:42:52.636 INFO:tasks.rados.rados.0.smithi076.stdout:1201: expect (ObjNum 460 snap 150 seq_num 460) 2024-09-21T09:42:52.636 INFO:tasks.rados.rados.0.smithi076.stdout:1202: rollback oid 6 current snap is 153 2024-09-21T09:42:52.636 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 147 2024-09-21T09:42:52.636 INFO:tasks.rados.rados.0.smithi076.stdout:1203: snap_create 2024-09-21T09:42:52.639 INFO:tasks.rados.rados.0.smithi076.stdout:1202: finishing rollback tid 0 to smithi07685715-6 2024-09-21T09:42:52.639 INFO:tasks.rados.rados.0.smithi076.stdout:1202: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:42:52.639 INFO:tasks.rados.rados.0.smithi076.stdout:1202: finishing rollback tid 2 to smithi07685715-6 2024-09-21T09:42:52.639 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 225 (ObjNum 426 snap 135 seq_num 426) dirty exists 2024-09-21T09:42:52.656 INFO:tasks.rados.rados.0.smithi076.stdout:1201: expect (ObjNum 460 snap 150 seq_num 460) 2024-09-21T09:42:53.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:52 smithi076 systemd[1]: Stopping Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:42:53.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:42:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:42:53] "GET /metrics HTTP/1.1" 200 33385 "" "Prometheus/2.51.0" 2024-09-21T09:42:53.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c[41302]: 2024-09-21T09:42:53.075+0000 7f0b470a9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:42:53.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c[41302]: 2024-09-21T09:42:53.075+0000 7f0b470a9640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:42:53.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:53 smithi076 podman[110772]: 2024-09-21 09:42:53.12843378 +0000 UTC m=+0.177998675 container died 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:42:53.786 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:53 smithi076 podman[110772]: 2024-09-21 09:42:53.575128951 +0000 UTC m=+0.624693845 container cleanup 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:42:53.786 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:53 smithi076 bash[110772]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c 2024-09-21T09:42:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:53 smithi076 podman[110789]: 2024-09-21 09:42:53.7850178 +0000 UTC m=+0.648555853 container remove 2906e77f2aff994ddf7ba6e8f1d2a2461bace0dc00baff574e802c04916a283b (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T09:42:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T09:42:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 systemd[1]: Stopped Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:42:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.c.service: Consumed 9.040s CPU time. 2024-09-21T09:42:54.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 systemd[1]: Starting Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:42:54.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 podman[110925]: 2024-09-21 09:42:54.357568746 +0000 UTC m=+0.015737073 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:42:54.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 podman[110925]: 2024-09-21 09:42:54.494702145 +0000 UTC m=+0.152870454 container create a49012ff5abf941d6ef55e86a2ff344112002060a04a0b2a1b5d0321fc2bba5b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-62da54b) 2024-09-21T09:42:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 podman[110925]: 2024-09-21 09:42:54.72214557 +0000 UTC m=+0.380313889 container init a49012ff5abf941d6ef55e86a2ff344112002060a04a0b2a1b5d0321fc2bba5b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:42:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 podman[110925]: 2024-09-21 09:42:54.726978417 +0000 UTC m=+0.385146727 container start a49012ff5abf941d6ef55e86a2ff344112002060a04a0b2a1b5d0321fc2bba5b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T09:42:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:42:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:42:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: pidfile_write: ignore empty --pid-file 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: load: jerasure load: lrc 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Git sha 0 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: DB SUMMARY 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: DB Session ID: LYX9KH09V66A27WBJAF2 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1411 Bytes 2024-09-21T09:42:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 14396582 ; 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.env: 0x560cff74f120 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.info_log: 0x560d00fcf700 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.statistics: (nil) 2024-09-21T09:42:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.use_fsync: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.db_log_dir: 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.wal_dir: 2024-09-21T09:42:55.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.write_buffer_manager: 0x560d00fd21e0 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:42:55.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.unordered_write: 0 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.row_cache: None 2024-09-21T09:42:55.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.wal_filter: None 2024-09-21T09:42:55.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.wal_compression: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:42:55.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_open_files: -1 2024-09-21T09:42:55.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Compression algorithms supported: 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kZSTD supported: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:42:55.132 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.merge_operator: 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_filter: None 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560d00fcf320) 2024-09-21T09:42:55.133 INFO:journalctl@ceph.mon.c.smithi076.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: index_type: 0 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: data_block_index_type: 0 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: index_shortening: 1 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: checksum: 4 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: no_block_cache: 0 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache: 0x560d00fc3610 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache_options: 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: capacity : 536870912 2024-09-21T09:42:55.134 INFO:journalctl@ceph.mon.c.smithi076.stdout: num_shard_bits : 4 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: strict_capacity_limit : 0 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_cache_compressed: (nil) 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: persistent_cache: (nil) 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_size: 4096 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_size_deviation: 10 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_restart_interval: 16 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: index_block_restart_interval: 1 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: metadata_block_size: 4096 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: partition_filters: 0 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: use_delta_encoding: 1 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: filter_policy: bloomfilter 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: whole_key_filtering: 1 2024-09-21T09:42:55.135 INFO:journalctl@ceph.mon.c.smithi076.stdout: verify_compression: 0 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: format_version: 5 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: enable_index_compression: 1 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: block_align: 0 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: max_auto_readahead_size: 262144 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: prepopulate_block_cache: 0 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression: NoCompression 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:42:55.136 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.num_levels: 7 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:42:55.137 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:42:55.138 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:42:55.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:42:55.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.ttl: 2592000 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:42:55.141 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 13388, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:42:55.142 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 581c3213-13f4-4179-98bb-d3d9d16183eb 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911774753996, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911774787768, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 8868747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13393, "largest_seqno": 19160, "table_properties": {"data_size": 8843640, "index_size": 17979, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 6085, "raw_key_size": 57184, "raw_average_key_size": 23, "raw_value_size": 8791530, "raw_average_value_size": 3641, "num_data_blocks": 774, "num_entries": 2414, "num_filter_entries": 2414, "num_deletions": 360, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911774, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "581c3213-13f4-4179-98bb-d3d9d16183eb", "db_session_id": "LYX9KH09V66A27WBJAF2", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911774787955, "job": 1, "event": "recovery_finished"} 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560d00ff2e00 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: DB pointer 0x560d010f4000 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** DB Stats ** 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:42:55.143 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: L0 1/0 8.46 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 582.6 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: L6 1/0 7.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Sum 2/0 16.21 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 582.6 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 582.6 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.144 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** Compaction Stats [default] ** 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 582.6 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Flush(GB): cumulative 0.008, interval 0.008 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Cumulative compaction: 0.01 GB write, 217.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Interval compaction: 0.01 GB write, 217.15 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:42:55.145 INFO:journalctl@ceph.mon.c.smithi076.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: Block cache BinnedLRUCache@0x560d00fc3610#7 capacity: 512.00 MB usage: 26.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,6.03 KB,0.00115037%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: starting mon.c rank 2 at public addrs [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] at bind addrs [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???) e3 preinit fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).mds e9 new map 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).mds e9 print_map 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: e9 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:42:55.146 INFO:journalctl@ceph.mon.c.smithi076.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: legacy client fscid: 1 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: Filesystem 'foo' (1) 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: fs_name foo 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: epoch 8 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: created 2024-09-21T09:36:39.879731+0000 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: modified 2024-09-21T09:36:51.753955+0000 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: tableserver 0 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: root 0 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: session_timeout 60 2024-09-21T09:42:55.147 INFO:journalctl@ceph.mon.c.smithi076.stdout: session_autoclose 300 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: max_file_size 1099511627776 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: max_xattr_size 65536 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: required_client_features {} 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: last_failure 0 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: last_failure_osd_epoch 0 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: max_mds 1 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: in 0 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: up {0=14631} 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: failed 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: damaged 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: stopped 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: data_pools [4] 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: metadata_pool 3 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: inline_data disabled 2024-09-21T09:42:55.148 INFO:journalctl@ceph.mon.c.smithi076.stdout: balancer 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: bal_rank_mask -1 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: standby_count_wanted 1 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: [mds.foo.smithi076.jymebx{0:14631} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: Standby daemons: 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout: [mds.foo.smithi191.dhrpzx{-1:24466} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).osd e372 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).osd e372 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).osd e372 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).osd e372 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 ceph-mon[110963]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-21T09:42:55.149 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 bash[110925]: a49012ff5abf941d6ef55e86a2ff344112002060a04a0b2a1b5d0321fc2bba5b 2024-09-21T09:42:55.150 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:42:54 smithi076 systemd[1]: Started Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:43:00.264 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (91s) 16s ago 7m 19.6M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (70s) 15s ago 7m 78.1M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (6m) 16s ago 6m 18.8M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (6m) 15s ago 6m 16.5M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (2m) 15s ago 11m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (3m) 16s ago 13m 487M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (27s) 16s ago 13m 35.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (11m) 15s ago 11m 62.7M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:43:00.873 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 starting - - - 2048M 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (2m) 16s ago 7m 10.0M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (2m) 15s ago 6m 11.4M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (11m) 16s ago 11m 97.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (10m) 16s ago 10m 77.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (10m) 16s ago 10m 96.4M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (9m) 16s ago 9m 74.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (9m) 15s ago 9m 87.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (9m) 15s ago 9m 98.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (8m) 15s ago 8m 70.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (8m) 15s ago 8m 70.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:43:00.874 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (111s) 15s ago 7m 39.3M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:43:01.061 INFO:tasks.rados.rados.0.smithi076.stdout:1201: done (2 left) 2024-09-21T09:43:01.061 INFO:tasks.rados.rados.0.smithi076.stdout:1202: done (1 left) 2024-09-21T09:43:01.061 INFO:tasks.rados.rados.0.smithi076.stdout:1203: done (0 left) 2024-09-21T09:43:01.061 INFO:tasks.rados.rados.0.smithi076.stdout:1204: write oid 29 current snap is 154 2024-09-21T09:43:01.061 INFO:tasks.rados.rados.0.smithi076.stdout:1204: seq_num 469 ranges {682483=789314,1944996=791497,3196560=554574} 2024-09-21T09:43:01.068 INFO:tasks.rados.rados.0.smithi076.stdout:1204: writing smithi07685715-29 from 682483 to 1471797 tid 1 2024-09-21T09:43:01.075 INFO:tasks.rados.rados.0.smithi076.stdout:1204: writing smithi07685715-29 from 1944996 to 2736493 tid 2 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi076.stdout:1204: writing smithi07685715-29 from 3196560 to 3751134 tid 3 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi076.stdout:1205: write oid 41 current snap is 154 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi076.stdout:1204: finishing write tid 1 to smithi07685715-29 2024-09-21T09:43:01.081 INFO:tasks.rados.rados.0.smithi076.stdout:1204: finishing write tid 2 to smithi07685715-29 2024-09-21T09:43:01.082 INFO:tasks.rados.rados.0.smithi076.stdout:1205: seq_num 470 ranges {699290=464563,1603478=535155,2823983=765846} 2024-09-21T09:43:01.087 INFO:tasks.rados.rados.0.smithi076.stdout:1205: writing smithi07685715-41 from 699290 to 1163853 tid 1 2024-09-21T09:43:01.093 INFO:tasks.rados.rados.0.smithi076.stdout:1205: writing smithi07685715-41 from 1603478 to 2138633 tid 2 2024-09-21T09:43:01.102 INFO:tasks.rados.rados.0.smithi076.stdout:1205: writing smithi07685715-41 from 2823983 to 3589829 tid 3 2024-09-21T09:43:01.102 INFO:tasks.rados.rados.0.smithi076.stdout:1206: read oid 24 snap -1 2024-09-21T09:43:01.103 INFO:tasks.rados.rados.0.smithi076.stdout:1206: expect deleted 2024-09-21T09:43:01.103 INFO:tasks.rados.rados.0.smithi076.stdout:1207: snap_remove snap 139 2024-09-21T09:43:01.252 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 1, 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:43:01.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 11, 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:43:01.254 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: mon.c calling monitor election 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: mon.b calling monitor election 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: mon.a calling monitor election 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: monmap epoch 3 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: last_changed 2024-09-21T09:31:20.913061+0000 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: min_mon_release 18 (reef) 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: election_strategy: 1 2024-09-21T09:43:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:43:01.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:43:01.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:43:01.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:43:01.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:43:01.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: mgrmap e39: y(active, since 20s), standbys: x 2024-09-21T09:43:01.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:01 smithi191 ceph-mon[33301]: overall HEALTH_OK 2024-09-21T09:43:01.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: mon.c calling monitor election 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: mon.b calling monitor election 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: mon.a calling monitor election 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: monmap epoch 3 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: last_changed 2024-09-21T09:31:20.913061+0000 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: min_mon_release 18 (reef) 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: election_strategy: 1 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:43:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: mgrmap e39: y(active, since 20s), standbys: x 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[110963]: overall HEALTH_OK 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: mon.c calling monitor election 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: mon.b calling monitor election 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: mon.a calling monitor election 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: monmap epoch 3 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: last_changed 2024-09-21T09:31:20.913061+0000 2024-09-21T09:43:01.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: min_mon_release 18 (reef) 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: election_strategy: 1 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: mgrmap e39: y(active, since 20s), standbys: x 2024-09-21T09:43:01.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:01 smithi076 ceph-mon[106482]: overall HEALTH_OK 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "1/3 daemons upgraded", 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:43:01.549 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1207: done (3 left) 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1208: snap_create 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: finishing write tid 3 to smithi07685715-29 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: finishing write tid 4 to smithi07685715-29 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: finishing write tid 5 to smithi07685715-29 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: oid 29 updating version 0 to 317 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: oid 29 updating version 317 to 318 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: oid 29 version 318 is already newer than 316 2024-09-21T09:43:02.067 INFO:tasks.rados.rados.0.smithi076.stdout:1204: oid 29 updating version 318 to 319 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 319 (ObjNum 469 snap 154 seq_num 469) dirty exists 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1204: left oid 29 (ObjNum 469 snap 154 seq_num 469) 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: finishing write tid 1 to smithi07685715-41 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: finishing write tid 2 to smithi07685715-41 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: finishing write tid 3 to smithi07685715-41 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: finishing write tid 4 to smithi07685715-41 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: finishing write tid 5 to smithi07685715-41 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: oid 41 updating version 0 to 582 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: oid 41 updating version 582 to 583 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: oid 41 updating version 583 to 584 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: oid 41 updating version 584 to 585 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 585 (ObjNum 470 snap 154 seq_num 470) dirty exists 2024-09-21T09:43:02.068 INFO:tasks.rados.rados.0.smithi076.stdout:1205: left oid 41 (ObjNum 470 snap 154 seq_num 470) 2024-09-21T09:43:02.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[106482]: pgmap v27: 89 pgs: 89 active+clean; 187 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 864 KiB/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:43:02.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[106482]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[106482]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[106482]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[106482]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:43:02.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3407295490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:02.360 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[110963]: pgmap v27: 89 pgs: 89 active+clean; 187 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 864 KiB/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:43:02.360 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[110963]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.360 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[110963]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.360 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[110963]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.360 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[110963]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:43:02.360 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3407295490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:02 smithi191 ceph-mon[33301]: pgmap v27: 89 pgs: 89 active+clean; 187 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 864 KiB/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:43:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:02 smithi191 ceph-mon[33301]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:02 smithi191 ceph-mon[33301]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:02 smithi191 ceph-mon[33301]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:02 smithi191 ceph-mon[33301]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:43:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:02 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/3407295490' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:03.076 INFO:tasks.rados.rados.0.smithi076.stdout:1204: done (3 left) 2024-09-21T09:43:03.076 INFO:tasks.rados.rados.0.smithi076.stdout:1205: done (2 left) 2024-09-21T09:43:03.076 INFO:tasks.rados.rados.0.smithi076.stdout:1206: done (1 left) 2024-09-21T09:43:03.076 INFO:tasks.rados.rados.0.smithi076.stdout:1208: done (0 left) 2024-09-21T09:43:03.076 INFO:tasks.rados.rados.0.smithi076.stdout:1209: read oid 37 snap -1 2024-09-21T09:43:03.076 INFO:tasks.rados.rados.0.smithi076.stdout:1209: expect deleted 2024-09-21T09:43:03.077 INFO:tasks.rados.rados.0.smithi076.stdout:1210: snap_remove snap 138 2024-09-21T09:43:03.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:03 smithi076 ceph-mon[106482]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:03.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:03 smithi076 ceph-mon[106482]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:43:03.348 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:43:03] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.51.0" 2024-09-21T09:43:03.348 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:03 smithi076 ceph-mon[110963]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:03.348 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:03 smithi076 ceph-mon[110963]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:43:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:03 smithi191 ceph-mon[33301]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:03 smithi191 ceph-mon[33301]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:43:03.628 INFO:tasks.rados.rados.0.smithi076.stdout:1210: done (1 left) 2024-09-21T09:43:03.628 INFO:tasks.rados.rados.0.smithi076.stdout:1211: rollback oid 41 current snap is 155 2024-09-21T09:43:03.628 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 147 2024-09-21T09:43:03.628 INFO:tasks.rados.rados.0.smithi076.stdout:1212: rollback oid 3 current snap is 155 2024-09-21T09:43:03.628 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 148 2024-09-21T09:43:03.628 INFO:tasks.rados.rados.0.smithi076.stdout:1213: snap_remove snap 140 2024-09-21T09:43:04.330 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:04 smithi191 ceph-mon[33301]: pgmap v30: 89 pgs: 89 active+clean; 192 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 635 KiB/s wr, 1 op/s 2024-09-21T09:43:04.330 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:04 smithi191 ceph-mon[33301]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:43:04.330 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:04 smithi191 ceph-mon[33301]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:43:04.330 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:04 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:04.330 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:04 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:04.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[106482]: pgmap v30: 89 pgs: 89 active+clean; 192 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 635 KiB/s wr, 1 op/s 2024-09-21T09:43:04.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[106482]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:43:04.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[106482]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:43:04.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:04.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:04.369 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[110963]: pgmap v30: 89 pgs: 89 active+clean; 192 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 635 KiB/s wr, 1 op/s 2024-09-21T09:43:04.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[110963]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:43:04.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[110963]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:43:04.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:04.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:04.612 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:43:04 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=infra.usagestats t=2024-09-21T09:43:04.327714427Z level=info msg="Usage stats are ready to report" 2024-09-21T09:43:04.632 INFO:tasks.rados.rados.0.smithi076.stdout:1213: done (3 left) 2024-09-21T09:43:04.632 INFO:tasks.rados.rados.0.smithi076.stdout:1214: write oid 32 current snap is 155 2024-09-21T09:43:04.632 INFO:tasks.rados.rados.0.smithi076.stdout:1214: seq_num 471 ranges {716097=539812,1945607=795166,3535053=381044} 2024-09-21T09:43:04.638 INFO:tasks.rados.rados.0.smithi076.stdout:1214: writing smithi07685715-32 from 716097 to 1255909 tid 1 2024-09-21T09:43:04.647 INFO:tasks.rados.rados.0.smithi076.stdout:1214: writing smithi07685715-32 from 1945607 to 2740773 tid 2 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:1214: writing smithi07685715-32 from 3535053 to 3916097 tid 3 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:1215: rollback oid 16 current snap is 155 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 154 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:1212: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:1211: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:43:04.654 INFO:tasks.rados.rados.0.smithi076.stdout:1211: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 590 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1214: finishing write tid 1 to smithi07685715-32 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1214: finishing write tid 2 to smithi07685715-32 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1209: done (4 left) 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1211: done (3 left) 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1212: done (2 left) 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1216: rollback oid 28 current snap is 155 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 28 to 127 2024-09-21T09:43:04.655 INFO:tasks.rados.rados.0.smithi076.stdout:1217: delete oid 20 current snap is 155 2024-09-21T09:43:04.656 INFO:tasks.rados.rados.0.smithi076.stdout:1214: finishing write tid 3 to smithi07685715-32 2024-09-21T09:43:04.656 INFO:tasks.rados.rados.0.smithi076.stdout:1217: done (3 left) 2024-09-21T09:43:04.657 INFO:tasks.rados.rados.0.smithi076.stdout:1218: write oid 8 current snap is 155 2024-09-21T09:43:04.657 INFO:tasks.rados.rados.0.smithi076.stdout:1218: seq_num 473 ranges {749711=406663,1778924=682482,3189899=716015} 2024-09-21T09:43:04.663 INFO:tasks.rados.rados.0.smithi076.stdout:1218: writing smithi07685715-8 from 749711 to 1156374 tid 1 2024-09-21T09:43:04.670 INFO:tasks.rados.rados.0.smithi076.stdout:1218: writing smithi07685715-8 from 1778924 to 2461406 tid 2 2024-09-21T09:43:04.678 INFO:tasks.rados.rados.0.smithi076.stdout:1218: writing smithi07685715-8 from 3189899 to 3905914 tid 3 2024-09-21T09:43:04.678 INFO:tasks.rados.rados.0.smithi076.stdout:1219: snap_remove snap 123 2024-09-21T09:43:04.952 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:04 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:43:04.801+0000 7f86bc22c640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:43:05.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:05 smithi191 ceph-mon[33301]: pgmap v33: 89 pgs: 89 active+clean; 192 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 953 KiB/s wr, 2 op/s 2024-09-21T09:43:05.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:05 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:05 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:05 smithi191 ceph-mon[33301]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:43:05.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[106482]: pgmap v33: 89 pgs: 89 active+clean; 192 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 953 KiB/s wr, 2 op/s 2024-09-21T09:43:05.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:05.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:05.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[106482]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:43:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[110963]: pgmap v33: 89 pgs: 89 active+clean; 192 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 953 KiB/s wr, 2 op/s 2024-09-21T09:43:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:05 smithi076 ceph-mon[110963]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:43:05.627 INFO:tasks.rados.rados.0.smithi076.stdout:1219: done (4 left) 2024-09-21T09:43:05.627 INFO:tasks.rados.rados.0.smithi076.stdout:1220: write oid 31 current snap is 155 2024-09-21T09:43:05.628 INFO:tasks.rados.rados.0.smithi076.stdout:1214: finishing write tid 4 to smithi07685715-32 2024-09-21T09:43:05.628 INFO:tasks.rados.rados.0.smithi076.stdout:1220: seq_num 474 ranges {766518=481912,1837406=542493,2933675=527287,3876584=89934} 2024-09-21T09:43:05.635 INFO:tasks.rados.rados.0.smithi076.stdout:1220: writing smithi07685715-31 from 766518 to 1248430 tid 1 2024-09-21T09:43:05.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: writing smithi07685715-31 from 1837406 to 2379899 tid 2 2024-09-21T09:43:05.648 INFO:tasks.rados.rados.0.smithi076.stdout:1220: writing smithi07685715-31 from 2933675 to 3460962 tid 3 2024-09-21T09:43:05.651 INFO:tasks.rados.rados.0.smithi076.stdout:1220: writing smithi07685715-31 from 3876584 to 3966518 tid 4 2024-09-21T09:43:05.651 INFO:tasks.rados.rados.0.smithi076.stdout:1221: rollback oid 25 current snap is 155 2024-09-21T09:43:05.651 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 148 2024-09-21T09:43:05.651 INFO:tasks.rados.rados.0.smithi076.stdout:1215: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:43:05.651 INFO:tasks.rados.rados.0.smithi076.stdout:1216: finishing rollback tid 0 to smithi07685715-28 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: finishing write tid 5 to smithi07685715-32 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: oid 32 updating version 0 to 458 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: oid 32 updating version 458 to 459 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: oid 32 updating version 459 to 460 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: oid 32 version 460 is already newer than 457 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 460 (ObjNum 471 snap 155 seq_num 471) dirty exists 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: left oid 32 (ObjNum 471 snap 155 seq_num 471) 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1215: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1216: finishing rollback tid 1 to smithi07685715-28 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1214: done (5 left) 2024-09-21T09:43:05.652 INFO:tasks.rados.rados.0.smithi076.stdout:1222: snap_remove snap 153 2024-09-21T09:43:06.640 INFO:tasks.rados.rados.0.smithi076.stdout:1222: done (5 left) 2024-09-21T09:43:06.640 INFO:tasks.rados.rados.0.smithi076.stdout:1223: delete oid 17 current snap is 155 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1216: finishing rollback tid 2 to smithi07685715-28 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 976 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1215: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 598 (ObjNum 446 snap 144 seq_num 446) dirty exists 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: finishing write tid 1 to smithi07685715-8 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: finishing write tid 2 to smithi07685715-8 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: finishing write tid 3 to smithi07685715-8 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: finishing write tid 4 to smithi07685715-8 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: finishing write tid 5 to smithi07685715-8 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: oid 8 updating version 0 to 981 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: oid 8 version 981 is already newer than 980 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: oid 8 version 981 is already newer than 978 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:1218: oid 8 version 981 is already newer than 979 2024-09-21T09:43:06.641 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 981 (ObjNum 473 snap 155 seq_num 473) dirty exists 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1218: left oid 8 (ObjNum 473 snap 155 seq_num 473) 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: finishing write tid 1 to smithi07685715-31 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: finishing write tid 2 to smithi07685715-31 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: finishing write tid 3 to smithi07685715-31 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: finishing write tid 4 to smithi07685715-31 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: finishing write tid 5 to smithi07685715-31 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: finishing write tid 6 to smithi07685715-31 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: oid 31 updating version 0 to 470 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: oid 31 version 470 is already newer than 466 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: oid 31 updating version 470 to 471 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: oid 31 updating version 471 to 472 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:1220: oid 31 version 472 is already newer than 469 2024-09-21T09:43:06.642 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 472 (ObjNum 474 snap 155 seq_num 474) dirty exists 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1220: left oid 31 (ObjNum 474 snap 155 seq_num 474) 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1221: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1221: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1221: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 258 (ObjNum 361 snap 110 seq_num 361) dirty exists 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1215: done (5 left) 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1216: done (4 left) 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1218: done (3 left) 2024-09-21T09:43:06.643 INFO:tasks.rados.rados.0.smithi076.stdout:1220: done (2 left) 2024-09-21T09:43:06.644 INFO:tasks.rados.rados.0.smithi076.stdout:1221: done (1 left) 2024-09-21T09:43:06.644 INFO:tasks.rados.rados.0.smithi076.stdout:1223: done (0 left) 2024-09-21T09:43:06.644 INFO:tasks.rados.rados.0.smithi076.stdout:1224: write oid 50 current snap is 155 2024-09-21T09:43:06.644 INFO:tasks.rados.rados.0.smithi076.stdout:1224: seq_num 476 ranges {400132=632410,1438017=546162} 2024-09-21T09:43:06.647 INFO:tasks.rados.rados.0.smithi076.stdout:1224: writing smithi07685715-50 from 400132 to 1032542 tid 1 2024-09-21T09:43:06.653 INFO:tasks.rados.rados.0.smithi076.stdout:1224: writing smithi07685715-50 from 1438017 to 1984179 tid 2 2024-09-21T09:43:06.653 INFO:tasks.rados.rados.0.smithi076.stdout:1225: read oid 31 snap 136 2024-09-21T09:43:06.653 INFO:tasks.rados.rados.0.smithi076.stdout:1225: expect (ObjNum 395 snap 123 seq_num 395) 2024-09-21T09:43:06.653 INFO:tasks.rados.rados.0.smithi076.stdout:1226: write oid 34 current snap is 155 2024-09-21T09:43:06.653 INFO:tasks.rados.rados.0.smithi076.stdout:1224: finishing write tid 1 to smithi07685715-50 2024-09-21T09:43:06.653 INFO:tasks.rados.rados.0.smithi076.stdout:1226: seq_num 477 ranges {416939=707659,1896499=120440} 2024-09-21T09:43:06.658 INFO:tasks.rados.rados.0.smithi076.stdout:1226: writing smithi07685715-34 from 416939 to 1124598 tid 1 2024-09-21T09:43:06.664 INFO:tasks.rados.rados.0.smithi076.stdout:1226: writing smithi07685715-34 from 1896499 to 2016939 tid 2 2024-09-21T09:43:06.664 INFO:tasks.rados.rados.0.smithi076.stdout:1227: write oid 10 current snap is 155 2024-09-21T09:43:06.664 INFO:tasks.rados.rados.0.smithi076.stdout:1227: seq_num 478 ranges {433746=782908,1954981=78765} 2024-09-21T09:43:06.670 INFO:tasks.rados.rados.0.smithi076.stdout:1227: writing smithi07685715-10 from 433746 to 1216654 tid 1 2024-09-21T09:43:06.675 INFO:tasks.rados.rados.0.smithi076.stdout:1227: writing smithi07685715-10 from 1954981 to 2033746 tid 2 2024-09-21T09:43:06.675 INFO:tasks.rados.rados.0.smithi076.stdout:1228: write oid 42 current snap is 155 2024-09-21T09:43:06.675 INFO:tasks.rados.rados.0.smithi076.stdout:1224: finishing write tid 2 to smithi07685715-50 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1224: finishing write tid 3 to smithi07685715-50 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1224: finishing write tid 4 to smithi07685715-50 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1224: oid 50 updating version 0 to 413 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1224: oid 50 version 413 is already newer than 411 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1224: oid 50 version 413 is already newer than 412 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 413 (ObjNum 476 snap 155 seq_num 476) dirty exists 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1224: left oid 50 (ObjNum 476 snap 155 seq_num 476) 2024-09-21T09:43:06.676 INFO:tasks.rados.rados.0.smithi076.stdout:1225: expect (ObjNum 395 snap 123 seq_num 395) 2024-09-21T09:43:06.780 INFO:tasks.rados.rados.0.smithi076.stdout:1226: finishing write tid 1 to smithi07685715-34 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: finishing write tid 2 to smithi07685715-34 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: finishing write tid 3 to smithi07685715-34 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: finishing write tid 4 to smithi07685715-34 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: oid 34 updating version 0 to 219 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: oid 34 updating version 219 to 220 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: oid 34 version 220 is already newer than 218 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 220 (ObjNum 477 snap 155 seq_num 477) dirty exists 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1226: left oid 34 (ObjNum 477 snap 155 seq_num 477) 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1227: finishing write tid 1 to smithi07685715-10 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1227: finishing write tid 2 to smithi07685715-10 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1227: finishing write tid 3 to smithi07685715-10 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1227: finishing write tid 4 to smithi07685715-10 2024-09-21T09:43:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:1227: oid 10 updating version 0 to 677 2024-09-21T09:43:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:1227: oid 10 updating version 677 to 678 2024-09-21T09:43:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:1227: oid 10 version 678 is already newer than 676 2024-09-21T09:43:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 678 (ObjNum 478 snap 155 seq_num 478) dirty exists 2024-09-21T09:43:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:1227: left oid 10 (ObjNum 478 snap 155 seq_num 478) 2024-09-21T09:43:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:1228: seq_num 479 ranges {450553=574510,1613463=437090} 2024-09-21T09:43:06.786 INFO:tasks.rados.rados.0.smithi076.stdout:1228: writing smithi07685715-42 from 450553 to 1025063 tid 1 2024-09-21T09:43:06.792 INFO:tasks.rados.rados.0.smithi076.stdout:1228: writing smithi07685715-42 from 1613463 to 2050553 tid 2 2024-09-21T09:43:06.792 INFO:tasks.rados.rados.0.smithi076.stdout:1224: done (4 left) 2024-09-21T09:43:06.792 INFO:tasks.rados.rados.0.smithi076.stdout:1225: done (3 left) 2024-09-21T09:43:06.792 INFO:tasks.rados.rados.0.smithi076.stdout:1226: done (2 left) 2024-09-21T09:43:06.792 INFO:tasks.rados.rados.0.smithi076.stdout:1227: done (1 left) 2024-09-21T09:43:06.792 INFO:tasks.rados.rados.0.smithi076.stdout:1229: snap_remove snap 136 2024-09-21T09:43:07.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:06 smithi191 ceph-mon[33301]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:43:07.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:06 smithi076 ceph-mon[106482]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:43:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:06 smithi076 ceph-mon[110963]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1229: done (1 left) 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1230: read oid 37 snap -1 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1230: expect deleted 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1231: delete oid 34 current snap is 155 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1228: finishing write tid 1 to smithi07685715-42 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1228: finishing write tid 2 to smithi07685715-42 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1228: finishing write tid 3 to smithi07685715-42 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1228: finishing write tid 4 to smithi07685715-42 2024-09-21T09:43:07.637 INFO:tasks.rados.rados.0.smithi076.stdout:1228: oid 42 updating version 0 to 394 2024-09-21T09:43:07.638 INFO:tasks.rados.rados.0.smithi076.stdout:1228: oid 42 version 394 is already newer than 392 2024-09-21T09:43:07.638 INFO:tasks.rados.rados.0.smithi076.stdout:1228: oid 42 version 394 is already newer than 393 2024-09-21T09:43:07.638 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 394 (ObjNum 479 snap 155 seq_num 479) dirty exists 2024-09-21T09:43:07.638 INFO:tasks.rados.rados.0.smithi076.stdout:1228: left oid 42 (ObjNum 479 snap 155 seq_num 479) 2024-09-21T09:43:07.640 INFO:tasks.rados.rados.0.smithi076.stdout:1228: done (2 left) 2024-09-21T09:43:07.640 INFO:tasks.rados.rados.0.smithi076.stdout:1230: done (1 left) 2024-09-21T09:43:07.640 INFO:tasks.rados.rados.0.smithi076.stdout:1231: done (0 left) 2024-09-21T09:43:07.640 INFO:tasks.rados.rados.0.smithi076.stdout:1232: write oid 2 current snap is 155 2024-09-21T09:43:07.640 INFO:tasks.rados.rados.0.smithi076.stdout:1232: seq_num 481 ranges {484167=725008,1730427=353740} 2024-09-21T09:43:07.648 INFO:tasks.rados.rados.0.smithi076.stdout:1232: writing smithi07685715-2 from 484167 to 1209175 tid 1 2024-09-21T09:43:07.654 INFO:tasks.rados.rados.0.smithi076.stdout:1232: writing smithi07685715-2 from 1730427 to 2084167 tid 2 2024-09-21T09:43:07.654 INFO:tasks.rados.rados.0.smithi076.stdout:1233: snap_remove snap 150 2024-09-21T09:43:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: pgmap v36: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:43:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:43:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:43:07.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: pgmap v36: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.877 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:43:07.878 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.879 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:43:07.880 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.881 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:43:07.882 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:43:07.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:07.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:07.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:07.883 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: pgmap v36: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:08.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:43:08.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:43:08.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:08.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:08.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:07 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:08.630 INFO:tasks.rados.rados.0.smithi076.stdout:1233: done (1 left) 2024-09-21T09:43:08.630 INFO:tasks.rados.rados.0.smithi076.stdout:1234: rollback oid 34 current snap is 155 2024-09-21T09:43:08.630 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 127 2024-09-21T09:43:08.630 INFO:tasks.rados.rados.0.smithi076.stdout:1232: finishing write tid 1 to smithi07685715-2 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: finishing write tid 2 to smithi07685715-2 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: finishing write tid 3 to smithi07685715-2 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: finishing write tid 4 to smithi07685715-2 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: oid 2 updating version 0 to 213 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: oid 2 updating version 213 to 215 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: oid 2 version 215 is already newer than 214 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 215 (ObjNum 481 snap 155 seq_num 481) dirty exists 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: left oid 2 (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1232: done (1 left) 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1235: read oid 15 snap -1 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1235: expect (ObjNum 413 snap 128 seq_num 413) 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:1236: rollback oid 31 current snap is 155 2024-09-21T09:43:08.631 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 154 2024-09-21T09:43:08.632 INFO:tasks.rados.rados.0.smithi076.stdout:1237: read oid 25 snap -1 2024-09-21T09:43:08.632 INFO:tasks.rados.rados.0.smithi076.stdout:1237: expect (ObjNum 361 snap 110 seq_num 361) 2024-09-21T09:43:08.632 INFO:tasks.rados.rados.0.smithi076.stdout:1238: rollback oid 17 current snap is 155 2024-09-21T09:43:08.632 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 152 2024-09-21T09:43:08.632 INFO:tasks.rados.rados.0.smithi076.stdout:1239: write oid 46 current snap is 155 2024-09-21T09:43:08.632 INFO:tasks.rados.rados.0.smithi076.stdout:1239: seq_num 482 ranges {500974=400257,1388909=557169} 2024-09-21T09:43:08.635 INFO:tasks.rados.rados.0.smithi076.stdout:1239: writing smithi07685715-46 from 500974 to 901231 tid 1 2024-09-21T09:43:08.642 INFO:tasks.rados.rados.0.smithi076.stdout:1239: writing smithi07685715-46 from 1388909 to 1946078 tid 2 2024-09-21T09:43:08.643 INFO:tasks.rados.rados.0.smithi076.stdout:1234: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:43:08.643 INFO:tasks.rados.rados.0.smithi076.stdout:1238: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:43:08.643 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 539 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:43:08.643 INFO:tasks.rados.rados.0.smithi076.stdout:1237: expect (ObjNum 361 snap 110 seq_num 361) 2024-09-21T09:43:08.732 INFO:tasks.rados.rados.0.smithi076.stdout:1234: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:43:08.732 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 227 (ObjNum 404 snap 124 seq_num 404) dirty exists 2024-09-21T09:43:08.732 INFO:tasks.rados.rados.0.smithi076.stdout:1239: finishing write tid 1 to smithi07685715-46 2024-09-21T09:43:08.732 INFO:tasks.rados.rados.0.smithi076.stdout:1235: expect (ObjNum 413 snap 128 seq_num 413) 2024-09-21T09:43:08.898 INFO:tasks.rados.rados.0.smithi076.stdout:1236: finishing rollback tid 0 to smithi07685715-31 2024-09-21T09:43:08.898 INFO:tasks.rados.rados.0.smithi076.stdout:1236: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:43:08.898 INFO:tasks.rados.rados.0.smithi076.stdout:1236: finishing rollback tid 2 to smithi07685715-31 2024-09-21T09:43:08.898 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 485 (ObjNum 441 snap 139 seq_num 441) dirty exists 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1239: finishing write tid 2 to smithi07685715-46 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1234: done (5 left) 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1235: done (4 left) 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1236: done (3 left) 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1237: done (2 left) 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1238: done (1 left) 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1240: read oid 32 snap -1 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1240: expect (ObjNum 471 snap 155 seq_num 471) 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1241: write oid 34 current snap is 155 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1239: finishing write tid 3 to smithi07685715-46 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1239: finishing write tid 4 to smithi07685715-46 2024-09-21T09:43:08.899 INFO:tasks.rados.rados.0.smithi076.stdout:1239: oid 46 updating version 0 to 967 2024-09-21T09:43:08.900 INFO:tasks.rados.rados.0.smithi076.stdout:1239: oid 46 version 967 is already newer than 966 2024-09-21T09:43:08.900 INFO:tasks.rados.rados.0.smithi076.stdout:1239: oid 46 updating version 967 to 968 2024-09-21T09:43:08.900 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 968 (ObjNum 482 snap 155 seq_num 482) dirty exists 2024-09-21T09:43:08.900 INFO:tasks.rados.rados.0.smithi076.stdout:1239: left oid 46 (ObjNum 482 snap 155 seq_num 482) 2024-09-21T09:43:08.900 INFO:tasks.rados.rados.0.smithi076.stdout:1241: seq_num 483 ranges {517781=475506,1731038=386743} 2024-09-21T09:43:08.903 INFO:tasks.rados.rados.0.smithi076.stdout:1241: writing smithi07685715-34 from 517781 to 993287 tid 1 2024-09-21T09:43:08.910 INFO:tasks.rados.rados.0.smithi076.stdout:1241: writing smithi07685715-34 from 1731038 to 2117781 tid 2 2024-09-21T09:43:08.910 INFO:tasks.rados.rados.0.smithi076.stdout:1239: done (2 left) 2024-09-21T09:43:08.910 INFO:tasks.rados.rados.0.smithi076.stdout:1242: write oid 8 current snap is 155 2024-09-21T09:43:08.910 INFO:tasks.rados.rados.0.smithi076.stdout:1240: expect (ObjNum 471 snap 155 seq_num 471) 2024-09-21T09:43:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all grafana 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all loki 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all promtail 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Finalizing container_image settings 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: Upgrade: Complete! 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:09.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:09.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:08 smithi191 ceph-mon[33301]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:43:09.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all grafana 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all loki 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all promtail 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Finalizing container_image settings 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: Upgrade: Complete! 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[106482]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all grafana 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all loki 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all promtail 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Finalizing container_image settings 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: Upgrade: Complete! 2024-09-21T09:43:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: osdmap e381: 8 total, 8 up, 8 in 2024-09-21T09:43:09.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:43:09.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:43:09.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:43:09.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:09.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:08 smithi076 ceph-mon[110963]: osdmap e382: 8 total, 8 up, 8 in 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: finishing write tid 1 to smithi07685715-34 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: finishing write tid 2 to smithi07685715-34 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: finishing write tid 3 to smithi07685715-34 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: finishing write tid 4 to smithi07685715-34 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: oid 34 updating version 0 to 229 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: oid 34 updating version 229 to 230 2024-09-21T09:43:09.151 INFO:tasks.rados.rados.0.smithi076.stdout:1241: oid 34 version 230 is already newer than 228 2024-09-21T09:43:09.152 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 230 (ObjNum 483 snap 155 seq_num 483) dirty exists 2024-09-21T09:43:09.152 INFO:tasks.rados.rados.0.smithi076.stdout:1241: left oid 34 (ObjNum 483 snap 155 seq_num 483) 2024-09-21T09:43:09.152 INFO:tasks.rados.rados.0.smithi076.stdout:1242: seq_num 484 ranges {534588=550755,1789520=345068} 2024-09-21T09:43:09.156 INFO:tasks.rados.rados.0.smithi076.stdout:1242: writing smithi07685715-8 from 534588 to 1085343 tid 1 2024-09-21T09:43:09.163 INFO:tasks.rados.rados.0.smithi076.stdout:1242: writing smithi07685715-8 from 1789520 to 2134588 tid 2 2024-09-21T09:43:09.163 INFO:tasks.rados.rados.0.smithi076.stdout:1240: done (2 left) 2024-09-21T09:43:09.163 INFO:tasks.rados.rados.0.smithi076.stdout:1241: done (1 left) 2024-09-21T09:43:09.163 INFO:tasks.rados.rados.0.smithi076.stdout:1243: write oid 47 current snap is 155 2024-09-21T09:43:09.163 INFO:tasks.rados.rados.0.smithi076.stdout:1242: finishing write tid 1 to smithi07685715-8 2024-09-21T09:43:09.163 INFO:tasks.rados.rados.0.smithi076.stdout:1243: seq_num 485 ranges {551395=626004,1848002=303393} 2024-09-21T09:43:09.169 INFO:tasks.rados.rados.0.smithi076.stdout:1243: writing smithi07685715-47 from 551395 to 1177399 tid 1 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1243: writing smithi07685715-47 from 1848002 to 2151395 tid 2 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1244: snap_create 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1242: finishing write tid 2 to smithi07685715-8 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1242: finishing write tid 3 to smithi07685715-8 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1242: finishing write tid 4 to smithi07685715-8 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1242: oid 8 updating version 0 to 1008 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1242: oid 8 version 1008 is already newer than 1006 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:1242: oid 8 version 1008 is already newer than 1007 2024-09-21T09:43:09.175 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1008 (ObjNum 484 snap 155 seq_num 484) dirty exists 2024-09-21T09:43:09.176 INFO:tasks.rados.rados.0.smithi076.stdout:1242: left oid 8 (ObjNum 484 snap 155 seq_num 484) 2024-09-21T09:43:09.176 INFO:tasks.rados.rados.0.smithi076.stdout:1243: finishing write tid 1 to smithi07685715-47 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:1243: finishing write tid 2 to smithi07685715-47 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:1243: finishing write tid 3 to smithi07685715-47 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:1243: finishing write tid 4 to smithi07685715-47 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:1243: oid 47 updating version 0 to 1010 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:1243: oid 47 updating version 1010 to 1012 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:1243: oid 47 version 1012 is already newer than 1011 2024-09-21T09:43:09.177 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1012 (ObjNum 485 snap 155 seq_num 485) dirty exists 2024-09-21T09:43:09.178 INFO:tasks.rados.rados.0.smithi076.stdout:1243: left oid 47 (ObjNum 485 snap 155 seq_num 485) 2024-09-21T09:43:09.718 INFO:tasks.rados.rados.0.smithi076.stdout:1242: done (2 left) 2024-09-21T09:43:09.718 INFO:tasks.rados.rados.0.smithi076.stdout:1243: done (1 left) 2024-09-21T09:43:09.718 INFO:tasks.rados.rados.0.smithi076.stdout:1244: done (0 left) 2024-09-21T09:43:09.718 INFO:tasks.rados.rados.0.smithi076.stdout:1245: delete oid 32 current snap is 156 2024-09-21T09:43:09.725 INFO:tasks.rados.rados.0.smithi076.stdout:1245: done (0 left) 2024-09-21T09:43:09.725 INFO:tasks.rados.rados.0.smithi076.stdout:1246: delete oid 41 current snap is 156 2024-09-21T09:43:09.728 INFO:tasks.rados.rados.0.smithi076.stdout:1246: done (0 left) 2024-09-21T09:43:09.728 INFO:tasks.rados.rados.0.smithi076.stdout:1247: snap_create 2024-09-21T09:43:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:09 smithi191 ceph-mon[33301]: pgmap v39: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:43:10.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:09 smithi076 ceph-mon[106482]: pgmap v39: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:43:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:09 smithi076 ceph-mon[110963]: pgmap v39: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:43:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:1247: done (0 left) 2024-09-21T09:43:10.728 INFO:tasks.rados.rados.0.smithi076.stdout:1248: delete oid 34 current snap is 157 2024-09-21T09:43:10.730 INFO:tasks.rados.rados.0.smithi076.stdout:1248: done (0 left) 2024-09-21T09:43:10.730 INFO:tasks.rados.rados.0.smithi076.stdout:1249: write oid 46 current snap is 157 2024-09-21T09:43:10.731 INFO:tasks.rados.rados.0.smithi076.stdout:1249: seq_num 489 ranges {618623=643353,1681930=536693} 2024-09-21T09:43:10.737 INFO:tasks.rados.rados.0.smithi076.stdout:1249: writing smithi07685715-46 from 618623 to 1261976 tid 1 2024-09-21T09:43:10.743 INFO:tasks.rados.rados.0.smithi076.stdout:1249: writing smithi07685715-46 from 1681930 to 2218623 tid 2 2024-09-21T09:43:10.743 INFO:tasks.rados.rados.0.smithi076.stdout:1250: write oid 48 current snap is 157 2024-09-21T09:43:10.743 INFO:tasks.rados.rados.0.smithi076.stdout:1249: finishing write tid 1 to smithi07685715-46 2024-09-21T09:43:10.743 INFO:tasks.rados.rados.0.smithi076.stdout:1250: seq_num 490 ranges {635430=718602,2140412=95018} 2024-09-21T09:43:10.750 INFO:tasks.rados.rados.0.smithi076.stdout:1250: writing smithi07685715-48 from 635430 to 1354032 tid 1 2024-09-21T09:43:10.756 INFO:tasks.rados.rados.0.smithi076.stdout:1250: writing smithi07685715-48 from 2140412 to 2235430 tid 2 2024-09-21T09:43:10.756 INFO:tasks.rados.rados.0.smithi076.stdout:1251: write oid 18 current snap is 157 2024-09-21T09:43:10.756 INFO:tasks.rados.rados.0.smithi076.stdout:1249: finishing write tid 2 to smithi07685715-46 2024-09-21T09:43:10.756 INFO:tasks.rados.rados.0.smithi076.stdout:1249: finishing write tid 3 to smithi07685715-46 2024-09-21T09:43:10.756 INFO:tasks.rados.rados.0.smithi076.stdout:1251: seq_num 491 ranges {652237=793851,2198894=53343} 2024-09-21T09:43:10.763 INFO:tasks.rados.rados.0.smithi076.stdout:1251: writing smithi07685715-18 from 652237 to 1446088 tid 1 2024-09-21T09:43:10.767 INFO:tasks.rados.rados.0.smithi076.stdout:1251: writing smithi07685715-18 from 2198894 to 2252237 tid 2 2024-09-21T09:43:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:1252: snap_remove snap 154 2024-09-21T09:43:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:10 smithi191 ceph-mon[33301]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:43:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:10 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:10 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:11.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:10 smithi076 ceph-mon[106482]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:43:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:10 smithi076 ceph-mon[110963]: osdmap e383: 8 total, 8 up, 8 in 2024-09-21T09:43:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:43:11.739 INFO:tasks.rados.rados.0.smithi076.stdout:1252: done (3 left) 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1253: write oid 6 current snap is 157 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1249: finishing write tid 4 to smithi07685715-46 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1249: oid 46 updating version 0 to 970 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1249: oid 46 updating version 970 to 972 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1249: oid 46 version 972 is already newer than 971 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 972 (ObjNum 489 snap 157 seq_num 489) dirty exists 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1249: left oid 46 (ObjNum 489 snap 157 seq_num 489) 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: finishing write tid 1 to smithi07685715-48 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: finishing write tid 2 to smithi07685715-48 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: finishing write tid 3 to smithi07685715-48 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: finishing write tid 4 to smithi07685715-48 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: oid 48 updating version 0 to 1016 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: oid 48 version 1016 is already newer than 1014 2024-09-21T09:43:11.740 INFO:tasks.rados.rados.0.smithi076.stdout:1250: oid 48 version 1016 is already newer than 1015 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1016 (ObjNum 490 snap 157 seq_num 490) dirty exists 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1250: left oid 48 (ObjNum 490 snap 157 seq_num 490) 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: finishing write tid 1 to smithi07685715-18 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: finishing write tid 2 to smithi07685715-18 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: finishing write tid 3 to smithi07685715-18 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: finishing write tid 4 to smithi07685715-18 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: oid 18 updating version 0 to 402 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: oid 18 updating version 402 to 404 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: oid 18 version 404 is already newer than 403 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 404 (ObjNum 491 snap 157 seq_num 491) dirty exists 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1251: left oid 18 (ObjNum 491 snap 157 seq_num 491) 2024-09-21T09:43:11.741 INFO:tasks.rados.rados.0.smithi076.stdout:1253: seq_num 492 ranges {669044=469100,1741023=459161} 2024-09-21T09:43:11.746 INFO:tasks.rados.rados.0.smithi076.stdout:1253: writing smithi07685715-6 from 669044 to 1138144 tid 1 2024-09-21T09:43:11.752 INFO:tasks.rados.rados.0.smithi076.stdout:1253: writing smithi07685715-6 from 1741023 to 2200184 tid 2 2024-09-21T09:43:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1249: done (3 left) 2024-09-21T09:43:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1250: done (2 left) 2024-09-21T09:43:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1251: done (1 left) 2024-09-21T09:43:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1254: snap_create 2024-09-21T09:43:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1253: finishing write tid 1 to smithi07685715-6 2024-09-21T09:43:11.755 INFO:tasks.rados.rados.0.smithi076.stdout:1253: finishing write tid 2 to smithi07685715-6 2024-09-21T09:43:11.755 INFO:tasks.rados.rados.0.smithi076.stdout:1253: finishing write tid 3 to smithi07685715-6 2024-09-21T09:43:11.755 INFO:tasks.rados.rados.0.smithi076.stdout:1253: finishing write tid 4 to smithi07685715-6 2024-09-21T09:43:11.756 INFO:tasks.rados.rados.0.smithi076.stdout:1253: oid 6 updating version 0 to 251 2024-09-21T09:43:11.756 INFO:tasks.rados.rados.0.smithi076.stdout:1253: oid 6 version 251 is already newer than 250 2024-09-21T09:43:11.756 INFO:tasks.rados.rados.0.smithi076.stdout:1253: oid 6 version 251 is already newer than 249 2024-09-21T09:43:11.756 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 251 (ObjNum 492 snap 157 seq_num 492) dirty exists 2024-09-21T09:43:11.756 INFO:tasks.rados.rados.0.smithi076.stdout:1253: left oid 6 (ObjNum 492 snap 157 seq_num 492) 2024-09-21T09:43:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:11 smithi191 ceph-mon[33301]: pgmap v42: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:11 smithi191 ceph-mon[33301]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:43:12.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:11 smithi076 ceph-mon[110963]: pgmap v42: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:12.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:11 smithi076 ceph-mon[110963]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:43:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:11 smithi076 ceph-mon[106482]: pgmap v42: 89 pgs: 89 active+clean; 194 MiB data, 661 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:11 smithi076 ceph-mon[106482]: osdmap e384: 8 total, 8 up, 8 in 2024-09-21T09:43:12.742 INFO:tasks.rados.rados.0.smithi076.stdout:1253: done (1 left) 2024-09-21T09:43:12.742 INFO:tasks.rados.rados.0.smithi076.stdout:1254: done (0 left) 2024-09-21T09:43:12.742 INFO:tasks.rados.rados.0.smithi076.stdout:1255: snap_remove snap 152 2024-09-21T09:43:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:12 smithi191 ceph-mon[33301]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:43:13.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:12 smithi076 ceph-mon[110963]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:43:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:12 smithi076 ceph-mon[106482]: osdmap e385: 8 total, 8 up, 8 in 2024-09-21T09:43:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:43:13] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.51.0" 2024-09-21T09:43:13.629 INFO:tasks.rados.rados.0.smithi076.stdout:1255: done (0 left) 2024-09-21T09:43:13.630 INFO:tasks.rados.rados.0.smithi076.stdout:1256: read oid 18 snap -1 2024-09-21T09:43:13.630 INFO:tasks.rados.rados.0.smithi076.stdout:1256: expect (ObjNum 491 snap 157 seq_num 491) 2024-09-21T09:43:13.630 INFO:tasks.rados.rados.0.smithi076.stdout:1257: snap_remove snap 127 2024-09-21T09:43:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:13 smithi191 ceph-mon[33301]: pgmap v45: 89 pgs: 89 active+clean; 200 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-09-21T09:43:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:13 smithi191 ceph-mon[33301]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:43:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:13 smithi191 ceph-mon[33301]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:43:14.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-mon[106482]: pgmap v45: 89 pgs: 89 active+clean; 200 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-09-21T09:43:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-mon[106482]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:43:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-mon[106482]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:43:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-mon[110963]: pgmap v45: 89 pgs: 89 active+clean; 200 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-09-21T09:43:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-mon[110963]: osdmap e386: 8 total, 8 up, 8 in 2024-09-21T09:43:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:13 smithi076 ceph-mon[110963]: osdmap e387: 8 total, 8 up, 8 in 2024-09-21T09:43:14.630 INFO:tasks.rados.rados.0.smithi076.stdout:1257: done (1 left) 2024-09-21T09:43:14.630 INFO:tasks.rados.rados.0.smithi076.stdout:1258: rollback oid 32 current snap is 158 2024-09-21T09:43:14.630 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 148 2024-09-21T09:43:14.630 INFO:tasks.rados.rados.0.smithi076.stdout:1256: expect (ObjNum 491 snap 157 seq_num 491) 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:1258: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:1258: finishing rollback tid 2 to smithi07685715-32 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 501 (ObjNum 424 snap 132 seq_num 424) dirty exists 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:1256: done (1 left) 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:1258: done (0 left) 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:1259: write oid 37 current snap is 158 2024-09-21T09:43:14.766 INFO:tasks.rados.rados.0.smithi076.stdout:1259: seq_num 493 ranges {685851=544349,1799505=486346} 2024-09-21T09:43:14.772 INFO:tasks.rados.rados.0.smithi076.stdout:1259: writing smithi07685715-37 from 685851 to 1230200 tid 1 2024-09-21T09:43:14.778 INFO:tasks.rados.rados.0.smithi076.stdout:1259: writing smithi07685715-37 from 1799505 to 2285851 tid 2 2024-09-21T09:43:14.778 INFO:tasks.rados.rados.0.smithi076.stdout:1260: write oid 39 current snap is 158 2024-09-21T09:43:14.778 INFO:tasks.rados.rados.0.smithi076.stdout:1259: finishing write tid 1 to smithi07685715-37 2024-09-21T09:43:14.778 INFO:tasks.rados.rados.0.smithi076.stdout:1260: seq_num 494 ranges {702658=619598,1857987=444671} 2024-09-21T09:43:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1260: writing smithi07685715-39 from 702658 to 1322256 tid 1 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1260: writing smithi07685715-39 from 1857987 to 2302658 tid 2 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1261: delete oid 38 current snap is 158 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: finishing write tid 2 to smithi07685715-37 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: finishing write tid 3 to smithi07685715-37 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: finishing write tid 4 to smithi07685715-37 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: oid 37 updating version 0 to 343 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: oid 37 version 343 is already newer than 342 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: oid 37 updating version 343 to 344 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 344 (ObjNum 493 snap 158 seq_num 493) dirty exists 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1259: left oid 37 (ObjNum 493 snap 158 seq_num 493) 2024-09-21T09:43:14.791 INFO:tasks.rados.rados.0.smithi076.stdout:1260: finishing write tid 1 to smithi07685715-39 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:1259: done (2 left) 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:1261: done (1 left) 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:1262: rollback oid 25 current snap is 158 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 157 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:1263: read oid 7 snap -1 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:1263: expect (ObjNum 455 snap 149 seq_num 455) 2024-09-21T09:43:14.793 INFO:tasks.rados.rados.0.smithi076.stdout:1264: delete oid 33 current snap is 158 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: finishing write tid 2 to smithi07685715-39 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: finishing write tid 3 to smithi07685715-39 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: finishing write tid 4 to smithi07685715-39 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: oid 39 updating version 0 to 987 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: oid 39 updating version 987 to 989 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: oid 39 version 989 is already newer than 988 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 989 (ObjNum 494 snap 158 seq_num 494) dirty exists 2024-09-21T09:43:14.794 INFO:tasks.rados.rados.0.smithi076.stdout:1260: left oid 39 (ObjNum 494 snap 158 seq_num 494) 2024-09-21T09:43:14.795 INFO:tasks.rados.rados.0.smithi076.stdout:1262: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:43:14.795 INFO:tasks.rados.rados.0.smithi076.stdout:1262: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:1262: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 278 (ObjNum 361 snap 110 seq_num 361) dirty exists 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:1260: done (3 left) 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:1262: done (2 left) 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:1264: done (1 left) 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:1265: write oid 34 current snap is 158 2024-09-21T09:43:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:1265: seq_num 497 ranges {753079=561698,2033433=319646} 2024-09-21T09:43:14.803 INFO:tasks.rados.rados.0.smithi076.stdout:1265: writing smithi07685715-34 from 753079 to 1314777 tid 1 2024-09-21T09:43:14.808 INFO:tasks.rados.rados.0.smithi076.stdout:1265: writing smithi07685715-34 from 2033433 to 2353079 tid 2 2024-09-21T09:43:14.809 INFO:tasks.rados.rados.0.smithi076.stdout:1266: snap_create 2024-09-21T09:43:14.809 INFO:tasks.rados.rados.0.smithi076.stdout:1263: expect (ObjNum 455 snap 149 seq_num 455) 2024-09-21T09:43:14.923 INFO:tasks.rados.rados.0.smithi076.stdout:1265: finishing write tid 1 to smithi07685715-34 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: finishing write tid 2 to smithi07685715-34 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: finishing write tid 3 to smithi07685715-34 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: finishing write tid 4 to smithi07685715-34 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: oid 34 updating version 0 to 241 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: oid 34 version 241 is already newer than 239 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: oid 34 version 241 is already newer than 240 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 241 (ObjNum 497 snap 158 seq_num 497) dirty exists 2024-09-21T09:43:14.924 INFO:tasks.rados.rados.0.smithi076.stdout:1265: left oid 34 (ObjNum 497 snap 158 seq_num 497) 2024-09-21T09:43:15.745 INFO:tasks.rados.rados.0.smithi076.stdout:1263: done (2 left) 2024-09-21T09:43:15.808 INFO:tasks.rados.rados.0.smithi076.stdout:1265: done (1 left) 2024-09-21T09:43:15.808 INFO:tasks.rados.rados.0.smithi076.stdout:1266: done (0 left) 2024-09-21T09:43:15.809 INFO:tasks.rados.rados.0.smithi076.stdout:1267: write oid 44 current snap is 159 2024-09-21T09:43:15.809 INFO:tasks.rados.rados.0.smithi076.stdout:1267: seq_num 498 ranges {769886=636947,2091915=277971} 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1267: writing smithi07685715-44 from 769886 to 1406833 tid 1 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1267: writing smithi07685715-44 from 2091915 to 2369886 tid 2 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1268: read oid 49 snap 156 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1268: expect (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1269: rollback oid 32 current snap is 159 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 155 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1270: read oid 5 snap -1 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1270: expect (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:43:15.810 INFO:tasks.rados.rados.0.smithi076.stdout:1271: snap_remove snap 148 2024-09-21T09:43:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:15 smithi191 ceph-mon[33301]: pgmap v48: 89 pgs: 89 active+clean; 200 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-09-21T09:43:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:15 smithi191 ceph-mon[33301]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:43:16.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:15 smithi076 ceph-mon[110963]: pgmap v48: 89 pgs: 89 active+clean; 200 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-09-21T09:43:16.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:15 smithi076 ceph-mon[110963]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:43:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:15 smithi076 ceph-mon[106482]: pgmap v48: 89 pgs: 89 active+clean; 200 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 2.7 MiB/s wr, 24 op/s 2024-09-21T09:43:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:15 smithi076 ceph-mon[106482]: osdmap e388: 8 total, 8 up, 8 in 2024-09-21T09:43:16.756 INFO:tasks.rados.rados.0.smithi076.stdout:1271: done (4 left) 2024-09-21T09:43:16.756 INFO:tasks.rados.rados.0.smithi076.stdout:1272: write oid 21 current snap is 159 2024-09-21T09:43:16.756 INFO:tasks.rados.rados.0.smithi076.stdout:1272: seq_num 499 ranges {786693=712196,2150397=236296} 2024-09-21T09:43:16.765 INFO:tasks.rados.rados.0.smithi076.stdout:1272: writing smithi07685715-21 from 786693 to 1498889 tid 1 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1272: writing smithi07685715-21 from 2150397 to 2386693 tid 2 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1273: rollback oid 31 current snap is 159 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 158 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1267: finishing write tid 1 to smithi07685715-44 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1269: finishing rollback tid 0 to smithi07685715-32 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1269: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1269: finishing rollback tid 2 to smithi07685715-32 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 511 (ObjNum 471 snap 155 seq_num 471) dirty exists 2024-09-21T09:43:16.771 INFO:tasks.rados.rados.0.smithi076.stdout:1267: finishing write tid 2 to smithi07685715-44 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1267: finishing write tid 3 to smithi07685715-44 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1267: finishing write tid 4 to smithi07685715-44 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1267: oid 44 updating version 0 to 257 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1267: oid 44 updating version 257 to 258 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1267: oid 44 updating version 258 to 259 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 259 (ObjNum 498 snap 159 seq_num 498) dirty exists 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1267: left oid 44 (ObjNum 498 snap 159 seq_num 498) 2024-09-21T09:43:16.772 INFO:tasks.rados.rados.0.smithi076.stdout:1270: expect (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:43:16.957 INFO:tasks.rados.rados.0.smithi076.stdout:1268: expect (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:43:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:16 smithi191 ceph-mon[33301]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:43:17.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:16 smithi076 ceph-mon[106482]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:43:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:16 smithi076 ceph-mon[110963]: osdmap e389: 8 total, 8 up, 8 in 2024-09-21T09:43:17.175 INFO:tasks.rados.rados.0.smithi076.stdout:1272: finishing write tid 1 to smithi07685715-21 2024-09-21T09:43:17.175 INFO:tasks.rados.rados.0.smithi076.stdout:1272: finishing write tid 2 to smithi07685715-21 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1272: finishing write tid 3 to smithi07685715-21 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1273: finishing rollback tid 0 to smithi07685715-31 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1273: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1272: finishing write tid 4 to smithi07685715-21 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1272: oid 21 updating version 0 to 999 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1272: oid 21 updating version 999 to 1000 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1272: oid 21 version 1000 is already newer than 998 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1000 (ObjNum 499 snap 159 seq_num 499) dirty exists 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1272: left oid 21 (ObjNum 499 snap 159 seq_num 499) 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1273: finishing rollback tid 2 to smithi07685715-31 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 521 (ObjNum 441 snap 139 seq_num 441) dirty exists 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1267: done (5 left) 2024-09-21T09:43:17.176 INFO:tasks.rados.rados.0.smithi076.stdout:1268: done (4 left) 2024-09-21T09:43:17.177 INFO:tasks.rados.rados.0.smithi076.stdout:1269: done (3 left) 2024-09-21T09:43:17.177 INFO:tasks.rados.rados.0.smithi076.stdout:1270: done (2 left) 2024-09-21T09:43:17.177 INFO:tasks.rados.rados.0.smithi076.stdout:1272: done (1 left) 2024-09-21T09:43:17.177 INFO:tasks.rados.rados.0.smithi076.stdout:1273: done (0 left) 2024-09-21T09:43:17.177 INFO:tasks.rados.rados.0.smithi076.stdout:1274: write oid 34 current snap is 159 2024-09-21T09:43:17.177 INFO:tasks.rados.rados.0.smithi076.stdout:1274: seq_num 500 ranges {403500=787445,1692526=473837} 2024-09-21T09:43:17.181 INFO:tasks.rados.rados.0.smithi076.stdout:1274: writing smithi07685715-34 from 403500 to 1190945 tid 1 2024-09-21T09:43:17.187 INFO:tasks.rados.rados.0.smithi076.stdout:1274: writing smithi07685715-34 from 1692526 to 2166363 tid 2 2024-09-21T09:43:17.187 INFO:tasks.rados.rados.0.smithi076.stdout:1275: delete oid 30 current snap is 159 2024-09-21T09:43:17.187 INFO:tasks.rados.rados.0.smithi076.stdout:1274: finishing write tid 1 to smithi07685715-34 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi076.stdout:1275: done (1 left) 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi076.stdout:1276: read oid 22 snap 143 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi076.stdout:1276: expect deleted 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi076.stdout:1277: write oid 4 current snap is 159 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi076.stdout:1274: finishing write tid 2 to smithi07685715-34 2024-09-21T09:43:17.189 INFO:tasks.rados.rados.0.smithi076.stdout:1277: seq_num 502 ranges {437114=654296,1525843=477506} 2024-09-21T09:43:17.195 INFO:tasks.rados.rados.0.smithi076.stdout:1277: writing smithi07685715-4 from 437114 to 1091410 tid 1 2024-09-21T09:43:17.200 INFO:tasks.rados.rados.0.smithi076.stdout:1277: writing smithi07685715-4 from 1525843 to 2003349 tid 2 2024-09-21T09:43:17.200 INFO:tasks.rados.rados.0.smithi076.stdout:1278: write oid 37 current snap is 159 2024-09-21T09:43:17.200 INFO:tasks.rados.rados.0.smithi076.stdout:1278: seq_num 503 ranges {453921=729545,1584325=621164} 2024-09-21T09:43:17.206 INFO:tasks.rados.rados.0.smithi076.stdout:1278: writing smithi07685715-37 from 453921 to 1183466 tid 1 2024-09-21T09:43:17.212 INFO:tasks.rados.rados.0.smithi076.stdout:1278: writing smithi07685715-37 from 1584325 to 2205489 tid 2 2024-09-21T09:43:17.212 INFO:tasks.rados.rados.0.smithi076.stdout:1279: snap_create 2024-09-21T09:43:17.212 INFO:tasks.rados.rados.0.smithi076.stdout:1274: finishing write tid 3 to smithi07685715-34 2024-09-21T09:43:17.212 INFO:tasks.rados.rados.0.smithi076.stdout:1274: finishing write tid 4 to smithi07685715-34 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:1274: oid 34 updating version 0 to 245 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:1274: oid 34 updating version 245 to 247 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:1274: oid 34 version 247 is already newer than 246 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 247 (ObjNum 500 snap 159 seq_num 500) dirty exists 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:1274: left oid 34 (ObjNum 500 snap 159 seq_num 500) 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:1277: finishing write tid 1 to smithi07685715-4 2024-09-21T09:43:17.213 INFO:tasks.rados.rados.0.smithi076.stdout:1277: finishing write tid 2 to smithi07685715-4 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:1277: finishing write tid 3 to smithi07685715-4 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:1277: finishing write tid 4 to smithi07685715-4 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:1277: oid 4 updating version 0 to 419 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:1277: oid 4 version 419 is already newer than 417 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:1277: oid 4 version 419 is already newer than 418 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 419 (ObjNum 502 snap 159 seq_num 502) dirty exists 2024-09-21T09:43:17.214 INFO:tasks.rados.rados.0.smithi076.stdout:1277: left oid 4 (ObjNum 502 snap 159 seq_num 502) 2024-09-21T09:43:17.215 INFO:tasks.rados.rados.0.smithi076.stdout:1278: finishing write tid 1 to smithi07685715-37 2024-09-21T09:43:17.216 INFO:tasks.rados.rados.0.smithi076.stdout:1278: finishing write tid 2 to smithi07685715-37 2024-09-21T09:43:17.216 INFO:tasks.rados.rados.0.smithi076.stdout:1278: finishing write tid 3 to smithi07685715-37 2024-09-21T09:43:17.216 INFO:tasks.rados.rados.0.smithi076.stdout:1278: finishing write tid 4 to smithi07685715-37 2024-09-21T09:43:17.216 INFO:tasks.rados.rados.0.smithi076.stdout:1278: oid 37 updating version 0 to 351 2024-09-21T09:43:17.216 INFO:tasks.rados.rados.0.smithi076.stdout:1278: oid 37 version 351 is already newer than 350 2024-09-21T09:43:17.216 INFO:tasks.rados.rados.0.smithi076.stdout:1278: oid 37 updating version 351 to 352 2024-09-21T09:43:17.217 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 352 (ObjNum 503 snap 159 seq_num 503) dirty exists 2024-09-21T09:43:17.217 INFO:tasks.rados.rados.0.smithi076.stdout:1278: left oid 37 (ObjNum 503 snap 159 seq_num 503) 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1274: done (4 left) 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1276: done (3 left) 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1277: done (2 left) 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1278: done (1 left) 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1279: done (0 left) 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1280: rollback oid 42 current snap is 160 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 156 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1281: rollback oid 39 current snap is 160 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 158 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1282: write oid 21 current snap is 160 2024-09-21T09:43:17.757 INFO:tasks.rados.rados.0.smithi076.stdout:1282: seq_num 504 ranges {470728=404794,1526454=481175} 2024-09-21T09:43:17.761 INFO:tasks.rados.rados.0.smithi076.stdout:1282: writing smithi07685715-21 from 470728 to 875522 tid 1 2024-09-21T09:43:17.768 INFO:tasks.rados.rados.0.smithi076.stdout:1282: writing smithi07685715-21 from 1526454 to 2007629 tid 2 2024-09-21T09:43:17.768 INFO:tasks.rados.rados.0.smithi076.stdout:1283: read oid 50 snap -1 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1283: expect (ObjNum 476 snap 155 seq_num 476) 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1284: write oid 38 current snap is 160 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1280: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1280: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1280: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 422 (ObjNum 479 snap 155 seq_num 479) dirty exists 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1281: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1281: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1281: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1004 (ObjNum 494 snap 158 seq_num 494) dirty exists 2024-09-21T09:43:17.769 INFO:tasks.rados.rados.0.smithi076.stdout:1284: seq_num 505 ranges {487535=480043,1584936=624833} 2024-09-21T09:43:17.773 INFO:tasks.rados.rados.0.smithi076.stdout:1284: writing smithi07685715-38 from 487535 to 967578 tid 1 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1284: writing smithi07685715-38 from 1584936 to 2209769 tid 2 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1280: done (4 left) 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1281: done (3 left) 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1285: read oid 8 snap -1 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1285: expect (ObjNum 484 snap 155 seq_num 484) 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1286: write oid 19 current snap is 160 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1282: finishing write tid 1 to smithi07685715-21 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1282: finishing write tid 2 to smithi07685715-21 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1282: finishing write tid 3 to smithi07685715-21 2024-09-21T09:43:17.781 INFO:tasks.rados.rados.0.smithi076.stdout:1282: finishing write tid 4 to smithi07685715-21 2024-09-21T09:43:17.782 INFO:tasks.rados.rados.0.smithi076.stdout:1282: oid 21 updating version 0 to 1007 2024-09-21T09:43:17.782 INFO:tasks.rados.rados.0.smithi076.stdout:1282: oid 21 version 1007 is already newer than 1006 2024-09-21T09:43:17.782 INFO:tasks.rados.rados.0.smithi076.stdout:1282: oid 21 updating version 1007 to 1008 2024-09-21T09:43:17.782 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1008 (ObjNum 504 snap 160 seq_num 504) dirty exists 2024-09-21T09:43:17.782 INFO:tasks.rados.rados.0.smithi076.stdout:1282: left oid 21 (ObjNum 504 snap 160 seq_num 504) 2024-09-21T09:43:17.782 INFO:tasks.rados.rados.0.smithi076.stdout:1283: expect (ObjNum 476 snap 155 seq_num 476) 2024-09-21T09:43:17.842 INFO:tasks.rados.rados.0.smithi076.stdout:1284: finishing write tid 1 to smithi07685715-38 2024-09-21T09:43:17.842 INFO:tasks.rados.rados.0.smithi076.stdout:1284: finishing write tid 2 to smithi07685715-38 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1284: finishing write tid 3 to smithi07685715-38 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1284: finishing write tid 4 to smithi07685715-38 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1284: oid 38 updating version 0 to 715 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1284: oid 38 version 715 is already newer than 714 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1284: oid 38 updating version 715 to 716 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 716 (ObjNum 505 snap 160 seq_num 505) dirty exists 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1284: left oid 38 (ObjNum 505 snap 160 seq_num 505) 2024-09-21T09:43:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:1285: expect (ObjNum 484 snap 155 seq_num 484) 2024-09-21T09:43:17.965 INFO:tasks.rados.rados.0.smithi076.stdout:1286: seq_num 506 ranges {504342=555292,1643418=484844} 2024-09-21T09:43:17.970 INFO:tasks.rados.rados.0.smithi076.stdout:1286: writing smithi07685715-19 from 504342 to 1059634 tid 1 2024-09-21T09:43:17.976 INFO:tasks.rados.rados.0.smithi076.stdout:1286: writing smithi07685715-19 from 1643418 to 2128262 tid 2 2024-09-21T09:43:17.976 INFO:tasks.rados.rados.0.smithi076.stdout:1282: done (4 left) 2024-09-21T09:43:17.976 INFO:tasks.rados.rados.0.smithi076.stdout:1283: done (3 left) 2024-09-21T09:43:17.977 INFO:tasks.rados.rados.0.smithi076.stdout:1284: done (2 left) 2024-09-21T09:43:17.977 INFO:tasks.rados.rados.0.smithi076.stdout:1285: done (1 left) 2024-09-21T09:43:17.977 INFO:tasks.rados.rados.0.smithi076.stdout:1287: write oid 26 current snap is 160 2024-09-21T09:43:17.977 INFO:tasks.rados.rados.0.smithi076.stdout:1286: finishing write tid 1 to smithi07685715-19 2024-09-21T09:43:17.977 INFO:tasks.rados.rados.0.smithi076.stdout:1287: seq_num 507 ranges {521149=630541,1701900=628502} 2024-09-21T09:43:17.982 INFO:tasks.rados.rados.0.smithi076.stdout:1287: writing smithi07685715-26 from 521149 to 1151690 tid 1 2024-09-21T09:43:17.989 INFO:tasks.rados.rados.0.smithi076.stdout:1287: writing smithi07685715-26 from 1701900 to 2330402 tid 2 2024-09-21T09:43:17.989 INFO:tasks.rados.rados.0.smithi076.stdout:1286: finishing write tid 2 to smithi07685715-19 2024-09-21T09:43:17.989 INFO:tasks.rados.rados.0.smithi076.stdout:1286: finishing write tid 3 to smithi07685715-19 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1286: finishing write tid 4 to smithi07685715-19 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1286: oid 19 updating version 0 to 525 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1286: oid 19 version 525 is already newer than 524 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1286: oid 19 version 525 is already newer than 523 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 525 (ObjNum 506 snap 160 seq_num 506) dirty exists 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1286: left oid 19 (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1287: finishing write tid 1 to smithi07685715-26 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1288: read oid 18 snap 147 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1288: expect (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1286: done (2 left) 2024-09-21T09:43:17.990 INFO:tasks.rados.rados.0.smithi076.stdout:1289: snap_create 2024-09-21T09:43:17.992 INFO:tasks.rados.rados.0.smithi076.stdout:1288: expect (ObjNum 380 snap 119 seq_num 380) 2024-09-21T09:43:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:17 smithi191 ceph-mon[33301]: pgmap v51: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:43:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:17 smithi191 ceph-mon[33301]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:43:18.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:17 smithi076 ceph-mon[110963]: pgmap v51: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:43:18.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:17 smithi076 ceph-mon[110963]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:43:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:17 smithi076 ceph-mon[106482]: pgmap v51: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:43:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:17 smithi076 ceph-mon[106482]: osdmap e390: 8 total, 8 up, 8 in 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: finishing write tid 2 to smithi07685715-26 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: finishing write tid 3 to smithi07685715-26 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: finishing write tid 4 to smithi07685715-26 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: oid 26 updating version 0 to 499 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: oid 26 updating version 499 to 501 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: oid 26 version 501 is already newer than 500 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 501 (ObjNum 507 snap 160 seq_num 507) dirty exists 2024-09-21T09:43:18.144 INFO:tasks.rados.rados.0.smithi076.stdout:1287: left oid 26 (ObjNum 507 snap 160 seq_num 507) 2024-09-21T09:43:18.630 INFO:tasks.rados.rados.0.smithi076.stdout:1287: done (2 left) 2024-09-21T09:43:18.630 INFO:tasks.rados.rados.0.smithi076.stdout:1288: done (1 left) 2024-09-21T09:43:18.630 INFO:tasks.rados.rados.0.smithi076.stdout:1289: done (0 left) 2024-09-21T09:43:18.630 INFO:tasks.rados.rados.0.smithi076.stdout:1290: snap_create 2024-09-21T09:43:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:18 smithi191 ceph-mon[33301]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:43:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:18 smithi191 ceph-mon[33301]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:43:19.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:18 smithi076 ceph-mon[106482]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:43:19.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:18 smithi076 ceph-mon[106482]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:43:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:18 smithi076 ceph-mon[110963]: osdmap e391: 8 total, 8 up, 8 in 2024-09-21T09:43:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:18 smithi076 ceph-mon[110963]: osdmap e392: 8 total, 8 up, 8 in 2024-09-21T09:43:19.632 INFO:tasks.rados.rados.0.smithi076.stdout:1290: done (0 left) 2024-09-21T09:43:19.632 INFO:tasks.rados.rados.0.smithi076.stdout:1291: read oid 13 snap -1 2024-09-21T09:43:19.632 INFO:tasks.rados.rados.0.smithi076.stdout:1291: expect deleted 2024-09-21T09:43:19.632 INFO:tasks.rados.rados.0.smithi076.stdout:1292: read oid 44 snap -1 2024-09-21T09:43:19.632 INFO:tasks.rados.rados.0.smithi076.stdout:1292: expect (ObjNum 498 snap 159 seq_num 498) 2024-09-21T09:43:19.632 INFO:tasks.rados.rados.0.smithi076.stdout:1293: snap_remove snap 147 2024-09-21T09:43:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:19 smithi191 ceph-mon[33301]: pgmap v54: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:43:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:19 smithi191 ceph-mon[33301]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:43:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:19 smithi076 ceph-mon[106482]: pgmap v54: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:43:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:19 smithi076 ceph-mon[106482]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:43:20.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:19 smithi076 ceph-mon[110963]: pgmap v54: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:43:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:19 smithi076 ceph-mon[110963]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:43:20.635 INFO:tasks.rados.rados.0.smithi076.stdout:1293: done (2 left) 2024-09-21T09:43:20.635 INFO:tasks.rados.rados.0.smithi076.stdout:1294: read oid 35 snap -1 2024-09-21T09:43:20.635 INFO:tasks.rados.rados.0.smithi076.stdout:1294: expect (ObjNum 451 snap 147 seq_num 451) 2024-09-21T09:43:20.635 INFO:tasks.rados.rados.0.smithi076.stdout:1295: rollback oid 29 current snap is 162 2024-09-21T09:43:20.635 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 132 2024-09-21T09:43:20.635 INFO:tasks.rados.rados.0.smithi076.stdout:1292: expect (ObjNum 498 snap 159 seq_num 498) 2024-09-21T09:43:20.749 INFO:tasks.rados.rados.0.smithi076.stdout:1295: finishing rollback tid 0 to smithi07685715-29 2024-09-21T09:43:20.749 INFO:tasks.rados.rados.0.smithi076.stdout:1295: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:43:20.749 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 355 (ObjNum 65 snap 0 seq_num 864441680) dirty dne 2024-09-21T09:43:20.749 INFO:tasks.rados.rados.0.smithi076.stdout:1294: expect (ObjNum 451 snap 147 seq_num 451) 2024-09-21T09:43:20.866 INFO:tasks.rados.rados.0.smithi076.stdout:1291: done (3 left) 2024-09-21T09:43:20.866 INFO:tasks.rados.rados.0.smithi076.stdout:1292: done (2 left) 2024-09-21T09:43:20.866 INFO:tasks.rados.rados.0.smithi076.stdout:1294: done (1 left) 2024-09-21T09:43:20.867 INFO:tasks.rados.rados.0.smithi076.stdout:1295: done (0 left) 2024-09-21T09:43:20.867 INFO:tasks.rados.rados.0.smithi076.stdout:1296: write oid 30 current snap is 162 2024-09-21T09:43:20.867 INFO:tasks.rados.rados.0.smithi076.stdout:1296: seq_num 508 ranges {537956=705790,1644029=772160} 2024-09-21T09:43:20.873 INFO:tasks.rados.rados.0.smithi076.stdout:1296: writing smithi07685715-30 from 537956 to 1243746 tid 1 2024-09-21T09:43:20.879 INFO:tasks.rados.rados.0.smithi076.stdout:1296: writing smithi07685715-30 from 1644029 to 2416189 tid 2 2024-09-21T09:43:20.880 INFO:tasks.rados.rados.0.smithi076.stdout:1297: write oid 33 current snap is 162 2024-09-21T09:43:20.880 INFO:tasks.rados.rados.0.smithi076.stdout:1296: finishing write tid 1 to smithi07685715-30 2024-09-21T09:43:20.880 INFO:tasks.rados.rados.0.smithi076.stdout:1297: seq_num 509 ranges {554763=781039,2102511=452252} 2024-09-21T09:43:20.887 INFO:tasks.rados.rados.0.smithi076.stdout:1297: writing smithi07685715-33 from 554763 to 1335802 tid 1 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1297: writing smithi07685715-33 from 2102511 to 2554763 tid 2 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1298: write oid 10 current snap is 162 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1296: finishing write tid 2 to smithi07685715-30 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1296: finishing write tid 3 to smithi07685715-30 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1296: finishing write tid 4 to smithi07685715-30 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1296: oid 30 updating version 0 to 1061 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1296: oid 30 version 1061 is already newer than 1059 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:1296: oid 30 version 1061 is already newer than 1060 2024-09-21T09:43:20.894 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1061 (ObjNum 508 snap 162 seq_num 508) dirty exists 2024-09-21T09:43:20.895 INFO:tasks.rados.rados.0.smithi076.stdout:1296: left oid 30 (ObjNum 508 snap 162 seq_num 508) 2024-09-21T09:43:20.895 INFO:tasks.rados.rados.0.smithi076.stdout:1297: finishing write tid 1 to smithi07685715-33 2024-09-21T09:43:20.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: seq_num 510 ranges {571570=572641,1877346=694224} 2024-09-21T09:43:20.900 INFO:tasks.rados.rados.0.smithi076.stdout:1298: writing smithi07685715-10 from 571570 to 1144211 tid 1 2024-09-21T09:43:20.908 INFO:tasks.rados.rados.0.smithi076.stdout:1298: writing smithi07685715-10 from 1877346 to 2571570 tid 2 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1296: done (2 left) 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1299: read oid 26 snap -1 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1299: expect (ObjNum 507 snap 160 seq_num 507) 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1300: rollback oid 7 current snap is 162 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 132 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1301: rollback oid 46 current snap is 162 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 156 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1297: finishing write tid 2 to smithi07685715-33 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1297: finishing write tid 3 to smithi07685715-33 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1297: finishing write tid 4 to smithi07685715-33 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1297: oid 33 updating version 0 to 285 2024-09-21T09:43:20.909 INFO:tasks.rados.rados.0.smithi076.stdout:1297: oid 33 updating version 285 to 286 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1297: oid 33 version 286 is already newer than 284 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 286 (ObjNum 509 snap 162 seq_num 509) dirty exists 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1297: left oid 33 (ObjNum 509 snap 162 seq_num 509) 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1298: finishing write tid 1 to smithi07685715-10 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1297: done (4 left) 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1302: read oid 36 snap 157 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1302: expect (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1303: read oid 14 snap -1 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1303: expect (ObjNum 459 snap 150 seq_num 459) 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1304: write oid 45 current snap is 162 2024-09-21T09:43:20.910 INFO:tasks.rados.rados.0.smithi076.stdout:1304: seq_num 511 ranges {588377=647890,1935828=635840} 2024-09-21T09:43:20.915 INFO:tasks.rados.rados.0.smithi076.stdout:1304: writing smithi07685715-45 from 588377 to 1236267 tid 1 2024-09-21T09:43:20.923 INFO:tasks.rados.rados.0.smithi076.stdout:1304: writing smithi07685715-45 from 1935828 to 2571668 tid 2 2024-09-21T09:43:20.923 INFO:tasks.rados.rados.0.smithi076.stdout:1305: read oid 2 snap -1 2024-09-21T09:43:20.923 INFO:tasks.rados.rados.0.smithi076.stdout:1305: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:20.923 INFO:tasks.rados.rados.0.smithi076.stdout:1306: read oid 12 snap 155 2024-09-21T09:43:20.923 INFO:tasks.rados.rados.0.smithi076.stdout:1306: expect deleted 2024-09-21T09:43:20.923 INFO:tasks.rados.rados.0.smithi076.stdout:1307: snap_remove snap 151 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:1307: done (9 left) 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:1308: write oid 1 current snap is 162 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:1301: finishing rollback tid 0 to smithi07685715-46 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:1301: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:1301: finishing rollback tid 2 to smithi07685715-46 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1020 (ObjNum 482 snap 155 seq_num 482) dirty exists 2024-09-21T09:43:21.766 INFO:tasks.rados.rados.0.smithi076.stdout:1299: expect (ObjNum 507 snap 160 seq_num 507) 2024-09-21T09:43:21.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: finishing write tid 2 to smithi07685715-10 2024-09-21T09:43:21.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: finishing write tid 3 to smithi07685715-10 2024-09-21T09:43:21.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: finishing write tid 4 to smithi07685715-10 2024-09-21T09:43:21.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: oid 10 updating version 0 to 726 2024-09-21T09:43:21.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: oid 10 version 726 is already newer than 725 2024-09-21T09:43:21.895 INFO:tasks.rados.rados.0.smithi076.stdout:1298: oid 10 version 726 is already newer than 724 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 726 (ObjNum 510 snap 162 seq_num 510) dirty exists 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:1298: left oid 10 (ObjNum 510 snap 162 seq_num 510) 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:1300: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:1300: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:1300: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 730 (ObjNum 396 snap 123 seq_num 396) dirty exists 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:1304: finishing write tid 1 to smithi07685715-45 2024-09-21T09:43:21.896 INFO:tasks.rados.rados.0.smithi076.stdout:1302: expect (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:43:21.976 INFO:tasks.rados.rados.0.smithi076.stdout:1303: expect (ObjNum 459 snap 150 seq_num 459) 2024-09-21T09:43:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:21 smithi191 ceph-mon[33301]: pgmap v57: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:21 smithi191 ceph-mon[33301]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:43:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:21 smithi076 ceph-mon[106482]: pgmap v57: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:21 smithi076 ceph-mon[106482]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:43:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:21 smithi076 ceph-mon[110963]: pgmap v57: 89 pgs: 89 active+clean; 189 MiB data, 651 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:21 smithi076 ceph-mon[110963]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: finishing write tid 2 to smithi07685715-45 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: finishing write tid 3 to smithi07685715-45 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: finishing write tid 4 to smithi07685715-45 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: oid 45 updating version 0 to 267 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: oid 45 updating version 267 to 268 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: oid 45 updating version 268 to 269 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 269 (ObjNum 511 snap 162 seq_num 511) dirty exists 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1304: left oid 45 (ObjNum 511 snap 162 seq_num 511) 2024-09-21T09:43:22.203 INFO:tasks.rados.rados.0.smithi076.stdout:1305: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:22.312 INFO:tasks.rados.rados.0.smithi076.stdout:1308: seq_num 512 ranges {605184=723139,1877957=727227} 2024-09-21T09:43:22.319 INFO:tasks.rados.rados.0.smithi076.stdout:1308: writing smithi07685715-1 from 605184 to 1328323 tid 1 2024-09-21T09:43:22.326 INFO:tasks.rados.rados.0.smithi076.stdout:1308: writing smithi07685715-1 from 1877957 to 2605184 tid 2 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1298: done (9 left) 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1299: done (8 left) 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1300: done (7 left) 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1301: done (6 left) 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1302: done (5 left) 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1303: done (4 left) 2024-09-21T09:43:22.327 INFO:tasks.rados.rados.0.smithi076.stdout:1304: done (3 left) 2024-09-21T09:43:22.328 INFO:tasks.rados.rados.0.smithi076.stdout:1305: done (2 left) 2024-09-21T09:43:22.328 INFO:tasks.rados.rados.0.smithi076.stdout:1306: done (1 left) 2024-09-21T09:43:22.328 INFO:tasks.rados.rados.0.smithi076.stdout:1309: snap_create 2024-09-21T09:43:22.328 INFO:tasks.rados.rados.0.smithi076.stdout:1308: finishing write tid 1 to smithi07685715-1 2024-09-21T09:43:22.330 INFO:tasks.rados.rados.0.smithi076.stdout:1308: finishing write tid 2 to smithi07685715-1 2024-09-21T09:43:22.330 INFO:tasks.rados.rados.0.smithi076.stdout:1308: finishing write tid 3 to smithi07685715-1 2024-09-21T09:43:22.330 INFO:tasks.rados.rados.0.smithi076.stdout:1308: finishing write tid 4 to smithi07685715-1 2024-09-21T09:43:22.331 INFO:tasks.rados.rados.0.smithi076.stdout:1308: oid 1 updating version 0 to 1028 2024-09-21T09:43:22.331 INFO:tasks.rados.rados.0.smithi076.stdout:1308: oid 1 version 1028 is already newer than 1026 2024-09-21T09:43:22.331 INFO:tasks.rados.rados.0.smithi076.stdout:1308: oid 1 version 1028 is already newer than 1027 2024-09-21T09:43:22.331 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 1028 (ObjNum 512 snap 162 seq_num 512) dirty exists 2024-09-21T09:43:22.331 INFO:tasks.rados.rados.0.smithi076.stdout:1308: left oid 1 (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:22.771 INFO:tasks.rados.rados.0.smithi076.stdout:1308: done (1 left) 2024-09-21T09:43:22.771 INFO:tasks.rados.rados.0.smithi076.stdout:1309: done (0 left) 2024-09-21T09:43:22.771 INFO:tasks.rados.rados.0.smithi076.stdout:1310: write oid 25 current snap is 163 2024-09-21T09:43:22.771 INFO:tasks.rados.rados.0.smithi076.stdout:1310: seq_num 513 ranges {621991=798388,1936439=639509} 2024-09-21T09:43:22.778 INFO:tasks.rados.rados.0.smithi076.stdout:1310: writing smithi07685715-25 from 621991 to 1420379 tid 1 2024-09-21T09:43:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1310: writing smithi07685715-25 from 1936439 to 2575948 tid 2 2024-09-21T09:43:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1311: read oid 21 snap -1 2024-09-21T09:43:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1311: expect (ObjNum 504 snap 160 seq_num 504) 2024-09-21T09:43:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1312: rollback oid 5 current snap is 163 2024-09-21T09:43:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 143 2024-09-21T09:43:22.785 INFO:tasks.rados.rados.0.smithi076.stdout:1310: finishing write tid 1 to smithi07685715-25 2024-09-21T09:43:22.786 INFO:tasks.rados.rados.0.smithi076.stdout:1313: write oid 26 current snap is 163 2024-09-21T09:43:22.786 INFO:tasks.rados.rados.0.smithi076.stdout:1313: seq_num 514 ranges {638798=473637,1594921=783167} 2024-09-21T09:43:22.791 INFO:tasks.rados.rados.0.smithi076.stdout:1313: writing smithi07685715-26 from 638798 to 1112435 tid 1 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1313: writing smithi07685715-26 from 1594921 to 2378088 tid 2 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1314: write oid 42 current snap is 163 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: finishing write tid 2 to smithi07685715-25 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: finishing write tid 3 to smithi07685715-25 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: finishing write tid 4 to smithi07685715-25 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: oid 25 updating version 0 to 291 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: oid 25 version 291 is already newer than 290 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: oid 25 updating version 291 to 292 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 292 (ObjNum 513 snap 163 seq_num 513) dirty exists 2024-09-21T09:43:22.799 INFO:tasks.rados.rados.0.smithi076.stdout:1310: left oid 25 (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:43:22.800 INFO:tasks.rados.rados.0.smithi076.stdout:1311: expect (ObjNum 504 snap 160 seq_num 504) 2024-09-21T09:43:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:1312: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1312: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1312: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1032 (ObjNum 444 snap 141 seq_num 444) dirty exists 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: finishing write tid 1 to smithi07685715-26 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: finishing write tid 2 to smithi07685715-26 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: finishing write tid 3 to smithi07685715-26 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: finishing write tid 4 to smithi07685715-26 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: oid 26 updating version 0 to 508 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: oid 26 updating version 508 to 509 2024-09-21T09:43:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:1313: oid 26 version 509 is already newer than 507 2024-09-21T09:43:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 509 (ObjNum 514 snap 163 seq_num 514) dirty exists 2024-09-21T09:43:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:1313: left oid 26 (ObjNum 514 snap 163 seq_num 514) 2024-09-21T09:43:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:1314: seq_num 515 ranges {655605=548886,1653403=643178} 2024-09-21T09:43:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:1314: writing smithi07685715-42 from 655605 to 1204491 tid 1 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1314: writing smithi07685715-42 from 1653403 to 2296581 tid 2 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1310: done (4 left) 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1311: done (3 left) 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1312: done (2 left) 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1313: done (1 left) 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1315: snap_create 2024-09-21T09:43:22.921 INFO:tasks.rados.rados.0.smithi076.stdout:1314: finishing write tid 1 to smithi07685715-42 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: finishing write tid 2 to smithi07685715-42 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: finishing write tid 3 to smithi07685715-42 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: finishing write tid 4 to smithi07685715-42 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: oid 42 updating version 0 to 430 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: oid 42 updating version 430 to 431 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: oid 42 updating version 431 to 432 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 432 (ObjNum 515 snap 163 seq_num 515) dirty exists 2024-09-21T09:43:22.924 INFO:tasks.rados.rados.0.smithi076.stdout:1314: left oid 42 (ObjNum 515 snap 163 seq_num 515) 2024-09-21T09:43:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:22 smithi191 ceph-mon[33301]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:43:23.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:22 smithi076 ceph-mon[110963]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:43:23.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:22 smithi076 ceph-mon[106482]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:43:23.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:43:23] "GET /metrics HTTP/1.1" 200 36128 "" "Prometheus/2.51.0" 2024-09-21T09:43:23.633 INFO:tasks.rados.rados.0.smithi076.stdout:1314: done (1 left) 2024-09-21T09:43:23.633 INFO:tasks.rados.rados.0.smithi076.stdout:1315: done (0 left) 2024-09-21T09:43:23.634 INFO:tasks.rados.rados.0.smithi076.stdout:1316: read oid 24 snap -1 2024-09-21T09:43:23.634 INFO:tasks.rados.rados.0.smithi076.stdout:1316: expect deleted 2024-09-21T09:43:23.634 INFO:tasks.rados.rados.0.smithi076.stdout:1317: delete oid 31 current snap is 164 2024-09-21T09:43:23.636 INFO:tasks.rados.rados.0.smithi076.stdout:1316: done (1 left) 2024-09-21T09:43:23.636 INFO:tasks.rados.rados.0.smithi076.stdout:1317: done (0 left) 2024-09-21T09:43:23.636 INFO:tasks.rados.rados.0.smithi076.stdout:1318: delete oid 41 current snap is 164 2024-09-21T09:43:23.640 INFO:tasks.rados.rados.0.smithi076.stdout:1318: done (0 left) 2024-09-21T09:43:23.640 INFO:tasks.rados.rados.0.smithi076.stdout:1319: snap_remove snap 160 2024-09-21T09:43:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:23 smithi191 ceph-mon[33301]: pgmap v60: 89 pgs: 89 active+clean; 213 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:43:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:23 smithi191 ceph-mon[33301]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:43:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:23 smithi191 ceph-mon[33301]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:43:24.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-mon[106482]: pgmap v60: 89 pgs: 89 active+clean; 213 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:43:24.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-mon[106482]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:43:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-mon[106482]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:43:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-mon[110963]: pgmap v60: 89 pgs: 89 active+clean; 213 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:43:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-mon[110963]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:43:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:23 smithi076 ceph-mon[110963]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:43:24.633 INFO:tasks.rados.rados.0.smithi076.stdout:1319: done (0 left) 2024-09-21T09:43:24.634 INFO:tasks.rados.rados.0.smithi076.stdout:1320: snap_create 2024-09-21T09:43:25.635 INFO:tasks.rados.rados.0.smithi076.stdout:1320: done (0 left) 2024-09-21T09:43:25.635 INFO:tasks.rados.rados.0.smithi076.stdout:1321: read oid 32 snap -1 2024-09-21T09:43:25.635 INFO:tasks.rados.rados.0.smithi076.stdout:1321: expect (ObjNum 471 snap 155 seq_num 471) 2024-09-21T09:43:25.635 INFO:tasks.rados.rados.0.smithi076.stdout:1322: rollback oid 49 current snap is 165 2024-09-21T09:43:25.635 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 162 2024-09-21T09:43:25.635 INFO:tasks.rados.rados.0.smithi076.stdout:1323: delete oid 50 current snap is 165 2024-09-21T09:43:25.644 INFO:tasks.rados.rados.0.smithi076.stdout:1323: done (2 left) 2024-09-21T09:43:25.644 INFO:tasks.rados.rados.0.smithi076.stdout:1324: read oid 37 snap 155 2024-09-21T09:43:25.644 INFO:tasks.rados.rados.0.smithi076.stdout:1324: expect deleted 2024-09-21T09:43:25.644 INFO:tasks.rados.rados.0.smithi076.stdout:1325: delete oid 28 current snap is 165 2024-09-21T09:43:25.645 INFO:tasks.rados.rados.0.smithi076.stdout:1321: expect (ObjNum 471 snap 155 seq_num 471) 2024-09-21T09:43:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:1322: finishing rollback tid 0 to smithi07685715-49 2024-09-21T09:43:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:1322: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:43:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:1322: finishing rollback tid 2 to smithi07685715-49 2024-09-21T09:43:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 1075 (ObjNum 450 snap 147 seq_num 450) dirty exists 2024-09-21T09:43:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:1321: done (3 left) 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1322: done (2 left) 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1324: done (1 left) 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1325: done (0 left) 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1326: read oid 3 snap 164 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1326: expect deleted 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1327: rollback oid 19 current snap is 165 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 164 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1328: write oid 23 current snap is 165 2024-09-21T09:43:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:1328: seq_num 520 ranges {739640=641484,1829460=794174} 2024-09-21T09:43:25.894 INFO:tasks.rados.rados.0.smithi076.stdout:1328: writing smithi07685715-23 from 739640 to 1381124 tid 1 2024-09-21T09:43:25.901 INFO:tasks.rados.rados.0.smithi076.stdout:1328: writing smithi07685715-23 from 1829460 to 2623634 tid 2 2024-09-21T09:43:25.901 INFO:tasks.rados.rados.0.smithi076.stdout:1329: snap_create 2024-09-21T09:43:25.901 INFO:tasks.rados.rados.0.smithi076.stdout:1327: finishing rollback tid 0 to smithi07685715-19 2024-09-21T09:43:25.902 INFO:tasks.rados.rados.0.smithi076.stdout:1327: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:43:25.902 INFO:tasks.rados.rados.0.smithi076.stdout:1327: finishing rollback tid 2 to smithi07685715-19 2024-09-21T09:43:25.902 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 545 (ObjNum 506 snap 160 seq_num 506) dirty exists 2024-09-21T09:43:25.902 INFO:tasks.rados.rados.0.smithi076.stdout:1328: finishing write tid 1 to smithi07685715-23 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: finishing write tid 2 to smithi07685715-23 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: finishing write tid 3 to smithi07685715-23 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: finishing write tid 4 to smithi07685715-23 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: oid 23 updating version 0 to 1080 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: oid 23 version 1080 is already newer than 1079 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: oid 23 version 1080 is already newer than 1078 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1080 (ObjNum 520 snap 165 seq_num 520) dirty exists 2024-09-21T09:43:25.906 INFO:tasks.rados.rados.0.smithi076.stdout:1328: left oid 23 (ObjNum 520 snap 165 seq_num 520) 2024-09-21T09:43:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:25 smithi191 ceph-mon[33301]: pgmap v63: 89 pgs: 89 active+clean; 213 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:43:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:25 smithi191 ceph-mon[33301]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:43:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:25 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:25 smithi076 ceph-mon[106482]: pgmap v63: 89 pgs: 89 active+clean; 213 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:43:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:25 smithi076 ceph-mon[106482]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:43:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:25 smithi076 ceph-mon[110963]: pgmap v63: 89 pgs: 89 active+clean; 213 MiB data, 684 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 37 op/s 2024-09-21T09:43:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:25 smithi076 ceph-mon[110963]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:43:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:26.644 INFO:tasks.rados.rados.0.smithi076.stdout:1326: done (3 left) 2024-09-21T09:43:26.645 INFO:tasks.rados.rados.0.smithi076.stdout:1327: done (2 left) 2024-09-21T09:43:26.645 INFO:tasks.rados.rados.0.smithi076.stdout:1328: done (1 left) 2024-09-21T09:43:26.645 INFO:tasks.rados.rados.0.smithi076.stdout:1329: done (0 left) 2024-09-21T09:43:26.645 INFO:tasks.rados.rados.0.smithi076.stdout:1330: delete oid 41 current snap is 166 2024-09-21T09:43:26.648 INFO:tasks.rados.rados.0.smithi076.stdout:1330: done (0 left) 2024-09-21T09:43:26.648 INFO:tasks.rados.rados.0.smithi076.stdout:1331: delete oid 45 current snap is 166 2024-09-21T09:43:26.650 INFO:tasks.rados.rados.0.smithi076.stdout:1331: done (0 left) 2024-09-21T09:43:26.650 INFO:tasks.rados.rados.0.smithi076.stdout:1332: snap_create 2024-09-21T09:43:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:26 smithi191 ceph-mon[33301]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:43:27.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:26 smithi076 ceph-mon[106482]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:43:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:26 smithi076 ceph-mon[110963]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:43:27.651 INFO:tasks.rados.rados.0.smithi076.stdout:1332: done (0 left) 2024-09-21T09:43:27.651 INFO:tasks.rados.rados.0.smithi076.stdout:1333: write oid 34 current snap is 167 2024-09-21T09:43:27.651 INFO:tasks.rados.rados.0.smithi076.stdout:1333: seq_num 523 ranges {790061=467231,2004906=541501} 2024-09-21T09:43:27.658 INFO:tasks.rados.rados.0.smithi076.stdout:1333: writing smithi07685715-34 from 790061 to 1257292 tid 1 2024-09-21T09:43:27.665 INFO:tasks.rados.rados.0.smithi076.stdout:1333: writing smithi07685715-34 from 2004906 to 2546407 tid 2 2024-09-21T09:43:27.665 INFO:tasks.rados.rados.0.smithi076.stdout:1334: read oid 50 snap -1 2024-09-21T09:43:27.665 INFO:tasks.rados.rados.0.smithi076.stdout:1334: expect deleted 2024-09-21T09:43:27.665 INFO:tasks.rados.rados.0.smithi076.stdout:1335: read oid 19 snap -1 2024-09-21T09:43:27.665 INFO:tasks.rados.rados.0.smithi076.stdout:1335: expect (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:27.666 INFO:tasks.rados.rados.0.smithi076.stdout:1336: snap_remove snap 162 2024-09-21T09:43:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:27 smithi191 ceph-mon[33301]: pgmap v66: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:43:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:27 smithi191 ceph-mon[33301]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:43:28.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:27 smithi076 ceph-mon[106482]: pgmap v66: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:43:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:27 smithi076 ceph-mon[106482]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:43:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:27 smithi076 ceph-mon[110963]: pgmap v66: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:43:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:27 smithi076 ceph-mon[110963]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:43:28.634 INFO:tasks.rados.rados.0.smithi076.stdout:1336: done (3 left) 2024-09-21T09:43:28.634 INFO:tasks.rados.rados.0.smithi076.stdout:1337: write oid 8 current snap is 167 2024-09-21T09:43:28.634 INFO:tasks.rados.rados.0.smithi076.stdout:1337: seq_num 524 ranges {406868=542480,1547035=401512,2690354=116514} 2024-09-21T09:43:28.639 INFO:tasks.rados.rados.0.smithi076.stdout:1337: writing smithi07685715-8 from 406868 to 949348 tid 1 2024-09-21T09:43:28.645 INFO:tasks.rados.rados.0.smithi076.stdout:1337: writing smithi07685715-8 from 1547035 to 1948547 tid 2 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1337: writing smithi07685715-8 from 2690354 to 2806868 tid 3 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1338: read oid 1 snap -1 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1338: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1339: read oid 43 snap -1 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1339: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1340: read oid 21 snap -1 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1340: expect (ObjNum 504 snap 160 seq_num 504) 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1341: snap_create 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1333: finishing write tid 1 to smithi07685715-34 2024-09-21T09:43:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1333: finishing write tid 2 to smithi07685715-34 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1333: finishing write tid 3 to smithi07685715-34 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1333: finishing write tid 4 to smithi07685715-34 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1333: oid 34 updating version 0 to 254 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1333: oid 34 updating version 254 to 255 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1333: oid 34 version 255 is already newer than 253 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 255 (ObjNum 523 snap 167 seq_num 523) dirty exists 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1333: left oid 34 (ObjNum 523 snap 167 seq_num 523) 2024-09-21T09:43:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1335: expect (ObjNum 506 snap 160 seq_num 506) 2024-09-21T09:43:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1337: finishing write tid 1 to smithi07685715-8 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: finishing write tid 2 to smithi07685715-8 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: finishing write tid 3 to smithi07685715-8 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: finishing write tid 4 to smithi07685715-8 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: finishing write tid 5 to smithi07685715-8 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: oid 8 updating version 0 to 1088 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: oid 8 version 1088 is already newer than 1082 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: oid 8 updating version 1088 to 1089 2024-09-21T09:43:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1337: oid 8 version 1089 is already newer than 1087 2024-09-21T09:43:28.839 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1089 (ObjNum 524 snap 167 seq_num 524) dirty exists 2024-09-21T09:43:28.839 INFO:tasks.rados.rados.0.smithi076.stdout:1337: left oid 8 (ObjNum 524 snap 167 seq_num 524) 2024-09-21T09:43:28.839 INFO:tasks.rados.rados.0.smithi076.stdout:1339: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:43:28.937 INFO:tasks.rados.rados.0.smithi076.stdout:1338: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:29.058 INFO:tasks.rados.rados.0.smithi076.stdout:1340: expect (ObjNum 504 snap 160 seq_num 504) 2024-09-21T09:43:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:28 smithi191 ceph-mon[33301]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:43:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:28 smithi191 ceph-mon[33301]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:43:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:28 smithi076 ceph-mon[106482]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:43:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:28 smithi076 ceph-mon[106482]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:43:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:28 smithi076 ceph-mon[110963]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:43:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:28 smithi076 ceph-mon[110963]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:43:29.654 INFO:tasks.rados.rados.0.smithi076.stdout:1333: done (7 left) 2024-09-21T09:43:29.654 INFO:tasks.rados.rados.0.smithi076.stdout:1334: done (6 left) 2024-09-21T09:43:29.654 INFO:tasks.rados.rados.0.smithi076.stdout:1335: done (5 left) 2024-09-21T09:43:29.654 INFO:tasks.rados.rados.0.smithi076.stdout:1337: done (4 left) 2024-09-21T09:43:29.655 INFO:tasks.rados.rados.0.smithi076.stdout:1338: done (3 left) 2024-09-21T09:43:29.655 INFO:tasks.rados.rados.0.smithi076.stdout:1339: done (2 left) 2024-09-21T09:43:29.655 INFO:tasks.rados.rados.0.smithi076.stdout:1340: done (1 left) 2024-09-21T09:43:29.655 INFO:tasks.rados.rados.0.smithi076.stdout:1341: done (0 left) 2024-09-21T09:43:29.655 INFO:tasks.rados.rados.0.smithi076.stdout:1342: write oid 19 current snap is 168 2024-09-21T09:43:29.655 INFO:tasks.rados.rados.0.smithi076.stdout:1342: seq_num 525 ranges {423675=734082,1721870=545170,2717777=105898} 2024-09-21T09:43:29.660 INFO:tasks.rados.rados.0.smithi076.stdout:1342: writing smithi07685715-19 from 423675 to 1157757 tid 1 2024-09-21T09:43:29.666 INFO:tasks.rados.rados.0.smithi076.stdout:1342: writing smithi07685715-19 from 1721870 to 2267040 tid 2 2024-09-21T09:43:29.670 INFO:tasks.rados.rados.0.smithi076.stdout:1342: writing smithi07685715-19 from 2717777 to 2823675 tid 3 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:1343: rollback oid 15 current snap is 168 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 143 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:1342: finishing write tid 1 to smithi07685715-19 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:1344: read oid 38 snap -1 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:1344: expect (ObjNum 505 snap 160 seq_num 505) 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:1345: read oid 2 snap -1 2024-09-21T09:43:29.671 INFO:tasks.rados.rados.0.smithi076.stdout:1345: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:29.672 INFO:tasks.rados.rados.0.smithi076.stdout:1346: delete oid 5 current snap is 168 2024-09-21T09:43:29.672 INFO:tasks.rados.rados.0.smithi076.stdout:1345: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:29.783 INFO:tasks.rados.rados.0.smithi076.stdout:1343: finishing rollback tid 0 to smithi07685715-15 2024-09-21T09:43:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:1343: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:43:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:1343: finishing rollback tid 2 to smithi07685715-15 2024-09-21T09:43:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 277 (ObjNum 413 snap 128 seq_num 413) dirty exists 2024-09-21T09:43:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:1342: finishing write tid 2 to smithi07685715-19 2024-09-21T09:43:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:1342: finishing write tid 3 to smithi07685715-19 2024-09-21T09:43:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:1342: finishing write tid 4 to smithi07685715-19 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:1342: finishing write tid 5 to smithi07685715-19 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:1342: oid 19 updating version 0 to 552 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:1342: oid 19 updating version 552 to 553 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:1342: oid 19 updating version 553 to 554 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:1342: oid 19 version 554 is already newer than 551 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 554 (ObjNum 525 snap 168 seq_num 525) dirty exists 2024-09-21T09:43:29.785 INFO:tasks.rados.rados.0.smithi076.stdout:1342: left oid 19 (ObjNum 525 snap 168 seq_num 525) 2024-09-21T09:43:29.786 INFO:tasks.rados.rados.0.smithi076.stdout:1344: expect (ObjNum 505 snap 160 seq_num 505) 2024-09-21T09:43:29.863 INFO:tasks.rados.rados.0.smithi076.stdout:1342: done (4 left) 2024-09-21T09:43:29.863 INFO:tasks.rados.rados.0.smithi076.stdout:1343: done (3 left) 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1344: done (2 left) 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1345: done (1 left) 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1346: done (0 left) 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1347: read oid 2 snap 161 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1347: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1348: read oid 36 snap -1 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1348: expect (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1349: rollback oid 26 current snap is 168 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 159 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1350: rollback oid 40 current snap is 168 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 164 2024-09-21T09:43:29.864 INFO:tasks.rados.rados.0.smithi076.stdout:1351: snap_remove snap 158 2024-09-21T09:43:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:29 smithi191 ceph-mon[33301]: pgmap v69: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:43:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:29 smithi191 ceph-mon[33301]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:43:30.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:29 smithi076 ceph-mon[106482]: pgmap v69: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:43:30.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:29 smithi076 ceph-mon[106482]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:43:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:29 smithi076 ceph-mon[110963]: pgmap v69: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T09:43:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:29 smithi076 ceph-mon[110963]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:43:30.674 INFO:tasks.rados.rados.0.smithi076.stdout:1351: done (4 left) 2024-09-21T09:43:30.674 INFO:tasks.rados.rados.0.smithi076.stdout:1352: delete oid 31 current snap is 168 2024-09-21T09:43:30.675 INFO:tasks.rados.rados.0.smithi076.stdout:1347: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1349: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1349: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1349: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 517 (ObjNum 464 snap 152 seq_num 464) dirty exists 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1350: finishing rollback tid 0 to smithi07685715-40 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1347: done (4 left) 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1349: done (3 left) 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1352: done (2 left) 2024-09-21T09:43:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:1353: snap_remove snap 157 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1353: done (2 left) 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1354: read oid 33 snap 132 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1354: expect deleted 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1355: write oid 46 current snap is 168 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1350: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1350: finishing rollback tid 2 to smithi07685715-40 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 627 (ObjNum 282 snap 88 seq_num 282) dirty exists 2024-09-21T09:43:31.680 INFO:tasks.rados.rados.0.smithi076.stdout:1348: expect (ObjNum 398 snap 123 seq_num 398) 2024-09-21T09:43:31.767 INFO:tasks.rados.rados.0.smithi076.stdout:1355: seq_num 528 ranges {474096=559829,1780963=408850,2800046=74050} 2024-09-21T09:43:31.773 INFO:tasks.rados.rados.0.smithi076.stdout:1355: writing smithi07685715-46 from 474096 to 1033925 tid 1 2024-09-21T09:43:31.779 INFO:tasks.rados.rados.0.smithi076.stdout:1355: writing smithi07685715-46 from 1780963 to 2189813 tid 2 2024-09-21T09:43:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:1355: writing smithi07685715-46 from 2800046 to 2874096 tid 3 2024-09-21T09:43:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:1348: done (3 left) 2024-09-21T09:43:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:1350: done (2 left) 2024-09-21T09:43:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:1354: done (1 left) 2024-09-21T09:43:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:1356: write oid 28 current snap is 168 2024-09-21T09:43:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:1355: finishing write tid 1 to smithi07685715-46 2024-09-21T09:43:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:1355: finishing write tid 2 to smithi07685715-46 2024-09-21T09:43:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:1356: seq_num 529 ranges {490903=635078,1839445=552508} 2024-09-21T09:43:31.789 INFO:tasks.rados.rados.0.smithi076.stdout:1356: writing smithi07685715-28 from 490903 to 1125981 tid 1 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1356: writing smithi07685715-28 from 1839445 to 2391953 tid 2 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1357: read oid 22 snap -1 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1357: expect (ObjNum 453 snap 148 seq_num 453) 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1358: read oid 1 snap -1 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1358: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1359: write oid 37 current snap is 168 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1355: finishing write tid 3 to smithi07685715-46 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1355: finishing write tid 4 to smithi07685715-46 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1355: finishing write tid 5 to smithi07685715-46 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1355: oid 46 updating version 0 to 1059 2024-09-21T09:43:31.797 INFO:tasks.rados.rados.0.smithi076.stdout:1355: oid 46 updating version 1059 to 1060 2024-09-21T09:43:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:1355: oid 46 updating version 1060 to 1061 2024-09-21T09:43:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:1355: oid 46 version 1061 is already newer than 1058 2024-09-21T09:43:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1061 (ObjNum 528 snap 168 seq_num 528) dirty exists 2024-09-21T09:43:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:1355: left oid 46 (ObjNum 528 snap 168 seq_num 528) 2024-09-21T09:43:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:1356: finishing write tid 1 to smithi07685715-28 2024-09-21T09:43:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:1359: seq_num 530 ranges {507710=710327,1897927=412519,2854892=52818} 2024-09-21T09:43:31.803 INFO:tasks.rados.rados.0.smithi076.stdout:1359: writing smithi07685715-37 from 507710 to 1218037 tid 1 2024-09-21T09:43:31.809 INFO:tasks.rados.rados.0.smithi076.stdout:1359: writing smithi07685715-37 from 1897927 to 2310446 tid 2 2024-09-21T09:43:31.812 INFO:tasks.rados.rados.0.smithi076.stdout:1359: writing smithi07685715-37 from 2854892 to 2907710 tid 3 2024-09-21T09:43:31.813 INFO:tasks.rados.rados.0.smithi076.stdout:1355: done (4 left) 2024-09-21T09:43:31.813 INFO:tasks.rados.rados.0.smithi076.stdout:1360: snap_remove snap 155 2024-09-21T09:43:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:31 smithi191 ceph-mon[33301]: pgmap v72: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:31 smithi191 ceph-mon[33301]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:43:32.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:31 smithi076 ceph-mon[106482]: pgmap v72: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:32.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:31 smithi076 ceph-mon[106482]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:43:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:31 smithi076 ceph-mon[110963]: pgmap v72: 89 pgs: 89 active+clean; 213 MiB data, 701 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:43:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:31 smithi076 ceph-mon[110963]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:43:32.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1360: done (4 left) 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1361: rollback oid 18 current snap is 168 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 18 to 167 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1356: finishing write tid 2 to smithi07685715-28 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1356: finishing write tid 3 to smithi07685715-28 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1356: finishing write tid 4 to smithi07685715-28 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1356: oid 28 updating version 0 to 1108 2024-09-21T09:43:32.688 INFO:tasks.rados.rados.0.smithi076.stdout:1356: oid 28 version 1108 is already newer than 1107 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1356: oid 28 version 1108 is already newer than 1106 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 1108 (ObjNum 529 snap 168 seq_num 529) dirty exists 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1356: left oid 28 (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1362: read oid 11 snap -1 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1362: expect deleted 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1356: done (5 left) 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1363: write oid 10 current snap is 168 2024-09-21T09:43:32.689 INFO:tasks.rados.rados.0.smithi076.stdout:1363: seq_num 531 ranges {524517=785576,1956409=556177} 2024-09-21T09:43:32.697 INFO:tasks.rados.rados.0.smithi076.stdout:1363: writing smithi07685715-10 from 524517 to 1310093 tid 1 2024-09-21T09:43:32.704 INFO:tasks.rados.rados.0.smithi076.stdout:1363: writing smithi07685715-10 from 1956409 to 2512586 tid 2 2024-09-21T09:43:32.704 INFO:tasks.rados.rados.0.smithi076.stdout:1364: write oid 45 current snap is 168 2024-09-21T09:43:32.704 INFO:tasks.rados.rados.0.smithi076.stdout:1364: seq_num 532 ranges {541324=460825,1498538=416188,2393385=547939} 2024-09-21T09:43:32.709 INFO:tasks.rados.rados.0.smithi076.stdout:1364: writing smithi07685715-45 from 541324 to 1002149 tid 1 2024-09-21T09:43:32.714 INFO:tasks.rados.rados.0.smithi076.stdout:1364: writing smithi07685715-45 from 1498538 to 1914726 tid 2 2024-09-21T09:43:32.720 INFO:tasks.rados.rados.0.smithi076.stdout:1364: writing smithi07685715-45 from 2393385 to 2941324 tid 3 2024-09-21T09:43:32.720 INFO:tasks.rados.rados.0.smithi076.stdout:1365: delete oid 32 current snap is 168 2024-09-21T09:43:32.720 INFO:tasks.rados.rados.0.smithi076.stdout:1358: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:32.765 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:32.848 INFO:tasks.rados.rados.0.smithi076.stdout:1357: expect (ObjNum 453 snap 148 seq_num 453) 2024-09-21T09:43:32.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:32 smithi076 ceph-mon[106482]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:43:32.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:32 smithi076 ceph-mon[106482]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:32 smithi076 ceph-mon[110963]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:43:32.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:32 smithi076 ceph-mon[110963]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1359: finishing write tid 1 to smithi07685715-37 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1359: finishing write tid 2 to smithi07685715-37 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1359: finishing write tid 3 to smithi07685715-37 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1357: done (7 left) 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1358: done (6 left) 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1365: done (5 left) 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1366: read oid 16 snap -1 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1366: expect (ObjNum 446 snap 144 seq_num 446) 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1367: read oid 2 snap 167 2024-09-21T09:43:32.965 INFO:tasks.rados.rados.0.smithi076.stdout:1367: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1368: snap_create 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: finishing write tid 4 to smithi07685715-37 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: finishing write tid 5 to smithi07685715-37 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: oid 37 updating version 0 to 376 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: oid 37 version 376 is already newer than 374 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: oid 37 version 376 is already newer than 373 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: oid 37 version 376 is already newer than 375 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 376 (ObjNum 530 snap 168 seq_num 530) dirty exists 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1359: left oid 37 (ObjNum 530 snap 168 seq_num 530) 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1361: finishing rollback tid 0 to smithi07685715-18 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1361: finishing rollback tid 1 to smithi07685715-18 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:1361: finishing rollback tid 2 to smithi07685715-18 2024-09-21T09:43:32.966 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 444 (ObjNum 491 snap 157 seq_num 491) dirty exists 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: finishing write tid 1 to smithi07685715-10 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: finishing write tid 2 to smithi07685715-10 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: finishing write tid 3 to smithi07685715-10 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: finishing write tid 4 to smithi07685715-10 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: oid 10 updating version 0 to 761 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: oid 10 version 761 is already newer than 760 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: oid 10 updating version 761 to 762 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 762 (ObjNum 531 snap 168 seq_num 531) dirty exists 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1363: left oid 10 (ObjNum 531 snap 168 seq_num 531) 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1364: finishing write tid 1 to smithi07685715-45 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1364: finishing write tid 2 to smithi07685715-45 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1364: finishing write tid 3 to smithi07685715-45 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1364: finishing write tid 4 to smithi07685715-45 2024-09-21T09:43:32.967 INFO:tasks.rados.rados.0.smithi076.stdout:1364: finishing write tid 5 to smithi07685715-45 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:1364: oid 45 updating version 0 to 287 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:1364: oid 45 version 287 is already newer than 286 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:1364: oid 45 version 287 is already newer than 285 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:1364: oid 45 version 287 is already newer than 284 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 287 (ObjNum 532 snap 168 seq_num 532) dirty exists 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:1364: left oid 45 (ObjNum 532 snap 168 seq_num 532) 2024-09-21T09:43:32.968 INFO:tasks.rados.rados.0.smithi076.stdout:1367: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:43:33.082 INFO:tasks.rados.rados.0.smithi076.stdout:1366: expect (ObjNum 446 snap 144 seq_num 446) 2024-09-21T09:43:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:32 smithi191 ceph-mon[33301]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:43:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:32 smithi191 ceph-mon[33301]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:33.584 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:43:33] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.51.0" 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1359: done (7 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1361: done (6 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1362: done (5 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1363: done (4 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1364: done (3 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1366: done (2 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1367: done (1 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1368: done (0 left) 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1369: read oid 17 snap -1 2024-09-21T09:43:33.635 INFO:tasks.rados.rados.0.smithi076.stdout:1369: expect deleted 2024-09-21T09:43:33.636 INFO:tasks.rados.rados.0.smithi076.stdout:1370: write oid 41 current snap is 169 2024-09-21T09:43:33.636 INFO:tasks.rados.rados.0.smithi076.stdout:1370: seq_num 534 ranges {574938=727676,1731855=703504,2848231=126707} 2024-09-21T09:43:33.641 INFO:tasks.rados.rados.0.smithi076.stdout:1370: writing smithi07685715-41 from 574938 to 1302614 tid 1 2024-09-21T09:43:33.648 INFO:tasks.rados.rados.0.smithi076.stdout:1370: writing smithi07685715-41 from 1731855 to 2435359 tid 2 2024-09-21T09:43:33.651 INFO:tasks.rados.rados.0.smithi076.stdout:1370: writing smithi07685715-41 from 2848231 to 2974938 tid 3 2024-09-21T09:43:33.651 INFO:tasks.rados.rados.0.smithi076.stdout:1371: snap_create 2024-09-21T09:43:33.652 INFO:tasks.rados.rados.0.smithi076.stdout:1370: finishing write tid 1 to smithi07685715-41 2024-09-21T09:43:33.652 INFO:tasks.rados.rados.0.smithi076.stdout:1370: finishing write tid 2 to smithi07685715-41 2024-09-21T09:43:33.653 INFO:tasks.rados.rados.0.smithi076.stdout:1370: finishing write tid 3 to smithi07685715-41 2024-09-21T09:43:33.653 INFO:tasks.rados.rados.0.smithi076.stdout:1370: finishing write tid 4 to smithi07685715-41 2024-09-21T09:43:33.653 INFO:tasks.rados.rados.0.smithi076.stdout:1370: finishing write tid 5 to smithi07685715-41 2024-09-21T09:43:33.654 INFO:tasks.rados.rados.0.smithi076.stdout:1370: oid 41 updating version 0 to 634 2024-09-21T09:43:33.654 INFO:tasks.rados.rados.0.smithi076.stdout:1370: oid 41 updating version 634 to 635 2024-09-21T09:43:33.654 INFO:tasks.rados.rados.0.smithi076.stdout:1370: oid 41 updating version 635 to 637 2024-09-21T09:43:33.654 INFO:tasks.rados.rados.0.smithi076.stdout:1370: oid 41 version 637 is already newer than 636 2024-09-21T09:43:33.654 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 637 (ObjNum 534 snap 169 seq_num 534) dirty exists 2024-09-21T09:43:33.654 INFO:tasks.rados.rados.0.smithi076.stdout:1370: left oid 41 (ObjNum 534 snap 169 seq_num 534) 2024-09-21T09:43:33.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-mon[106482]: pgmap v75: 89 pgs: 89 active+clean; 215 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 844 KiB/s wr, 23 op/s 2024-09-21T09:43:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-mon[106482]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:43:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-mon[106482]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:43:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-mon[110963]: pgmap v75: 89 pgs: 89 active+clean; 215 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 844 KiB/s wr, 23 op/s 2024-09-21T09:43:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-mon[110963]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:43:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:33 smithi076 ceph-mon[110963]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:43:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:33 smithi191 ceph-mon[33301]: pgmap v75: 89 pgs: 89 active+clean; 215 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 844 KiB/s wr, 23 op/s 2024-09-21T09:43:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:33 smithi191 ceph-mon[33301]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:43:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:33 smithi191 ceph-mon[33301]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:43:34.640 INFO:tasks.rados.rados.0.smithi076.stdout:1369: done (2 left) 2024-09-21T09:43:34.640 INFO:tasks.rados.rados.0.smithi076.stdout:1370: done (1 left) 2024-09-21T09:43:34.640 INFO:tasks.rados.rados.0.smithi076.stdout:1371: done (0 left) 2024-09-21T09:43:34.640 INFO:tasks.rados.rados.0.smithi076.stdout:1372: write oid 10 current snap is 170 2024-09-21T09:43:34.640 INFO:tasks.rados.rados.0.smithi076.stdout:1372: seq_num 535 ranges {591745=402925,1790337=563515} 2024-09-21T09:43:34.645 INFO:tasks.rados.rados.0.smithi076.stdout:1372: writing smithi07685715-10 from 591745 to 994670 tid 1 2024-09-21T09:43:34.653 INFO:tasks.rados.rados.0.smithi076.stdout:1372: writing smithi07685715-10 from 1790337 to 2353852 tid 2 2024-09-21T09:43:34.653 INFO:tasks.rados.rados.0.smithi076.stdout:1373: delete oid 29 current snap is 170 2024-09-21T09:43:34.653 INFO:tasks.rados.rados.0.smithi076.stdout:1372: finishing write tid 1 to smithi07685715-10 2024-09-21T09:43:34.654 INFO:tasks.rados.rados.0.smithi076.stdout:1373: done (1 left) 2024-09-21T09:43:34.654 INFO:tasks.rados.rados.0.smithi076.stdout:1374: read oid 32 snap -1 2024-09-21T09:43:34.655 INFO:tasks.rados.rados.0.smithi076.stdout:1374: expect deleted 2024-09-21T09:43:34.655 INFO:tasks.rados.rados.0.smithi076.stdout:1375: snap_create 2024-09-21T09:43:34.656 INFO:tasks.rados.rados.0.smithi076.stdout:1372: finishing write tid 2 to smithi07685715-10 2024-09-21T09:43:34.656 INFO:tasks.rados.rados.0.smithi076.stdout:1372: finishing write tid 3 to smithi07685715-10 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi076.stdout:1372: finishing write tid 4 to smithi07685715-10 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi076.stdout:1372: oid 10 updating version 0 to 764 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi076.stdout:1372: oid 10 updating version 764 to 766 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi076.stdout:1372: oid 10 version 766 is already newer than 765 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 766 (ObjNum 535 snap 170 seq_num 535) dirty exists 2024-09-21T09:43:34.657 INFO:tasks.rados.rados.0.smithi076.stdout:1372: left oid 10 (ObjNum 535 snap 170 seq_num 535) 2024-09-21T09:43:35.644 INFO:tasks.rados.rados.0.smithi076.stdout:1372: done (2 left) 2024-09-21T09:43:35.644 INFO:tasks.rados.rados.0.smithi076.stdout:1374: done (1 left) 2024-09-21T09:43:35.644 INFO:tasks.rados.rados.0.smithi076.stdout:1375: done (0 left) 2024-09-21T09:43:35.644 INFO:tasks.rados.rados.0.smithi076.stdout:1376: read oid 41 snap -1 2024-09-21T09:43:35.645 INFO:tasks.rados.rados.0.smithi076.stdout:1376: expect (ObjNum 534 snap 169 seq_num 534) 2024-09-21T09:43:35.645 INFO:tasks.rados.rados.0.smithi076.stdout:1377: delete oid 20 current snap is 171 2024-09-21T09:43:35.648 INFO:tasks.rados.rados.0.smithi076.stdout:1377: done (1 left) 2024-09-21T09:43:35.648 INFO:tasks.rados.rados.0.smithi076.stdout:1378: write oid 17 current snap is 171 2024-09-21T09:43:35.648 INFO:tasks.rados.rados.0.smithi076.stdout:1378: seq_num 538 ranges {642166=628672,1849430=710842} 2024-09-21T09:43:35.656 INFO:tasks.rados.rados.0.smithi076.stdout:1378: writing smithi07685715-17 from 642166 to 1270838 tid 1 2024-09-21T09:43:35.664 INFO:tasks.rados.rados.0.smithi076.stdout:1378: writing smithi07685715-17 from 1849430 to 2560272 tid 2 2024-09-21T09:43:35.664 INFO:tasks.rados.rados.0.smithi076.stdout:1379: write oid 8 current snap is 171 2024-09-21T09:43:35.664 INFO:tasks.rados.rados.0.smithi076.stdout:1379: seq_num 539 ranges {658973=703921,1907912=570853,2985346=73627} 2024-09-21T09:43:35.671 INFO:tasks.rados.rados.0.smithi076.stdout:1379: writing smithi07685715-8 from 658973 to 1362894 tid 1 2024-09-21T09:43:35.678 INFO:tasks.rados.rados.0.smithi076.stdout:1379: writing smithi07685715-8 from 1907912 to 2478765 tid 2 2024-09-21T09:43:35.681 INFO:tasks.rados.rados.0.smithi076.stdout:1379: writing smithi07685715-8 from 2985346 to 3058973 tid 3 2024-09-21T09:43:35.682 INFO:tasks.rados.rados.0.smithi076.stdout:1380: rollback oid 40 current snap is 171 2024-09-21T09:43:35.682 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 163 2024-09-21T09:43:35.682 INFO:tasks.rados.rados.0.smithi076.stdout:1376: expect (ObjNum 534 snap 169 seq_num 534) 2024-09-21T09:43:35.776 INFO:tasks.rados.rados.0.smithi076.stdout:1378: finishing write tid 1 to smithi07685715-17 2024-09-21T09:43:35.776 INFO:tasks.rados.rados.0.smithi076.stdout:1376: done (3 left) 2024-09-21T09:43:35.776 INFO:tasks.rados.rados.0.smithi076.stdout:1381: write oid 45 current snap is 171 2024-09-21T09:43:35.776 INFO:tasks.rados.rados.0.smithi076.stdout:1378: finishing write tid 2 to smithi07685715-17 2024-09-21T09:43:35.776 INFO:tasks.rados.rados.0.smithi076.stdout:1378: finishing write tid 3 to smithi07685715-17 2024-09-21T09:43:35.776 INFO:tasks.rados.rados.0.smithi076.stdout:1381: seq_num 540 ranges {675780=495523,1682747=714511,3012769=63011} 2024-09-21T09:43:35.781 INFO:tasks.rados.rados.0.smithi076.stdout:1381: writing smithi07685715-45 from 675780 to 1171303 tid 1 2024-09-21T09:43:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1381: writing smithi07685715-45 from 1682747 to 2397258 tid 2 2024-09-21T09:43:35.792 INFO:tasks.rados.rados.0.smithi076.stdout:1381: writing smithi07685715-45 from 3012769 to 3075780 tid 3 2024-09-21T09:43:35.792 INFO:tasks.rados.rados.0.smithi076.stdout:1382: snap_create 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1378: finishing write tid 4 to smithi07685715-17 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1378: oid 17 updating version 0 to 769 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1378: oid 17 version 769 is already newer than 768 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1378: oid 17 version 769 is already newer than 767 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 769 (ObjNum 538 snap 171 seq_num 538) dirty exists 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1378: left oid 17 (ObjNum 538 snap 171 seq_num 538) 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: finishing write tid 1 to smithi07685715-8 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: finishing write tid 2 to smithi07685715-8 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: finishing write tid 3 to smithi07685715-8 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: finishing write tid 4 to smithi07685715-8 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1380: finishing rollback tid 0 to smithi07685715-40 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: finishing write tid 5 to smithi07685715-8 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: oid 8 updating version 0 to 1121 2024-09-21T09:43:35.793 INFO:tasks.rados.rados.0.smithi076.stdout:1379: oid 8 updating version 1121 to 1123 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1379: oid 8 version 1123 is already newer than 1120 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1379: oid 8 version 1123 is already newer than 1122 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1123 (ObjNum 539 snap 171 seq_num 539) dirty exists 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1379: left oid 8 (ObjNum 539 snap 171 seq_num 539) 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1380: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1380: finishing rollback tid 2 to smithi07685715-40 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 641 (ObjNum 282 snap 88 seq_num 282) dirty exists 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1381: finishing write tid 1 to smithi07685715-45 2024-09-21T09:43:35.794 INFO:tasks.rados.rados.0.smithi076.stdout:1381: finishing write tid 2 to smithi07685715-45 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: finishing write tid 3 to smithi07685715-45 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: finishing write tid 4 to smithi07685715-45 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: finishing write tid 5 to smithi07685715-45 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: oid 45 updating version 0 to 291 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: oid 45 version 291 is already newer than 289 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: oid 45 updating version 291 to 292 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: oid 45 version 292 is already newer than 290 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 292 (ObjNum 540 snap 171 seq_num 540) dirty exists 2024-09-21T09:43:35.795 INFO:tasks.rados.rados.0.smithi076.stdout:1381: left oid 45 (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:35 smithi191 ceph-mon[33301]: pgmap v78: 89 pgs: 89 active+clean; 215 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 844 KiB/s wr, 23 op/s 2024-09-21T09:43:36.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:35 smithi191 ceph-mon[33301]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:43:36.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:35 smithi076 ceph-mon[106482]: pgmap v78: 89 pgs: 89 active+clean; 215 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 844 KiB/s wr, 23 op/s 2024-09-21T09:43:36.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:35 smithi076 ceph-mon[106482]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:43:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:35 smithi076 ceph-mon[110963]: pgmap v78: 89 pgs: 89 active+clean; 215 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 844 KiB/s wr, 23 op/s 2024-09-21T09:43:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:35 smithi076 ceph-mon[110963]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:43:36.648 INFO:tasks.rados.rados.0.smithi076.stdout:1378: done (4 left) 2024-09-21T09:43:36.648 INFO:tasks.rados.rados.0.smithi076.stdout:1379: done (3 left) 2024-09-21T09:43:36.649 INFO:tasks.rados.rados.0.smithi076.stdout:1380: done (2 left) 2024-09-21T09:43:36.649 INFO:tasks.rados.rados.0.smithi076.stdout:1381: done (1 left) 2024-09-21T09:43:36.649 INFO:tasks.rados.rados.0.smithi076.stdout:1382: done (0 left) 2024-09-21T09:43:36.649 INFO:tasks.rados.rados.0.smithi076.stdout:1383: delete oid 36 current snap is 172 2024-09-21T09:43:36.653 INFO:tasks.rados.rados.0.smithi076.stdout:1383: done (0 left) 2024-09-21T09:43:36.653 INFO:tasks.rados.rados.0.smithi076.stdout:1384: read oid 1 snap 168 2024-09-21T09:43:36.653 INFO:tasks.rados.rados.0.smithi076.stdout:1384: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:36.653 INFO:tasks.rados.rados.0.smithi076.stdout:1385: write oid 35 current snap is 172 2024-09-21T09:43:36.653 INFO:tasks.rados.rados.0.smithi076.stdout:1385: seq_num 542 ranges {709394=646021,2083358=718180} 2024-09-21T09:43:36.660 INFO:tasks.rados.rados.0.smithi076.stdout:1385: writing smithi07685715-35 from 709394 to 1355415 tid 1 2024-09-21T09:43:36.668 INFO:tasks.rados.rados.0.smithi076.stdout:1385: writing smithi07685715-35 from 2083358 to 2801538 tid 2 2024-09-21T09:43:36.668 INFO:tasks.rados.rados.0.smithi076.stdout:1386: read oid 45 snap -1 2024-09-21T09:43:36.668 INFO:tasks.rados.rados.0.smithi076.stdout:1386: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:36.668 INFO:tasks.rados.rados.0.smithi076.stdout:1387: write oid 41 current snap is 172 2024-09-21T09:43:36.668 INFO:tasks.rados.rados.0.smithi076.stdout:1387: seq_num 543 ranges {726201=721270,2141840=578191} 2024-09-21T09:43:36.676 INFO:tasks.rados.rados.0.smithi076.stdout:1387: writing smithi07685715-41 from 726201 to 1447471 tid 1 2024-09-21T09:43:36.683 INFO:tasks.rados.rados.0.smithi076.stdout:1387: writing smithi07685715-41 from 2141840 to 2720031 tid 2 2024-09-21T09:43:36.683 INFO:tasks.rados.rados.0.smithi076.stdout:1388: delete oid 24 current snap is 172 2024-09-21T09:43:36.683 INFO:tasks.rados.rados.0.smithi076.stdout:1384: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:43:36.802 INFO:tasks.rados.rados.0.smithi076.stdout:1385: finishing write tid 1 to smithi07685715-35 2024-09-21T09:43:36.802 INFO:tasks.rados.rados.0.smithi076.stdout:1385: finishing write tid 2 to smithi07685715-35 2024-09-21T09:43:36.802 INFO:tasks.rados.rados.0.smithi076.stdout:1385: finishing write tid 3 to smithi07685715-35 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:1385: finishing write tid 4 to smithi07685715-35 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:1385: oid 35 updating version 0 to 1069 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:1385: oid 35 updating version 1069 to 1071 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:1385: oid 35 version 1071 is already newer than 1070 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1071 (ObjNum 542 snap 172 seq_num 542) dirty exists 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:1385: left oid 35 (ObjNum 542 snap 172 seq_num 542) 2024-09-21T09:43:36.803 INFO:tasks.rados.rados.0.smithi076.stdout:1386: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:36 smithi076 ceph-mon[106482]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:43:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:36 smithi076 ceph-mon[110963]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:43:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:1387: finishing write tid 1 to smithi07685715-41 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: finishing write tid 2 to smithi07685715-41 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: finishing write tid 3 to smithi07685715-41 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: finishing write tid 4 to smithi07685715-41 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: oid 41 updating version 0 to 644 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: oid 41 version 644 is already newer than 643 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: oid 41 updating version 644 to 645 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 645 (ObjNum 543 snap 172 seq_num 543) dirty exists 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: left oid 41 (ObjNum 543 snap 172 seq_num 543) 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1384: done (4 left) 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1385: done (3 left) 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1386: done (2 left) 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1387: done (1 left) 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1388: done (0 left) 2024-09-21T09:43:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:1389: snap_remove snap 168 2024-09-21T09:43:37.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:36 smithi191 ceph-mon[33301]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:43:37.653 INFO:tasks.rados.rados.0.smithi076.stdout:1389: done (0 left) 2024-09-21T09:43:37.653 INFO:tasks.rados.rados.0.smithi076.stdout:1390: snap_create 2024-09-21T09:43:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:37 smithi191 ceph-mon[33301]: pgmap v81: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-21T09:43:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:37 smithi191 ceph-mon[33301]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:43:38.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:37 smithi076 ceph-mon[110963]: pgmap v81: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-21T09:43:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:37 smithi076 ceph-mon[110963]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:43:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:37 smithi076 ceph-mon[106482]: pgmap v81: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-21T09:43:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:37 smithi076 ceph-mon[106482]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:43:38.640 INFO:tasks.rados.rados.0.smithi076.stdout:1390: done (0 left) 2024-09-21T09:43:38.641 INFO:tasks.rados.rados.0.smithi076.stdout:1391: delete oid 44 current snap is 173 2024-09-21T09:43:38.644 INFO:tasks.rados.rados.0.smithi076.stdout:1391: done (0 left) 2024-09-21T09:43:38.644 INFO:tasks.rados.rados.0.smithi076.stdout:1392: write oid 12 current snap is 173 2024-09-21T09:43:38.644 INFO:tasks.rados.rados.0.smithi076.stdout:1392: seq_num 546 ranges {776622=547017,1800933=725518,2944601=232021} 2024-09-21T09:43:38.650 INFO:tasks.rados.rados.0.smithi076.stdout:1392: writing smithi07685715-12 from 776622 to 1323639 tid 1 2024-09-21T09:43:38.657 INFO:tasks.rados.rados.0.smithi076.stdout:1392: writing smithi07685715-12 from 1800933 to 2526451 tid 2 2024-09-21T09:43:38.662 INFO:tasks.rados.rados.0.smithi076.stdout:1392: writing smithi07685715-12 from 2944601 to 3176622 tid 3 2024-09-21T09:43:38.662 INFO:tasks.rados.rados.0.smithi076.stdout:1393: read oid 40 snap -1 2024-09-21T09:43:38.663 INFO:tasks.rados.rados.0.smithi076.stdout:1393: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:43:38.663 INFO:tasks.rados.rados.0.smithi076.stdout:1394: rollback oid 2 current snap is 173 2024-09-21T09:43:38.663 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 161 2024-09-21T09:43:38.663 INFO:tasks.rados.rados.0.smithi076.stdout:1392: finishing write tid 1 to smithi07685715-12 2024-09-21T09:43:38.663 INFO:tasks.rados.rados.0.smithi076.stdout:1395: rollback oid 42 current snap is 173 2024-09-21T09:43:38.663 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 169 2024-09-21T09:43:38.664 INFO:tasks.rados.rados.0.smithi076.stdout:1396: read oid 45 snap -1 2024-09-21T09:43:38.664 INFO:tasks.rados.rados.0.smithi076.stdout:1396: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:38.664 INFO:tasks.rados.rados.0.smithi076.stdout:1397: write oid 23 current snap is 173 2024-09-21T09:43:38.664 INFO:tasks.rados.rados.0.smithi076.stdout:1397: seq_num 547 ranges {793429=622266,1859415=469176,2972024=221405} 2024-09-21T09:43:38.671 INFO:tasks.rados.rados.0.smithi076.stdout:1397: writing smithi07685715-23 from 793429 to 1415695 tid 1 2024-09-21T09:43:38.676 INFO:tasks.rados.rados.0.smithi076.stdout:1397: writing smithi07685715-23 from 1859415 to 2328591 tid 2 2024-09-21T09:43:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:1397: writing smithi07685715-23 from 2972024 to 3193429 tid 3 2024-09-21T09:43:38.681 INFO:tasks.rados.rados.0.smithi076.stdout:1398: snap_remove snap 171 2024-09-21T09:43:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:38 smithi191 ceph-mon[33301]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:43:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:38 smithi191 ceph-mon[33301]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:43:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:38 smithi076 ceph-mon[106482]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:43:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:38 smithi076 ceph-mon[106482]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:43:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:38 smithi076 ceph-mon[110963]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:43:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:38 smithi076 ceph-mon[110963]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:43:39.674 INFO:tasks.rados.rados.0.smithi076.stdout:1398: done (6 left) 2024-09-21T09:43:39.674 INFO:tasks.rados.rados.0.smithi076.stdout:1399: delete oid 19 current snap is 173 2024-09-21T09:43:39.675 INFO:tasks.rados.rados.0.smithi076.stdout:1392: finishing write tid 2 to smithi07685715-12 2024-09-21T09:43:39.675 INFO:tasks.rados.rados.0.smithi076.stdout:1396: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1395: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1394: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1395: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1395: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 452 (ObjNum 515 snap 163 seq_num 515) dirty exists 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1392: finishing write tid 3 to smithi07685715-12 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1395: done (6 left) 2024-09-21T09:43:39.812 INFO:tasks.rados.rados.0.smithi076.stdout:1396: done (5 left) 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1399: done (4 left) 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1400: read oid 43 snap -1 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1400: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1401: read oid 5 snap -1 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1401: expect deleted 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1402: write oid 26 current snap is 173 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1392: finishing write tid 4 to smithi07685715-12 2024-09-21T09:43:39.813 INFO:tasks.rados.rados.0.smithi076.stdout:1394: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1392: finishing write tid 5 to smithi07685715-12 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1392: oid 12 updating version 0 to 527 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1392: oid 12 version 527 is already newer than 526 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1392: oid 12 version 527 is already newer than 524 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1392: oid 12 version 527 is already newer than 525 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 527 (ObjNum 546 snap 173 seq_num 546) dirty exists 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1392: left oid 12 (ObjNum 546 snap 173 seq_num 546) 2024-09-21T09:43:39.814 INFO:tasks.rados.rados.0.smithi076.stdout:1394: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:43:39.815 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 233 (ObjNum 481 snap 155 seq_num 481) dirty exists 2024-09-21T09:43:39.815 INFO:tasks.rados.rados.0.smithi076.stdout:1393: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:43:39.932 INFO:tasks.rados.rados.0.smithi076.stdout:1397: finishing write tid 1 to smithi07685715-23 2024-09-21T09:43:39.932 INFO:tasks.rados.rados.0.smithi076.stdout:1397: finishing write tid 2 to smithi07685715-23 2024-09-21T09:43:39.932 INFO:tasks.rados.rados.0.smithi076.stdout:1397: finishing write tid 3 to smithi07685715-23 2024-09-21T09:43:39.932 INFO:tasks.rados.rados.0.smithi076.stdout:1397: finishing write tid 4 to smithi07685715-23 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1397: finishing write tid 5 to smithi07685715-23 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1397: oid 23 updating version 0 to 1130 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1397: oid 23 version 1130 is already newer than 1128 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1397: oid 23 version 1130 is already newer than 1129 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1397: oid 23 version 1130 is already newer than 1127 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1130 (ObjNum 547 snap 173 seq_num 547) dirty exists 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1397: left oid 23 (ObjNum 547 snap 173 seq_num 547) 2024-09-21T09:43:39.933 INFO:tasks.rados.rados.0.smithi076.stdout:1402: seq_num 549 ranges {427043=489117,1576379=472845,2626870=600173} 2024-09-21T09:43:39.938 INFO:tasks.rados.rados.0.smithi076.stdout:1402: writing smithi07685715-26 from 427043 to 916160 tid 1 2024-09-21T09:43:39.943 INFO:tasks.rados.rados.0.smithi076.stdout:1402: writing smithi07685715-26 from 1576379 to 2049224 tid 2 2024-09-21T09:43:39.950 INFO:tasks.rados.rados.0.smithi076.stdout:1402: writing smithi07685715-26 from 2626870 to 3227043 tid 3 2024-09-21T09:43:39.950 INFO:tasks.rados.rados.0.smithi076.stdout:1392: done (6 left) 2024-09-21T09:43:39.950 INFO:tasks.rados.rados.0.smithi076.stdout:1393: done (5 left) 2024-09-21T09:43:39.950 INFO:tasks.rados.rados.0.smithi076.stdout:1394: done (4 left) 2024-09-21T09:43:39.950 INFO:tasks.rados.rados.0.smithi076.stdout:1397: done (3 left) 2024-09-21T09:43:39.950 INFO:tasks.rados.rados.0.smithi076.stdout:1403: snap_remove snap 166 2024-09-21T09:43:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:39 smithi191 ceph-mon[33301]: pgmap v84: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-21T09:43:40.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:39 smithi076 ceph-mon[106482]: pgmap v84: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-21T09:43:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:39 smithi076 ceph-mon[110963]: pgmap v84: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-21T09:43:40.677 INFO:tasks.rados.rados.0.smithi076.stdout:1403: done (3 left) 2024-09-21T09:43:40.677 INFO:tasks.rados.rados.0.smithi076.stdout:1404: snap_remove snap 172 2024-09-21T09:43:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:40 smithi191 ceph-mon[33301]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:43:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:40 smithi076 ceph-mon[106482]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:43:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:40 smithi076 ceph-mon[110963]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:43:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:41.679 INFO:tasks.rados.rados.0.smithi076.stdout:1404: done (3 left) 2024-09-21T09:43:41.679 INFO:tasks.rados.rados.0.smithi076.stdout:1405: write oid 35 current snap is 173 2024-09-21T09:43:41.680 INFO:tasks.rados.rados.0.smithi076.stdout:1405: seq_num 550 ranges {443850=564366,1634861=732856,2770646=473204} 2024-09-21T09:43:41.685 INFO:tasks.rados.rados.0.smithi076.stdout:1405: writing smithi07685715-35 from 443850 to 1008216 tid 1 2024-09-21T09:43:41.693 INFO:tasks.rados.rados.0.smithi076.stdout:1405: writing smithi07685715-35 from 1634861 to 2367717 tid 2 2024-09-21T09:43:41.698 INFO:tasks.rados.rados.0.smithi076.stdout:1405: writing smithi07685715-35 from 2770646 to 3243850 tid 3 2024-09-21T09:43:41.698 INFO:tasks.rados.rados.0.smithi076.stdout:1406: snap_create 2024-09-21T09:43:41.698 INFO:tasks.rados.rados.0.smithi076.stdout:1400: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: finishing write tid 1 to smithi07685715-26 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: finishing write tid 2 to smithi07685715-26 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: finishing write tid 3 to smithi07685715-26 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: finishing write tid 4 to smithi07685715-26 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: finishing write tid 5 to smithi07685715-26 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: oid 26 updating version 0 to 531 2024-09-21T09:43:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:1402: oid 26 version 531 is already newer than 529 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1402: oid 26 updating version 531 to 532 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1402: oid 26 version 532 is already newer than 530 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 532 (ObjNum 549 snap 173 seq_num 549) dirty exists 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1402: left oid 26 (ObjNum 549 snap 173 seq_num 549) 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: finishing write tid 1 to smithi07685715-35 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: finishing write tid 2 to smithi07685715-35 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: finishing write tid 3 to smithi07685715-35 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: finishing write tid 4 to smithi07685715-35 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: finishing write tid 5 to smithi07685715-35 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: oid 35 updating version 0 to 1083 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: oid 35 version 1083 is already newer than 1082 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: oid 35 updating version 1083 to 1084 2024-09-21T09:43:41.798 INFO:tasks.rados.rados.0.smithi076.stdout:1405: oid 35 updating version 1084 to 1085 2024-09-21T09:43:41.799 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1085 (ObjNum 550 snap 173 seq_num 550) dirty exists 2024-09-21T09:43:41.799 INFO:tasks.rados.rados.0.smithi076.stdout:1405: left oid 35 (ObjNum 550 snap 173 seq_num 550) 2024-09-21T09:43:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:41 smithi191 ceph-mon[33301]: pgmap v87: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:43:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:41 smithi191 ceph-mon[33301]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:43:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:41 smithi076 ceph-mon[106482]: pgmap v87: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:43:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:41 smithi076 ceph-mon[106482]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:43:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:41 smithi076 ceph-mon[110963]: pgmap v87: 89 pgs: 89 active+clean; 233 MiB data, 744 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:43:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:41 smithi076 ceph-mon[110963]: osdmap e414: 8 total, 8 up, 8 in 2024-09-21T09:43:42.693 INFO:tasks.rados.rados.0.smithi076.stdout:1400: done (4 left) 2024-09-21T09:43:42.693 INFO:tasks.rados.rados.0.smithi076.stdout:1401: done (3 left) 2024-09-21T09:43:42.693 INFO:tasks.rados.rados.0.smithi076.stdout:1402: done (2 left) 2024-09-21T09:43:42.693 INFO:tasks.rados.rados.0.smithi076.stdout:1405: done (1 left) 2024-09-21T09:43:42.693 INFO:tasks.rados.rados.0.smithi076.stdout:1406: done (0 left) 2024-09-21T09:43:42.694 INFO:tasks.rados.rados.0.smithi076.stdout:1407: write oid 24 current snap is 174 2024-09-21T09:43:42.694 INFO:tasks.rados.rados.0.smithi076.stdout:1407: seq_num 551 ranges {460657=639615,1693343=476514,2681716=578941} 2024-09-21T09:43:42.699 INFO:tasks.rados.rados.0.smithi076.stdout:1407: writing smithi07685715-24 from 460657 to 1100272 tid 1 2024-09-21T09:43:42.705 INFO:tasks.rados.rados.0.smithi076.stdout:1407: writing smithi07685715-24 from 1693343 to 2169857 tid 2 2024-09-21T09:43:42.711 INFO:tasks.rados.rados.0.smithi076.stdout:1407: writing smithi07685715-24 from 2681716 to 3260657 tid 3 2024-09-21T09:43:42.711 INFO:tasks.rados.rados.0.smithi076.stdout:1408: read oid 49 snap -1 2024-09-21T09:43:42.711 INFO:tasks.rados.rados.0.smithi076.stdout:1408: expect (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:43:42.711 INFO:tasks.rados.rados.0.smithi076.stdout:1409: write oid 9 current snap is 174 2024-09-21T09:43:42.712 INFO:tasks.rados.rados.0.smithi076.stdout:1409: seq_num 552 ranges {477464=714864,1751825=736525,3225492=51972} 2024-09-21T09:43:42.718 INFO:tasks.rados.rados.0.smithi076.stdout:1409: writing smithi07685715-9 from 477464 to 1192328 tid 1 2024-09-21T09:43:42.725 INFO:tasks.rados.rados.0.smithi076.stdout:1409: writing smithi07685715-9 from 1751825 to 2488350 tid 2 2024-09-21T09:43:42.730 INFO:tasks.rados.rados.0.smithi076.stdout:1409: writing smithi07685715-9 from 3225492 to 3277464 tid 3 2024-09-21T09:43:42.730 INFO:tasks.rados.rados.0.smithi076.stdout:1410: read oid 12 snap 163 2024-09-21T09:43:42.730 INFO:tasks.rados.rados.0.smithi076.stdout:1410: expect deleted 2024-09-21T09:43:42.730 INFO:tasks.rados.rados.0.smithi076.stdout:1411: delete oid 10 current snap is 174 2024-09-21T09:43:42.730 INFO:tasks.rados.rados.0.smithi076.stdout:1407: finishing write tid 1 to smithi07685715-24 2024-09-21T09:43:42.730 INFO:tasks.rados.rados.0.smithi076.stdout:1407: finishing write tid 2 to smithi07685715-24 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: finishing write tid 3 to smithi07685715-24 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: finishing write tid 4 to smithi07685715-24 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: finishing write tid 5 to smithi07685715-24 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: oid 24 updating version 0 to 151 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: oid 24 updating version 151 to 152 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: oid 24 updating version 152 to 153 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: oid 24 version 153 is already newer than 150 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 153 (ObjNum 551 snap 174 seq_num 551) dirty exists 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1407: left oid 24 (ObjNum 551 snap 174 seq_num 551) 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1409: finishing write tid 1 to smithi07685715-9 2024-09-21T09:43:42.731 INFO:tasks.rados.rados.0.smithi076.stdout:1408: expect (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:43:42.930 INFO:tasks.rados.rados.0.smithi076.stdout:1409: finishing write tid 2 to smithi07685715-9 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: finishing write tid 3 to smithi07685715-9 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: finishing write tid 4 to smithi07685715-9 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: finishing write tid 5 to smithi07685715-9 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: oid 9 updating version 0 to 537 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: oid 9 updating version 537 to 540 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: oid 9 version 540 is already newer than 538 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: oid 9 version 540 is already newer than 539 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 540 (ObjNum 552 snap 174 seq_num 552) dirty exists 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: left oid 9 (ObjNum 552 snap 174 seq_num 552) 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1407: done (4 left) 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1408: done (3 left) 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1409: done (2 left) 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1410: done (1 left) 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1411: done (0 left) 2024-09-21T09:43:42.931 INFO:tasks.rados.rados.0.smithi076.stdout:1412: snap_remove snap 163 2024-09-21T09:43:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:42 smithi191 ceph-mon[33301]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:43:43.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:42 smithi076 ceph-mon[106482]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:43:43.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:42 smithi076 ceph-mon[110963]: osdmap e415: 8 total, 8 up, 8 in 2024-09-21T09:43:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:43:43] "GET /metrics HTTP/1.1" 200 36132 "" "Prometheus/2.51.0" 2024-09-21T09:43:43.642 INFO:tasks.rados.rados.0.smithi076.stdout:1412: done (0 left) 2024-09-21T09:43:43.643 INFO:tasks.rados.rados.0.smithi076.stdout:1413: write oid 21 current snap is 174 2024-09-21T09:43:43.643 INFO:tasks.rados.rados.0.smithi076.stdout:1413: seq_num 554 ranges {511078=465362,1752436=740194,3163985=147093} 2024-09-21T09:43:43.648 INFO:tasks.rados.rados.0.smithi076.stdout:1413: writing smithi07685715-21 from 511078 to 976440 tid 1 2024-09-21T09:43:43.657 INFO:tasks.rados.rados.0.smithi076.stdout:1413: writing smithi07685715-21 from 1752436 to 2492630 tid 2 2024-09-21T09:43:43.662 INFO:tasks.rados.rados.0.smithi076.stdout:1413: writing smithi07685715-21 from 3163985 to 3311078 tid 3 2024-09-21T09:43:43.662 INFO:tasks.rados.rados.0.smithi076.stdout:1414: read oid 25 snap -1 2024-09-21T09:43:43.662 INFO:tasks.rados.rados.0.smithi076.stdout:1414: expect (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:43:43.662 INFO:tasks.rados.rados.0.smithi076.stdout:1415: delete oid 19 current snap is 174 2024-09-21T09:43:43.662 INFO:tasks.rados.rados.0.smithi076.stdout:1413: finishing write tid 1 to smithi07685715-21 2024-09-21T09:43:43.662 INFO:tasks.rados.rados.0.smithi076.stdout:1413: finishing write tid 2 to smithi07685715-21 2024-09-21T09:43:43.663 INFO:tasks.rados.rados.0.smithi076.stdout:1415: done (2 left) 2024-09-21T09:43:43.663 INFO:tasks.rados.rados.0.smithi076.stdout:1416: delete oid 46 current snap is 174 2024-09-21T09:43:43.665 INFO:tasks.rados.rados.0.smithi076.stdout:1413: finishing write tid 3 to smithi07685715-21 2024-09-21T09:43:43.666 INFO:tasks.rados.rados.0.smithi076.stdout:1414: expect (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:43:43.807 INFO:tasks.rados.rados.0.smithi076.stdout:1413: finishing write tid 4 to smithi07685715-21 2024-09-21T09:43:43.807 INFO:tasks.rados.rados.0.smithi076.stdout:1413: finishing write tid 5 to smithi07685715-21 2024-09-21T09:43:43.807 INFO:tasks.rados.rados.0.smithi076.stdout:1413: oid 21 updating version 0 to 1095 2024-09-21T09:43:43.807 INFO:tasks.rados.rados.0.smithi076.stdout:1413: oid 21 version 1095 is already newer than 1093 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1413: oid 21 version 1095 is already newer than 1094 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1413: oid 21 updating version 1095 to 1096 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1096 (ObjNum 554 snap 174 seq_num 554) dirty exists 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1413: left oid 21 (ObjNum 554 snap 174 seq_num 554) 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1413: done (2 left) 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1414: done (1 left) 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1416: done (0 left) 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1417: write oid 34 current snap is 174 2024-09-21T09:43:43.808 INFO:tasks.rados.rados.0.smithi076.stdout:1417: seq_num 557 ranges {561499=407462,1527882=487521,2729901=557217} 2024-09-21T09:43:43.813 INFO:tasks.rados.rados.0.smithi076.stdout:1417: writing smithi07685715-34 from 561499 to 968961 tid 1 2024-09-21T09:43:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:1417: writing smithi07685715-34 from 1527882 to 2015403 tid 2 2024-09-21T09:43:43.825 INFO:tasks.rados.rados.0.smithi076.stdout:1417: writing smithi07685715-34 from 2729901 to 3287118 tid 3 2024-09-21T09:43:43.825 INFO:tasks.rados.rados.0.smithi076.stdout:1418: write oid 8 current snap is 174 2024-09-21T09:43:43.825 INFO:tasks.rados.rados.0.smithi076.stdout:1418: seq_num 558 ranges {578306=482711,1586364=631179,2757324=484842} 2024-09-21T09:43:43.831 INFO:tasks.rados.rados.0.smithi076.stdout:1418: writing smithi07685715-8 from 578306 to 1061017 tid 1 2024-09-21T09:43:43.837 INFO:tasks.rados.rados.0.smithi076.stdout:1418: writing smithi07685715-8 from 1586364 to 2217543 tid 2 2024-09-21T09:43:43.843 INFO:tasks.rados.rados.0.smithi076.stdout:1418: writing smithi07685715-8 from 2757324 to 3242166 tid 3 2024-09-21T09:43:43.843 INFO:tasks.rados.rados.0.smithi076.stdout:1419: read oid 28 snap -1 2024-09-21T09:43:43.843 INFO:tasks.rados.rados.0.smithi076.stdout:1419: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:43:43.843 INFO:tasks.rados.rados.0.smithi076.stdout:1420: snap_remove snap 165 2024-09-21T09:43:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:43 smithi191 ceph-mon[33301]: pgmap v90: 89 pgs: 89 active+clean; 241 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:43:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:43 smithi191 ceph-mon[33301]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:43:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:43 smithi191 ceph-mon[33301]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:43:44.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-mon[106482]: pgmap v90: 89 pgs: 89 active+clean; 241 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:43:44.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-mon[106482]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:43:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-mon[106482]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:43:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-mon[110963]: pgmap v90: 89 pgs: 89 active+clean; 241 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:43:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-mon[110963]: osdmap e416: 8 total, 8 up, 8 in 2024-09-21T09:43:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:43 smithi076 ceph-mon[110963]: osdmap e417: 8 total, 8 up, 8 in 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: finishing write tid 1 to smithi07685715-34 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: finishing write tid 2 to smithi07685715-34 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: finishing write tid 3 to smithi07685715-34 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: finishing write tid 4 to smithi07685715-34 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: finishing write tid 5 to smithi07685715-34 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: oid 34 updating version 0 to 268 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: oid 34 updating version 268 to 270 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: oid 34 version 270 is already newer than 269 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: oid 34 version 270 is already newer than 267 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 270 (ObjNum 557 snap 174 seq_num 557) dirty exists 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1417: left oid 34 (ObjNum 557 snap 174 seq_num 557) 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1418: finishing write tid 1 to smithi07685715-8 2024-09-21T09:43:44.698 INFO:tasks.rados.rados.0.smithi076.stdout:1418: finishing write tid 2 to smithi07685715-8 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: finishing write tid 3 to smithi07685715-8 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: finishing write tid 4 to smithi07685715-8 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: finishing write tid 5 to smithi07685715-8 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: oid 8 updating version 0 to 1146 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: oid 8 updating version 1146 to 1147 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: oid 8 updating version 1147 to 1149 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: oid 8 version 1149 is already newer than 1148 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1149 (ObjNum 558 snap 174 seq_num 558) dirty exists 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1418: left oid 8 (ObjNum 558 snap 174 seq_num 558) 2024-09-21T09:43:44.699 INFO:tasks.rados.rados.0.smithi076.stdout:1419: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:43:44.796 INFO:tasks.rados.rados.0.smithi076.stdout:1417: done (3 left) 2024-09-21T09:43:44.796 INFO:tasks.rados.rados.0.smithi076.stdout:1418: done (2 left) 2024-09-21T09:43:44.796 INFO:tasks.rados.rados.0.smithi076.stdout:1419: done (1 left) 2024-09-21T09:43:44.796 INFO:tasks.rados.rados.0.smithi076.stdout:1420: done (0 left) 2024-09-21T09:43:44.796 INFO:tasks.rados.rados.0.smithi076.stdout:1421: read oid 38 snap -1 2024-09-21T09:43:44.797 INFO:tasks.rados.rados.0.smithi076.stdout:1421: expect (ObjNum 505 snap 160 seq_num 505) 2024-09-21T09:43:44.797 INFO:tasks.rados.rados.0.smithi076.stdout:1422: snap_remove snap 167 2024-09-21T09:43:45.711 INFO:tasks.rados.rados.0.smithi076.stdout:1422: done (1 left) 2024-09-21T09:43:45.711 INFO:tasks.rados.rados.0.smithi076.stdout:1423: snap_create 2024-09-21T09:43:45.711 INFO:tasks.rados.rados.0.smithi076.stdout:1421: expect (ObjNum 505 snap 160 seq_num 505) 2024-09-21T09:43:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:45 smithi191 ceph-mon[33301]: pgmap v93: 89 pgs: 89 active+clean; 241 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:43:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:45 smithi191 ceph-mon[33301]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:43:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:45 smithi076 ceph-mon[106482]: pgmap v93: 89 pgs: 89 active+clean; 241 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:43:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:45 smithi076 ceph-mon[106482]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:43:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:45 smithi076 ceph-mon[110963]: pgmap v93: 89 pgs: 89 active+clean; 241 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:43:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:45 smithi076 ceph-mon[110963]: osdmap e418: 8 total, 8 up, 8 in 2024-09-21T09:43:46.716 INFO:tasks.rados.rados.0.smithi076.stdout:1421: done (1 left) 2024-09-21T09:43:46.717 INFO:tasks.rados.rados.0.smithi076.stdout:1423: done (0 left) 2024-09-21T09:43:46.717 INFO:tasks.rados.rados.0.smithi076.stdout:1424: delete oid 36 current snap is 175 2024-09-21T09:43:46.720 INFO:tasks.rados.rados.0.smithi076.stdout:1424: done (0 left) 2024-09-21T09:43:46.720 INFO:tasks.rados.rados.0.smithi076.stdout:1425: read oid 29 snap 156 2024-09-21T09:43:46.720 INFO:tasks.rados.rados.0.smithi076.stdout:1425: expect (ObjNum 469 snap 154 seq_num 469) 2024-09-21T09:43:46.720 INFO:tasks.rados.rados.0.smithi076.stdout:1426: write oid 27 current snap is 175 2024-09-21T09:43:46.720 INFO:tasks.rados.rados.0.smithi076.stdout:1426: seq_num 560 ranges {611920=633209,1703328=634848,2812170=507386} 2024-09-21T09:43:46.727 INFO:tasks.rados.rados.0.smithi076.stdout:1426: writing smithi07685715-27 from 611920 to 1245129 tid 1 2024-09-21T09:43:46.735 INFO:tasks.rados.rados.0.smithi076.stdout:1426: writing smithi07685715-27 from 1703328 to 2338176 tid 2 2024-09-21T09:43:46.741 INFO:tasks.rados.rados.0.smithi076.stdout:1426: writing smithi07685715-27 from 2812170 to 3319556 tid 3 2024-09-21T09:43:46.741 INFO:tasks.rados.rados.0.smithi076.stdout:1427: read oid 43 snap -1 2024-09-21T09:43:46.741 INFO:tasks.rados.rados.0.smithi076.stdout:1427: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:43:46.741 INFO:tasks.rados.rados.0.smithi076.stdout:1428: delete oid 44 current snap is 175 2024-09-21T09:43:46.741 INFO:tasks.rados.rados.0.smithi076.stdout:1425: expect (ObjNum 469 snap 154 seq_num 469) 2024-09-21T09:43:46.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:46 smithi076 ceph-mon[106482]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:43:46.872 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:46 smithi076 ceph-mon[110963]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: finishing write tid 1 to smithi07685715-27 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: finishing write tid 2 to smithi07685715-27 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: finishing write tid 3 to smithi07685715-27 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: finishing write tid 4 to smithi07685715-27 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: finishing write tid 5 to smithi07685715-27 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: oid 27 updating version 0 to 659 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: oid 27 version 659 is already newer than 658 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: oid 27 updating version 659 to 660 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: oid 27 updating version 660 to 661 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 661 (ObjNum 560 snap 175 seq_num 560) dirty exists 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1426: left oid 27 (ObjNum 560 snap 175 seq_num 560) 2024-09-21T09:43:46.965 INFO:tasks.rados.rados.0.smithi076.stdout:1427: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:43:47.062 INFO:tasks.rados.rados.0.smithi076.stdout:1425: done (3 left) 2024-09-21T09:43:47.063 INFO:tasks.rados.rados.0.smithi076.stdout:1426: done (2 left) 2024-09-21T09:43:47.063 INFO:tasks.rados.rados.0.smithi076.stdout:1427: done (1 left) 2024-09-21T09:43:47.063 INFO:tasks.rados.rados.0.smithi076.stdout:1428: done (0 left) 2024-09-21T09:43:47.063 INFO:tasks.rados.rados.0.smithi076.stdout:1429: write oid 9 current snap is 175 2024-09-21T09:43:47.063 INFO:tasks.rados.rados.0.smithi076.stdout:1429: seq_num 562 ranges {645534=783707,2103939=638517,3150663=294871} 2024-09-21T09:43:47.070 INFO:tasks.rados.rados.0.smithi076.stdout:1429: writing smithi07685715-9 from 645534 to 1429241 tid 1 2024-09-21T09:43:47.077 INFO:tasks.rados.rados.0.smithi076.stdout:1429: writing smithi07685715-9 from 2103939 to 2742456 tid 2 2024-09-21T09:43:47.081 INFO:tasks.rados.rados.0.smithi076.stdout:1429: writing smithi07685715-9 from 3150663 to 3445534 tid 3 2024-09-21T09:43:47.082 INFO:tasks.rados.rados.0.smithi076.stdout:1430: snap_remove snap 161 2024-09-21T09:43:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:46 smithi191 ceph-mon[33301]: osdmap e419: 8 total, 8 up, 8 in 2024-09-21T09:43:47.718 INFO:tasks.rados.rados.0.smithi076.stdout:1429: finishing write tid 1 to smithi07685715-9 2024-09-21T09:43:47.718 INFO:tasks.rados.rados.0.smithi076.stdout:1429: finishing write tid 2 to smithi07685715-9 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: finishing write tid 3 to smithi07685715-9 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: finishing write tid 4 to smithi07685715-9 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: finishing write tid 5 to smithi07685715-9 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: oid 9 updating version 0 to 550 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: oid 9 version 550 is already newer than 548 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: oid 9 updating version 550 to 551 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: oid 9 version 551 is already newer than 549 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 551 (ObjNum 562 snap 175 seq_num 562) dirty exists 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: left oid 9 (ObjNum 562 snap 175 seq_num 562) 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1429: done (1 left) 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1430: done (0 left) 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1431: read oid 4 snap -1 2024-09-21T09:43:47.719 INFO:tasks.rados.rados.0.smithi076.stdout:1431: expect (ObjNum 502 snap 159 seq_num 502) 2024-09-21T09:43:47.720 INFO:tasks.rados.rados.0.smithi076.stdout:1432: write oid 21 current snap is 175 2024-09-21T09:43:47.720 INFO:tasks.rados.rados.0.smithi076.stdout:1432: seq_num 563 ranges {662341=575309,1878774=498528,2894439=457555} 2024-09-21T09:43:47.726 INFO:tasks.rados.rados.0.smithi076.stdout:1432: writing smithi07685715-21 from 662341 to 1237650 tid 1 2024-09-21T09:43:47.732 INFO:tasks.rados.rados.0.smithi076.stdout:1432: writing smithi07685715-21 from 1878774 to 2377302 tid 2 2024-09-21T09:43:47.738 INFO:tasks.rados.rados.0.smithi076.stdout:1432: writing smithi07685715-21 from 2894439 to 3351994 tid 3 2024-09-21T09:43:47.738 INFO:tasks.rados.rados.0.smithi076.stdout:1433: write oid 9 current snap is 175 2024-09-21T09:43:47.738 INFO:tasks.rados.rados.0.smithi076.stdout:1433: seq_num 564 ranges {679148=650558,1937256=642186,3321862=157286} 2024-09-21T09:43:47.745 INFO:tasks.rados.rados.0.smithi076.stdout:1433: writing smithi07685715-9 from 679148 to 1329706 tid 1 2024-09-21T09:43:47.752 INFO:tasks.rados.rados.0.smithi076.stdout:1433: writing smithi07685715-9 from 1937256 to 2579442 tid 2 2024-09-21T09:43:47.757 INFO:tasks.rados.rados.0.smithi076.stdout:1433: writing smithi07685715-9 from 3321862 to 3479148 tid 3 2024-09-21T09:43:47.757 INFO:tasks.rados.rados.0.smithi076.stdout:1434: read oid 27 snap -1 2024-09-21T09:43:47.757 INFO:tasks.rados.rados.0.smithi076.stdout:1434: expect (ObjNum 560 snap 175 seq_num 560) 2024-09-21T09:43:47.757 INFO:tasks.rados.rados.0.smithi076.stdout:1435: write oid 5 current snap is 175 2024-09-21T09:43:47.757 INFO:tasks.rados.rados.0.smithi076.stdout:1431: expect (ObjNum 502 snap 159 seq_num 502) 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: finishing write tid 1 to smithi07685715-21 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: finishing write tid 2 to smithi07685715-21 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: finishing write tid 3 to smithi07685715-21 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: finishing write tid 4 to smithi07685715-21 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: finishing write tid 5 to smithi07685715-21 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: oid 21 updating version 0 to 1125 2024-09-21T09:43:47.901 INFO:tasks.rados.rados.0.smithi076.stdout:1432: oid 21 updating version 1125 to 1128 2024-09-21T09:43:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:1432: oid 21 updating version 1128 to 1129 2024-09-21T09:43:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:1432: oid 21 version 1129 is already newer than 1120 2024-09-21T09:43:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1129 (ObjNum 563 snap 175 seq_num 563) dirty exists 2024-09-21T09:43:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:1432: left oid 21 (ObjNum 563 snap 175 seq_num 563) 2024-09-21T09:43:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:1433: finishing write tid 1 to smithi07685715-9 2024-09-21T09:43:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:1435: seq_num 565 ranges {695955=725807,1879385=502197,2949285=480099} 2024-09-21T09:43:47.908 INFO:tasks.rados.rados.0.smithi076.stdout:1435: writing smithi07685715-5 from 695955 to 1421762 tid 1 2024-09-21T09:43:47.914 INFO:tasks.rados.rados.0.smithi076.stdout:1435: writing smithi07685715-5 from 1879385 to 2381582 tid 2 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1435: writing smithi07685715-5 from 2949285 to 3429384 tid 3 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1431: done (4 left) 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1432: done (3 left) 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1436: rollback oid 10 current snap is 175 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 173 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1433: finishing write tid 2 to smithi07685715-9 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1433: finishing write tid 3 to smithi07685715-9 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1433: finishing write tid 4 to smithi07685715-9 2024-09-21T09:43:47.920 INFO:tasks.rados.rados.0.smithi076.stdout:1433: finishing write tid 5 to smithi07685715-9 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:1433: oid 9 updating version 0 to 557 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:1433: oid 9 version 557 is already newer than 555 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:1433: oid 9 version 557 is already newer than 556 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:1433: oid 9 version 557 is already newer than 554 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 557 (ObjNum 564 snap 175 seq_num 564) dirty exists 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:1433: left oid 9 (ObjNum 564 snap 175 seq_num 564) 2024-09-21T09:43:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:1434: expect (ObjNum 560 snap 175 seq_num 560) 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1435: finishing write tid 1 to smithi07685715-5 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1435: finishing write tid 2 to smithi07685715-5 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1436: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1436: finishing rollback tid 2 to smithi07685715-10 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 787 (ObjNum 535 snap 170 seq_num 535) dirty exists 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1433: done (3 left) 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1434: done (2 left) 2024-09-21T09:43:48.026 INFO:tasks.rados.rados.0.smithi076.stdout:1436: done (1 left) 2024-09-21T09:43:48.027 INFO:tasks.rados.rados.0.smithi076.stdout:1437: write oid 42 current snap is 175 2024-09-21T09:43:48.027 INFO:tasks.rados.rados.0.smithi076.stdout:1435: finishing write tid 3 to smithi07685715-5 2024-09-21T09:43:48.027 INFO:tasks.rados.rados.0.smithi076.stdout:1437: seq_num 566 ranges {712762=401056,1537867=645855,2860355=652407} 2024-09-21T09:43:48.032 INFO:tasks.rados.rados.0.smithi076.stdout:1437: writing smithi07685715-42 from 712762 to 1113818 tid 1 2024-09-21T09:43:48.037 INFO:tasks.rados.rados.0.smithi076.stdout:1437: writing smithi07685715-42 from 1537867 to 2183722 tid 2 2024-09-21T09:43:48.045 INFO:tasks.rados.rados.0.smithi076.stdout:1437: writing smithi07685715-42 from 2860355 to 3512762 tid 3 2024-09-21T09:43:48.045 INFO:tasks.rados.rados.0.smithi076.stdout:1438: write oid 38 current snap is 175 2024-09-21T09:43:48.045 INFO:tasks.rados.rados.0.smithi076.stdout:1438: seq_num 567 ranges {729569=476305,1996349=505866,3004131=525438} 2024-09-21T09:43:48.051 INFO:tasks.rados.rados.0.smithi076.stdout:1438: writing smithi07685715-38 from 729569 to 1205874 tid 1 2024-09-21T09:43:48.058 INFO:tasks.rados.rados.0.smithi076.stdout:1438: writing smithi07685715-38 from 1996349 to 2502215 tid 2 2024-09-21T09:43:48.064 INFO:tasks.rados.rados.0.smithi076.stdout:1438: writing smithi07685715-38 from 3004131 to 3529569 tid 3 2024-09-21T09:43:48.064 INFO:tasks.rados.rados.0.smithi076.stdout:1439: snap_remove snap 169 2024-09-21T09:43:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:47 smithi191 ceph-mon[33301]: pgmap v96: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:43:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:47 smithi191 ceph-mon[33301]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:43:48.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:47 smithi076 ceph-mon[106482]: pgmap v96: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:43:48.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:47 smithi076 ceph-mon[106482]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:43:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:47 smithi076 ceph-mon[110963]: pgmap v96: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:43:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:47 smithi076 ceph-mon[110963]: osdmap e420: 8 total, 8 up, 8 in 2024-09-21T09:43:48.638 INFO:tasks.rados.rados.0.smithi076.stdout:1439: done (3 left) 2024-09-21T09:43:48.638 INFO:tasks.rados.rados.0.smithi076.stdout:1440: read oid 31 snap -1 2024-09-21T09:43:48.638 INFO:tasks.rados.rados.0.smithi076.stdout:1440: expect deleted 2024-09-21T09:43:48.638 INFO:tasks.rados.rados.0.smithi076.stdout:1441: rollback oid 23 current snap is 175 2024-09-21T09:43:48.638 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 159 2024-09-21T09:43:48.639 INFO:tasks.rados.rados.0.smithi076.stdout:1442: rollback oid 46 current snap is 175 2024-09-21T09:43:48.639 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 164 2024-09-21T09:43:48.639 INFO:tasks.rados.rados.0.smithi076.stdout:1443: write oid 22 current snap is 175 2024-09-21T09:43:48.639 INFO:tasks.rados.rados.0.smithi076.stdout:1443: seq_num 568 ranges {746376=551554,2054831=649524,3315201=231175} 2024-09-21T09:43:48.646 INFO:tasks.rados.rados.0.smithi076.stdout:1443: writing smithi07685715-22 from 746376 to 1297930 tid 1 2024-09-21T09:43:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1443: writing smithi07685715-22 from 2054831 to 2704355 tid 2 2024-09-21T09:43:48.660 INFO:tasks.rados.rados.0.smithi076.stdout:1443: writing smithi07685715-22 from 3315201 to 3546376 tid 3 2024-09-21T09:43:48.660 INFO:tasks.rados.rados.0.smithi076.stdout:1444: rollback oid 9 current snap is 175 2024-09-21T09:43:48.660 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 159 2024-09-21T09:43:48.660 INFO:tasks.rados.rados.0.smithi076.stdout:1435: finishing write tid 4 to smithi07685715-5 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1435: finishing write tid 5 to smithi07685715-5 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1435: oid 5 updating version 0 to 1131 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1435: oid 5 version 1131 is already newer than 1130 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1435: oid 5 updating version 1131 to 1133 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1435: oid 5 version 1133 is already newer than 1132 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1133 (ObjNum 565 snap 175 seq_num 565) dirty exists 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1435: left oid 5 (ObjNum 565 snap 175 seq_num 565) 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: finishing write tid 1 to smithi07685715-42 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: finishing write tid 2 to smithi07685715-42 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: finishing write tid 3 to smithi07685715-42 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: finishing write tid 4 to smithi07685715-42 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: finishing write tid 5 to smithi07685715-42 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: oid 42 updating version 0 to 479 2024-09-21T09:43:48.661 INFO:tasks.rados.rados.0.smithi076.stdout:1437: oid 42 updating version 479 to 480 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1437: oid 42 updating version 480 to 481 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1437: oid 42 version 481 is already newer than 478 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 481 (ObjNum 566 snap 175 seq_num 566) dirty exists 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1437: left oid 42 (ObjNum 566 snap 175 seq_num 566) 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: finishing write tid 1 to smithi07685715-38 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: finishing write tid 2 to smithi07685715-38 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: finishing write tid 3 to smithi07685715-38 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: finishing write tid 4 to smithi07685715-38 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: finishing write tid 5 to smithi07685715-38 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: oid 38 updating version 0 to 789 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: oid 38 updating version 789 to 791 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: oid 38 version 791 is already newer than 790 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:1438: oid 38 updating version 791 to 792 2024-09-21T09:43:48.662 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 792 (ObjNum 567 snap 175 seq_num 567) dirty exists 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1438: left oid 38 (ObjNum 567 snap 175 seq_num 567) 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1442: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1442: finishing rollback tid 2 to smithi07685715-46 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1135 (ObjNum 482 snap 155 seq_num 482) dirty exists 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1441: finishing rollback tid 0 to smithi07685715-23 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1435: done (7 left) 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1437: done (6 left) 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1438: done (5 left) 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1440: done (4 left) 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1442: done (3 left) 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1445: rollback oid 5 current snap is 175 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 143 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1441: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1166 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:43:48.663 INFO:tasks.rados.rados.0.smithi076.stdout:1443: finishing write tid 1 to smithi07685715-22 2024-09-21T09:43:48.664 INFO:tasks.rados.rados.0.smithi076.stdout:1443: finishing write tid 2 to smithi07685715-22 2024-09-21T09:43:48.664 INFO:tasks.rados.rados.0.smithi076.stdout:1441: done (3 left) 2024-09-21T09:43:48.664 INFO:tasks.rados.rados.0.smithi076.stdout:1446: read oid 40 snap -1 2024-09-21T09:43:48.664 INFO:tasks.rados.rados.0.smithi076.stdout:1446: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:43:48.664 INFO:tasks.rados.rados.0.smithi076.stdout:1447: snap_remove snap 156 2024-09-21T09:43:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:48 smithi191 ceph-mon[33301]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:43:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:48 smithi191 ceph-mon[33301]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:43:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:48 smithi076 ceph-mon[106482]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:43:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:48 smithi076 ceph-mon[106482]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:43:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:48 smithi076 ceph-mon[110963]: osdmap e421: 8 total, 8 up, 8 in 2024-09-21T09:43:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:48 smithi076 ceph-mon[110963]: osdmap e422: 8 total, 8 up, 8 in 2024-09-21T09:43:49.641 INFO:tasks.rados.rados.0.smithi076.stdout:1447: done (4 left) 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1448: read oid 30 snap -1 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1448: expect (ObjNum 508 snap 162 seq_num 508) 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1444: finishing rollback tid 0 to smithi07685715-9 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1444: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 561 (ObjNum 33 snap 0 seq_num 830383920) dirty dne 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1445: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1445: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1445: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1144 (ObjNum 444 snap 141 seq_num 444) dirty exists 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1443: finishing write tid 3 to smithi07685715-22 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1444: done (4 left) 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1445: done (3 left) 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1449: snap_create 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1443: finishing write tid 4 to smithi07685715-22 2024-09-21T09:43:49.642 INFO:tasks.rados.rados.0.smithi076.stdout:1443: finishing write tid 5 to smithi07685715-22 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1443: oid 22 updating version 0 to 672 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1443: oid 22 version 672 is already newer than 671 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1443: oid 22 version 672 is already newer than 669 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1443: oid 22 version 672 is already newer than 670 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 672 (ObjNum 568 snap 175 seq_num 568) dirty exists 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1443: left oid 22 (ObjNum 568 snap 175 seq_num 568) 2024-09-21T09:43:49.643 INFO:tasks.rados.rados.0.smithi076.stdout:1446: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:43:49.767 INFO:tasks.rados.rados.0.smithi076.stdout:1448: expect (ObjNum 508 snap 162 seq_num 508) 2024-09-21T09:43:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:49 smithi191 ceph-mon[33301]: pgmap v99: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:43:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:49 smithi191 ceph-mon[33301]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:43:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:49 smithi076 ceph-mon[106482]: pgmap v99: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:43:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:49 smithi076 ceph-mon[106482]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:43:50.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:49 smithi076 ceph-mon[110963]: pgmap v99: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:43:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:49 smithi076 ceph-mon[110963]: osdmap e423: 8 total, 8 up, 8 in 2024-09-21T09:43:50.642 INFO:tasks.rados.rados.0.smithi076.stdout:1443: done (3 left) 2024-09-21T09:43:50.642 INFO:tasks.rados.rados.0.smithi076.stdout:1446: done (2 left) 2024-09-21T09:43:50.642 INFO:tasks.rados.rados.0.smithi076.stdout:1448: done (1 left) 2024-09-21T09:43:50.642 INFO:tasks.rados.rados.0.smithi076.stdout:1449: done (0 left) 2024-09-21T09:43:50.642 INFO:tasks.rados.rados.0.smithi076.stdout:1450: snap_create 2024-09-21T09:43:51.649 INFO:tasks.rados.rados.0.smithi076.stdout:1450: done (0 left) 2024-09-21T09:43:51.649 INFO:tasks.rados.rados.0.smithi076.stdout:1451: write oid 17 current snap is 177 2024-09-21T09:43:51.649 INFO:tasks.rados.rados.0.smithi076.stdout:1451: seq_num 569 ranges {763183=626803,1996960=793182,3226271=336912} 2024-09-21T09:43:51.657 INFO:tasks.rados.rados.0.smithi076.stdout:1451: writing smithi07685715-17 from 763183 to 1389986 tid 1 2024-09-21T09:43:51.664 INFO:tasks.rados.rados.0.smithi076.stdout:1451: writing smithi07685715-17 from 1996960 to 2790142 tid 2 2024-09-21T09:43:51.669 INFO:tasks.rados.rados.0.smithi076.stdout:1451: writing smithi07685715-17 from 3226271 to 3563183 tid 3 2024-09-21T09:43:51.669 INFO:tasks.rados.rados.0.smithi076.stdout:1452: delete oid 4 current snap is 177 2024-09-21T09:43:51.669 INFO:tasks.rados.rados.0.smithi076.stdout:1451: finishing write tid 1 to smithi07685715-17 2024-09-21T09:43:51.669 INFO:tasks.rados.rados.0.smithi076.stdout:1451: finishing write tid 2 to smithi07685715-17 2024-09-21T09:43:51.672 INFO:tasks.rados.rados.0.smithi076.stdout:1452: done (1 left) 2024-09-21T09:43:51.672 INFO:tasks.rados.rados.0.smithi076.stdout:1453: snap_create 2024-09-21T09:43:51.672 INFO:tasks.rados.rados.0.smithi076.stdout:1451: finishing write tid 3 to smithi07685715-17 2024-09-21T09:43:51.672 INFO:tasks.rados.rados.0.smithi076.stdout:1451: finishing write tid 4 to smithi07685715-17 2024-09-21T09:43:51.672 INFO:tasks.rados.rados.0.smithi076.stdout:1451: finishing write tid 5 to smithi07685715-17 2024-09-21T09:43:51.673 INFO:tasks.rados.rados.0.smithi076.stdout:1451: oid 17 updating version 0 to 807 2024-09-21T09:43:51.673 INFO:tasks.rados.rados.0.smithi076.stdout:1451: oid 17 version 807 is already newer than 806 2024-09-21T09:43:51.673 INFO:tasks.rados.rados.0.smithi076.stdout:1451: oid 17 version 807 is already newer than 804 2024-09-21T09:43:51.673 INFO:tasks.rados.rados.0.smithi076.stdout:1451: oid 17 version 807 is already newer than 805 2024-09-21T09:43:51.673 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 807 (ObjNum 569 snap 177 seq_num 569) dirty exists 2024-09-21T09:43:51.673 INFO:tasks.rados.rados.0.smithi076.stdout:1451: left oid 17 (ObjNum 569 snap 177 seq_num 569) 2024-09-21T09:43:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:51 smithi191 ceph-mon[33301]: pgmap v102: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:43:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:51 smithi191 ceph-mon[33301]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:43:52.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:51 smithi076 ceph-mon[106482]: pgmap v102: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:43:52.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:51 smithi076 ceph-mon[106482]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:43:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:51 smithi076 ceph-mon[110963]: pgmap v102: 89 pgs: 89 active+clean; 254 MiB data, 817 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:43:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:51 smithi076 ceph-mon[110963]: osdmap e424: 8 total, 8 up, 8 in 2024-09-21T09:43:52.663 INFO:tasks.rados.rados.0.smithi076.stdout:1451: done (1 left) 2024-09-21T09:43:52.663 INFO:tasks.rados.rados.0.smithi076.stdout:1453: done (0 left) 2024-09-21T09:43:52.663 INFO:tasks.rados.rados.0.smithi076.stdout:1454: snap_create 2024-09-21T09:43:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:52 smithi191 ceph-mon[33301]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:43:53.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:52 smithi076 ceph-mon[106482]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:43:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:52 smithi076 ceph-mon[110963]: osdmap e425: 8 total, 8 up, 8 in 2024-09-21T09:43:53.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:43:53] "GET /metrics HTTP/1.1" 200 36132 "" "Prometheus/2.51.0" 2024-09-21T09:43:53.641 INFO:tasks.rados.rados.0.smithi076.stdout:1454: done (0 left) 2024-09-21T09:43:53.641 INFO:tasks.rados.rados.0.smithi076.stdout:1455: rollback oid 17 current snap is 179 2024-09-21T09:43:53.641 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 175 2024-09-21T09:43:53.641 INFO:tasks.rados.rados.0.smithi076.stdout:1456: read oid 29 snap -1 2024-09-21T09:43:53.641 INFO:tasks.rados.rados.0.smithi076.stdout:1456: expect deleted 2024-09-21T09:43:53.641 INFO:tasks.rados.rados.0.smithi076.stdout:1457: snap_create 2024-09-21T09:43:53.645 INFO:tasks.rados.rados.0.smithi076.stdout:1455: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:43:53.645 INFO:tasks.rados.rados.0.smithi076.stdout:1455: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:43:53.645 INFO:tasks.rados.rados.0.smithi076.stdout:1455: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:43:53.646 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 811 (ObjNum 538 snap 171 seq_num 538) dirty exists 2024-09-21T09:43:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:53 smithi191 ceph-mon[33301]: pgmap v105: 89 pgs: 89 active+clean; 263 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-09-21T09:43:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:53 smithi191 ceph-mon[33301]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:43:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:53 smithi191 ceph-mon[33301]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:43:54.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-mon[106482]: pgmap v105: 89 pgs: 89 active+clean; 263 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-09-21T09:43:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-mon[106482]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:43:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-mon[106482]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:43:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-mon[110963]: pgmap v105: 89 pgs: 89 active+clean; 263 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-09-21T09:43:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-mon[110963]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:43:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:53 smithi076 ceph-mon[110963]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:43:54.647 INFO:tasks.rados.rados.0.smithi076.stdout:1455: done (2 left) 2024-09-21T09:43:54.647 INFO:tasks.rados.rados.0.smithi076.stdout:1456: done (1 left) 2024-09-21T09:43:54.647 INFO:tasks.rados.rados.0.smithi076.stdout:1457: done (0 left) 2024-09-21T09:43:54.648 INFO:tasks.rados.rados.0.smithi076.stdout:1458: delete oid 6 current snap is 180 2024-09-21T09:43:54.652 INFO:tasks.rados.rados.0.smithi076.stdout:1458: done (0 left) 2024-09-21T09:43:54.652 INFO:tasks.rados.rados.0.smithi076.stdout:1459: rollback oid 25 current snap is 180 2024-09-21T09:43:54.652 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 173 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1460: read oid 45 snap -1 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1460: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1461: read oid 21 snap -1 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1461: expect (ObjNum 563 snap 175 seq_num 563) 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1462: read oid 46 snap -1 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1462: expect (ObjNum 482 snap 155 seq_num 482) 2024-09-21T09:43:54.653 INFO:tasks.rados.rados.0.smithi076.stdout:1463: rollback oid 40 current snap is 180 2024-09-21T09:43:54.654 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 177 2024-09-21T09:43:54.654 INFO:tasks.rados.rados.0.smithi076.stdout:1464: snap_remove snap 176 2024-09-21T09:43:55.644 INFO:tasks.rados.rados.0.smithi076.stdout:1464: done (5 left) 2024-09-21T09:43:55.644 INFO:tasks.rados.rados.0.smithi076.stdout:1465: read oid 49 snap -1 2024-09-21T09:43:55.644 INFO:tasks.rados.rados.0.smithi076.stdout:1465: expect (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:43:55.644 INFO:tasks.rados.rados.0.smithi076.stdout:1466: snap_remove snap 132 2024-09-21T09:43:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:55 smithi191 ceph-mon[33301]: pgmap v108: 89 pgs: 89 active+clean; 263 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-09-21T09:43:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:55 smithi191 ceph-mon[33301]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:43:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:55 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:55 smithi076 ceph-mon[106482]: pgmap v108: 89 pgs: 89 active+clean; 263 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-09-21T09:43:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:55 smithi076 ceph-mon[106482]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:43:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:55 smithi076 ceph-mon[110963]: pgmap v108: 89 pgs: 89 active+clean; 263 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 3.1 MiB/s wr, 26 op/s 2024-09-21T09:43:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:55 smithi076 ceph-mon[110963]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:43:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:43:56.647 INFO:tasks.rados.rados.0.smithi076.stdout:1466: done (6 left) 2024-09-21T09:43:56.647 INFO:tasks.rados.rados.0.smithi076.stdout:1467: snap_remove snap 174 2024-09-21T09:43:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:56 smithi191 ceph-mon[33301]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:43:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:56 smithi191 ceph-mon[33301]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:43:57.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:56 smithi076 ceph-mon[110963]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:43:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:56 smithi076 ceph-mon[110963]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:43:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:56 smithi076 ceph-mon[106482]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:43:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:56 smithi076 ceph-mon[106482]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:43:57.648 INFO:tasks.rados.rados.0.smithi076.stdout:1467: done (6 left) 2024-09-21T09:43:57.648 INFO:tasks.rados.rados.0.smithi076.stdout:1468: read oid 7 snap -1 2024-09-21T09:43:57.648 INFO:tasks.rados.rados.0.smithi076.stdout:1468: expect (ObjNum 396 snap 123 seq_num 396) 2024-09-21T09:43:57.648 INFO:tasks.rados.rados.0.smithi076.stdout:1469: snap_remove snap 164 2024-09-21T09:43:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:57 smithi191 ceph-mon[33301]: pgmap v111: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 429 KiB/s wr, 11 op/s 2024-09-21T09:43:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:57 smithi191 ceph-mon[33301]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:43:58.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:57 smithi076 ceph-mon[106482]: pgmap v111: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 429 KiB/s wr, 11 op/s 2024-09-21T09:43:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:57 smithi076 ceph-mon[106482]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:43:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:57 smithi076 ceph-mon[110963]: pgmap v111: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 429 KiB/s wr, 11 op/s 2024-09-21T09:43:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:57 smithi076 ceph-mon[110963]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:43:58.643 INFO:tasks.rados.rados.0.smithi076.stdout:1469: done (7 left) 2024-09-21T09:43:58.643 INFO:tasks.rados.rados.0.smithi076.stdout:1470: delete oid 19 current snap is 180 2024-09-21T09:43:58.643 INFO:tasks.rados.rados.0.smithi076.stdout:1459: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:43:58.643 INFO:tasks.rados.rados.0.smithi076.stdout:1460: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1459: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1463: finishing rollback tid 0 to smithi07685715-40 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1460: done (7 left) 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1470: done (6 left) 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1471: write oid 47 current snap is 180 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1459: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 310 (ObjNum 513 snap 163 seq_num 513) dirty exists 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1463: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:1463: finishing rollback tid 2 to smithi07685715-40 2024-09-21T09:43:58.783 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 680 (ObjNum 282 snap 88 seq_num 282) dirty exists 2024-09-21T09:43:58.784 INFO:tasks.rados.rados.0.smithi076.stdout:1461: expect (ObjNum 563 snap 175 seq_num 563) 2024-09-21T09:43:58.995 INFO:tasks.rados.rados.0.smithi076.stdout:1462: expect (ObjNum 482 snap 155 seq_num 482) 2024-09-21T09:43:59.114 INFO:tasks.rados.rados.0.smithi076.stdout:1465: expect (ObjNum 450 snap 147 seq_num 450) 2024-09-21T09:43:59.312 INFO:tasks.rados.rados.0.smithi076.stdout:1468: expect (ObjNum 396 snap 123 seq_num 396) 2024-09-21T09:43:59.456 INFO:tasks.rados.rados.0.smithi076.stdout:1471: seq_num 573 ranges {430411=644152,1830888=400520,2935963=402981} 2024-09-21T09:43:59.462 INFO:tasks.rados.rados.0.smithi076.stdout:1471: writing smithi07685715-47 from 430411 to 1074563 tid 1 2024-09-21T09:43:59.468 INFO:tasks.rados.rados.0.smithi076.stdout:1471: writing smithi07685715-47 from 1830888 to 2231408 tid 2 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1471: writing smithi07685715-47 from 2935963 to 3338944 tid 3 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1459: done (6 left) 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1461: done (5 left) 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1462: done (4 left) 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1463: done (3 left) 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1465: done (2 left) 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1468: done (1 left) 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1472: read oid 3 snap -1 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1472: expect deleted 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1473: snap_create 2024-09-21T09:43:59.475 INFO:tasks.rados.rados.0.smithi076.stdout:1471: finishing write tid 1 to smithi07685715-47 2024-09-21T09:43:59.476 INFO:tasks.rados.rados.0.smithi076.stdout:1471: finishing write tid 2 to smithi07685715-47 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: finishing write tid 3 to smithi07685715-47 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: finishing write tid 4 to smithi07685715-47 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: finishing write tid 5 to smithi07685715-47 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: oid 47 updating version 0 to 1202 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: oid 47 version 1202 is already newer than 1201 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: oid 47 version 1202 is already newer than 1200 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: oid 47 updating version 1202 to 1203 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1203 (ObjNum 573 snap 180 seq_num 573) dirty exists 2024-09-21T09:43:59.478 INFO:tasks.rados.rados.0.smithi076.stdout:1471: left oid 47 (ObjNum 573 snap 180 seq_num 573) 2024-09-21T09:43:59.671 INFO:tasks.rados.rados.0.smithi076.stdout:1471: done (2 left) 2024-09-21T09:43:59.671 INFO:tasks.rados.rados.0.smithi076.stdout:1472: done (1 left) 2024-09-21T09:43:59.671 INFO:tasks.rados.rados.0.smithi076.stdout:1473: done (0 left) 2024-09-21T09:43:59.672 INFO:tasks.rados.rados.0.smithi076.stdout:1474: write oid 32 current snap is 181 2024-09-21T09:43:59.672 INFO:tasks.rados.rados.0.smithi076.stdout:1474: seq_num 574 ranges {447218=719401,1889370=660531,2963386=614253} 2024-09-21T09:43:59.677 INFO:tasks.rados.rados.0.smithi076.stdout:1474: writing smithi07685715-32 from 447218 to 1166619 tid 1 2024-09-21T09:43:59.685 INFO:tasks.rados.rados.0.smithi076.stdout:1474: writing smithi07685715-32 from 1889370 to 2549901 tid 2 2024-09-21T09:43:59.691 INFO:tasks.rados.rados.0.smithi076.stdout:1474: writing smithi07685715-32 from 2963386 to 3577639 tid 3 2024-09-21T09:43:59.691 INFO:tasks.rados.rados.0.smithi076.stdout:1475: snap_create 2024-09-21T09:43:59.691 INFO:tasks.rados.rados.0.smithi076.stdout:1474: finishing write tid 1 to smithi07685715-32 2024-09-21T09:43:59.691 INFO:tasks.rados.rados.0.smithi076.stdout:1474: finishing write tid 2 to smithi07685715-32 2024-09-21T09:43:59.693 INFO:tasks.rados.rados.0.smithi076.stdout:1474: finishing write tid 3 to smithi07685715-32 2024-09-21T09:43:59.693 INFO:tasks.rados.rados.0.smithi076.stdout:1474: finishing write tid 4 to smithi07685715-32 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:1474: finishing write tid 5 to smithi07685715-32 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:1474: oid 32 updating version 0 to 622 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:1474: oid 32 updating version 622 to 623 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:1474: oid 32 version 623 is already newer than 621 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:1474: oid 32 version 623 is already newer than 620 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 623 (ObjNum 574 snap 181 seq_num 574) dirty exists 2024-09-21T09:43:59.694 INFO:tasks.rados.rados.0.smithi076.stdout:1474: left oid 32 (ObjNum 574 snap 181 seq_num 574) 2024-09-21T09:44:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:59 smithi191 ceph-mon[33301]: pgmap v114: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 429 KiB/s wr, 11 op/s 2024-09-21T09:44:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:43:59 smithi191 ceph-mon[33301]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:44:00.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:59 smithi076 ceph-mon[106482]: pgmap v114: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 429 KiB/s wr, 11 op/s 2024-09-21T09:44:00.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:43:59 smithi076 ceph-mon[106482]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:44:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:59 smithi076 ceph-mon[110963]: pgmap v114: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 429 KiB/s wr, 11 op/s 2024-09-21T09:44:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:43:59 smithi076 ceph-mon[110963]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:44:00.675 INFO:tasks.rados.rados.0.smithi076.stdout:1474: done (1 left) 2024-09-21T09:44:00.675 INFO:tasks.rados.rados.0.smithi076.stdout:1475: done (0 left) 2024-09-21T09:44:00.675 INFO:tasks.rados.rados.0.smithi076.stdout:1476: write oid 12 current snap is 182 2024-09-21T09:44:00.675 INFO:tasks.rados.rados.0.smithi076.stdout:1476: seq_num 575 ranges {464025=794650,1947852=404189,2990809=425525} 2024-09-21T09:44:00.681 INFO:tasks.rados.rados.0.smithi076.stdout:1476: writing smithi07685715-12 from 464025 to 1258675 tid 1 2024-09-21T09:44:00.687 INFO:tasks.rados.rados.0.smithi076.stdout:1476: writing smithi07685715-12 from 1947852 to 2352041 tid 2 2024-09-21T09:44:00.693 INFO:tasks.rados.rados.0.smithi076.stdout:1476: writing smithi07685715-12 from 2990809 to 3416334 tid 3 2024-09-21T09:44:00.694 INFO:tasks.rados.rados.0.smithi076.stdout:1477: snap_create 2024-09-21T09:44:00.694 INFO:tasks.rados.rados.0.smithi076.stdout:1476: finishing write tid 1 to smithi07685715-12 2024-09-21T09:44:00.694 INFO:tasks.rados.rados.0.smithi076.stdout:1476: finishing write tid 2 to smithi07685715-12 2024-09-21T09:44:00.696 INFO:tasks.rados.rados.0.smithi076.stdout:1476: finishing write tid 3 to smithi07685715-12 2024-09-21T09:44:00.696 INFO:tasks.rados.rados.0.smithi076.stdout:1476: finishing write tid 4 to smithi07685715-12 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:1476: finishing write tid 5 to smithi07685715-12 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:1476: oid 12 updating version 0 to 573 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:1476: oid 12 updating version 573 to 574 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:1476: oid 12 version 574 is already newer than 571 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:1476: oid 12 version 574 is already newer than 572 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 574 (ObjNum 575 snap 182 seq_num 575) dirty exists 2024-09-21T09:44:00.697 INFO:tasks.rados.rados.0.smithi076.stdout:1476: left oid 12 (ObjNum 575 snap 182 seq_num 575) 2024-09-21T09:44:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:00 smithi191 ceph-mon[33301]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:44:01.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:00 smithi076 ceph-mon[106482]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:44:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:00 smithi076 ceph-mon[110963]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:44:01.675 INFO:tasks.rados.rados.0.smithi076.stdout:1476: done (1 left) 2024-09-21T09:44:01.676 INFO:tasks.rados.rados.0.smithi076.stdout:1477: done (0 left) 2024-09-21T09:44:01.676 INFO:tasks.rados.rados.0.smithi076.stdout:1478: read oid 36 snap -1 2024-09-21T09:44:01.676 INFO:tasks.rados.rados.0.smithi076.stdout:1478: expect deleted 2024-09-21T09:44:01.676 INFO:tasks.rados.rados.0.smithi076.stdout:1479: write oid 10 current snap is 183 2024-09-21T09:44:01.676 INFO:tasks.rados.rados.0.smithi076.stdout:1479: seq_num 576 ranges {480832=469899,1606334=664200,3018232=662600} 2024-09-21T09:44:01.681 INFO:tasks.rados.rados.0.smithi076.stdout:1479: writing smithi07685715-10 from 480832 to 950731 tid 1 2024-09-21T09:44:01.688 INFO:tasks.rados.rados.0.smithi076.stdout:1479: writing smithi07685715-10 from 1606334 to 2270534 tid 2 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:1479: writing smithi07685715-10 from 3018232 to 3680832 tid 3 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:1480: rollback oid 48 current snap is 183 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 180 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:1479: finishing write tid 1 to smithi07685715-10 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:1479: finishing write tid 2 to smithi07685715-10 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:1478: done (2 left) 2024-09-21T09:44:01.697 INFO:tasks.rados.rados.0.smithi076.stdout:1481: snap_remove snap 170 2024-09-21T09:44:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:01 smithi191 ceph-mon[33301]: pgmap v117: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:01 smithi191 ceph-mon[33301]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:44:02.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:01 smithi076 ceph-mon[106482]: pgmap v117: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:02.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:01 smithi076 ceph-mon[106482]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:44:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:01 smithi076 ceph-mon[110963]: pgmap v117: 89 pgs: 89 active+clean; 266 MiB data, 840 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:01 smithi076 ceph-mon[110963]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:44:02.687 INFO:tasks.rados.rados.0.smithi076.stdout:1481: done (2 left) 2024-09-21T09:44:02.687 INFO:tasks.rados.rados.0.smithi076.stdout:1482: snap_remove snap 181 2024-09-21T09:44:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:02 smithi076 ceph-mon[106482]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:44:03.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:02 smithi076 ceph-mon[110963]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:44:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:02 smithi191 ceph-mon[33301]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:44:03.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:44:03] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.51.0" 2024-09-21T09:44:03.643 INFO:tasks.rados.rados.0.smithi076.stdout:1482: done (2 left) 2024-09-21T09:44:03.643 INFO:tasks.rados.rados.0.smithi076.stdout:1483: read oid 47 snap -1 2024-09-21T09:44:03.643 INFO:tasks.rados.rados.0.smithi076.stdout:1483: expect (ObjNum 573 snap 180 seq_num 573) 2024-09-21T09:44:03.644 INFO:tasks.rados.rados.0.smithi076.stdout:1484: write oid 35 current snap is 183 2024-09-21T09:44:03.644 INFO:tasks.rados.rados.0.smithi076.stdout:1479: finishing write tid 3 to smithi07685715-10 2024-09-21T09:44:03.644 INFO:tasks.rados.rados.0.smithi076.stdout:1484: seq_num 577 ranges {497639=545148,1548463=407858,2529302=564422,3528143=169496} 2024-09-21T09:44:03.648 INFO:tasks.rados.rados.0.smithi076.stdout:1484: writing smithi07685715-35 from 497639 to 1042787 tid 1 2024-09-21T09:44:03.654 INFO:tasks.rados.rados.0.smithi076.stdout:1484: writing smithi07685715-35 from 1548463 to 1956321 tid 2 2024-09-21T09:44:03.661 INFO:tasks.rados.rados.0.smithi076.stdout:1484: writing smithi07685715-35 from 2529302 to 3093724 tid 3 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1484: writing smithi07685715-35 from 3528143 to 3697639 tid 4 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1485: read oid 43 snap -1 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1485: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1486: snap_create 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1479: finishing write tid 4 to smithi07685715-10 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1480: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1479: finishing write tid 5 to smithi07685715-10 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1479: oid 10 updating version 0 to 836 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1479: oid 10 version 836 is already newer than 834 2024-09-21T09:44:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1479: oid 10 version 836 is already newer than 835 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1479: oid 10 version 836 is already newer than 833 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 836 (ObjNum 576 snap 183 seq_num 576) dirty exists 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1479: left oid 10 (ObjNum 576 snap 183 seq_num 576) 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1480: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1480: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1207 (ObjNum 490 snap 157 seq_num 490) dirty exists 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1484: finishing write tid 1 to smithi07685715-35 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1484: finishing write tid 2 to smithi07685715-35 2024-09-21T09:44:03.666 INFO:tasks.rados.rados.0.smithi076.stdout:1483: expect (ObjNum 573 snap 180 seq_num 573) 2024-09-21T09:44:03.904 INFO:tasks.rados.rados.0.smithi076.stdout:1484: finishing write tid 3 to smithi07685715-35 2024-09-21T09:44:03.904 INFO:tasks.rados.rados.0.smithi076.stdout:1484: finishing write tid 4 to smithi07685715-35 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: finishing write tid 5 to smithi07685715-35 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: finishing write tid 6 to smithi07685715-35 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: oid 35 updating version 0 to 1182 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: oid 35 updating version 1182 to 1184 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: oid 35 version 1184 is already newer than 1180 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: oid 35 version 1184 is already newer than 1183 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:1484: oid 35 version 1184 is already newer than 1181 2024-09-21T09:44:03.905 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1184 (ObjNum 577 snap 183 seq_num 577) dirty exists 2024-09-21T09:44:03.906 INFO:tasks.rados.rados.0.smithi076.stdout:1484: left oid 35 (ObjNum 577 snap 183 seq_num 577) 2024-09-21T09:44:03.906 INFO:tasks.rados.rados.0.smithi076.stdout:1485: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:44:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:03 smithi191 ceph-mon[33301]: pgmap v120: 89 pgs: 89 active+clean; 242 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:44:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:03 smithi191 ceph-mon[33301]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:44:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:03 smithi191 ceph-mon[33301]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:44:04.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-mon[106482]: pgmap v120: 89 pgs: 89 active+clean; 242 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:44:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-mon[106482]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:44:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-mon[106482]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:44:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-mon[110963]: pgmap v120: 89 pgs: 89 active+clean; 242 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:44:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-mon[110963]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:44:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:03 smithi076 ceph-mon[110963]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1479: done (5 left) 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1480: done (4 left) 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1483: done (3 left) 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1484: done (2 left) 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1485: done (1 left) 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1486: done (0 left) 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1487: write oid 7 current snap is 184 2024-09-21T09:44:04.644 INFO:tasks.rados.rados.0.smithi076.stdout:1487: seq_num 578 ranges {514446=736750,1723298=667869,3073078=641368} 2024-09-21T09:44:04.650 INFO:tasks.rados.rados.0.smithi076.stdout:1487: writing smithi07685715-7 from 514446 to 1251196 tid 1 2024-09-21T09:44:04.657 INFO:tasks.rados.rados.0.smithi076.stdout:1487: writing smithi07685715-7 from 1723298 to 2391167 tid 2 2024-09-21T09:44:04.664 INFO:tasks.rados.rados.0.smithi076.stdout:1487: writing smithi07685715-7 from 3073078 to 3714446 tid 3 2024-09-21T09:44:04.665 INFO:tasks.rados.rados.0.smithi076.stdout:1488: write oid 21 current snap is 184 2024-09-21T09:44:04.665 INFO:tasks.rados.rados.0.smithi076.stdout:1487: finishing write tid 1 to smithi07685715-7 2024-09-21T09:44:04.665 INFO:tasks.rados.rados.0.smithi076.stdout:1487: finishing write tid 2 to smithi07685715-7 2024-09-21T09:44:04.665 INFO:tasks.rados.rados.0.smithi076.stdout:1488: seq_num 579 ranges {531253=411999,1381780=411527,2300501=586966,3376974=354279} 2024-09-21T09:44:04.669 INFO:tasks.rados.rados.0.smithi076.stdout:1488: writing smithi07685715-21 from 531253 to 943252 tid 1 2024-09-21T09:44:04.674 INFO:tasks.rados.rados.0.smithi076.stdout:1488: writing smithi07685715-21 from 1381780 to 1793307 tid 2 2024-09-21T09:44:04.680 INFO:tasks.rados.rados.0.smithi076.stdout:1488: writing smithi07685715-21 from 2300501 to 2887467 tid 3 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1488: writing smithi07685715-21 from 3376974 to 3731253 tid 4 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1489: write oid 44 current snap is 184 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: finishing write tid 3 to smithi07685715-7 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: finishing write tid 4 to smithi07685715-7 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: finishing write tid 5 to smithi07685715-7 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: oid 7 updating version 0 to 845 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: oid 7 updating version 845 to 847 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: oid 7 version 847 is already newer than 846 2024-09-21T09:44:04.686 INFO:tasks.rados.rados.0.smithi076.stdout:1487: oid 7 version 847 is already newer than 844 2024-09-21T09:44:04.687 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 847 (ObjNum 578 snap 184 seq_num 578) dirty exists 2024-09-21T09:44:04.687 INFO:tasks.rados.rados.0.smithi076.stdout:1487: left oid 7 (ObjNum 578 snap 184 seq_num 578) 2024-09-21T09:44:04.687 INFO:tasks.rados.rados.0.smithi076.stdout:1488: finishing write tid 1 to smithi07685715-21 2024-09-21T09:44:04.687 INFO:tasks.rados.rados.0.smithi076.stdout:1488: finishing write tid 2 to smithi07685715-21 2024-09-21T09:44:04.687 INFO:tasks.rados.rados.0.smithi076.stdout:1488: finishing write tid 3 to smithi07685715-21 2024-09-21T09:44:04.687 INFO:tasks.rados.rados.0.smithi076.stdout:1489: seq_num 580 ranges {548060=487248,1440262=671538,2844277=798238} 2024-09-21T09:44:04.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: writing smithi07685715-44 from 548060 to 1035308 tid 1 2024-09-21T09:44:04.697 INFO:tasks.rados.rados.0.smithi076.stdout:1489: writing smithi07685715-44 from 1440262 to 2111800 tid 2 2024-09-21T09:44:04.705 INFO:tasks.rados.rados.0.smithi076.stdout:1489: writing smithi07685715-44 from 2844277 to 3642515 tid 3 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1487: done (2 left) 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1490: delete oid 48 current snap is 184 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: finishing write tid 4 to smithi07685715-21 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: finishing write tid 5 to smithi07685715-21 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: finishing write tid 6 to smithi07685715-21 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: oid 21 updating version 0 to 1186 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: oid 21 updating version 1186 to 1190 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: oid 21 version 1190 is already newer than 1187 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: oid 21 version 1190 is already newer than 1189 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: oid 21 version 1190 is already newer than 1188 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1190 (ObjNum 579 snap 184 seq_num 579) dirty exists 2024-09-21T09:44:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1488: left oid 21 (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:44:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1489: finishing write tid 1 to smithi07685715-44 2024-09-21T09:44:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1489: finishing write tid 2 to smithi07685715-44 2024-09-21T09:44:04.708 INFO:tasks.rados.rados.0.smithi076.stdout:1488: done (2 left) 2024-09-21T09:44:04.708 INFO:tasks.rados.rados.0.smithi076.stdout:1490: done (1 left) 2024-09-21T09:44:04.708 INFO:tasks.rados.rados.0.smithi076.stdout:1491: write oid 39 current snap is 184 2024-09-21T09:44:04.708 INFO:tasks.rados.rados.0.smithi076.stdout:1491: seq_num 582 ranges {581674=637746,1840873=558854,3066417=537135} 2024-09-21T09:44:04.714 INFO:tasks.rados.rados.0.smithi076.stdout:1491: writing smithi07685715-39 from 581674 to 1219420 tid 1 2024-09-21T09:44:04.721 INFO:tasks.rados.rados.0.smithi076.stdout:1491: writing smithi07685715-39 from 1840873 to 2399727 tid 2 2024-09-21T09:44:04.727 INFO:tasks.rados.rados.0.smithi076.stdout:1491: writing smithi07685715-39 from 3066417 to 3603552 tid 3 2024-09-21T09:44:04.727 INFO:tasks.rados.rados.0.smithi076.stdout:1492: snap_remove snap 180 2024-09-21T09:44:05.690 INFO:tasks.rados.rados.0.smithi076.stdout:1492: done (2 left) 2024-09-21T09:44:05.690 INFO:tasks.rados.rados.0.smithi076.stdout:1493: read oid 41 snap -1 2024-09-21T09:44:05.690 INFO:tasks.rados.rados.0.smithi076.stdout:1493: expect (ObjNum 543 snap 172 seq_num 543) 2024-09-21T09:44:05.690 INFO:tasks.rados.rados.0.smithi076.stdout:1494: write oid 6 current snap is 184 2024-09-21T09:44:05.690 INFO:tasks.rados.rados.0.smithi076.stdout:1489: finishing write tid 3 to smithi07685715-44 2024-09-21T09:44:05.690 INFO:tasks.rados.rados.0.smithi076.stdout:1489: finishing write tid 4 to smithi07685715-44 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: finishing write tid 5 to smithi07685715-44 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: oid 44 updating version 0 to 320 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: oid 44 version 320 is already newer than 319 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: oid 44 updating version 320 to 321 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: oid 44 updating version 321 to 322 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 322 (ObjNum 580 snap 184 seq_num 580) dirty exists 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1489: left oid 44 (ObjNum 580 snap 184 seq_num 580) 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1491: finishing write tid 1 to smithi07685715-39 2024-09-21T09:44:05.691 INFO:tasks.rados.rados.0.smithi076.stdout:1494: seq_num 583 ranges {598481=712995,1899355=418865,3093840=704641} 2024-09-21T09:44:05.697 INFO:tasks.rados.rados.0.smithi076.stdout:1494: writing smithi07685715-6 from 598481 to 1311476 tid 1 2024-09-21T09:44:05.702 INFO:tasks.rados.rados.0.smithi076.stdout:1494: writing smithi07685715-6 from 1899355 to 2318220 tid 2 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1494: writing smithi07685715-6 from 3093840 to 3798481 tid 3 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1489: done (3 left) 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1495: snap_create 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: finishing write tid 2 to smithi07685715-39 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: finishing write tid 3 to smithi07685715-39 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: finishing write tid 4 to smithi07685715-39 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: finishing write tid 5 to smithi07685715-39 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: oid 39 updating version 0 to 1192 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: oid 39 updating version 1192 to 1195 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: oid 39 version 1195 is already newer than 1193 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: oid 39 version 1195 is already newer than 1194 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1195 (ObjNum 582 snap 184 seq_num 582) dirty exists 2024-09-21T09:44:05.712 INFO:tasks.rados.rados.0.smithi076.stdout:1491: left oid 39 (ObjNum 582 snap 184 seq_num 582) 2024-09-21T09:44:05.713 INFO:tasks.rados.rados.0.smithi076.stdout:1494: finishing write tid 1 to smithi07685715-6 2024-09-21T09:44:05.713 INFO:tasks.rados.rados.0.smithi076.stdout:1494: finishing write tid 2 to smithi07685715-6 2024-09-21T09:44:05.713 INFO:tasks.rados.rados.0.smithi076.stdout:1493: expect (ObjNum 543 snap 172 seq_num 543) 2024-09-21T09:44:05.851 INFO:tasks.rados.rados.0.smithi076.stdout:1494: finishing write tid 3 to smithi07685715-6 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: finishing write tid 4 to smithi07685715-6 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: finishing write tid 5 to smithi07685715-6 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: oid 6 updating version 0 to 324 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: oid 6 updating version 324 to 325 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: oid 6 updating version 325 to 326 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: oid 6 version 326 is already newer than 323 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 326 (ObjNum 583 snap 184 seq_num 583) dirty exists 2024-09-21T09:44:05.852 INFO:tasks.rados.rados.0.smithi076.stdout:1494: left oid 6 (ObjNum 583 snap 184 seq_num 583) 2024-09-21T09:44:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:05 smithi191 ceph-mon[33301]: pgmap v123: 89 pgs: 89 active+clean; 242 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:44:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:05 smithi191 ceph-mon[33301]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:44:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:05 smithi076 ceph-mon[106482]: pgmap v123: 89 pgs: 89 active+clean; 242 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:44:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:05 smithi076 ceph-mon[106482]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:44:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:05 smithi076 ceph-mon[110963]: pgmap v123: 89 pgs: 89 active+clean; 242 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-09-21T09:44:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:05 smithi076 ceph-mon[110963]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:44:06.703 INFO:tasks.rados.rados.0.smithi076.stdout:1491: done (3 left) 2024-09-21T09:44:06.703 INFO:tasks.rados.rados.0.smithi076.stdout:1493: done (2 left) 2024-09-21T09:44:06.703 INFO:tasks.rados.rados.0.smithi076.stdout:1494: done (1 left) 2024-09-21T09:44:06.703 INFO:tasks.rados.rados.0.smithi076.stdout:1495: done (0 left) 2024-09-21T09:44:06.703 INFO:tasks.rados.rados.0.smithi076.stdout:1496: snap_create 2024-09-21T09:44:07.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:06 smithi191 ceph-mon[33301]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:44:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:06 smithi076 ceph-mon[106482]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:44:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:06 smithi076 ceph-mon[110963]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:44:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1496: done (0 left) 2024-09-21T09:44:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1497: snap_create 2024-09-21T09:44:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:07 smithi191 ceph-mon[33301]: pgmap v126: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 18 op/s 2024-09-21T09:44:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:07 smithi191 ceph-mon[33301]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:44:08.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:07 smithi076 ceph-mon[106482]: pgmap v126: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 18 op/s 2024-09-21T09:44:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:07 smithi076 ceph-mon[106482]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:44:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:07 smithi076 ceph-mon[110963]: pgmap v126: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 18 op/s 2024-09-21T09:44:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:07 smithi076 ceph-mon[110963]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:44:08.644 INFO:tasks.rados.rados.0.smithi076.stdout:1497: done (0 left) 2024-09-21T09:44:08.644 INFO:tasks.rados.rados.0.smithi076.stdout:1498: snap_remove snap 177 2024-09-21T09:44:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:08 smithi191 ceph-mon[33301]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:44:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:08 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:08 smithi191 ceph-mon[33301]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:44:09.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:08 smithi076 ceph-mon[106482]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:44:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:08 smithi076 ceph-mon[106482]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:44:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:08 smithi076 ceph-mon[110963]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:44:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:08 smithi076 ceph-mon[110963]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:44:09.644 INFO:tasks.rados.rados.0.smithi076.stdout:1498: done (0 left) 2024-09-21T09:44:09.645 INFO:tasks.rados.rados.0.smithi076.stdout:1499: snap_create 2024-09-21T09:44:10.096 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:09 smithi191 ceph-mon[33301]: pgmap v129: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 18 op/s 2024-09-21T09:44:10.096 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:09 smithi191 ceph-mon[33301]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:44:10.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:09 smithi076 ceph-mon[106482]: pgmap v129: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 18 op/s 2024-09-21T09:44:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:09 smithi076 ceph-mon[106482]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:44:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:09 smithi076 ceph-mon[110963]: pgmap v129: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 18 op/s 2024-09-21T09:44:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:09 smithi076 ceph-mon[110963]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:44:10.647 INFO:tasks.rados.rados.0.smithi076.stdout:1499: done (0 left) 2024-09-21T09:44:10.647 INFO:tasks.rados.rados.0.smithi076.stdout:1500: snap_remove snap 183 2024-09-21T09:44:11.044 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:11.045 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:10 smithi076 ceph-mon[106482]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:44:11.045 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:11.045 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:10 smithi076 ceph-mon[110963]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:44:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:10 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:10 smithi191 ceph-mon[33301]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:44:11.648 INFO:tasks.rados.rados.0.smithi076.stdout:1500: done (0 left) 2024-09-21T09:44:11.648 INFO:tasks.rados.rados.0.smithi076.stdout:1501: rollback oid 50 current snap is 188 2024-09-21T09:44:11.648 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 178 2024-09-21T09:44:11.649 INFO:tasks.rados.rados.0.smithi076.stdout:1502: write oid 24 current snap is 188 2024-09-21T09:44:11.649 INFO:tasks.rados.rados.0.smithi076.stdout:1502: seq_num 584 ranges {615288=788244,1957837=562523,3121263=559679} 2024-09-21T09:44:11.656 INFO:tasks.rados.rados.0.smithi076.stdout:1502: writing smithi07685715-24 from 615288 to 1403532 tid 1 2024-09-21T09:44:11.662 INFO:tasks.rados.rados.0.smithi076.stdout:1502: writing smithi07685715-24 from 1957837 to 2520360 tid 2 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:1502: writing smithi07685715-24 from 3121263 to 3680942 tid 3 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:1503: write oid 49 current snap is 188 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:1501: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 425 (ObjNum 81 snap 0 seq_num 0) dirty dne 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:1502: finishing write tid 1 to smithi07685715-24 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:1502: finishing write tid 2 to smithi07685715-24 2024-09-21T09:44:11.669 INFO:tasks.rados.rados.0.smithi076.stdout:1503: seq_num 585 ranges {632095=463493,1616319=422534,2748686=487304} 2024-09-21T09:44:11.674 INFO:tasks.rados.rados.0.smithi076.stdout:1503: writing smithi07685715-49 from 632095 to 1095588 tid 1 2024-09-21T09:44:11.680 INFO:tasks.rados.rados.0.smithi076.stdout:1503: writing smithi07685715-49 from 1616319 to 2038853 tid 2 2024-09-21T09:44:11.686 INFO:tasks.rados.rados.0.smithi076.stdout:1503: writing smithi07685715-49 from 2748686 to 3235990 tid 3 2024-09-21T09:44:11.687 INFO:tasks.rados.rados.0.smithi076.stdout:1501: done (2 left) 2024-09-21T09:44:11.687 INFO:tasks.rados.rados.0.smithi076.stdout:1504: snap_remove snap 143 2024-09-21T09:44:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:11 smithi191 ceph-mon[33301]: pgmap v132: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:11 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:11 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:11 smithi191 ceph-mon[33301]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[106482]: pgmap v132: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[106482]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[110963]: pgmap v132: 89 pgs: 89 active+clean; 252 MiB data, 839 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:11 smithi076 ceph-mon[110963]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1504: done (2 left) 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1505: delete oid 46 current snap is 188 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1502: finishing write tid 3 to smithi07685715-24 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1502: finishing write tid 4 to smithi07685715-24 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1502: finishing write tid 5 to smithi07685715-24 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1502: oid 24 updating version 0 to 158 2024-09-21T09:44:12.651 INFO:tasks.rados.rados.0.smithi076.stdout:1502: oid 24 updating version 158 to 160 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1502: oid 24 version 160 is already newer than 157 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1502: oid 24 version 160 is already newer than 159 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 160 (ObjNum 584 snap 188 seq_num 584) dirty exists 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1502: left oid 24 (ObjNum 584 snap 188 seq_num 584) 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: finishing write tid 1 to smithi07685715-49 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: finishing write tid 2 to smithi07685715-49 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: finishing write tid 3 to smithi07685715-49 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: finishing write tid 4 to smithi07685715-49 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: finishing write tid 5 to smithi07685715-49 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: oid 49 updating version 0 to 1232 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: oid 49 version 1232 is already newer than 1230 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: oid 49 version 1232 is already newer than 1229 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:1503: oid 49 version 1232 is already newer than 1231 2024-09-21T09:44:12.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 1232 (ObjNum 585 snap 188 seq_num 585) dirty exists 2024-09-21T09:44:12.653 INFO:tasks.rados.rados.0.smithi076.stdout:1503: left oid 49 (ObjNum 585 snap 188 seq_num 585) 2024-09-21T09:44:12.659 INFO:tasks.rados.rados.0.smithi076.stdout:1502: done (2 left) 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1503: done (1 left) 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1505: done (0 left) 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1506: read oid 1 snap -1 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1506: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1507: read oid 47 snap -1 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1507: expect (ObjNum 573 snap 180 seq_num 573) 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1508: read oid 16 snap -1 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1508: expect (ObjNum 446 snap 144 seq_num 446) 2024-09-21T09:44:12.660 INFO:tasks.rados.rados.0.smithi076.stdout:1509: snap_remove snap 182 2024-09-21T09:44:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:13 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:13 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:13 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:13 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:13 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:13 smithi191 ceph-mon[33301]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:44:13.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:44:13] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.51.0" 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[110963]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:13.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:13.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:13 smithi076 ceph-mon[106482]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:44:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:1509: done (3 left) 2024-09-21T09:44:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:1510: write oid 50 current snap is 188 2024-09-21T09:44:13.648 INFO:tasks.rados.rados.0.smithi076.stdout:1510: seq_num 587 ranges {665709=730344,2133283=426203,3203532=509848} 2024-09-21T09:44:13.655 INFO:tasks.rados.rados.0.smithi076.stdout:1510: writing smithi07685715-50 from 665709 to 1396053 tid 1 2024-09-21T09:44:13.662 INFO:tasks.rados.rados.0.smithi076.stdout:1510: writing smithi07685715-50 from 2133283 to 2559486 tid 2 2024-09-21T09:44:13.669 INFO:tasks.rados.rados.0.smithi076.stdout:1510: writing smithi07685715-50 from 3203532 to 3713380 tid 3 2024-09-21T09:44:13.669 INFO:tasks.rados.rados.0.smithi076.stdout:1511: delete oid 42 current snap is 188 2024-09-21T09:44:13.669 INFO:tasks.rados.rados.0.smithi076.stdout:1506: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:44:13.790 INFO:tasks.rados.rados.0.smithi076.stdout:1507: expect (ObjNum 573 snap 180 seq_num 573) 2024-09-21T09:44:14.026 INFO:tasks.rados.rados.0.smithi076.stdout:1508: expect (ObjNum 446 snap 144 seq_num 446) 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: finishing write tid 1 to smithi07685715-50 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: finishing write tid 2 to smithi07685715-50 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: finishing write tid 3 to smithi07685715-50 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: finishing write tid 4 to smithi07685715-50 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: finishing write tid 5 to smithi07685715-50 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: oid 50 updating version 0 to 494 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: oid 50 updating version 494 to 495 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: oid 50 updating version 495 to 497 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:1510: oid 50 version 497 is already newer than 496 2024-09-21T09:44:14.228 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 497 (ObjNum 587 snap 188 seq_num 587) dirty exists 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1510: left oid 50 (ObjNum 587 snap 188 seq_num 587) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1506: done (4 left) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1507: done (3 left) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1508: done (2 left) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1510: done (1 left) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1511: done (0 left) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1512: read oid 34 snap -1 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1512: expect (ObjNum 557 snap 174 seq_num 557) 2024-09-21T09:44:14.229 INFO:tasks.rados.rados.0.smithi076.stdout:1513: snap_create 2024-09-21T09:44:14.234 INFO:tasks.rados.rados.0.smithi076.stdout:1512: expect (ObjNum 557 snap 174 seq_num 557) 2024-09-21T09:44:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:14 smithi191 ceph-mon[33301]: pgmap v135: 89 pgs: 89 active+clean; 252 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-09-21T09:44:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:14 smithi191 ceph-mon[33301]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:44:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:14 smithi191 ceph-mon[33301]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:44:14.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:14 smithi076 ceph-mon[106482]: pgmap v135: 89 pgs: 89 active+clean; 252 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-09-21T09:44:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:14 smithi076 ceph-mon[106482]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:44:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:14 smithi076 ceph-mon[106482]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:44:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:14 smithi076 ceph-mon[110963]: pgmap v135: 89 pgs: 89 active+clean; 252 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-09-21T09:44:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:14 smithi076 ceph-mon[110963]: mgrmap e40: y(active, since 92s), standbys: x 2024-09-21T09:44:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:14 smithi076 ceph-mon[110963]: osdmap e447: 8 total, 8 up, 8 in 2024-09-21T09:44:15.057 INFO:tasks.rados.rados.0.smithi076.stdout:1512: done (1 left) 2024-09-21T09:44:15.057 INFO:tasks.rados.rados.0.smithi076.stdout:1513: done (0 left) 2024-09-21T09:44:15.058 INFO:tasks.rados.rados.0.smithi076.stdout:1514: delete oid 15 current snap is 189 2024-09-21T09:44:15.061 INFO:tasks.rados.rados.0.smithi076.stdout:1514: done (0 left) 2024-09-21T09:44:15.061 INFO:tasks.rados.rados.0.smithi076.stdout:1515: read oid 24 snap -1 2024-09-21T09:44:15.061 INFO:tasks.rados.rados.0.smithi076.stdout:1515: expect (ObjNum 584 snap 188 seq_num 584) 2024-09-21T09:44:15.061 INFO:tasks.rados.rados.0.smithi076.stdout:1516: read oid 34 snap 173 2024-09-21T09:44:15.061 INFO:tasks.rados.rados.0.smithi076.stdout:1516: expect (ObjNum 523 snap 167 seq_num 523) 2024-09-21T09:44:15.061 INFO:tasks.rados.rados.0.smithi076.stdout:1517: snap_remove snap 185 2024-09-21T09:44:16.063 INFO:tasks.rados.rados.0.smithi076.stdout:1517: done (2 left) 2024-09-21T09:44:16.063 INFO:tasks.rados.rados.0.smithi076.stdout:1518: write oid 29 current snap is 189 2024-09-21T09:44:16.063 INFO:tasks.rados.rados.0.smithi076.stdout:1518: seq_num 590 ranges {716130=556091,1792376=573530,2769448=460017,3821956=94174} 2024-09-21T09:44:16.069 INFO:tasks.rados.rados.0.smithi076.stdout:1518: writing smithi07685715-29 from 716130 to 1272221 tid 1 2024-09-21T09:44:16.076 INFO:tasks.rados.rados.0.smithi076.stdout:1518: writing smithi07685715-29 from 1792376 to 2365906 tid 2 2024-09-21T09:44:16.081 INFO:tasks.rados.rados.0.smithi076.stdout:1518: writing smithi07685715-29 from 2769448 to 3229465 tid 3 2024-09-21T09:44:16.085 INFO:tasks.rados.rados.0.smithi076.stdout:1518: writing smithi07685715-29 from 3821956 to 3916130 tid 4 2024-09-21T09:44:16.085 INFO:tasks.rados.rados.0.smithi076.stdout:1519: read oid 31 snap -1 2024-09-21T09:44:16.085 INFO:tasks.rados.rados.0.smithi076.stdout:1519: expect deleted 2024-09-21T09:44:16.085 INFO:tasks.rados.rados.0.smithi076.stdout:1520: snap_remove snap 188 2024-09-21T09:44:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:16 smithi191 ceph-mon[33301]: pgmap v138: 89 pgs: 89 active+clean; 252 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-09-21T09:44:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:16 smithi191 ceph-mon[33301]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:44:16.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:16 smithi076 ceph-mon[106482]: pgmap v138: 89 pgs: 89 active+clean; 252 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-09-21T09:44:16.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:16 smithi076 ceph-mon[106482]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:44:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:16 smithi076 ceph-mon[110963]: pgmap v138: 89 pgs: 89 active+clean; 252 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-09-21T09:44:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:16 smithi076 ceph-mon[110963]: osdmap e448: 8 total, 8 up, 8 in 2024-09-21T09:44:17.068 INFO:tasks.rados.rados.0.smithi076.stdout:1520: done (4 left) 2024-09-21T09:44:17.069 INFO:tasks.rados.rados.0.smithi076.stdout:1521: write oid 4 current snap is 189 2024-09-21T09:44:17.069 INFO:tasks.rados.rados.0.smithi076.stdout:1521: seq_num 591 ranges {732937=631340,1850858=433541,2796871=671289,3888195=44742} 2024-09-21T09:44:17.076 INFO:tasks.rados.rados.0.smithi076.stdout:1521: writing smithi07685715-4 from 732937 to 1364277 tid 1 2024-09-21T09:44:17.081 INFO:tasks.rados.rados.0.smithi076.stdout:1521: writing smithi07685715-4 from 1850858 to 2284399 tid 2 2024-09-21T09:44:17.087 INFO:tasks.rados.rados.0.smithi076.stdout:1521: writing smithi07685715-4 from 2796871 to 3468160 tid 3 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi076.stdout:1521: writing smithi07685715-4 from 3888195 to 3932937 tid 4 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi076.stdout:1522: read oid 7 snap -1 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi076.stdout:1522: expect (ObjNum 578 snap 184 seq_num 578) 2024-09-21T09:44:17.090 INFO:tasks.rados.rados.0.smithi076.stdout:1523: snap_remove snap 184 2024-09-21T09:44:17.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:17 smithi191 ceph-mon[33301]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:44:17.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:17 smithi076 ceph-mon[106482]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:44:17.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:17 smithi076 ceph-mon[110963]: osdmap e449: 8 total, 8 up, 8 in 2024-09-21T09:44:18.071 INFO:tasks.rados.rados.0.smithi076.stdout:1523: done (6 left) 2024-09-21T09:44:18.071 INFO:tasks.rados.rados.0.smithi076.stdout:1524: rollback oid 42 current snap is 189 2024-09-21T09:44:18.071 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 179 2024-09-21T09:44:18.071 INFO:tasks.rados.rados.0.smithi076.stdout:1516: expect (ObjNum 523 snap 167 seq_num 523) 2024-09-21T09:44:18.218 INFO:tasks.rados.rados.0.smithi076.stdout:1515: expect (ObjNum 584 snap 188 seq_num 584) 2024-09-21T09:44:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:18 smithi191 ceph-mon[33301]: pgmap v141: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-21T09:44:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:18 smithi191 ceph-mon[33301]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:44:18.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:18 smithi076 ceph-mon[106482]: pgmap v141: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-21T09:44:18.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:18 smithi076 ceph-mon[106482]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:44:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:18 smithi076 ceph-mon[110963]: pgmap v141: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-21T09:44:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:18 smithi076 ceph-mon[110963]: osdmap e450: 8 total, 8 up, 8 in 2024-09-21T09:44:18.385 INFO:tasks.rados.rados.0.smithi076.stdout:1518: finishing write tid 1 to smithi07685715-29 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: finishing write tid 2 to smithi07685715-29 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: finishing write tid 3 to smithi07685715-29 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: finishing write tid 4 to smithi07685715-29 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: finishing write tid 5 to smithi07685715-29 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: finishing write tid 6 to smithi07685715-29 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: oid 29 updating version 0 to 407 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: oid 29 version 407 is already newer than 404 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: oid 29 version 407 is already newer than 405 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: oid 29 version 407 is already newer than 406 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: oid 29 version 407 is already newer than 403 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 407 (ObjNum 590 snap 189 seq_num 590) dirty exists 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1518: left oid 29 (ObjNum 590 snap 189 seq_num 590) 2024-09-21T09:44:18.386 INFO:tasks.rados.rados.0.smithi076.stdout:1521: finishing write tid 1 to smithi07685715-4 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1515: done (6 left) 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1516: done (5 left) 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1518: done (4 left) 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1519: done (3 left) 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1525: write oid 41 current snap is 189 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: finishing write tid 2 to smithi07685715-4 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: finishing write tid 3 to smithi07685715-4 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: finishing write tid 4 to smithi07685715-4 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: finishing write tid 5 to smithi07685715-4 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: finishing write tid 6 to smithi07685715-4 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: oid 4 updating version 0 to 501 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: oid 4 version 501 is already newer than 500 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: oid 4 updating version 501 to 502 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: oid 4 version 502 is already newer than 498 2024-09-21T09:44:18.387 INFO:tasks.rados.rados.0.smithi076.stdout:1521: oid 4 version 502 is already newer than 499 2024-09-21T09:44:18.388 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 502 (ObjNum 591 snap 189 seq_num 591) dirty exists 2024-09-21T09:44:18.388 INFO:tasks.rados.rados.0.smithi076.stdout:1521: left oid 4 (ObjNum 591 snap 189 seq_num 591) 2024-09-21T09:44:18.388 INFO:tasks.rados.rados.0.smithi076.stdout:1522: expect (ObjNum 578 snap 184 seq_num 578) 2024-09-21T09:44:18.589 INFO:tasks.rados.rados.0.smithi076.stdout:1524: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:44:18.589 INFO:tasks.rados.rados.0.smithi076.stdout:1524: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:44:18.589 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 507 (ObjNum 566 snap 175 seq_num 566) dirty exists 2024-09-21T09:44:18.589 INFO:tasks.rados.rados.0.smithi076.stdout:1525: seq_num 592 ranges {749744=706589,1909340=577199,3224294=482561} 2024-09-21T09:44:18.596 INFO:tasks.rados.rados.0.smithi076.stdout:1525: writing smithi07685715-41 from 749744 to 1456333 tid 1 2024-09-21T09:44:18.602 INFO:tasks.rados.rados.0.smithi076.stdout:1525: writing smithi07685715-41 from 1909340 to 2486539 tid 2 2024-09-21T09:44:18.608 INFO:tasks.rados.rados.0.smithi076.stdout:1525: writing smithi07685715-41 from 3224294 to 3706855 tid 3 2024-09-21T09:44:18.608 INFO:tasks.rados.rados.0.smithi076.stdout:1521: done (3 left) 2024-09-21T09:44:18.608 INFO:tasks.rados.rados.0.smithi076.stdout:1522: done (2 left) 2024-09-21T09:44:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:1524: done (1 left) 2024-09-21T09:44:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:1526: write oid 37 current snap is 189 2024-09-21T09:44:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:1525: finishing write tid 1 to smithi07685715-41 2024-09-21T09:44:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:1525: finishing write tid 2 to smithi07685715-41 2024-09-21T09:44:18.609 INFO:tasks.rados.rados.0.smithi076.stdout:1526: seq_num 593 ranges {766551=498191,1684175=720857,2851717=693833} 2024-09-21T09:44:18.615 INFO:tasks.rados.rados.0.smithi076.stdout:1526: writing smithi07685715-37 from 766551 to 1264742 tid 1 2024-09-21T09:44:18.621 INFO:tasks.rados.rados.0.smithi076.stdout:1526: writing smithi07685715-37 from 1684175 to 2405032 tid 2 2024-09-21T09:44:18.627 INFO:tasks.rados.rados.0.smithi076.stdout:1526: writing smithi07685715-37 from 2851717 to 3545550 tid 3 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1527: write oid 30 current snap is 189 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: finishing write tid 3 to smithi07685715-41 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: finishing write tid 4 to smithi07685715-41 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: finishing write tid 5 to smithi07685715-41 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: oid 41 updating version 0 to 709 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: oid 41 version 709 is already newer than 708 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: oid 41 updating version 709 to 710 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: oid 41 updating version 710 to 711 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 711 (ObjNum 592 snap 189 seq_num 592) dirty exists 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1525: left oid 41 (ObjNum 592 snap 189 seq_num 592) 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1526: finishing write tid 1 to smithi07685715-37 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1526: finishing write tid 2 to smithi07685715-37 2024-09-21T09:44:18.628 INFO:tasks.rados.rados.0.smithi076.stdout:1527: seq_num 594 ranges {783358=573440,2026304=580868,3279140=621458} 2024-09-21T09:44:18.635 INFO:tasks.rados.rados.0.smithi076.stdout:1527: writing smithi07685715-30 from 783358 to 1356798 tid 1 2024-09-21T09:44:18.641 INFO:tasks.rados.rados.0.smithi076.stdout:1527: writing smithi07685715-30 from 2026304 to 2607172 tid 2 2024-09-21T09:44:18.649 INFO:tasks.rados.rados.0.smithi076.stdout:1527: writing smithi07685715-30 from 3279140 to 3900598 tid 3 2024-09-21T09:44:18.649 INFO:tasks.rados.rados.0.smithi076.stdout:1525: done (2 left) 2024-09-21T09:44:18.649 INFO:tasks.rados.rados.0.smithi076.stdout:1528: snap_remove snap 173 2024-09-21T09:44:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:19 smithi191 ceph-mon[33301]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:44:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:19 smithi191 ceph-mon[33301]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:44:19.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:19 smithi076 ceph-mon[110963]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:44:19.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:19 smithi076 ceph-mon[110963]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:44:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:19 smithi076 ceph-mon[106482]: osdmap e451: 8 total, 8 up, 8 in 2024-09-21T09:44:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:19 smithi076 ceph-mon[106482]: osdmap e452: 8 total, 8 up, 8 in 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: finishing write tid 3 to smithi07685715-37 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1528: done (2 left) 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1529: delete oid 7 current snap is 189 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: finishing write tid 4 to smithi07685715-37 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: finishing write tid 5 to smithi07685715-37 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: oid 37 updating version 0 to 411 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: oid 37 version 411 is already newer than 409 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: oid 37 version 411 is already newer than 410 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: oid 37 updating version 411 to 412 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 412 (ObjNum 593 snap 189 seq_num 593) dirty exists 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1526: left oid 37 (ObjNum 593 snap 189 seq_num 593) 2024-09-21T09:44:19.650 INFO:tasks.rados.rados.0.smithi076.stdout:1527: finishing write tid 1 to smithi07685715-30 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: finishing write tid 2 to smithi07685715-30 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: finishing write tid 3 to smithi07685715-30 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: finishing write tid 4 to smithi07685715-30 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: finishing write tid 5 to smithi07685715-30 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: oid 30 updating version 0 to 1254 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: oid 30 version 1254 is already newer than 1252 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: oid 30 updating version 1254 to 1255 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: oid 30 version 1255 is already newer than 1253 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1255 (ObjNum 594 snap 189 seq_num 594) dirty exists 2024-09-21T09:44:19.651 INFO:tasks.rados.rados.0.smithi076.stdout:1527: left oid 30 (ObjNum 594 snap 189 seq_num 594) 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1526: done (2 left) 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1527: done (1 left) 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1529: done (0 left) 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1530: read oid 40 snap -1 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1530: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1531: write oid 14 current snap is 189 2024-09-21T09:44:19.657 INFO:tasks.rados.rados.0.smithi076.stdout:1531: seq_num 596 ranges {416972=723938,1743268=273704} 2024-09-21T09:44:19.665 INFO:tasks.rados.rados.0.smithi076.stdout:1531: writing smithi07685715-14 from 416972 to 1140910 tid 1 2024-09-21T09:44:19.670 INFO:tasks.rados.rados.0.smithi076.stdout:1531: writing smithi07685715-14 from 1743268 to 2016972 tid 2 2024-09-21T09:44:19.670 INFO:tasks.rados.rados.0.smithi076.stdout:1532: snap_create 2024-09-21T09:44:19.670 INFO:tasks.rados.rados.0.smithi076.stdout:1530: expect (ObjNum 282 snap 88 seq_num 282) 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: finishing write tid 1 to smithi07685715-14 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: finishing write tid 2 to smithi07685715-14 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: finishing write tid 3 to smithi07685715-14 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: finishing write tid 4 to smithi07685715-14 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: oid 14 updating version 0 to 882 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: oid 14 updating version 882 to 883 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: oid 14 version 883 is already newer than 881 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 883 (ObjNum 596 snap 189 seq_num 596) dirty exists 2024-09-21T09:44:19.792 INFO:tasks.rados.rados.0.smithi076.stdout:1531: left oid 14 (ObjNum 596 snap 189 seq_num 596) 2024-09-21T09:44:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:20 smithi191 ceph-mon[33301]: pgmap v144: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-21T09:44:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:20 smithi191 ceph-mon[33301]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:44:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:20 smithi076 ceph-mon[106482]: pgmap v144: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-21T09:44:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:20 smithi076 ceph-mon[106482]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:44:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:20 smithi076 ceph-mon[110963]: pgmap v144: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-21T09:44:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:20 smithi076 ceph-mon[110963]: osdmap e453: 8 total, 8 up, 8 in 2024-09-21T09:44:20.651 INFO:tasks.rados.rados.0.smithi076.stdout:1530: done (2 left) 2024-09-21T09:44:20.652 INFO:tasks.rados.rados.0.smithi076.stdout:1531: done (1 left) 2024-09-21T09:44:20.652 INFO:tasks.rados.rados.0.smithi076.stdout:1532: done (0 left) 2024-09-21T09:44:20.652 INFO:tasks.rados.rados.0.smithi076.stdout:1533: write oid 26 current snap is 190 2024-09-21T09:44:20.652 INFO:tasks.rados.rados.0.smithi076.stdout:1533: seq_num 597 ranges {433779=799187,1801750=232029} 2024-09-21T09:44:20.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: writing smithi07685715-26 from 433779 to 1232966 tid 1 2024-09-21T09:44:20.663 INFO:tasks.rados.rados.0.smithi076.stdout:1533: writing smithi07685715-26 from 1801750 to 2033779 tid 2 2024-09-21T09:44:20.663 INFO:tasks.rados.rados.0.smithi076.stdout:1533: finishing write tid 1 to smithi07685715-26 2024-09-21T09:44:20.663 INFO:tasks.rados.rados.0.smithi076.stdout:1534: snap_remove snap 159 2024-09-21T09:44:21.657 INFO:tasks.rados.rados.0.smithi076.stdout:1534: done (1 left) 2024-09-21T09:44:21.657 INFO:tasks.rados.rados.0.smithi076.stdout:1535: delete oid 22 current snap is 190 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: finishing write tid 2 to smithi07685715-26 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: finishing write tid 3 to smithi07685715-26 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: finishing write tid 4 to smithi07685715-26 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: oid 26 updating version 0 to 592 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: oid 26 version 592 is already newer than 590 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: oid 26 version 592 is already newer than 591 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 592 (ObjNum 597 snap 190 seq_num 597) dirty exists 2024-09-21T09:44:21.658 INFO:tasks.rados.rados.0.smithi076.stdout:1533: left oid 26 (ObjNum 597 snap 190 seq_num 597) 2024-09-21T09:44:21.668 INFO:tasks.rados.rados.0.smithi076.stdout:1533: done (1 left) 2024-09-21T09:44:21.668 INFO:tasks.rados.rados.0.smithi076.stdout:1535: done (0 left) 2024-09-21T09:44:21.668 INFO:tasks.rados.rados.0.smithi076.stdout:1536: snap_create 2024-09-21T09:44:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:21 smithi191 ceph-mon[33301]: pgmap v147: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:21 smithi191 ceph-mon[33301]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:44:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:21 smithi076 ceph-mon[106482]: pgmap v147: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:21 smithi076 ceph-mon[106482]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:44:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:21 smithi076 ceph-mon[110963]: pgmap v147: 89 pgs: 89 active+clean; 205 MiB data, 750 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:44:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:21 smithi076 ceph-mon[110963]: osdmap e454: 8 total, 8 up, 8 in 2024-09-21T09:44:22.665 INFO:tasks.rados.rados.0.smithi076.stdout:1536: done (0 left) 2024-09-21T09:44:22.666 INFO:tasks.rados.rados.0.smithi076.stdout:1537: snap_remove snap 186 2024-09-21T09:44:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:22 smithi191 ceph-mon[33301]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:44:23.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:22 smithi076 ceph-mon[106482]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:44:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:22 smithi076 ceph-mon[110963]: osdmap e455: 8 total, 8 up, 8 in 2024-09-21T09:44:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:44:23] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.51.0" 2024-09-21T09:44:23.650 INFO:tasks.rados.rados.0.smithi076.stdout:1537: done (0 left) 2024-09-21T09:44:23.650 INFO:tasks.rados.rados.0.smithi076.stdout:1538: read oid 45 snap -1 2024-09-21T09:44:23.650 INFO:tasks.rados.rados.0.smithi076.stdout:1538: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:44:23.650 INFO:tasks.rados.rados.0.smithi076.stdout:1539: snap_remove snap 187 2024-09-21T09:44:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:23 smithi191 ceph-mon[33301]: pgmap v150: 89 pgs: 89 active+clean; 210 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:44:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:23 smithi191 ceph-mon[33301]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:44:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:23 smithi191 ceph-mon[33301]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:44:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-mon[106482]: pgmap v150: 89 pgs: 89 active+clean; 210 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:44:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-mon[106482]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:44:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-mon[106482]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:44:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-mon[110963]: pgmap v150: 89 pgs: 89 active+clean; 210 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:44:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-mon[110963]: osdmap e456: 8 total, 8 up, 8 in 2024-09-21T09:44:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:23 smithi076 ceph-mon[110963]: osdmap e457: 8 total, 8 up, 8 in 2024-09-21T09:44:24.651 INFO:tasks.rados.rados.0.smithi076.stdout:1539: done (1 left) 2024-09-21T09:44:24.652 INFO:tasks.rados.rados.0.smithi076.stdout:1540: read oid 38 snap -1 2024-09-21T09:44:24.652 INFO:tasks.rados.rados.0.smithi076.stdout:1540: expect (ObjNum 567 snap 175 seq_num 567) 2024-09-21T09:44:24.652 INFO:tasks.rados.rados.0.smithi076.stdout:1541: snap_remove snap 178 2024-09-21T09:44:25.668 INFO:tasks.rados.rados.0.smithi076.stdout:1541: done (2 left) 2024-09-21T09:44:25.668 INFO:tasks.rados.rados.0.smithi076.stdout:1542: read oid 15 snap -1 2024-09-21T09:44:25.668 INFO:tasks.rados.rados.0.smithi076.stdout:1542: expect deleted 2024-09-21T09:44:25.668 INFO:tasks.rados.rados.0.smithi076.stdout:1543: read oid 28 snap 190 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1543: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1544: read oid 7 snap 190 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1544: expect deleted 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1545: read oid 29 snap 179 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1545: expect deleted 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1546: read oid 21 snap -1 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1546: expect (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1547: read oid 46 snap 190 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1547: expect deleted 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1548: read oid 25 snap -1 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1548: expect (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1549: delete oid 47 current snap is 191 2024-09-21T09:44:25.669 INFO:tasks.rados.rados.0.smithi076.stdout:1538: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:44:25.809 INFO:tasks.rados.rados.0.smithi076.stdout:1540: expect (ObjNum 567 snap 175 seq_num 567) 2024-09-21T09:44:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:1548: expect (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:44:26.092 INFO:tasks.rados.rados.0.smithi076.stdout:1546: expect (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:44:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:25 smithi191 ceph-mon[33301]: pgmap v153: 89 pgs: 89 active+clean; 210 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:44:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:25 smithi191 ceph-mon[33301]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:44:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:25 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:26.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:25 smithi076 ceph-mon[106482]: pgmap v153: 89 pgs: 89 active+clean; 210 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:44:26.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:25 smithi076 ceph-mon[106482]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:44:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:25 smithi076 ceph-mon[110963]: pgmap v153: 89 pgs: 89 active+clean; 210 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:44:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:25 smithi076 ceph-mon[110963]: osdmap e458: 8 total, 8 up, 8 in 2024-09-21T09:44:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:26.363 INFO:tasks.rados.rados.0.smithi076.stdout:1543: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1538: done (9 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1540: done (8 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1542: done (7 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1543: done (6 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1544: done (5 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1545: done (4 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1546: done (3 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1547: done (2 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1548: done (1 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1549: done (0 left) 2024-09-21T09:44:26.458 INFO:tasks.rados.rados.0.smithi076.stdout:1550: read oid 10 snap -1 2024-09-21T09:44:26.459 INFO:tasks.rados.rados.0.smithi076.stdout:1550: expect (ObjNum 576 snap 183 seq_num 576) 2024-09-21T09:44:26.459 INFO:tasks.rados.rados.0.smithi076.stdout:1551: write oid 27 current snap is 191 2024-09-21T09:44:26.459 INFO:tasks.rados.rados.0.smithi076.stdout:1551: seq_num 600 ranges {484200=624934,1860843=223357} 2024-09-21T09:44:26.463 INFO:tasks.rados.rados.0.smithi076.stdout:1551: writing smithi07685715-27 from 484200 to 1109134 tid 1 2024-09-21T09:44:26.469 INFO:tasks.rados.rados.0.smithi076.stdout:1551: writing smithi07685715-27 from 1860843 to 2084200 tid 2 2024-09-21T09:44:26.469 INFO:tasks.rados.rados.0.smithi076.stdout:1552: read oid 7 snap -1 2024-09-21T09:44:26.469 INFO:tasks.rados.rados.0.smithi076.stdout:1552: expect deleted 2024-09-21T09:44:26.469 INFO:tasks.rados.rados.0.smithi076.stdout:1553: write oid 8 current snap is 191 2024-09-21T09:44:26.469 INFO:tasks.rados.rados.0.smithi076.stdout:1553: seq_num 601 ranges {501007=416536,1635678=465329} 2024-09-21T09:44:26.474 INFO:tasks.rados.rados.0.smithi076.stdout:1553: writing smithi07685715-8 from 501007 to 917543 tid 1 2024-09-21T09:44:26.480 INFO:tasks.rados.rados.0.smithi076.stdout:1553: writing smithi07685715-8 from 1635678 to 2101007 tid 2 2024-09-21T09:44:26.481 INFO:tasks.rados.rados.0.smithi076.stdout:1554: delete oid 30 current snap is 191 2024-09-21T09:44:26.481 INFO:tasks.rados.rados.0.smithi076.stdout:1550: expect (ObjNum 576 snap 183 seq_num 576) 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1551: finishing write tid 1 to smithi07685715-27 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1550: done (4 left) 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1554: done (3 left) 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1555: read oid 37 snap -1 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1555: expect (ObjNum 593 snap 189 seq_num 593) 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1556: read oid 3 snap -1 2024-09-21T09:44:26.754 INFO:tasks.rados.rados.0.smithi076.stdout:1556: expect deleted 2024-09-21T09:44:26.755 INFO:tasks.rados.rados.0.smithi076.stdout:1557: snap_remove snap 189 2024-09-21T09:44:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:26 smithi191 ceph-mon[33301]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:44:27.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:26 smithi076 ceph-mon[106482]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:44:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:26 smithi076 ceph-mon[110963]: osdmap e459: 8 total, 8 up, 8 in 2024-09-21T09:44:27.678 INFO:tasks.rados.rados.0.smithi076.stdout:1557: done (5 left) 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1558: delete oid 39 current snap is 191 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: finishing write tid 2 to smithi07685715-27 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: finishing write tid 3 to smithi07685715-27 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: finishing write tid 4 to smithi07685715-27 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: oid 27 updating version 0 to 739 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: oid 27 updating version 739 to 741 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: oid 27 version 741 is already newer than 740 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 741 (ObjNum 600 snap 191 seq_num 600) dirty exists 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1551: left oid 27 (ObjNum 600 snap 191 seq_num 600) 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1553: finishing write tid 1 to smithi07685715-8 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1553: finishing write tid 2 to smithi07685715-8 2024-09-21T09:44:27.679 INFO:tasks.rados.rados.0.smithi076.stdout:1553: finishing write tid 3 to smithi07685715-8 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:1553: finishing write tid 4 to smithi07685715-8 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:1553: oid 8 updating version 0 to 1297 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:1553: oid 8 updating version 1297 to 1299 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:1553: oid 8 version 1299 is already newer than 1298 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1299 (ObjNum 601 snap 191 seq_num 601) dirty exists 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:1553: left oid 8 (ObjNum 601 snap 191 seq_num 601) 2024-09-21T09:44:27.680 INFO:tasks.rados.rados.0.smithi076.stdout:1555: expect (ObjNum 593 snap 189 seq_num 593) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1551: done (5 left) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1552: done (4 left) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1553: done (3 left) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1555: done (2 left) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1556: done (1 left) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1558: done (0 left) 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1559: write oid 29 current snap is 191 2024-09-21T09:44:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:1559: seq_num 604 ranges {551428=642283,1694771=456657} 2024-09-21T09:44:27.938 INFO:tasks.rados.rados.0.smithi076.stdout:1559: writing smithi07685715-29 from 551428 to 1193711 tid 1 2024-09-21T09:44:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:1559: writing smithi07685715-29 from 1694771 to 2151428 tid 2 2024-09-21T09:44:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:1560: snap_remove snap 190 2024-09-21T09:44:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:27 smithi191 ceph-mon[33301]: pgmap v156: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:44:28.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:27 smithi076 ceph-mon[106482]: pgmap v156: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:44:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:27 smithi076 ceph-mon[110963]: pgmap v156: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 10 op/s 2024-09-21T09:44:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1560: done (1 left) 2024-09-21T09:44:28.651 INFO:tasks.rados.rados.0.smithi076.stdout:1561: read oid 46 snap -1 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1561: expect deleted 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1562: rollback oid 22 current snap is 191 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 175 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: finishing write tid 1 to smithi07685715-29 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: finishing write tid 2 to smithi07685715-29 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: finishing write tid 3 to smithi07685715-29 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: finishing write tid 4 to smithi07685715-29 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: oid 29 updating version 0 to 428 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: oid 29 version 428 is already newer than 427 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: oid 29 version 428 is already newer than 426 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 428 (ObjNum 604 snap 191 seq_num 604) dirty exists 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: left oid 29 (ObjNum 604 snap 191 seq_num 604) 2024-09-21T09:44:28.652 INFO:tasks.rados.rados.0.smithi076.stdout:1559: done (2 left) 2024-09-21T09:44:28.653 INFO:tasks.rados.rados.0.smithi076.stdout:1563: read oid 29 snap -1 2024-09-21T09:44:28.653 INFO:tasks.rados.rados.0.smithi076.stdout:1563: expect (ObjNum 604 snap 191 seq_num 604) 2024-09-21T09:44:28.653 INFO:tasks.rados.rados.0.smithi076.stdout:1564: read oid 12 snap -1 2024-09-21T09:44:28.653 INFO:tasks.rados.rados.0.smithi076.stdout:1564: expect (ObjNum 575 snap 182 seq_num 575) 2024-09-21T09:44:28.653 INFO:tasks.rados.rados.0.smithi076.stdout:1565: snap_create 2024-09-21T09:44:28.661 INFO:tasks.rados.rados.0.smithi076.stdout:1562: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:44:28.661 INFO:tasks.rados.rados.0.smithi076.stdout:1562: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:44:28.661 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 747 (ObjNum 568 snap 175 seq_num 568) dirty exists 2024-09-21T09:44:28.661 INFO:tasks.rados.rados.0.smithi076.stdout:1564: expect (ObjNum 575 snap 182 seq_num 575) 2024-09-21T09:44:28.829 INFO:tasks.rados.rados.0.smithi076.stdout:1563: expect (ObjNum 604 snap 191 seq_num 604) 2024-09-21T09:44:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:28 smithi191 ceph-mon[33301]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:44:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:28 smithi191 ceph-mon[33301]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:44:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:28 smithi076 ceph-mon[106482]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:44:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:28 smithi076 ceph-mon[106482]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:44:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:28 smithi076 ceph-mon[110963]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:44:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:28 smithi076 ceph-mon[110963]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:44:29.652 INFO:tasks.rados.rados.0.smithi076.stdout:1561: done (4 left) 2024-09-21T09:44:29.652 INFO:tasks.rados.rados.0.smithi076.stdout:1562: done (3 left) 2024-09-21T09:44:29.652 INFO:tasks.rados.rados.0.smithi076.stdout:1563: done (2 left) 2024-09-21T09:44:29.653 INFO:tasks.rados.rados.0.smithi076.stdout:1564: done (1 left) 2024-09-21T09:44:29.653 INFO:tasks.rados.rados.0.smithi076.stdout:1565: done (0 left) 2024-09-21T09:44:29.653 INFO:tasks.rados.rados.0.smithi076.stdout:1566: snap_create 2024-09-21T09:44:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:29 smithi191 ceph-mon[33301]: pgmap v158: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:44:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:29 smithi191 ceph-mon[33301]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:44:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:29 smithi076 ceph-mon[106482]: pgmap v158: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:44:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:29 smithi076 ceph-mon[106482]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:44:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:29 smithi076 ceph-mon[110963]: pgmap v158: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:44:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:29 smithi076 ceph-mon[110963]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:44:30.654 INFO:tasks.rados.rados.0.smithi076.stdout:1566: done (0 left) 2024-09-21T09:44:30.654 INFO:tasks.rados.rados.0.smithi076.stdout:1567: rollback oid 6 current snap is 193 2024-09-21T09:44:30.654 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 175 2024-09-21T09:44:30.654 INFO:tasks.rados.rados.0.smithi076.stdout:1568: write oid 32 current snap is 193 2024-09-21T09:44:30.654 INFO:tasks.rados.rados.0.smithi076.stdout:1568: seq_num 605 ranges {568235=717532,1753253=414982} 2024-09-21T09:44:30.660 INFO:tasks.rados.rados.0.smithi076.stdout:1568: writing smithi07685715-32 from 568235 to 1285767 tid 1 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:1568: writing smithi07685715-32 from 1753253 to 2168235 tid 2 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:1569: rollback oid 46 current snap is 193 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 179 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:1567: finishing rollback tid 0 to smithi07685715-6 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:1567: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:1567: finishing rollback tid 2 to smithi07685715-6 2024-09-21T09:44:30.666 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 344 (ObjNum 492 snap 157 seq_num 492) dirty exists 2024-09-21T09:44:30.667 INFO:tasks.rados.rados.0.smithi076.stdout:1568: finishing write tid 1 to smithi07685715-32 2024-09-21T09:44:30.667 INFO:tasks.rados.rados.0.smithi076.stdout:1567: done (2 left) 2024-09-21T09:44:30.667 INFO:tasks.rados.rados.0.smithi076.stdout:1570: read oid 10 snap 192 2024-09-21T09:44:30.667 INFO:tasks.rados.rados.0.smithi076.stdout:1570: expect (ObjNum 576 snap 183 seq_num 576) 2024-09-21T09:44:30.667 INFO:tasks.rados.rados.0.smithi076.stdout:1571: snap_create 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: finishing write tid 2 to smithi07685715-32 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: finishing write tid 3 to smithi07685715-32 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: finishing write tid 4 to smithi07685715-32 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: oid 32 updating version 0 to 639 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: oid 32 updating version 639 to 640 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: oid 32 updating version 640 to 641 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 641 (ObjNum 605 snap 193 seq_num 605) dirty exists 2024-09-21T09:44:30.671 INFO:tasks.rados.rados.0.smithi076.stdout:1568: left oid 32 (ObjNum 605 snap 193 seq_num 605) 2024-09-21T09:44:30.672 INFO:tasks.rados.rados.0.smithi076.stdout:1569: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:44:30.672 INFO:tasks.rados.rados.0.smithi076.stdout:1569: finishing rollback tid 2 to smithi07685715-46 2024-09-21T09:44:30.672 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1275 (ObjNum 482 snap 155 seq_num 482) dirty exists 2024-09-21T09:44:30.672 INFO:tasks.rados.rados.0.smithi076.stdout:1570: expect (ObjNum 576 snap 183 seq_num 576) 2024-09-21T09:44:31.665 INFO:tasks.rados.rados.0.smithi076.stdout:1568: done (3 left) 2024-09-21T09:44:31.665 INFO:tasks.rados.rados.0.smithi076.stdout:1569: done (2 left) 2024-09-21T09:44:31.665 INFO:tasks.rados.rados.0.smithi076.stdout:1570: done (1 left) 2024-09-21T09:44:31.665 INFO:tasks.rados.rados.0.smithi076.stdout:1571: done (0 left) 2024-09-21T09:44:31.665 INFO:tasks.rados.rados.0.smithi076.stdout:1572: read oid 17 snap 175 2024-09-21T09:44:31.665 INFO:tasks.rados.rados.0.smithi076.stdout:1572: expect (ObjNum 538 snap 171 seq_num 538) 2024-09-21T09:44:31.666 INFO:tasks.rados.rados.0.smithi076.stdout:1573: read oid 36 snap -1 2024-09-21T09:44:31.666 INFO:tasks.rados.rados.0.smithi076.stdout:1573: expect deleted 2024-09-21T09:44:31.666 INFO:tasks.rados.rados.0.smithi076.stdout:1574: snap_remove snap 192 2024-09-21T09:44:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:31 smithi191 ceph-mon[33301]: pgmap v161: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:44:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:31 smithi191 ceph-mon[33301]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:44:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:31 smithi076 ceph-mon[106482]: pgmap v161: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:44:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:31 smithi076 ceph-mon[106482]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:44:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:31 smithi076 ceph-mon[110963]: pgmap v161: 89 pgs: 89 active+clean; 155 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:44:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:31 smithi076 ceph-mon[110963]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:44:32.670 INFO:tasks.rados.rados.0.smithi076.stdout:1574: done (2 left) 2024-09-21T09:44:32.671 INFO:tasks.rados.rados.0.smithi076.stdout:1575: snap_remove snap 179 2024-09-21T09:44:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:32 smithi076 ceph-mon[106482]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:44:33.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:32 smithi076 ceph-mon[110963]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:44:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:32 smithi191 ceph-mon[33301]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:44:33.294 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:44:33] "GET /metrics HTTP/1.1" 200 36131 "" "Prometheus/2.51.0" 2024-09-21T09:44:33.651 INFO:tasks.rados.rados.0.smithi076.stdout:1575: done (2 left) 2024-09-21T09:44:33.652 INFO:tasks.rados.rados.0.smithi076.stdout:1576: write oid 7 current snap is 194 2024-09-21T09:44:33.652 INFO:tasks.rados.rados.0.smithi076.stdout:1576: seq_num 606 ranges {585042=792781,2095382=89660} 2024-09-21T09:44:33.660 INFO:tasks.rados.rados.0.smithi076.stdout:1576: writing smithi07685715-7 from 585042 to 1377823 tid 1 2024-09-21T09:44:33.665 INFO:tasks.rados.rados.0.smithi076.stdout:1576: writing smithi07685715-7 from 2095382 to 2185042 tid 2 2024-09-21T09:44:33.665 INFO:tasks.rados.rados.0.smithi076.stdout:1572: expect (ObjNum 538 snap 171 seq_num 538) 2024-09-21T09:44:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:1576: finishing write tid 1 to smithi07685715-7 2024-09-21T09:44:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:1576: finishing write tid 2 to smithi07685715-7 2024-09-21T09:44:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:1576: finishing write tid 3 to smithi07685715-7 2024-09-21T09:44:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:1576: finishing write tid 4 to smithi07685715-7 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1576: oid 7 updating version 0 to 910 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1576: oid 7 updating version 910 to 914 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1576: oid 7 version 914 is already newer than 913 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 914 (ObjNum 606 snap 194 seq_num 606) dirty exists 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1576: left oid 7 (ObjNum 606 snap 194 seq_num 606) 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1572: done (2 left) 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1573: done (1 left) 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1576: done (0 left) 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1577: write oid 4 current snap is 194 2024-09-21T09:44:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:1577: seq_num 607 ranges {601849=468030,1753864=447985} 2024-09-21T09:44:33.774 INFO:tasks.rados.rados.0.smithi076.stdout:1577: writing smithi07685715-4 from 601849 to 1069879 tid 1 2024-09-21T09:44:33.780 INFO:tasks.rados.rados.0.smithi076.stdout:1577: writing smithi07685715-4 from 1753864 to 2201849 tid 2 2024-09-21T09:44:33.780 INFO:tasks.rados.rados.0.smithi076.stdout:1578: write oid 42 current snap is 194 2024-09-21T09:44:33.780 INFO:tasks.rados.rados.0.smithi076.stdout:1577: finishing write tid 1 to smithi07685715-4 2024-09-21T09:44:33.780 INFO:tasks.rados.rados.0.smithi076.stdout:1578: seq_num 608 ranges {618656=543279,1812346=406310} 2024-09-21T09:44:33.786 INFO:tasks.rados.rados.0.smithi076.stdout:1578: writing smithi07685715-42 from 618656 to 1161935 tid 1 2024-09-21T09:44:33.792 INFO:tasks.rados.rados.0.smithi076.stdout:1578: writing smithi07685715-42 from 1812346 to 2218656 tid 2 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1579: read oid 14 snap -1 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1579: expect (ObjNum 596 snap 189 seq_num 596) 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1580: rollback oid 20 current snap is 194 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 175 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: finishing write tid 2 to smithi07685715-4 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: finishing write tid 3 to smithi07685715-4 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: finishing write tid 4 to smithi07685715-4 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: oid 4 updating version 0 to 512 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: oid 4 updating version 512 to 514 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: oid 4 version 514 is already newer than 513 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 514 (ObjNum 607 snap 194 seq_num 607) dirty exists 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: left oid 4 (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1578: finishing write tid 1 to smithi07685715-42 2024-09-21T09:44:33.793 INFO:tasks.rados.rados.0.smithi076.stdout:1577: done (3 left) 2024-09-21T09:44:33.794 INFO:tasks.rados.rados.0.smithi076.stdout:1581: snap_remove snap 193 2024-09-21T09:44:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:33 smithi191 ceph-mon[33301]: pgmap v164: 89 pgs: 89 active+clean; 157 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 968 KiB/s wr, 18 op/s 2024-09-21T09:44:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:33 smithi191 ceph-mon[33301]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:44:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:33 smithi191 ceph-mon[33301]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:44:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-mon[106482]: pgmap v164: 89 pgs: 89 active+clean; 157 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 968 KiB/s wr, 18 op/s 2024-09-21T09:44:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-mon[106482]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:44:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-mon[106482]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:44:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-mon[110963]: pgmap v164: 89 pgs: 89 active+clean; 157 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 968 KiB/s wr, 18 op/s 2024-09-21T09:44:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-mon[110963]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:44:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:33 smithi076 ceph-mon[110963]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:44:34.674 INFO:tasks.rados.rados.0.smithi076.stdout:1581: done (3 left) 2024-09-21T09:44:34.674 INFO:tasks.rados.rados.0.smithi076.stdout:1582: snap_create 2024-09-21T09:44:34.674 INFO:tasks.rados.rados.0.smithi076.stdout:1580: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:44:34.674 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 0 (ObjNum 65 snap 0 seq_num 864388368) dirty dne 2024-09-21T09:44:34.674 INFO:tasks.rados.rados.0.smithi076.stdout:1578: finishing write tid 2 to smithi07685715-42 2024-09-21T09:44:34.674 INFO:tasks.rados.rados.0.smithi076.stdout:1578: finishing write tid 3 to smithi07685715-42 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:1578: finishing write tid 4 to smithi07685715-42 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:1578: oid 42 updating version 0 to 525 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:1578: oid 42 updating version 525 to 526 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:1578: oid 42 updating version 526 to 527 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 527 (ObjNum 608 snap 194 seq_num 608) dirty exists 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:1578: left oid 42 (ObjNum 608 snap 194 seq_num 608) 2024-09-21T09:44:34.675 INFO:tasks.rados.rados.0.smithi076.stdout:1579: expect (ObjNum 596 snap 189 seq_num 596) 2024-09-21T09:44:35.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-21T09:44:35.285 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:35.676 INFO:tasks.rados.rados.0.smithi076.stdout:1578: done (3 left) 2024-09-21T09:44:35.676 INFO:tasks.rados.rados.0.smithi076.stdout:1579: done (2 left) 2024-09-21T09:44:35.676 INFO:tasks.rados.rados.0.smithi076.stdout:1580: done (1 left) 2024-09-21T09:44:35.676 INFO:tasks.rados.rados.0.smithi076.stdout:1582: done (0 left) 2024-09-21T09:44:35.676 INFO:tasks.rados.rados.0.smithi076.stdout:1583: snap_remove snap 194 2024-09-21T09:44:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:35 smithi191 ceph-mon[33301]: pgmap v167: 89 pgs: 89 active+clean; 157 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 968 KiB/s wr, 18 op/s 2024-09-21T09:44:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:35 smithi191 ceph-mon[33301]: osdmap e467: 8 total, 8 up, 8 in 2024-09-21T09:44:36.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:35 smithi076 ceph-mon[106482]: pgmap v167: 89 pgs: 89 active+clean; 157 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 968 KiB/s wr, 18 op/s 2024-09-21T09:44:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:35 smithi076 ceph-mon[106482]: osdmap e467: 8 total, 8 up, 8 in 2024-09-21T09:44:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:35 smithi076 ceph-mon[110963]: pgmap v167: 89 pgs: 89 active+clean; 157 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 968 KiB/s wr, 18 op/s 2024-09-21T09:44:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:35 smithi076 ceph-mon[110963]: osdmap e467: 8 total, 8 up, 8 in 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1583: done (0 left) 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1584: read oid 14 snap -1 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1584: expect (ObjNum 596 snap 189 seq_num 596) 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1585: read oid 5 snap -1 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1585: expect (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1586: rollback oid 23 current snap is 195 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 175 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1587: read oid 1 snap 191 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1587: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:44:36.678 INFO:tasks.rados.rados.0.smithi076.stdout:1588: write oid 22 current snap is 195 2024-09-21T09:44:36.679 INFO:tasks.rados.rados.0.smithi076.stdout:1588: seq_num 609 ranges {635463=734881,1987181=248282} 2024-09-21T09:44:36.687 INFO:tasks.rados.rados.0.smithi076.stdout:1588: writing smithi07685715-22 from 635463 to 1370344 tid 1 2024-09-21T09:44:36.692 INFO:tasks.rados.rados.0.smithi076.stdout:1588: writing smithi07685715-22 from 1987181 to 2235463 tid 2 2024-09-21T09:44:36.692 INFO:tasks.rados.rados.0.smithi076.stdout:1589: write oid 11 current snap is 195 2024-09-21T09:44:36.692 INFO:tasks.rados.rados.0.smithi076.stdout:1589: seq_num 610 ranges {652270=410130,1529310=493867} 2024-09-21T09:44:36.698 INFO:tasks.rados.rados.0.smithi076.stdout:1589: writing smithi07685715-11 from 652270 to 1062400 tid 1 2024-09-21T09:44:36.704 INFO:tasks.rados.rados.0.smithi076.stdout:1589: writing smithi07685715-11 from 1529310 to 2023177 tid 2 2024-09-21T09:44:36.704 INFO:tasks.rados.rados.0.smithi076.stdout:1590: write oid 27 current snap is 195 2024-09-21T09:44:36.704 INFO:tasks.rados.rados.0.smithi076.stdout:1590: seq_num 611 ranges {669077=485379,1587792=681285} 2024-09-21T09:44:36.710 INFO:tasks.rados.rados.0.smithi076.stdout:1590: writing smithi07685715-27 from 669077 to 1154456 tid 1 2024-09-21T09:44:36.717 INFO:tasks.rados.rados.0.smithi076.stdout:1590: writing smithi07685715-27 from 1587792 to 2269077 tid 2 2024-09-21T09:44:36.717 INFO:tasks.rados.rados.0.smithi076.stdout:1591: delete oid 8 current snap is 195 2024-09-21T09:44:36.717 INFO:tasks.rados.rados.0.smithi076.stdout:1586: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:44:36.717 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 0 (ObjNum 609 snap 0 seq_num 1682286601) dirty dne 2024-09-21T09:44:36.717 INFO:tasks.rados.rados.0.smithi076.stdout:1584: expect (ObjNum 596 snap 189 seq_num 596) 2024-09-21T09:44:36.847 INFO:tasks.rados.rados.0.smithi076.stdout:1588: finishing write tid 1 to smithi07685715-22 2024-09-21T09:44:36.847 INFO:tasks.rados.rados.0.smithi076.stdout:1588: finishing write tid 2 to smithi07685715-22 2024-09-21T09:44:36.847 INFO:tasks.rados.rados.0.smithi076.stdout:1588: finishing write tid 3 to smithi07685715-22 2024-09-21T09:44:36.847 INFO:tasks.rados.rados.0.smithi076.stdout:1588: finishing write tid 4 to smithi07685715-22 2024-09-21T09:44:36.847 INFO:tasks.rados.rados.0.smithi076.stdout:1588: oid 22 updating version 0 to 761 2024-09-21T09:44:36.847 INFO:tasks.rados.rados.0.smithi076.stdout:1588: oid 22 version 761 is already newer than 760 2024-09-21T09:44:36.848 INFO:tasks.rados.rados.0.smithi076.stdout:1588: oid 22 version 761 is already newer than 759 2024-09-21T09:44:36.848 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 761 (ObjNum 609 snap 195 seq_num 609) dirty exists 2024-09-21T09:44:36.848 INFO:tasks.rados.rados.0.smithi076.stdout:1588: left oid 22 (ObjNum 609 snap 195 seq_num 609) 2024-09-21T09:44:36.848 INFO:tasks.rados.rados.0.smithi076.stdout:1585: expect (ObjNum 444 snap 141 seq_num 444) 2024-09-21T09:44:36.969 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:36 smithi076 ceph-mon[106482]: osdmap e468: 8 total, 8 up, 8 in 2024-09-21T09:44:36.969 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:36 smithi076 ceph-mon[106482]: osdmap e469: 8 total, 8 up, 8 in 2024-09-21T09:44:36.970 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:36 smithi076 ceph-mon[110963]: osdmap e468: 8 total, 8 up, 8 in 2024-09-21T09:44:36.970 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:36 smithi076 ceph-mon[110963]: osdmap e469: 8 total, 8 up, 8 in 2024-09-21T09:44:36.970 INFO:teuthology.orchestra.run.smithi076.stdout:wait for majority of mons to upgrade 2024-09-21T09:44:37.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:36 smithi191 ceph-mon[33301]: osdmap e468: 8 total, 8 up, 8 in 2024-09-21T09:44:37.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:36 smithi191 ceph-mon[33301]: osdmap e469: 8 total, 8 up, 8 in 2024-09-21T09:44:37.035 INFO:tasks.rados.rados.0.smithi076.stdout:1587: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:44:37.155 INFO:tasks.rados.rados.0.smithi076.stdout:1589: finishing write tid 1 to smithi07685715-11 2024-09-21T09:44:37.155 INFO:tasks.rados.rados.0.smithi076.stdout:1589: finishing write tid 2 to smithi07685715-11 2024-09-21T09:44:37.155 INFO:tasks.rados.rados.0.smithi076.stdout:1589: finishing write tid 3 to smithi07685715-11 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1589: finishing write tid 4 to smithi07685715-11 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1589: oid 11 updating version 0 to 1286 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1589: oid 11 version 1286 is already newer than 1285 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1589: oid 11 version 1286 is already newer than 1284 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 1286 (ObjNum 610 snap 195 seq_num 610) dirty exists 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1589: left oid 11 (ObjNum 610 snap 195 seq_num 610) 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1590: finishing write tid 1 to smithi07685715-27 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1584: done (7 left) 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1585: done (6 left) 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1586: done (5 left) 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1587: done (4 left) 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1588: done (3 left) 2024-09-21T09:44:37.156 INFO:tasks.rados.rados.0.smithi076.stdout:1589: done (2 left) 2024-09-21T09:44:37.157 INFO:tasks.rados.rados.0.smithi076.stdout:1591: done (1 left) 2024-09-21T09:44:37.157 INFO:tasks.rados.rados.0.smithi076.stdout:1592: write oid 20 current snap is 195 2024-09-21T09:44:37.157 INFO:tasks.rados.rados.0.smithi076.stdout:1590: finishing write tid 2 to smithi07685715-27 2024-09-21T09:44:37.157 INFO:tasks.rados.rados.0.smithi076.stdout:1590: finishing write tid 3 to smithi07685715-27 2024-09-21T09:44:37.157 INFO:tasks.rados.rados.0.smithi076.stdout:1592: seq_num 613 ranges {702691=635877,2104756=197935} 2024-09-21T09:44:37.161 INFO:tasks.rados.rados.0.smithi076.stdout:1592: writing smithi07685715-20 from 702691 to 1338568 tid 1 2024-09-21T09:44:37.167 INFO:tasks.rados.rados.0.smithi076.stdout:1592: writing smithi07685715-20 from 2104756 to 2302691 tid 2 2024-09-21T09:44:37.167 INFO:tasks.rados.rados.0.smithi076.stdout:1593: read oid 44 snap 191 2024-09-21T09:44:37.167 INFO:tasks.rados.rados.0.smithi076.stdout:1593: expect (ObjNum 580 snap 184 seq_num 580) 2024-09-21T09:44:37.167 INFO:tasks.rados.rados.0.smithi076.stdout:1594: delete oid 5 current snap is 195 2024-09-21T09:44:37.167 INFO:tasks.rados.rados.0.smithi076.stdout:1590: finishing write tid 4 to smithi07685715-27 2024-09-21T09:44:37.168 INFO:tasks.rados.rados.0.smithi076.stdout:1590: oid 27 updating version 0 to 764 2024-09-21T09:44:37.168 INFO:tasks.rados.rados.0.smithi076.stdout:1590: oid 27 updating version 764 to 765 2024-09-21T09:44:37.168 INFO:tasks.rados.rados.0.smithi076.stdout:1590: oid 27 version 765 is already newer than 763 2024-09-21T09:44:37.168 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 765 (ObjNum 611 snap 195 seq_num 611) dirty exists 2024-09-21T09:44:37.168 INFO:tasks.rados.rados.0.smithi076.stdout:1590: left oid 27 (ObjNum 611 snap 195 seq_num 611) 2024-09-21T09:44:37.168 INFO:tasks.rados.rados.0.smithi076.stdout:1592: finishing write tid 1 to smithi07685715-20 2024-09-21T09:44:37.170 INFO:tasks.rados.rados.0.smithi076.stdout:1592: finishing write tid 2 to smithi07685715-20 2024-09-21T09:44:37.170 INFO:tasks.rados.rados.0.smithi076.stdout:1590: done (3 left) 2024-09-21T09:44:37.170 INFO:tasks.rados.rados.0.smithi076.stdout:1594: done (2 left) 2024-09-21T09:44:37.170 INFO:tasks.rados.rados.0.smithi076.stdout:1595: write oid 11 current snap is 195 2024-09-21T09:44:37.170 INFO:tasks.rados.rados.0.smithi076.stdout:1595: seq_num 615 ranges {736305=786375,2105367=230938} 2024-09-21T09:44:37.178 INFO:tasks.rados.rados.0.smithi076.stdout:1595: writing smithi07685715-11 from 736305 to 1522680 tid 1 2024-09-21T09:44:37.182 INFO:tasks.rados.rados.0.smithi076.stdout:1595: writing smithi07685715-11 from 2105367 to 2336305 tid 2 2024-09-21T09:44:37.183 INFO:tasks.rados.rados.0.smithi076.stdout:1596: snap_remove snap 175 2024-09-21T09:44:37.701 INFO:tasks.rados.rados.0.smithi076.stdout:1596: done (3 left) 2024-09-21T09:44:37.701 INFO:tasks.rados.rados.0.smithi076.stdout:1597: delete oid 36 current snap is 195 2024-09-21T09:44:37.701 INFO:tasks.rados.rados.0.smithi076.stdout:1592: finishing write tid 3 to smithi07685715-20 2024-09-21T09:44:37.701 INFO:tasks.rados.rados.0.smithi076.stdout:1592: finishing write tid 4 to smithi07685715-20 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:1592: oid 20 updating version 0 to 433 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:1592: oid 20 updating version 433 to 434 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:1592: oid 20 updating version 434 to 435 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 435 (ObjNum 613 snap 195 seq_num 613) dirty exists 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:1592: left oid 20 (ObjNum 613 snap 195 seq_num 613) 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:1595: finishing write tid 1 to smithi07685715-11 2024-09-21T09:44:37.702 INFO:tasks.rados.rados.0.smithi076.stdout:1593: expect (ObjNum 580 snap 184 seq_num 580) 2024-09-21T09:44:37.824 INFO:tasks.rados.rados.0.smithi076.stdout:1595: finishing write tid 2 to smithi07685715-11 2024-09-21T09:44:37.824 INFO:tasks.rados.rados.0.smithi076.stdout:1595: finishing write tid 3 to smithi07685715-11 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1595: finishing write tid 4 to smithi07685715-11 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1595: oid 11 updating version 0 to 1289 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1595: oid 11 updating version 1289 to 1291 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1595: oid 11 version 1291 is already newer than 1290 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 1291 (ObjNum 615 snap 195 seq_num 615) dirty exists 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1595: left oid 11 (ObjNum 615 snap 195 seq_num 615) 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1592: done (3 left) 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1593: done (2 left) 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1595: done (1 left) 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1597: done (0 left) 2024-09-21T09:44:37.825 INFO:tasks.rados.rados.0.smithi076.stdout:1598: snap_remove snap 191 2024-09-21T09:44:37.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-21T09:44:38.073 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:37 smithi191 ceph-mon[33301]: pgmap v170: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 672 KiB/s wr, 8 op/s 2024-09-21T09:44:38.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:37 smithi076 ceph-mon[106482]: pgmap v170: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 672 KiB/s wr, 8 op/s 2024-09-21T09:44:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:37 smithi076 ceph-mon[110963]: pgmap v170: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 672 KiB/s wr, 8 op/s 2024-09-21T09:44:38.657 INFO:tasks.rados.rados.0.smithi076.stdout:1598: done (0 left) 2024-09-21T09:44:38.658 INFO:tasks.rados.rados.0.smithi076.stdout:1599: snap_create 2024-09-21T09:44:38.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:38 smithi076 ceph-mon[106482]: osdmap e470: 8 total, 8 up, 8 in 2024-09-21T09:44:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:38 smithi076 ceph-mon[106482]: osdmap e471: 8 total, 8 up, 8 in 2024-09-21T09:44:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:38 smithi076 ceph-mon[110963]: osdmap e470: 8 total, 8 up, 8 in 2024-09-21T09:44:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:38 smithi076 ceph-mon[110963]: osdmap e471: 8 total, 8 up, 8 in 2024-09-21T09:44:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:38 smithi191 ceph-mon[33301]: osdmap e470: 8 total, 8 up, 8 in 2024-09-21T09:44:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:38 smithi191 ceph-mon[33301]: osdmap e471: 8 total, 8 up, 8 in 2024-09-21T09:44:39.654 INFO:tasks.rados.rados.0.smithi076.stdout:1599: done (0 left) 2024-09-21T09:44:39.654 INFO:tasks.rados.rados.0.smithi076.stdout:1600: rollback oid 25 current snap is 196 2024-09-21T09:44:39.654 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 195 2024-09-21T09:44:39.654 INFO:tasks.rados.rados.0.smithi076.stdout:1601: rollback oid 38 current snap is 196 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 195 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:1602: rollback oid 43 current snap is 196 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 195 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:1603: rollback oid 6 current snap is 196 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 195 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:1604: read oid 2 snap -1 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:1604: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:44:39.655 INFO:tasks.rados.rados.0.smithi076.stdout:1605: delete oid 9 current snap is 196 2024-09-21T09:44:39.659 INFO:tasks.rados.rados.0.smithi076.stdout:1604: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1602: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1602: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1601: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1602: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 296 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1600: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1600: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:44:39.775 INFO:tasks.rados.rados.0.smithi076.stdout:1602: done (5 left) 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1604: done (4 left) 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1605: done (3 left) 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1606: write oid 13 current snap is 196 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1600: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 338 (ObjNum 513 snap 163 seq_num 513) dirty exists 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1601: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1601: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 926 (ObjNum 567 snap 175 seq_num 567) dirty exists 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1603: finishing rollback tid 0 to smithi07685715-6 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1603: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1603: finishing rollback tid 2 to smithi07685715-6 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 356 (ObjNum 492 snap 157 seq_num 492) dirty exists 2024-09-21T09:44:39.776 INFO:tasks.rados.rados.0.smithi076.stdout:1606: seq_num 618 ranges {786726=728475,2280813=105913} 2024-09-21T09:44:39.783 INFO:tasks.rados.rados.0.smithi076.stdout:1606: writing smithi07685715-13 from 786726 to 1515201 tid 1 2024-09-21T09:44:39.788 INFO:tasks.rados.rados.0.smithi076.stdout:1606: writing smithi07685715-13 from 2280813 to 2386726 tid 2 2024-09-21T09:44:39.788 INFO:tasks.rados.rados.0.smithi076.stdout:1600: done (3 left) 2024-09-21T09:44:39.788 INFO:tasks.rados.rados.0.smithi076.stdout:1601: done (2 left) 2024-09-21T09:44:39.788 INFO:tasks.rados.rados.0.smithi076.stdout:1603: done (1 left) 2024-09-21T09:44:39.789 INFO:tasks.rados.rados.0.smithi076.stdout:1607: rollback oid 3 current snap is 196 2024-09-21T09:44:39.789 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 195 2024-09-21T09:44:39.789 INFO:tasks.rados.rados.0.smithi076.stdout:1606: finishing write tid 1 to smithi07685715-13 2024-09-21T09:44:39.789 INFO:tasks.rados.rados.0.smithi076.stdout:1608: snap_create 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:1606: finishing write tid 2 to smithi07685715-13 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:1607: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 305 snap 0 seq_num 0) dirty dne 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:1606: finishing write tid 3 to smithi07685715-13 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:1606: finishing write tid 4 to smithi07685715-13 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:1606: oid 13 updating version 0 to 262 2024-09-21T09:44:39.790 INFO:tasks.rados.rados.0.smithi076.stdout:1606: oid 13 updating version 262 to 263 2024-09-21T09:44:39.791 INFO:tasks.rados.rados.0.smithi076.stdout:1606: oid 13 updating version 263 to 264 2024-09-21T09:44:39.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 264 (ObjNum 618 snap 196 seq_num 618) dirty exists 2024-09-21T09:44:39.791 INFO:tasks.rados.rados.0.smithi076.stdout:1606: left oid 13 (ObjNum 618 snap 196 seq_num 618) 2024-09-21T09:44:39.794 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:39 smithi076 ceph-mon[106482]: pgmap v173: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 672 KiB/s wr, 8 op/s 2024-09-21T09:44:39.795 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:39 smithi076 ceph-mon[106482]: osdmap e472: 8 total, 8 up, 8 in 2024-09-21T09:44:39.795 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:39 smithi076 ceph-mon[110963]: pgmap v173: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 672 KiB/s wr, 8 op/s 2024-09-21T09:44:39.795 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:39 smithi076 ceph-mon[110963]: osdmap e472: 8 total, 8 up, 8 in 2024-09-21T09:44:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:39 smithi191 ceph-mon[33301]: pgmap v173: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 672 KiB/s wr, 8 op/s 2024-09-21T09:44:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:39 smithi191 ceph-mon[33301]: osdmap e472: 8 total, 8 up, 8 in 2024-09-21T09:44:40.715 INFO:tasks.rados.rados.0.smithi076.stdout:1606: done (2 left) 2024-09-21T09:44:40.715 INFO:tasks.rados.rados.0.smithi076.stdout:1607: done (1 left) 2024-09-21T09:44:40.715 INFO:tasks.rados.rados.0.smithi076.stdout:1608: done (0 left) 2024-09-21T09:44:40.715 INFO:tasks.rados.rados.0.smithi076.stdout:1609: read oid 25 snap -1 2024-09-21T09:44:40.715 INFO:tasks.rados.rados.0.smithi076.stdout:1609: expect (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:44:40.715 INFO:tasks.rados.rados.0.smithi076.stdout:1610: snap_create 2024-09-21T09:44:40.717 INFO:tasks.rados.rados.0.smithi076.stdout:1609: expect (ObjNum 513 snap 163 seq_num 513) 2024-09-21T09:44:40.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:40 smithi076 ceph-mon[106482]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:40.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:40.959 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:40 smithi076 ceph-mon[110963]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:40.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:40 smithi191 ceph-mon[33301]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:40 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:41.716 INFO:tasks.rados.rados.0.smithi076.stdout:1609: done (1 left) 2024-09-21T09:44:41.716 INFO:tasks.rados.rados.0.smithi076.stdout:1610: done (0 left) 2024-09-21T09:44:41.716 INFO:tasks.rados.rados.0.smithi076.stdout:1611: delete oid 41 current snap is 198 2024-09-21T09:44:41.722 INFO:tasks.rados.rados.0.smithi076.stdout:1611: done (0 left) 2024-09-21T09:44:41.722 INFO:tasks.rados.rados.0.smithi076.stdout:1612: write oid 41 current snap is 198 2024-09-21T09:44:41.722 INFO:tasks.rados.rados.0.smithi076.stdout:1612: seq_num 620 ranges {420340=478973,1597777=512212} 2024-09-21T09:44:41.727 INFO:tasks.rados.rados.0.smithi076.stdout:1612: writing smithi07685715-41 from 420340 to 899313 tid 1 2024-09-21T09:44:41.734 INFO:tasks.rados.rados.0.smithi076.stdout:1612: writing smithi07685715-41 from 1597777 to 2109989 tid 2 2024-09-21T09:44:41.734 INFO:tasks.rados.rados.0.smithi076.stdout:1613: rollback oid 43 current snap is 198 2024-09-21T09:44:41.734 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 197 2024-09-21T09:44:41.734 INFO:tasks.rados.rados.0.smithi076.stdout:1612: finishing write tid 1 to smithi07685715-41 2024-09-21T09:44:41.734 INFO:tasks.rados.rados.0.smithi076.stdout:1614: snap_create 2024-09-21T09:44:41.736 INFO:tasks.rados.rados.0.smithi076.stdout:1613: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:44:41.737 INFO:tasks.rados.rados.0.smithi076.stdout:1613: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:44:41.737 INFO:tasks.rados.rados.0.smithi076.stdout:1613: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:44:41.737 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 300 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: finishing write tid 2 to smithi07685715-41 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: finishing write tid 3 to smithi07685715-41 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: finishing write tid 4 to smithi07685715-41 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: oid 41 updating version 0 to 782 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: oid 41 version 782 is already newer than 780 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: oid 41 version 782 is already newer than 781 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 782 (ObjNum 620 snap 198 seq_num 620) dirty exists 2024-09-21T09:44:41.738 INFO:tasks.rados.rados.0.smithi076.stdout:1612: left oid 41 (ObjNum 620 snap 198 seq_num 620) 2024-09-21T09:44:41.829 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:41 smithi076 ceph-mon[106482]: pgmap v176: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:44:41.830 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:41 smithi076 ceph-mon[106482]: osdmap e473: 8 total, 8 up, 8 in 2024-09-21T09:44:41.830 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:41 smithi076 ceph-mon[110963]: pgmap v176: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:44:41.830 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:41 smithi076 ceph-mon[110963]: osdmap e473: 8 total, 8 up, 8 in 2024-09-21T09:44:41.862 INFO:teuthology.orchestra.run.smithi076.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:41 smithi191 ceph-mon[33301]: pgmap v176: 89 pgs: 89 active+clean; 160 MiB data, 657 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:44:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:41 smithi191 ceph-mon[33301]: osdmap e473: 8 total, 8 up, 8 in 2024-09-21T09:44:42.662 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:44:42.732 INFO:tasks.rados.rados.0.smithi076.stdout:1612: done (2 left) 2024-09-21T09:44:42.732 INFO:tasks.rados.rados.0.smithi076.stdout:1613: done (1 left) 2024-09-21T09:44:42.732 INFO:tasks.rados.rados.0.smithi076.stdout:1614: done (0 left) 2024-09-21T09:44:42.732 INFO:tasks.rados.rados.0.smithi076.stdout:1615: write oid 37 current snap is 199 2024-09-21T09:44:42.732 INFO:tasks.rados.rados.0.smithi076.stdout:1615: seq_num 621 ranges {437147=554222,1656259=655870} 2024-09-21T09:44:42.738 INFO:tasks.rados.rados.0.smithi076.stdout:1615: writing smithi07685715-37 from 437147 to 991369 tid 1 2024-09-21T09:44:42.745 INFO:tasks.rados.rados.0.smithi076.stdout:1615: writing smithi07685715-37 from 1656259 to 2312129 tid 2 2024-09-21T09:44:42.746 INFO:tasks.rados.rados.0.smithi076.stdout:1616: snap_remove snap 196 2024-09-21T09:44:42.876 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:42.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: osdmap e474: 8 total, 8 up, 8 in 2024-09-21T09:44:42.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:42.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:42.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:42.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:42.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[106482]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: osdmap e474: 8 total, 8 up, 8 in 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:42.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:42 smithi076 ceph-mon[110963]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: osdmap e474: 8 total, 8 up, 8 in 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:42 smithi191 ceph-mon[33301]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:44:43.573 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:44:43] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.51.0" 2024-09-21T09:44:43.654 INFO:tasks.rados.rados.0.smithi076.stdout:1616: done (1 left) 2024-09-21T09:44:43.655 INFO:tasks.rados.rados.0.smithi076.stdout:1617: read oid 48 snap -1 2024-09-21T09:44:43.655 INFO:tasks.rados.rados.0.smithi076.stdout:1617: expect deleted 2024-09-21T09:44:43.655 INFO:tasks.rados.rados.0.smithi076.stdout:1618: read oid 19 snap -1 2024-09-21T09:44:43.655 INFO:tasks.rados.rados.0.smithi076.stdout:1618: expect deleted 2024-09-21T09:44:43.655 INFO:tasks.rados.rados.0.smithi076.stdout:1619: snap_remove snap 195 2024-09-21T09:44:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-mon[106482]: pgmap v179: 89 pgs: 89 active+clean; 102 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 23 op/s 2024-09-21T09:44:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-mon[106482]: osdmap e475: 8 total, 8 up, 8 in 2024-09-21T09:44:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-mon[106482]: osdmap e476: 8 total, 8 up, 8 in 2024-09-21T09:44:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-mon[110963]: pgmap v179: 89 pgs: 89 active+clean; 102 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 23 op/s 2024-09-21T09:44:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-mon[110963]: osdmap e475: 8 total, 8 up, 8 in 2024-09-21T09:44:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:43 smithi076 ceph-mon[110963]: osdmap e476: 8 total, 8 up, 8 in 2024-09-21T09:44:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:43 smithi191 ceph-mon[33301]: pgmap v179: 89 pgs: 89 active+clean; 102 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 23 op/s 2024-09-21T09:44:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:43 smithi191 ceph-mon[33301]: osdmap e475: 8 total, 8 up, 8 in 2024-09-21T09:44:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:43 smithi191 ceph-mon[33301]: osdmap e476: 8 total, 8 up, 8 in 2024-09-21T09:44:44.656 INFO:tasks.rados.rados.0.smithi076.stdout:1619: done (3 left) 2024-09-21T09:44:44.656 INFO:tasks.rados.rados.0.smithi076.stdout:1620: write oid 41 current snap is 199 2024-09-21T09:44:44.656 INFO:tasks.rados.rados.0.smithi076.stdout:1620: seq_num 622 ranges {453954=629471,1598388=515881} 2024-09-21T09:44:44.661 INFO:tasks.rados.rados.0.smithi076.stdout:1620: writing smithi07685715-41 from 453954 to 1083425 tid 1 2024-09-21T09:44:44.667 INFO:tasks.rados.rados.0.smithi076.stdout:1620: writing smithi07685715-41 from 1598388 to 2114269 tid 2 2024-09-21T09:44:44.667 INFO:tasks.rados.rados.0.smithi076.stdout:1621: snap_remove snap 198 2024-09-21T09:44:45.660 INFO:tasks.rados.rados.0.smithi076.stdout:1621: done (4 left) 2024-09-21T09:44:45.660 INFO:tasks.rados.rados.0.smithi076.stdout:1622: read oid 1 snap -1 2024-09-21T09:44:45.660 INFO:tasks.rados.rados.0.smithi076.stdout:1622: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:44:45.660 INFO:tasks.rados.rados.0.smithi076.stdout:1623: snap_create 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: finishing write tid 1 to smithi07685715-37 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: finishing write tid 2 to smithi07685715-37 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: finishing write tid 3 to smithi07685715-37 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: finishing write tid 4 to smithi07685715-37 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: oid 37 updating version 0 to 441 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: oid 37 version 441 is already newer than 440 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: oid 37 version 441 is already newer than 439 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 441 (ObjNum 621 snap 199 seq_num 621) dirty exists 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1615: left oid 37 (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1620: finishing write tid 1 to smithi07685715-41 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1620: finishing write tid 2 to smithi07685715-41 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1620: finishing write tid 3 to smithi07685715-41 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1620: finishing write tid 4 to smithi07685715-41 2024-09-21T09:44:45.661 INFO:tasks.rados.rados.0.smithi076.stdout:1620: oid 41 updating version 0 to 789 2024-09-21T09:44:45.662 INFO:tasks.rados.rados.0.smithi076.stdout:1620: oid 41 version 789 is already newer than 788 2024-09-21T09:44:45.662 INFO:tasks.rados.rados.0.smithi076.stdout:1620: oid 41 updating version 789 to 790 2024-09-21T09:44:45.662 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 790 (ObjNum 622 snap 199 seq_num 622) dirty exists 2024-09-21T09:44:45.662 INFO:tasks.rados.rados.0.smithi076.stdout:1620: left oid 41 (ObjNum 622 snap 199 seq_num 622) 2024-09-21T09:44:45.670 INFO:tasks.rados.rados.0.smithi076.stdout:1622: expect (ObjNum 512 snap 162 seq_num 512) 2024-09-21T09:44:45.928 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: pgmap v182: 89 pgs: 89 active+clean; 102 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 23 op/s 2024-09-21T09:44:45.928 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: pgmap v182: 89 pgs: 89 active+clean; 102 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 23 op/s 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:45.929 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:45.930 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:45.930 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:44:45.930 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: pgmap v182: 89 pgs: 89 active+clean; 102 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 23 op/s 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T09:44:45.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:45 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T09:44:46.125 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:44:46.663 INFO:tasks.rados.rados.0.smithi076.stdout:1615: done (5 left) 2024-09-21T09:44:46.663 INFO:tasks.rados.rados.0.smithi076.stdout:1617: done (4 left) 2024-09-21T09:44:46.663 INFO:tasks.rados.rados.0.smithi076.stdout:1618: done (3 left) 2024-09-21T09:44:46.663 INFO:tasks.rados.rados.0.smithi076.stdout:1620: done (2 left) 2024-09-21T09:44:46.663 INFO:tasks.rados.rados.0.smithi076.stdout:1622: done (1 left) 2024-09-21T09:44:46.664 INFO:tasks.rados.rados.0.smithi076.stdout:1623: done (0 left) 2024-09-21T09:44:46.664 INFO:tasks.rados.rados.0.smithi076.stdout:1624: snap_remove snap 197 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (3m) 35s ago 9m 19.6M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (2m) 34s ago 9m 80.4M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (8m) 35s ago 7m 19.7M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (7m) 34s ago 7m 16.9M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (4m) 34s ago 13m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (4m) 35s ago 14m 510M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (2m) 35s ago 14m 51.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (13m) 34s ago 13m 58.4M 2048M 18.2.4-871-g6a1bdf85 0adab276c97e dcb99d4acf72 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (112s) 35s ago 13m 38.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:44:46.696 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (4m) 35s ago 8m 12.2M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (3m) 34s ago 8m 10.8M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (12m) 35s ago 12m 106M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (12m) 35s ago 12m 83.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (12m) 35s ago 12m 104M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (11m) 35s ago 11m 79.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (11m) 34s ago 11m 96.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (10m) 34s ago 10m 108M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (10m) 34s ago 10m 76.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (10m) 34s ago 10m 78.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:44:46.697 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (3m) 34s ago 9m 43.6M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:44:46.858 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:44:46.858 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:44:46.859 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: Upgrade: Setting container_image for all mgr 2024-09-21T09:44:46.859 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:44:46.859 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:44:46.859 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:46.859 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:46 smithi191 ceph-mon[33301]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 1, 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:44:47.060 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 11, 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:44:47.061 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:44:47.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:44:47.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:44:47.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all mgr 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[110963]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all mgr 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:47.062 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:46 smithi076 ceph-mon[106482]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:44:47.351 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:44:47.351 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:44:47.351 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:44:47.351 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-21T09:44:47.351 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:44:47.352 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "2/3 daemons upgraded", 2024-09-21T09:44:47.352 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:44:47.352 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:44:47.352 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:44:47.660 INFO:tasks.rados.rados.0.smithi076.stdout:1624: done (0 left) 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1625: read oid 34 snap -1 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1625: expect (ObjNum 557 snap 174 seq_num 557) 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1626: read oid 35 snap -1 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1626: expect (ObjNum 577 snap 183 seq_num 577) 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1627: read oid 30 snap -1 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1627: expect deleted 2024-09-21T09:44:47.661 INFO:tasks.rados.rados.0.smithi076.stdout:1628: delete oid 14 current snap is 200 2024-09-21T09:44:47.665 INFO:tasks.rados.rados.0.smithi076.stdout:1625: expect (ObjNum 557 snap 174 seq_num 557) 2024-09-21T09:44:47.872 INFO:tasks.rados.rados.0.smithi076.stdout:1626: expect (ObjNum 577 snap 183 seq_num 577) 2024-09-21T09:44:48.069 INFO:tasks.rados.rados.0.smithi076.stdout:1625: done (3 left) 2024-09-21T09:44:48.069 INFO:tasks.rados.rados.0.smithi076.stdout:1626: done (2 left) 2024-09-21T09:44:48.069 INFO:tasks.rados.rados.0.smithi076.stdout:1627: done (1 left) 2024-09-21T09:44:48.069 INFO:tasks.rados.rados.0.smithi076.stdout:1628: done (0 left) 2024-09-21T09:44:48.069 INFO:tasks.rados.rados.0.smithi076.stdout:1629: delete oid 11 current snap is 200 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1629: done (0 left) 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1630: read oid 13 snap -1 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1630: expect (ObjNum 618 snap 196 seq_num 618) 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1631: read oid 46 snap -1 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1631: expect (ObjNum 482 snap 155 seq_num 482) 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1632: write oid 39 current snap is 200 2024-09-21T09:44:48.072 INFO:tasks.rados.rados.0.smithi076.stdout:1632: seq_num 625 ranges {504375=571571,1490187=663208} 2024-09-21T09:44:48.078 INFO:tasks.rados.rados.0.smithi076.stdout:1632: writing smithi07685715-39 from 504375 to 1075946 tid 1 2024-09-21T09:44:48.084 INFO:tasks.rados.rados.0.smithi076.stdout:1632: writing smithi07685715-39 from 1490187 to 2153395 tid 2 2024-09-21T09:44:48.084 INFO:tasks.rados.rados.0.smithi076.stdout:1633: delete oid 22 current snap is 200 2024-09-21T09:44:48.084 INFO:tasks.rados.rados.0.smithi076.stdout:1630: expect (ObjNum 618 snap 196 seq_num 618) 2024-09-21T09:44:48.164 INFO:tasks.rados.rados.0.smithi076.stdout:1631: expect (ObjNum 482 snap 155 seq_num 482) 2024-09-21T09:44:48.285 INFO:tasks.rados.rados.0.smithi076.stdout:1632: finishing write tid 1 to smithi07685715-39 2024-09-21T09:44:48.285 INFO:tasks.rados.rados.0.smithi076.stdout:1632: finishing write tid 2 to smithi07685715-39 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: finishing write tid 3 to smithi07685715-39 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: finishing write tid 4 to smithi07685715-39 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: oid 39 updating version 0 to 1307 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: oid 39 version 1307 is already newer than 1306 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: oid 39 updating version 1307 to 1308 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1308 (ObjNum 625 snap 200 seq_num 625) dirty exists 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: left oid 39 (ObjNum 625 snap 200 seq_num 625) 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1630: done (3 left) 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1631: done (2 left) 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1632: done (1 left) 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1633: done (0 left) 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1634: read oid 36 snap 199 2024-09-21T09:44:48.286 INFO:tasks.rados.rados.0.smithi076.stdout:1634: expect deleted 2024-09-21T09:44:48.287 INFO:tasks.rados.rados.0.smithi076.stdout:1635: read oid 50 snap -1 2024-09-21T09:44:48.287 INFO:tasks.rados.rados.0.smithi076.stdout:1635: expect (ObjNum 587 snap 188 seq_num 587) 2024-09-21T09:44:48.287 INFO:tasks.rados.rados.0.smithi076.stdout:1636: snap_create 2024-09-21T09:44:48.298 INFO:tasks.rados.rados.0.smithi076.stdout:1635: expect (ObjNum 587 snap 188 seq_num 587) 2024-09-21T09:44:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: pgmap v185: 89 pgs: 89 active+clean; 105 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 818 KiB/s wr, 6 op/s 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: Upgrade: Updating mon.b 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: Deploying daemon mon.b on smithi191 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: from='client.? 172.21.15.76:0/2506227787' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:47 smithi191 ceph-mon[33301]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:44:48.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: pgmap v185: 89 pgs: 89 active+clean; 105 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 818 KiB/s wr, 6 op/s 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: Upgrade: Updating mon.b 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: Deploying daemon mon.b on smithi191 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2506227787' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[106482]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:44:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: pgmap v185: 89 pgs: 89 active+clean; 105 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 818 KiB/s wr, 6 op/s 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: Upgrade: Updating mon.b 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: Deploying daemon mon.b on smithi191 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2506227787' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:44:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:47 smithi076 ceph-mon[110963]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1634: done (2 left) 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1635: done (1 left) 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1636: done (0 left) 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1637: read oid 48 snap -1 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1637: expect deleted 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1638: write oid 22 current snap is 201 2024-09-21T09:44:48.654 INFO:tasks.rados.rados.0.smithi076.stdout:1638: seq_num 627 ranges {537989=722069,1890798=647191} 2024-09-21T09:44:48.660 INFO:tasks.rados.rados.0.smithi076.stdout:1638: writing smithi07685715-22 from 537989 to 1260058 tid 1 2024-09-21T09:44:48.668 INFO:tasks.rados.rados.0.smithi076.stdout:1638: writing smithi07685715-22 from 1890798 to 2537989 tid 2 2024-09-21T09:44:48.668 INFO:tasks.rados.rados.0.smithi076.stdout:1639: snap_remove snap 199 2024-09-21T09:44:48.969 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:48 smithi191 systemd[1]: Stopping Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:44:48.969 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:48 smithi191 ceph-mon[33301]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:48.969 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:48 smithi191 ceph-mon[33301]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:44:49.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b[33278]: 2024-09-21T09:44:48.966+0000 7f0b03018640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:44:49.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b[33278]: 2024-09-21T09:44:48.966+0000 7f0b03018640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:44:49.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 podman[102019]: 2024-09-21 09:44:49.027798253 +0000 UTC m=+0.186133065 container died dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T09:44:49.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:48 smithi076 ceph-mon[106482]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:48 smithi076 ceph-mon[106482]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:44:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:48 smithi076 ceph-mon[110963]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:48 smithi076 ceph-mon[110963]: osdmap e481: 8 total, 8 up, 8 in 2024-09-21T09:44:49.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 podman[102019]: 2024-09-21 09:44:49.415404535 +0000 UTC m=+0.573739333 container cleanup dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:44:49.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 bash[102019]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b 2024-09-21T09:44:50.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 podman[102036]: 2024-09-21 09:44:49.684019779 +0000 UTC m=+0.659850509 container remove dcb99d4acf72b38110ce77473d202ba1b7d796368b835a4310947c5c04a6f72d (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T09:44:50.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.b.service: Deactivated successfully. 2024-09-21T09:44:50.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 systemd[1]: Stopped Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:44:50.002 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:49 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.b.service: Consumed 11.662s CPU time. 2024-09-21T09:44:50.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 systemd[1]: Starting Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:44:50.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 podman[102168]: 2024-09-21 09:44:50.113431317 +0000 UTC m=+0.016687746 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:44:50.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 podman[102168]: 2024-09-21 09:44:50.235902042 +0000 UTC m=+0.139158466 container create bb6de71151ea53bd9376ee82f3878ff4d138086270d3ca1fd65f4cd56a765428 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 podman[102168]: 2024-09-21 09:44:50.438932114 +0000 UTC m=+0.342188548 container init bb6de71151ea53bd9376ee82f3878ff4d138086270d3ca1fd65f4cd56a765428 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 podman[102168]: 2024-09-21 09:44:50.444285447 +0000 UTC m=+0.347541863 container start bb6de71151ea53bd9376ee82f3878ff4d138086270d3ca1fd65f4cd56a765428 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: pidfile_write: ignore empty --pid-file 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: load: jerasure load: lrc 2024-09-21T09:44:50.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Git sha 0 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: DB SUMMARY 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: DB Session ID: V4VXNE0GLJNIATLNGOGF 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1898 Bytes 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000033.sst 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000031.log size: 92923 ; 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:44:50.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:44:50.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:44:50.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:44:50.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.env: 0x5643a4f3c120 2024-09-21T09:44:50.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:44:50.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.info_log: 0x5643a73cd700 2024-09-21T09:44:50.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:44:50.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.statistics: (nil) 2024-09-21T09:44:50.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.use_fsync: 0 2024-09-21T09:44:50.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.db_log_dir: 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.wal_dir: 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:44:50.867 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.write_buffer_manager: 0x5643a73d01e0 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:44:50.868 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.unordered_write: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.row_cache: None 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.wal_filter: None 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.wal_compression: 0 2024-09-21T09:44:50.869 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:44:50.870 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_open_files: -1 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Compression algorithms supported: 2024-09-21T09:44:50.871 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kZSTD supported: 0 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.merge_operator: 2024-09-21T09:44:50.872 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_filter: None 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5643a73cd320) 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_type: 0 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_block_index_type: 0 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_shortening: 1 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: checksum: 4 2024-09-21T09:44:50.873 INFO:journalctl@ceph.mon.b.smithi191.stdout: no_block_cache: 0 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache: 0x5643a73c1610 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_options: 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: capacity : 536870912 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: num_shard_bits : 4 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: strict_capacity_limit : 0 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_cache_compressed: (nil) 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: persistent_cache: (nil) 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_size: 4096 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_size_deviation: 10 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_restart_interval: 16 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: index_block_restart_interval: 1 2024-09-21T09:44:50.874 INFO:journalctl@ceph.mon.b.smithi191.stdout: metadata_block_size: 4096 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: partition_filters: 0 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: use_delta_encoding: 1 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: filter_policy: bloomfilter 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: whole_key_filtering: 1 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: verify_compression: 0 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: format_version: 5 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: enable_index_compression: 1 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: block_align: 0 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_auto_readahead_size: 262144 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: prepopulate_block_cache: 0 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:44:50.875 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression: NoCompression 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.num_levels: 7 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:44:50.876 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:44:50.877 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:44:50.878 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:44:50.879 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.ttl: 2592000 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:44:50.880 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 35, last_sequence is 22640, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 31 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dab05661-a9f5-450c-9f5f-83c8d1de037e 2024-09-21T09:44:50.881 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911890471225, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911890471912, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 62624, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 22645, "largest_seqno": 22700, "table_properties": {"data_size": 61244, "index_size": 214, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 978, "raw_average_key_size": 25, "raw_value_size": 60324, "raw_average_value_size": 1587, "num_data_blocks": 9, "num_entries": 38, "num_filter_entries": 38, "num_deletions": 1, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726911890, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dab05661-a9f5-450c-9f5f-83c8d1de037e", "db_session_id": "V4VXNE0GLJNIATLNGOGF", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911890472008, "job": 1, "event": "recovery_finished"} 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5643a73f0e00 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: DB pointer 0x5643a74f2000 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** DB Stats ** 2024-09-21T09:44:50.882 INFO:journalctl@ceph.mon.b.smithi191.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** Compaction Stats [default] ** 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: L0 1/0 61.16 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 131.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: L6 1/0 13.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Sum 2/0 13.61 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 131.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:44:50.883 INFO:journalctl@ceph.mon.b.smithi191.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 131.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** Compaction Stats [default] ** 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 131.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:44:50.884 INFO:journalctl@ceph.mon.b.smithi191.stdout: Cumulative compaction: 0.00 GB write, 20.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: Interval compaction: 0.00 GB write, 20.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: Block cache BinnedLRUCache@0x5643a73c1610#7 capacity: 512.00 MB usage: 0.47 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.30 KB,5.66244e-05%) Misc(1,0.00 KB,0%) 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: starting mon.b rank 1 at public addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] at bind addrs [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???) e3 preinit fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).mds e9 new map 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).mds e9 print_map 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: e9 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:44:50.885 INFO:journalctl@ceph.mon.b.smithi191.stdout: legacy client fscid: 1 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: Filesystem 'foo' (1) 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: fs_name foo 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: epoch 8 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: created 2024-09-21T09:36:39.879731+0000 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: modified 2024-09-21T09:36:51.753955+0000 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: tableserver 0 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: root 0 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: session_timeout 60 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: session_autoclose 300 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_file_size 1099511627776 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_xattr_size 65536 2024-09-21T09:44:50.886 INFO:journalctl@ceph.mon.b.smithi191.stdout: required_client_features {} 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: last_failure 0 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: last_failure_osd_epoch 0 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: max_mds 1 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: in 0 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: up {0=14631} 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: failed 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: damaged 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: stopped 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: data_pools [4] 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: metadata_pool 3 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: inline_data disabled 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: balancer 2024-09-21T09:44:50.887 INFO:journalctl@ceph.mon.b.smithi191.stdout: bal_rank_mask -1 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: standby_count_wanted 1 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: [mds.foo.smithi076.jymebx{0:14631} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.76:6834/975532171,v1:172.21.15.76:6835/975532171] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: Standby daemons: 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout: [mds.foo.smithi191.dhrpzx{-1:24466} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.191:6832/3879233829,v1:172.21.15.191:6833/3879233829] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).osd e481 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).osd e481 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).osd e481 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).osd e481 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 ceph-mon[102206]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 bash[102168]: bb6de71151ea53bd9376ee82f3878ff4d138086270d3ca1fd65f4cd56a765428 2024-09-21T09:44:50.888 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:50 smithi191 systemd[1]: Started Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:44:52.063 INFO:tasks.rados.rados.0.smithi076.stdout:1639: done (2 left) 2024-09-21T09:44:52.063 INFO:tasks.rados.rados.0.smithi076.stdout:1640: read oid 28 snap -1 2024-09-21T09:44:52.064 INFO:tasks.rados.rados.0.smithi076.stdout:1640: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:44:52.064 INFO:tasks.rados.rados.0.smithi076.stdout:1641: snap_remove snap 200 2024-09-21T09:44:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:44:53] "GET /metrics HTTP/1.1" 200 36124 "" "Prometheus/2.51.0" 2024-09-21T09:44:55.873 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:44:55 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:44:55.484+0000 7f86bc22c640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1641: done (3 left) 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1642: delete oid 41 current snap is 201 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: finishing write tid 1 to smithi07685715-22 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: finishing write tid 2 to smithi07685715-22 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: finishing write tid 3 to smithi07685715-22 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: finishing write tid 4 to smithi07685715-22 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: oid 22 updating version 0 to 799 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: oid 22 version 799 is already newer than 798 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: oid 22 version 799 is already newer than 797 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 799 (ObjNum 627 snap 201 seq_num 627) dirty exists 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1638: left oid 22 (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:44:57.152 INFO:tasks.rados.rados.0.smithi076.stdout:1640: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:44:57.249 INFO:tasks.rados.rados.0.smithi076.stdout:1637: done (3 left) 2024-09-21T09:44:57.249 INFO:tasks.rados.rados.0.smithi076.stdout:1638: done (2 left) 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi076.stdout:1640: done (1 left) 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi076.stdout:1642: done (0 left) 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi076.stdout:1643: read oid 37 snap -1 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi076.stdout:1643: expect (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:44:57.250 INFO:tasks.rados.rados.0.smithi076.stdout:1644: snap_create 2024-09-21T09:44:57.258 INFO:tasks.rados.rados.0.smithi076.stdout:1643: expect (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mon.a calling monitor election 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mon.c calling monitor election 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mon.b calling monitor election 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: pgmap v192: 89 pgs: 89 active+clean; 104 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 456 KiB/s wr, 10 op/s 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: pgmap v193: 89 pgs: 89 active+clean; 104 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 386 KiB/s wr, 8 op/s 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: pgmap v194: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 318 KiB/s wr, 8 op/s 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: overall HEALTH_OK 2024-09-21T09:44:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mon.a calling monitor election 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: monmap epoch 4 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: last_changed 2024-09-21T09:44:52.059881+0000 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: min_mon_release 19 (squid) 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: election_strategy: 1 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:44:57.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:44:57.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: overall HEALTH_OK 2024-09-21T09:44:57.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:57 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mon.a calling monitor election 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mon.c calling monitor election 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mon.b calling monitor election 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: pgmap v192: 89 pgs: 89 active+clean; 104 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 456 KiB/s wr, 10 op/s 2024-09-21T09:44:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: pgmap v193: 89 pgs: 89 active+clean; 104 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 386 KiB/s wr, 8 op/s 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: pgmap v194: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 318 KiB/s wr, 8 op/s 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: overall HEALTH_OK 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mon.a calling monitor election 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: monmap epoch 4 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: last_changed 2024-09-21T09:44:52.059881+0000 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: min_mon_release 19 (squid) 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: election_strategy: 1 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:44:57.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: overall HEALTH_OK 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:44:57.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mon.a calling monitor election 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mon.c calling monitor election 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mon.b calling monitor election 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: pgmap v192: 89 pgs: 89 active+clean; 104 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 456 KiB/s wr, 10 op/s 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: pgmap v193: 89 pgs: 89 active+clean; 104 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 386 KiB/s wr, 8 op/s 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: pgmap v194: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 318 KiB/s wr, 8 op/s 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: overall HEALTH_OK 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mon.a calling monitor election 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: monmap epoch 4 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: last_changed 2024-09-21T09:44:52.059881+0000 2024-09-21T09:44:57.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: created 2024-09-21T09:29:50.086004+0000 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: min_mon_release 19 (squid) 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: election_strategy: 1 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: 0: [v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] mon.a 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: 1: [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] mon.b 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: 2: [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0] mon.c 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: fsmap foo:1 {0=foo.smithi076.jymebx=up:active} 1 up:standby 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: osdmap e482: 8 total, 8 up, 8 in 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: mgrmap e40: y(active, since 2m), standbys: x 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: overall HEALTH_OK 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:44:57.627 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:57 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:44:58.161 INFO:tasks.rados.rados.0.smithi076.stdout:1643: done (1 left) 2024-09-21T09:44:58.161 INFO:tasks.rados.rados.0.smithi076.stdout:1644: done (0 left) 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi076.stdout:1645: rollback oid 21 current snap is 202 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 201 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi076.stdout:1646: rollback oid 48 current snap is 202 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 201 2024-09-21T09:44:58.162 INFO:tasks.rados.rados.0.smithi076.stdout:1647: snap_create 2024-09-21T09:44:58.164 INFO:tasks.rados.rados.0.smithi076.stdout:1646: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:44:58.164 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi076.stdout:1645: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi076.stdout:1645: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi076.stdout:1645: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:44:58.167 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1314 (ObjNum 579 snap 184 seq_num 579) dirty exists 2024-09-21T09:44:58.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:58 smithi191 ceph-mon[102206]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:44:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:58 smithi076 ceph-mon[106482]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:44:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:58 smithi076 ceph-mon[110963]: osdmap e483: 8 total, 8 up, 8 in 2024-09-21T09:44:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:1645: done (2 left) 2024-09-21T09:44:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:1646: done (1 left) 2024-09-21T09:44:58.656 INFO:tasks.rados.rados.0.smithi076.stdout:1647: done (0 left) 2024-09-21T09:44:58.657 INFO:tasks.rados.rados.0.smithi076.stdout:1648: write oid 40 current snap is 203 2024-09-21T09:44:58.657 INFO:tasks.rados.rados.0.smithi076.stdout:1648: seq_num 629 ranges {571603=472567,1607762=670546} 2024-09-21T09:44:58.662 INFO:tasks.rados.rados.0.smithi076.stdout:1648: writing smithi07685715-40 from 571603 to 1044170 tid 1 2024-09-21T09:44:58.669 INFO:tasks.rados.rados.0.smithi076.stdout:1648: writing smithi07685715-40 from 1607762 to 2278308 tid 2 2024-09-21T09:44:58.669 INFO:tasks.rados.rados.0.smithi076.stdout:1649: rollback oid 8 current snap is 203 2024-09-21T09:44:58.669 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 201 2024-09-21T09:44:58.669 INFO:tasks.rados.rados.0.smithi076.stdout:1648: finishing write tid 1 to smithi07685715-40 2024-09-21T09:44:58.669 INFO:tasks.rados.rados.0.smithi076.stdout:1650: write oid 47 current snap is 203 2024-09-21T09:44:58.670 INFO:tasks.rados.rados.0.smithi076.stdout:1650: seq_num 630 ranges {588410=547816,1666244=414204,2484603=103807} 2024-09-21T09:44:58.675 INFO:tasks.rados.rados.0.smithi076.stdout:1650: writing smithi07685715-47 from 588410 to 1136226 tid 1 2024-09-21T09:44:58.681 INFO:tasks.rados.rados.0.smithi076.stdout:1650: writing smithi07685715-47 from 1666244 to 2080448 tid 2 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:1650: writing smithi07685715-47 from 2484603 to 2588410 tid 3 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:1651: read oid 7 snap -1 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:1651: expect (ObjNum 606 snap 194 seq_num 606) 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:1652: write oid 13 current snap is 203 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:1649: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 0 (ObjNum 129 snap 0 seq_num 864383584) dirty dne 2024-09-21T09:44:58.684 INFO:tasks.rados.rados.0.smithi076.stdout:1652: seq_num 631 ranges {605217=739418,2124726=480491} 2024-09-21T09:44:58.691 INFO:tasks.rados.rados.0.smithi076.stdout:1652: writing smithi07685715-13 from 605217 to 1344635 tid 1 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1652: writing smithi07685715-13 from 2124726 to 2605217 tid 2 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1649: done (4 left) 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1653: write oid 23 current snap is 203 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: finishing write tid 2 to smithi07685715-40 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: finishing write tid 3 to smithi07685715-40 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: finishing write tid 4 to smithi07685715-40 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: oid 40 updating version 0 to 806 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: oid 40 version 806 is already newer than 804 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: oid 40 version 806 is already newer than 805 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 806 (ObjNum 629 snap 203 seq_num 629) dirty exists 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1648: left oid 40 (ObjNum 629 snap 203 seq_num 629) 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1650: finishing write tid 1 to smithi07685715-47 2024-09-21T09:44:58.699 INFO:tasks.rados.rados.0.smithi076.stdout:1653: seq_num 632 ranges {622024=414667,1783208=417873} 2024-09-21T09:44:58.704 INFO:tasks.rados.rados.0.smithi076.stdout:1653: writing smithi07685715-23 from 622024 to 1036691 tid 1 2024-09-21T09:44:58.710 INFO:tasks.rados.rados.0.smithi076.stdout:1653: writing smithi07685715-23 from 1783208 to 2201081 tid 2 2024-09-21T09:44:58.710 INFO:tasks.rados.rados.0.smithi076.stdout:1648: done (4 left) 2024-09-21T09:44:58.710 INFO:tasks.rados.rados.0.smithi076.stdout:1654: read oid 36 snap -1 2024-09-21T09:44:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:1654: expect deleted 2024-09-21T09:44:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:1655: snap_remove snap 202 2024-09-21T09:44:59.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:59 smithi191 ceph-mon[102206]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:44:59.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:59 smithi191 ceph-mon[102206]: pgmap v197: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1 op/s 2024-09-21T09:44:59.453 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:44:59 smithi191 ceph-mon[102206]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:44:59.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:59 smithi076 ceph-mon[106482]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:44:59.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:59 smithi076 ceph-mon[106482]: pgmap v197: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1 op/s 2024-09-21T09:44:59.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:44:59 smithi076 ceph-mon[106482]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:44:59.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:59 smithi076 ceph-mon[110963]: osdmap e484: 8 total, 8 up, 8 in 2024-09-21T09:44:59.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:59 smithi076 ceph-mon[110963]: pgmap v197: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1 op/s 2024-09-21T09:44:59.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:44:59 smithi076 ceph-mon[110963]: osdmap e485: 8 total, 8 up, 8 in 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:1655: done (5 left) 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:1656: rollback oid 9 current snap is 203 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 201 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:1650: finishing write tid 2 to smithi07685715-47 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:1650: finishing write tid 3 to smithi07685715-47 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:1650: finishing write tid 4 to smithi07685715-47 2024-09-21T09:44:59.659 INFO:tasks.rados.rados.0.smithi076.stdout:1650: finishing write tid 5 to smithi07685715-47 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:1650: oid 47 updating version 0 to 1340 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:1650: oid 47 updating version 1340 to 1341 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:1650: oid 47 version 1341 is already newer than 1338 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:1650: oid 47 version 1341 is already newer than 1339 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1341 (ObjNum 630 snap 203 seq_num 630) dirty exists 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:1650: left oid 47 (ObjNum 630 snap 203 seq_num 630) 2024-09-21T09:44:59.660 INFO:tasks.rados.rados.0.smithi076.stdout:1651: expect (ObjNum 606 snap 194 seq_num 606) 2024-09-21T09:44:59.731 INFO:tasks.rados.rados.0.smithi076.stdout:1652: finishing write tid 1 to smithi07685715-13 2024-09-21T09:44:59.731 INFO:tasks.rados.rados.0.smithi076.stdout:1650: done (5 left) 2024-09-21T09:44:59.731 INFO:tasks.rados.rados.0.smithi076.stdout:1651: done (4 left) 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1657: read oid 35 snap -1 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1657: expect (ObjNum 577 snap 183 seq_num 577) 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1658: snap_create 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: finishing write tid 2 to smithi07685715-13 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: finishing write tid 3 to smithi07685715-13 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: finishing write tid 4 to smithi07685715-13 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: oid 13 updating version 0 to 267 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: oid 13 version 267 is already newer than 266 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: oid 13 updating version 267 to 268 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 268 (ObjNum 631 snap 203 seq_num 631) dirty exists 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1652: left oid 13 (ObjNum 631 snap 203 seq_num 631) 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1653: finishing write tid 1 to smithi07685715-23 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1653: finishing write tid 2 to smithi07685715-23 2024-09-21T09:44:59.732 INFO:tasks.rados.rados.0.smithi076.stdout:1653: finishing write tid 3 to smithi07685715-23 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:1653: finishing write tid 4 to smithi07685715-23 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:1653: oid 23 updating version 0 to 1344 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:1653: oid 23 version 1344 is already newer than 1343 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:1653: oid 23 version 1344 is already newer than 1342 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1344 (ObjNum 632 snap 203 seq_num 632) dirty exists 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:1653: left oid 23 (ObjNum 632 snap 203 seq_num 632) 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:1656: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:44:59.733 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 0 (ObjNum 289 snap 0 seq_num 1472411748) dirty dne 2024-09-21T09:44:59.741 INFO:tasks.rados.rados.0.smithi076.stdout:1657: expect (ObjNum 577 snap 183 seq_num 577) 2024-09-21T09:45:00.662 INFO:tasks.rados.rados.0.smithi076.stdout:1652: done (5 left) 2024-09-21T09:45:00.663 INFO:tasks.rados.rados.0.smithi076.stdout:1653: done (4 left) 2024-09-21T09:45:00.663 INFO:tasks.rados.rados.0.smithi076.stdout:1654: done (3 left) 2024-09-21T09:45:00.663 INFO:tasks.rados.rados.0.smithi076.stdout:1656: done (2 left) 2024-09-21T09:45:00.663 INFO:tasks.rados.rados.0.smithi076.stdout:1657: done (1 left) 2024-09-21T09:45:00.663 INFO:tasks.rados.rados.0.smithi076.stdout:1658: done (0 left) 2024-09-21T09:45:00.663 INFO:tasks.rados.rados.0.smithi076.stdout:1659: delete oid 27 current snap is 204 2024-09-21T09:45:00.665 INFO:tasks.rados.rados.0.smithi076.stdout:1659: done (0 left) 2024-09-21T09:45:00.665 INFO:tasks.rados.rados.0.smithi076.stdout:1660: delete oid 15 current snap is 204 2024-09-21T09:45:00.668 INFO:tasks.rados.rados.0.smithi076.stdout:1660: done (0 left) 2024-09-21T09:45:00.668 INFO:tasks.rados.rados.0.smithi076.stdout:1661: snap_remove snap 203 2024-09-21T09:45:01.056 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:00 smithi191 ceph-mon[102206]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:45:01.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:00 smithi076 ceph-mon[106482]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:45:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:00 smithi076 ceph-mon[110963]: osdmap e486: 8 total, 8 up, 8 in 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:1661: done (0 left) 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:1662: read oid 37 snap 201 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:1662: expect (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:1663: rollback oid 14 current snap is 204 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 201 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:1664: write oid 30 current snap is 204 2024-09-21T09:45:01.672 INFO:tasks.rados.rados.0.smithi076.stdout:1664: seq_num 635 ranges {672445=640414,1842301=681553} 2024-09-21T09:45:01.679 INFO:tasks.rados.rados.0.smithi076.stdout:1664: writing smithi07685715-30 from 672445 to 1312859 tid 1 2024-09-21T09:45:01.686 INFO:tasks.rados.rados.0.smithi076.stdout:1664: writing smithi07685715-30 from 1842301 to 2523854 tid 2 2024-09-21T09:45:01.686 INFO:tasks.rados.rados.0.smithi076.stdout:1665: delete oid 9 current snap is 204 2024-09-21T09:45:01.686 INFO:tasks.rados.rados.0.smithi076.stdout:1663: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:45:01.686 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 0 (ObjNum 97 snap 0 seq_num 864414384) dirty dne 2024-09-21T09:45:01.686 INFO:tasks.rados.rados.0.smithi076.stdout:1662: expect (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1664: finishing write tid 1 to smithi07685715-30 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1662: done (3 left) 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1663: done (2 left) 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1665: done (1 left) 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1666: snap_create 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1664: finishing write tid 2 to smithi07685715-30 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1664: finishing write tid 3 to smithi07685715-30 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1664: finishing write tid 4 to smithi07685715-30 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1664: oid 30 updating version 0 to 1347 2024-09-21T09:45:01.861 INFO:tasks.rados.rados.0.smithi076.stdout:1664: oid 30 version 1347 is already newer than 1346 2024-09-21T09:45:01.862 INFO:tasks.rados.rados.0.smithi076.stdout:1664: oid 30 version 1347 is already newer than 1345 2024-09-21T09:45:01.862 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1347 (ObjNum 635 snap 204 seq_num 635) dirty exists 2024-09-21T09:45:01.862 INFO:tasks.rados.rados.0.smithi076.stdout:1664: left oid 30 (ObjNum 635 snap 204 seq_num 635) 2024-09-21T09:45:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:01 smithi191 ceph-mon[102206]: pgmap v200: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:01 smithi191 ceph-mon[102206]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:45:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:02.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[106482]: pgmap v200: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:02.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[106482]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:45:02.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:02.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:02.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[110963]: pgmap v200: 89 pgs: 89 active+clean; 98 MiB data, 532 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:02.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[110963]: osdmap e487: 8 total, 8 up, 8 in 2024-09-21T09:45:02.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:02.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:02.686 INFO:tasks.rados.rados.0.smithi076.stdout:1664: done (1 left) 2024-09-21T09:45:02.686 INFO:tasks.rados.rados.0.smithi076.stdout:1666: done (0 left) 2024-09-21T09:45:02.686 INFO:tasks.rados.rados.0.smithi076.stdout:1667: write oid 33 current snap is 205 2024-09-21T09:45:02.686 INFO:tasks.rados.rados.0.smithi076.stdout:1667: seq_num 637 ranges {706059=790912,1959265=685222} 2024-09-21T09:45:02.694 INFO:tasks.rados.rados.0.smithi076.stdout:1667: writing smithi07685715-33 from 706059 to 1496971 tid 1 2024-09-21T09:45:02.701 INFO:tasks.rados.rados.0.smithi076.stdout:1667: writing smithi07685715-33 from 1959265 to 2644487 tid 2 2024-09-21T09:45:02.701 INFO:tasks.rados.rados.0.smithi076.stdout:1668: delete oid 15 current snap is 205 2024-09-21T09:45:02.701 INFO:tasks.rados.rados.0.smithi076.stdout:1667: finishing write tid 1 to smithi07685715-33 2024-09-21T09:45:02.704 INFO:tasks.rados.rados.0.smithi076.stdout:1667: finishing write tid 2 to smithi07685715-33 2024-09-21T09:45:02.704 INFO:tasks.rados.rados.0.smithi076.stdout:1668: done (1 left) 2024-09-21T09:45:02.704 INFO:tasks.rados.rados.0.smithi076.stdout:1669: write oid 49 current snap is 205 2024-09-21T09:45:02.704 INFO:tasks.rados.rados.0.smithi076.stdout:1667: finishing write tid 3 to smithi07685715-33 2024-09-21T09:45:02.704 INFO:tasks.rados.rados.0.smithi076.stdout:1669: seq_num 639 ranges {739673=657763,2076229=663444} 2024-09-21T09:45:02.711 INFO:tasks.rados.rados.0.smithi076.stdout:1669: writing smithi07685715-49 from 739673 to 1397436 tid 1 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1669: writing smithi07685715-49 from 2076229 to 2739673 tid 2 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1670: read oid 37 snap -1 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1670: expect (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1671: write oid 50 current snap is 205 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1667: finishing write tid 4 to smithi07685715-33 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1667: oid 33 updating version 0 to 343 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1667: oid 33 updating version 343 to 344 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1667: oid 33 version 344 is already newer than 342 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 344 (ObjNum 637 snap 205 seq_num 637) dirty exists 2024-09-21T09:45:02.720 INFO:tasks.rados.rados.0.smithi076.stdout:1667: left oid 33 (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:45:02.721 INFO:tasks.rados.rados.0.smithi076.stdout:1669: finishing write tid 1 to smithi07685715-49 2024-09-21T09:45:02.721 INFO:tasks.rados.rados.0.smithi076.stdout:1671: seq_num 640 ranges {756480=733012,2134711=432549} 2024-09-21T09:45:02.728 INFO:tasks.rados.rados.0.smithi076.stdout:1671: writing smithi07685715-50 from 756480 to 1489492 tid 1 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1671: writing smithi07685715-50 from 2134711 to 2567260 tid 2 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1667: done (3 left) 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1672: delete oid 7 current snap is 205 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1669: finishing write tid 2 to smithi07685715-49 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1669: finishing write tid 3 to smithi07685715-49 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1669: finishing write tid 4 to smithi07685715-49 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1669: oid 49 updating version 0 to 1349 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1669: oid 49 updating version 1349 to 1351 2024-09-21T09:45:02.734 INFO:tasks.rados.rados.0.smithi076.stdout:1669: oid 49 version 1351 is already newer than 1350 2024-09-21T09:45:02.735 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 1351 (ObjNum 639 snap 205 seq_num 639) dirty exists 2024-09-21T09:45:02.735 INFO:tasks.rados.rados.0.smithi076.stdout:1669: left oid 49 (ObjNum 639 snap 205 seq_num 639) 2024-09-21T09:45:02.735 INFO:tasks.rados.rados.0.smithi076.stdout:1670: expect (ObjNum 621 snap 199 seq_num 621) 2024-09-21T09:45:02.905 INFO:tasks.rados.rados.0.smithi076.stdout:1671: finishing write tid 1 to smithi07685715-50 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: finishing write tid 2 to smithi07685715-50 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: finishing write tid 3 to smithi07685715-50 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: finishing write tid 4 to smithi07685715-50 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: oid 50 updating version 0 to 525 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: oid 50 version 525 is already newer than 523 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: oid 50 version 525 is already newer than 524 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 525 (ObjNum 640 snap 205 seq_num 640) dirty exists 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: left oid 50 (ObjNum 640 snap 205 seq_num 640) 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1669: done (3 left) 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1670: done (2 left) 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1671: done (1 left) 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1672: done (0 left) 2024-09-21T09:45:02.906 INFO:tasks.rados.rados.0.smithi076.stdout:1673: snap_remove snap 204 2024-09-21T09:45:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:02 smithi076 ceph-mon[106482]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:45:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:03.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:02 smithi076 ceph-mon[110963]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:45:03.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:03.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:02 smithi191 ceph-mon[102206]: osdmap e488: 8 total, 8 up, 8 in 2024-09-21T09:45:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:03.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:45:03] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:45:03.659 INFO:tasks.rados.rados.0.smithi076.stdout:1673: done (0 left) 2024-09-21T09:45:03.659 INFO:tasks.rados.rados.0.smithi076.stdout:1674: rollback oid 23 current snap is 205 2024-09-21T09:45:03.659 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 201 2024-09-21T09:45:03.659 INFO:tasks.rados.rados.0.smithi076.stdout:1675: read oid 11 snap -1 2024-09-21T09:45:03.659 INFO:tasks.rados.rados.0.smithi076.stdout:1675: expect deleted 2024-09-21T09:45:03.659 INFO:tasks.rados.rados.0.smithi076.stdout:1676: delete oid 44 current snap is 205 2024-09-21T09:45:03.664 INFO:tasks.rados.rados.0.smithi076.stdout:1675: done (2 left) 2024-09-21T09:45:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1676: done (1 left) 2024-09-21T09:45:03.665 INFO:tasks.rados.rados.0.smithi076.stdout:1677: snap_create 2024-09-21T09:45:03.667 INFO:tasks.rados.rados.0.smithi076.stdout:1674: finishing rollback tid 0 to smithi07685715-23 2024-09-21T09:45:03.668 INFO:tasks.rados.rados.0.smithi076.stdout:1674: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:45:03.668 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1353 (ObjNum 65 snap 0 seq_num 1682113569) dirty dne 2024-09-21T09:45:03.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:03 smithi191 ceph-mon[102206]: pgmap v203: 89 pgs: 89 active+clean; 102 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:45:03.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:03 smithi191 ceph-mon[102206]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:45:03.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:03 smithi191 ceph-mon[102206]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:45:04.040 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-mon[106482]: pgmap v203: 89 pgs: 89 active+clean; 102 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:45:04.040 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-mon[106482]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:45:04.040 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-mon[106482]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:45:04.041 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-mon[110963]: pgmap v203: 89 pgs: 89 active+clean; 102 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:45:04.041 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-mon[110963]: osdmap e489: 8 total, 8 up, 8 in 2024-09-21T09:45:04.041 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:03 smithi076 ceph-mon[110963]: osdmap e490: 8 total, 8 up, 8 in 2024-09-21T09:45:04.658 INFO:tasks.rados.rados.0.smithi076.stdout:1674: done (1 left) 2024-09-21T09:45:04.659 INFO:tasks.rados.rados.0.smithi076.stdout:1677: done (0 left) 2024-09-21T09:45:04.659 INFO:tasks.rados.rados.0.smithi076.stdout:1678: snap_create 2024-09-21T09:45:05.663 INFO:tasks.rados.rados.0.smithi076.stdout:1678: done (0 left) 2024-09-21T09:45:05.663 INFO:tasks.rados.rados.0.smithi076.stdout:1679: snap_remove snap 201 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: pgmap v206: 89 pgs: 89 active+clean; 102 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: pgmap v206: 89 pgs: 89 active+clean; 102 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:05.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: pgmap v206: 89 pgs: 89 active+clean; 102 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:45:06.180 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: osdmap e491: 8 total, 8 up, 8 in 2024-09-21T09:45:06.180 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:06.181 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:06.667 INFO:tasks.rados.rados.0.smithi076.stdout:1679: done (0 left) 2024-09-21T09:45:06.667 INFO:tasks.rados.rados.0.smithi076.stdout:1680: snap_remove snap 205 2024-09-21T09:45:06.860 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[106482]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:45:06.860 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[106482]: Reconfiguring daemon mon.a on smithi076 2024-09-21T09:45:06.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[106482]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:45:06.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[106482]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:45:06.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[110963]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:45:06.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[110963]: Reconfiguring daemon mon.a on smithi076 2024-09-21T09:45:06.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[110963]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:45:06.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:06 smithi076 ceph-mon[110963]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:45:07.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:06 smithi191 ceph-mon[102206]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:45:07.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:06 smithi191 ceph-mon[102206]: Reconfiguring daemon mon.a on smithi076 2024-09-21T09:45:07.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:06 smithi191 ceph-mon[102206]: osdmap e492: 8 total, 8 up, 8 in 2024-09-21T09:45:07.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:06 smithi191 ceph-mon[102206]: osdmap e493: 8 total, 8 up, 8 in 2024-09-21T09:45:07.667 INFO:tasks.rados.rados.0.smithi076.stdout:1680: done (0 left) 2024-09-21T09:45:07.667 INFO:tasks.rados.rados.0.smithi076.stdout:1681: write oid 25 current snap is 207 2024-09-21T09:45:07.667 INFO:tasks.rados.rados.0.smithi076.stdout:1681: seq_num 643 ranges {406901=558759,1393804=579876,2608396=198505} 2024-09-21T09:45:07.671 INFO:tasks.rados.rados.0.smithi076.stdout:1681: writing smithi07685715-25 from 406901 to 965660 tid 1 2024-09-21T09:45:07.677 INFO:tasks.rados.rados.0.smithi076.stdout:1681: writing smithi07685715-25 from 1393804 to 1973680 tid 2 2024-09-21T09:45:07.682 INFO:tasks.rados.rados.0.smithi076.stdout:1681: writing smithi07685715-25 from 2608396 to 2806901 tid 3 2024-09-21T09:45:07.682 INFO:tasks.rados.rados.0.smithi076.stdout:1682: write oid 31 current snap is 207 2024-09-21T09:45:07.682 INFO:tasks.rados.rados.0.smithi076.stdout:1681: finishing write tid 1 to smithi07685715-25 2024-09-21T09:45:07.683 INFO:tasks.rados.rados.0.smithi076.stdout:1681: finishing write tid 2 to smithi07685715-25 2024-09-21T09:45:07.683 INFO:tasks.rados.rados.0.smithi076.stdout:1682: seq_num 644 ranges {423708=634008,1852286=439887,2752172=71536} 2024-09-21T09:45:07.688 INFO:tasks.rados.rados.0.smithi076.stdout:1682: writing smithi07685715-31 from 423708 to 1057716 tid 1 2024-09-21T09:45:07.695 INFO:tasks.rados.rados.0.smithi076.stdout:1682: writing smithi07685715-31 from 1852286 to 2292173 tid 2 2024-09-21T09:45:07.698 INFO:tasks.rados.rados.0.smithi076.stdout:1682: writing smithi07685715-31 from 2752172 to 2823708 tid 3 2024-09-21T09:45:07.698 INFO:tasks.rados.rados.0.smithi076.stdout:1683: snap_create 2024-09-21T09:45:07.698 INFO:tasks.rados.rados.0.smithi076.stdout:1681: finishing write tid 3 to smithi07685715-25 2024-09-21T09:45:07.698 INFO:tasks.rados.rados.0.smithi076.stdout:1681: finishing write tid 4 to smithi07685715-25 2024-09-21T09:45:07.698 INFO:tasks.rados.rados.0.smithi076.stdout:1681: finishing write tid 5 to smithi07685715-25 2024-09-21T09:45:07.698 INFO:tasks.rados.rados.0.smithi076.stdout:1681: oid 25 updating version 0 to 352 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1681: oid 25 updating version 352 to 353 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1681: oid 25 version 353 is already newer than 351 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1681: oid 25 version 353 is already newer than 350 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 353 (ObjNum 643 snap 207 seq_num 643) dirty exists 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1681: left oid 25 (ObjNum 643 snap 207 seq_num 643) 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1682: finishing write tid 1 to smithi07685715-31 2024-09-21T09:45:07.699 INFO:tasks.rados.rados.0.smithi076.stdout:1682: finishing write tid 2 to smithi07685715-31 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: finishing write tid 3 to smithi07685715-31 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: finishing write tid 4 to smithi07685715-31 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: finishing write tid 5 to smithi07685715-31 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: oid 31 updating version 0 to 646 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: oid 31 updating version 646 to 649 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: oid 31 version 649 is already newer than 647 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: oid 31 version 649 is already newer than 648 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 649 (ObjNum 644 snap 207 seq_num 644) dirty exists 2024-09-21T09:45:07.700 INFO:tasks.rados.rados.0.smithi076.stdout:1682: left oid 31 (ObjNum 644 snap 207 seq_num 644) 2024-09-21T09:45:08.275 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: pgmap v209: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: Reconfiguring daemon mgr.y on smithi076 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[106482]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:45:08.276 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: pgmap v209: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: Reconfiguring daemon mgr.y on smithi076 2024-09-21T09:45:08.277 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:07 smithi076 ceph-mon[110963]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:45:08.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: pgmap v209: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:45:08.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: Reconfiguring daemon mgr.y on smithi076 2024-09-21T09:45:08.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:07 smithi191 ceph-mon[102206]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:45:08.661 INFO:tasks.rados.rados.0.smithi076.stdout:1681: done (2 left) 2024-09-21T09:45:08.661 INFO:tasks.rados.rados.0.smithi076.stdout:1682: done (1 left) 2024-09-21T09:45:08.661 INFO:tasks.rados.rados.0.smithi076.stdout:1683: done (0 left) 2024-09-21T09:45:08.661 INFO:tasks.rados.rados.0.smithi076.stdout:1684: snap_remove snap 206 2024-09-21T09:45:09.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: pgmap v212: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: Reconfiguring daemon mon.c on smithi076 2024-09-21T09:45:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:09 smithi191 ceph-mon[102206]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: pgmap v212: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: Reconfiguring daemon mon.c on smithi076 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[106482]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: pgmap v212: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:09.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:09.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:45:09.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:09.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:09.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:09.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: Reconfiguring daemon mon.c on smithi076 2024-09-21T09:45:09.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:09 smithi076 ceph-mon[110963]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:45:09.662 INFO:tasks.rados.rados.0.smithi076.stdout:1684: done (0 left) 2024-09-21T09:45:09.662 INFO:tasks.rados.rados.0.smithi076.stdout:1685: delete oid 40 current snap is 208 2024-09-21T09:45:09.666 INFO:tasks.rados.rados.0.smithi076.stdout:1685: done (0 left) 2024-09-21T09:45:09.666 INFO:tasks.rados.rados.0.smithi076.stdout:1686: read oid 22 snap -1 2024-09-21T09:45:09.666 INFO:tasks.rados.rados.0.smithi076.stdout:1686: expect (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:45:09.666 INFO:tasks.rados.rados.0.smithi076.stdout:1687: write oid 35 current snap is 208 2024-09-21T09:45:09.666 INFO:tasks.rados.rados.0.smithi076.stdout:1687: seq_num 646 ranges {457322=784506,1969250=443556} 2024-09-21T09:45:09.673 INFO:tasks.rados.rados.0.smithi076.stdout:1687: writing smithi07685715-35 from 457322 to 1241828 tid 1 2024-09-21T09:45:09.680 INFO:tasks.rados.rados.0.smithi076.stdout:1687: writing smithi07685715-35 from 1969250 to 2412806 tid 2 2024-09-21T09:45:09.680 INFO:tasks.rados.rados.0.smithi076.stdout:1688: delete oid 20 current snap is 208 2024-09-21T09:45:09.680 INFO:tasks.rados.rados.0.smithi076.stdout:1686: expect (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1687: finishing write tid 1 to smithi07685715-35 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1687: finishing write tid 2 to smithi07685715-35 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1687: finishing write tid 3 to smithi07685715-35 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1686: done (2 left) 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1688: done (1 left) 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1689: write oid 18 current snap is 208 2024-09-21T09:45:09.762 INFO:tasks.rados.rados.0.smithi076.stdout:1687: finishing write tid 4 to smithi07685715-35 2024-09-21T09:45:09.763 INFO:tasks.rados.rados.0.smithi076.stdout:1687: oid 35 updating version 0 to 1318 2024-09-21T09:45:09.763 INFO:tasks.rados.rados.0.smithi076.stdout:1687: oid 35 updating version 1318 to 1320 2024-09-21T09:45:09.763 INFO:tasks.rados.rados.0.smithi076.stdout:1687: oid 35 version 1320 is already newer than 1319 2024-09-21T09:45:09.763 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1320 (ObjNum 646 snap 208 seq_num 646) dirty exists 2024-09-21T09:45:09.763 INFO:tasks.rados.rados.0.smithi076.stdout:1687: left oid 35 (ObjNum 646 snap 208 seq_num 646) 2024-09-21T09:45:09.763 INFO:tasks.rados.rados.0.smithi076.stdout:1689: seq_num 648 ranges {490936=651357,1686214=447225,2861864=29072} 2024-09-21T09:45:09.768 INFO:tasks.rados.rados.0.smithi076.stdout:1689: writing smithi07685715-18 from 490936 to 1142293 tid 1 2024-09-21T09:45:09.773 INFO:tasks.rados.rados.0.smithi076.stdout:1689: writing smithi07685715-18 from 1686214 to 2133439 tid 2 2024-09-21T09:45:09.778 INFO:tasks.rados.rados.0.smithi076.stdout:1689: writing smithi07685715-18 from 2861864 to 2890936 tid 3 2024-09-21T09:45:09.778 INFO:tasks.rados.rados.0.smithi076.stdout:1687: done (1 left) 2024-09-21T09:45:09.778 INFO:tasks.rados.rados.0.smithi076.stdout:1690: read oid 9 snap -1 2024-09-21T09:45:09.778 INFO:tasks.rados.rados.0.smithi076.stdout:1690: expect deleted 2024-09-21T09:45:09.778 INFO:tasks.rados.rados.0.smithi076.stdout:1691: write oid 23 current snap is 208 2024-09-21T09:45:09.779 INFO:tasks.rados.rados.0.smithi076.stdout:1689: finishing write tid 1 to smithi07685715-18 2024-09-21T09:45:09.779 INFO:tasks.rados.rados.0.smithi076.stdout:1689: finishing write tid 2 to smithi07685715-18 2024-09-21T09:45:09.779 INFO:tasks.rados.rados.0.smithi076.stdout:1691: seq_num 649 ranges {507743=726606,1744696=590883,2772934=134809} 2024-09-21T09:45:09.785 INFO:tasks.rados.rados.0.smithi076.stdout:1691: writing smithi07685715-23 from 507743 to 1234349 tid 1 2024-09-21T09:45:09.791 INFO:tasks.rados.rados.0.smithi076.stdout:1691: writing smithi07685715-23 from 1744696 to 2335579 tid 2 2024-09-21T09:45:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:1691: writing smithi07685715-23 from 2772934 to 2907743 tid 3 2024-09-21T09:45:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:1692: read oid 19 snap -1 2024-09-21T09:45:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:1692: expect deleted 2024-09-21T09:45:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:1689: finishing write tid 3 to smithi07685715-18 2024-09-21T09:45:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:1689: finishing write tid 4 to smithi07685715-18 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: finishing write tid 5 to smithi07685715-18 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: oid 18 updating version 0 to 537 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: oid 18 version 537 is already newer than 536 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: oid 18 version 537 is already newer than 535 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: oid 18 updating version 537 to 538 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 538 (ObjNum 648 snap 208 seq_num 648) dirty exists 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: left oid 18 (ObjNum 648 snap 208 seq_num 648) 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1691: finishing write tid 1 to smithi07685715-23 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1689: done (3 left) 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1690: done (2 left) 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1693: delete oid 16 current snap is 208 2024-09-21T09:45:09.796 INFO:tasks.rados.rados.0.smithi076.stdout:1691: finishing write tid 2 to smithi07685715-23 2024-09-21T09:45:09.797 INFO:tasks.rados.rados.0.smithi076.stdout:1692: done (2 left) 2024-09-21T09:45:09.797 INFO:tasks.rados.rados.0.smithi076.stdout:1693: done (1 left) 2024-09-21T09:45:09.797 INFO:tasks.rados.rados.0.smithi076.stdout:1694: write oid 1 current snap is 208 2024-09-21T09:45:09.797 INFO:tasks.rados.rados.0.smithi076.stdout:1694: seq_num 651 ranges {541357=477104,1745307=594552} 2024-09-21T09:45:09.802 INFO:tasks.rados.rados.0.smithi076.stdout:1694: writing smithi07685715-1 from 541357 to 1018461 tid 1 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1694: writing smithi07685715-1 from 1745307 to 2339859 tid 2 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1695: delete oid 37 current snap is 208 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1691: finishing write tid 3 to smithi07685715-23 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1691: finishing write tid 4 to smithi07685715-23 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1691: finishing write tid 5 to smithi07685715-23 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1691: oid 23 updating version 0 to 1361 2024-09-21T09:45:09.810 INFO:tasks.rados.rados.0.smithi076.stdout:1691: oid 23 version 1361 is already newer than 1359 2024-09-21T09:45:09.811 INFO:tasks.rados.rados.0.smithi076.stdout:1691: oid 23 version 1361 is already newer than 1360 2024-09-21T09:45:09.811 INFO:tasks.rados.rados.0.smithi076.stdout:1691: oid 23 version 1361 is already newer than 1358 2024-09-21T09:45:09.811 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1361 (ObjNum 649 snap 208 seq_num 649) dirty exists 2024-09-21T09:45:09.811 INFO:tasks.rados.rados.0.smithi076.stdout:1691: left oid 23 (ObjNum 649 snap 208 seq_num 649) 2024-09-21T09:45:09.811 INFO:tasks.rados.rados.0.smithi076.stdout:1694: finishing write tid 1 to smithi07685715-1 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1691: done (2 left) 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1695: done (1 left) 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1696: read oid 19 snap -1 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1696: expect deleted 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1697: snap_create 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1694: finishing write tid 2 to smithi07685715-1 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1694: finishing write tid 3 to smithi07685715-1 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1694: finishing write tid 4 to smithi07685715-1 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1694: oid 1 updating version 0 to 1322 2024-09-21T09:45:09.813 INFO:tasks.rados.rados.0.smithi076.stdout:1694: oid 1 updating version 1322 to 1323 2024-09-21T09:45:09.814 INFO:tasks.rados.rados.0.smithi076.stdout:1694: oid 1 updating version 1323 to 1324 2024-09-21T09:45:09.814 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 1324 (ObjNum 651 snap 208 seq_num 651) dirty exists 2024-09-21T09:45:09.814 INFO:tasks.rados.rados.0.smithi076.stdout:1694: left oid 1 (ObjNum 651 snap 208 seq_num 651) 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1694: done (2 left) 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1696: done (1 left) 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1697: done (0 left) 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1698: read oid 48 snap 208 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1698: expect deleted 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1699: rollback oid 3 current snap is 209 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 208 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1700: read oid 15 snap -1 2024-09-21T09:45:10.785 INFO:tasks.rados.rados.0.smithi076.stdout:1700: expect deleted 2024-09-21T09:45:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:1701: delete oid 42 current snap is 209 2024-09-21T09:45:10.788 INFO:tasks.rados.rados.0.smithi076.stdout:1699: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:45:10.788 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:45:10.790 INFO:tasks.rados.rados.0.smithi076.stdout:1698: done (3 left) 2024-09-21T09:45:10.790 INFO:tasks.rados.rados.0.smithi076.stdout:1699: done (2 left) 2024-09-21T09:45:10.790 INFO:tasks.rados.rados.0.smithi076.stdout:1700: done (1 left) 2024-09-21T09:45:10.790 INFO:tasks.rados.rados.0.smithi076.stdout:1701: done (0 left) 2024-09-21T09:45:10.791 INFO:tasks.rados.rados.0.smithi076.stdout:1702: snap_remove snap 207 2024-09-21T09:45:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.0 on smithi076 2024-09-21T09:45:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.0 on smithi076 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:45:11.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:45:11.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:11.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.0 on smithi076 2024-09-21T09:45:11.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:11.793 INFO:tasks.rados.rados.0.smithi076.stdout:1702: done (0 left) 2024-09-21T09:45:11.793 INFO:tasks.rados.rados.0.smithi076.stdout:1703: read oid 48 snap 208 2024-09-21T09:45:11.793 INFO:tasks.rados.rados.0.smithi076.stdout:1703: expect deleted 2024-09-21T09:45:11.793 INFO:tasks.rados.rados.0.smithi076.stdout:1704: read oid 31 snap -1 2024-09-21T09:45:11.794 INFO:tasks.rados.rados.0.smithi076.stdout:1704: expect (ObjNum 644 snap 207 seq_num 644) 2024-09-21T09:45:11.794 INFO:tasks.rados.rados.0.smithi076.stdout:1705: rollback oid 7 current snap is 209 2024-09-21T09:45:11.794 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 208 2024-09-21T09:45:11.794 INFO:tasks.rados.rados.0.smithi076.stdout:1706: snap_create 2024-09-21T09:45:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:1705: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:45:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 0 (ObjNum 129 snap 0 seq_num 1682319905) dirty dne 2024-09-21T09:45:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:1704: expect (ObjNum 644 snap 207 seq_num 644) 2024-09-21T09:45:11.867 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[106482]: pgmap v215: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:11.867 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[106482]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[110963]: pgmap v215: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[110963]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:45:11.868 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:11 smithi191 ceph-mon[102206]: pgmap v215: 89 pgs: 89 active+clean; 106 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:11 smithi191 ceph-mon[102206]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:45:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:45:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1703: done (3 left) 2024-09-21T09:45:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1704: done (2 left) 2024-09-21T09:45:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1705: done (1 left) 2024-09-21T09:45:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1706: done (0 left) 2024-09-21T09:45:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1707: write oid 47 current snap is 210 2024-09-21T09:45:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1707: seq_num 654 ranges {591778=419204,1637106=741879,2910049=81729} 2024-09-21T09:45:12.806 INFO:tasks.rados.rados.0.smithi076.stdout:1707: writing smithi07685715-47 from 591778 to 1010982 tid 1 2024-09-21T09:45:12.814 INFO:tasks.rados.rados.0.smithi076.stdout:1707: writing smithi07685715-47 from 1637106 to 2378985 tid 2 2024-09-21T09:45:12.818 INFO:tasks.rados.rados.0.smithi076.stdout:1707: writing smithi07685715-47 from 2910049 to 2991778 tid 3 2024-09-21T09:45:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:1708: snap_create 2024-09-21T09:45:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:1707: finishing write tid 1 to smithi07685715-47 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: finishing write tid 2 to smithi07685715-47 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: finishing write tid 3 to smithi07685715-47 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: finishing write tid 4 to smithi07685715-47 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: finishing write tid 5 to smithi07685715-47 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: oid 47 updating version 0 to 1365 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: oid 47 version 1365 is already newer than 1363 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: oid 47 updating version 1365 to 1366 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: oid 47 version 1366 is already newer than 1364 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1366 (ObjNum 654 snap 210 seq_num 654) dirty exists 2024-09-21T09:45:12.821 INFO:tasks.rados.rados.0.smithi076.stdout:1707: left oid 47 (ObjNum 654 snap 210 seq_num 654) 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:12 smithi191 ceph-mon[102206]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:45:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:12 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.1 on smithi076 2024-09-21T09:45:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:12 smithi191 ceph-mon[102206]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:45:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:12 smithi076 ceph-mon[106482]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:45:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:12 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.1 on smithi076 2024-09-21T09:45:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:12 smithi076 ceph-mon[106482]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:45:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:12 smithi076 ceph-mon[110963]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:45:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:12 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.1 on smithi076 2024-09-21T09:45:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:12 smithi076 ceph-mon[110963]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:45:13.400 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:45:13] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.51.0" 2024-09-21T09:45:13.665 INFO:tasks.rados.rados.0.smithi076.stdout:1707: done (1 left) 2024-09-21T09:45:13.665 INFO:tasks.rados.rados.0.smithi076.stdout:1708: done (0 left) 2024-09-21T09:45:13.665 INFO:tasks.rados.rados.0.smithi076.stdout:1709: write oid 37 current snap is 211 2024-09-21T09:45:13.665 INFO:tasks.rados.rados.0.smithi076.stdout:1709: seq_num 655 ranges {608585=494453,1579235=601890,2821119=187466} 2024-09-21T09:45:13.671 INFO:tasks.rados.rados.0.smithi076.stdout:1709: writing smithi07685715-37 from 608585 to 1103038 tid 1 2024-09-21T09:45:13.677 INFO:tasks.rados.rados.0.smithi076.stdout:1709: writing smithi07685715-37 from 1579235 to 2181125 tid 2 2024-09-21T09:45:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: writing smithi07685715-37 from 2821119 to 3008585 tid 3 2024-09-21T09:45:13.682 INFO:tasks.rados.rados.0.smithi076.stdout:1710: snap_remove snap 208 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: pgmap v218: 89 pgs: 89 active+clean; 100 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.2 on smithi076 2024-09-21T09:45:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:13 smithi191 ceph-mon[102206]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: pgmap v218: 89 pgs: 89 active+clean; 100 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.2 on smithi076 2024-09-21T09:45:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[110963]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: pgmap v218: 89 pgs: 89 active+clean; 100 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.2 on smithi076 2024-09-21T09:45:14.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:13 smithi076 ceph-mon[106482]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:45:14.663 INFO:tasks.rados.rados.0.smithi076.stdout:1710: done (1 left) 2024-09-21T09:45:14.664 INFO:tasks.rados.rados.0.smithi076.stdout:1711: write oid 38 current snap is 211 2024-09-21T09:45:14.664 INFO:tasks.rados.rados.0.smithi076.stdout:1709: finishing write tid 1 to smithi07685715-37 2024-09-21T09:45:14.664 INFO:tasks.rados.rados.0.smithi076.stdout:1711: seq_num 656 ranges {625392=569702,1637717=745548,2848542=176850} 2024-09-21T09:45:14.670 INFO:tasks.rados.rados.0.smithi076.stdout:1711: writing smithi07685715-38 from 625392 to 1195094 tid 1 2024-09-21T09:45:14.676 INFO:tasks.rados.rados.0.smithi076.stdout:1711: writing smithi07685715-38 from 1637717 to 2383265 tid 2 2024-09-21T09:45:14.680 INFO:tasks.rados.rados.0.smithi076.stdout:1711: writing smithi07685715-38 from 2848542 to 3025392 tid 3 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1712: read oid 9 snap -1 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1712: expect deleted 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1713: read oid 4 snap -1 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1713: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1714: delete oid 47 current snap is 211 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: finishing write tid 2 to smithi07685715-37 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: finishing write tid 3 to smithi07685715-37 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: finishing write tid 4 to smithi07685715-37 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: finishing write tid 5 to smithi07685715-37 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: oid 37 updating version 0 to 461 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: oid 37 version 461 is already newer than 460 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: oid 37 version 461 is already newer than 458 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: oid 37 version 461 is already newer than 459 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 461 (ObjNum 655 snap 211 seq_num 655) dirty exists 2024-09-21T09:45:14.681 INFO:tasks.rados.rados.0.smithi076.stdout:1709: left oid 37 (ObjNum 655 snap 211 seq_num 655) 2024-09-21T09:45:14.682 INFO:tasks.rados.rados.0.smithi076.stdout:1711: finishing write tid 1 to smithi07685715-38 2024-09-21T09:45:14.682 INFO:tasks.rados.rados.0.smithi076.stdout:1711: finishing write tid 2 to smithi07685715-38 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: finishing write tid 3 to smithi07685715-38 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: finishing write tid 4 to smithi07685715-38 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: finishing write tid 5 to smithi07685715-38 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: oid 38 updating version 0 to 943 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: oid 38 version 943 is already newer than 942 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: oid 38 version 943 is already newer than 941 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: oid 38 version 943 is already newer than 940 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 943 (ObjNum 656 snap 211 seq_num 656) dirty exists 2024-09-21T09:45:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1711: left oid 38 (ObjNum 656 snap 211 seq_num 656) 2024-09-21T09:45:14.689 INFO:tasks.rados.rados.0.smithi076.stdout:1713: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1709: done (4 left) 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1711: done (3 left) 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1712: done (2 left) 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1713: done (1 left) 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1714: done (0 left) 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1715: write oid 47 current snap is 211 2024-09-21T09:45:14.785 INFO:tasks.rados.rados.0.smithi076.stdout:1715: seq_num 658 ranges {659006=720200,2154681=749217} 2024-09-21T09:45:14.792 INFO:tasks.rados.rados.0.smithi076.stdout:1715: writing smithi07685715-47 from 659006 to 1379206 tid 1 2024-09-21T09:45:14.801 INFO:tasks.rados.rados.0.smithi076.stdout:1715: writing smithi07685715-47 from 2154681 to 2903898 tid 2 2024-09-21T09:45:14.801 INFO:tasks.rados.rados.0.smithi076.stdout:1716: delete oid 40 current snap is 211 2024-09-21T09:45:14.801 INFO:tasks.rados.rados.0.smithi076.stdout:1715: finishing write tid 1 to smithi07685715-47 2024-09-21T09:45:14.803 INFO:tasks.rados.rados.0.smithi076.stdout:1716: done (1 left) 2024-09-21T09:45:14.803 INFO:tasks.rados.rados.0.smithi076.stdout:1717: snap_create 2024-09-21T09:45:14.805 INFO:tasks.rados.rados.0.smithi076.stdout:1715: finishing write tid 2 to smithi07685715-47 2024-09-21T09:45:14.805 INFO:tasks.rados.rados.0.smithi076.stdout:1715: finishing write tid 3 to smithi07685715-47 2024-09-21T09:45:14.805 INFO:tasks.rados.rados.0.smithi076.stdout:1715: finishing write tid 4 to smithi07685715-47 2024-09-21T09:45:14.805 INFO:tasks.rados.rados.0.smithi076.stdout:1715: oid 47 updating version 0 to 1371 2024-09-21T09:45:14.805 INFO:tasks.rados.rados.0.smithi076.stdout:1715: oid 47 updating version 1371 to 1373 2024-09-21T09:45:14.806 INFO:tasks.rados.rados.0.smithi076.stdout:1715: oid 47 version 1373 is already newer than 1372 2024-09-21T09:45:14.806 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1373 (ObjNum 658 snap 211 seq_num 658) dirty exists 2024-09-21T09:45:14.806 INFO:tasks.rados.rados.0.smithi076.stdout:1715: left oid 47 (ObjNum 658 snap 211 seq_num 658) 2024-09-21T09:45:15.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.3 on smithi076 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: pgmap v221: 89 pgs: 89 active+clean; 100 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:45:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:15 smithi191 ceph-mon[102206]: osdmap e501: 8 total, 8 up, 8 in 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.3 on smithi076 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: pgmap v221: 89 pgs: 89 active+clean; 100 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[110963]: osdmap e501: 8 total, 8 up, 8 in 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:45:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:45:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.3 on smithi076 2024-09-21T09:45:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: pgmap v221: 89 pgs: 89 active+clean; 100 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.8 MiB/s wr, 12 op/s 2024-09-21T09:45:15.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:15 smithi076 ceph-mon[106482]: osdmap e501: 8 total, 8 up, 8 in 2024-09-21T09:45:15.666 INFO:tasks.rados.rados.0.smithi076.stdout:1715: done (1 left) 2024-09-21T09:45:15.666 INFO:tasks.rados.rados.0.smithi076.stdout:1717: done (0 left) 2024-09-21T09:45:15.666 INFO:tasks.rados.rados.0.smithi076.stdout:1718: snap_remove snap 210 2024-09-21T09:45:16.670 INFO:tasks.rados.rados.0.smithi076.stdout:1718: done (0 left) 2024-09-21T09:45:16.670 INFO:tasks.rados.rados.0.smithi076.stdout:1719: delete oid 29 current snap is 212 2024-09-21T09:45:16.679 INFO:tasks.rados.rados.0.smithi076.stdout:1719: done (0 left) 2024-09-21T09:45:16.679 INFO:tasks.rados.rados.0.smithi076.stdout:1720: rollback oid 42 current snap is 212 2024-09-21T09:45:16.680 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 209 2024-09-21T09:45:16.680 INFO:tasks.rados.rados.0.smithi076.stdout:1721: write oid 17 current snap is 212 2024-09-21T09:45:16.680 INFO:tasks.rados.rados.0.smithi076.stdout:1721: seq_num 661 ranges {709427=545947,1813774=612897,2985657=123770} 2024-09-21T09:45:16.686 INFO:tasks.rados.rados.0.smithi076.stdout:1721: writing smithi07685715-17 from 709427 to 1255374 tid 1 2024-09-21T09:45:16.692 INFO:tasks.rados.rados.0.smithi076.stdout:1721: writing smithi07685715-17 from 1813774 to 2426671 tid 2 2024-09-21T09:45:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1721: writing smithi07685715-17 from 2985657 to 3109427 tid 3 2024-09-21T09:45:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1722: write oid 23 current snap is 212 2024-09-21T09:45:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:1720: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:45:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:1721: finishing write tid 1 to smithi07685715-17 2024-09-21T09:45:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:1721: finishing write tid 2 to smithi07685715-17 2024-09-21T09:45:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:1722: seq_num 662 ranges {726234=737549,1988609=756555} 2024-09-21T09:45:16.704 INFO:tasks.rados.rados.0.smithi076.stdout:1722: writing smithi07685715-23 from 726234 to 1463783 tid 1 2024-09-21T09:45:16.711 INFO:tasks.rados.rados.0.smithi076.stdout:1722: writing smithi07685715-23 from 1988609 to 2745164 tid 2 2024-09-21T09:45:16.711 INFO:tasks.rados.rados.0.smithi076.stdout:1720: done (2 left) 2024-09-21T09:45:16.711 INFO:tasks.rados.rados.0.smithi076.stdout:1723: write oid 47 current snap is 212 2024-09-21T09:45:16.712 INFO:tasks.rados.rados.0.smithi076.stdout:1721: finishing write tid 3 to smithi07685715-17 2024-09-21T09:45:16.712 INFO:tasks.rados.rados.0.smithi076.stdout:1723: seq_num 663 ranges {743041=412798,1930738=616566,3040503=102538} 2024-09-21T09:45:16.718 INFO:tasks.rados.rados.0.smithi076.stdout:1723: writing smithi07685715-47 from 743041 to 1155839 tid 1 2024-09-21T09:45:16.725 INFO:tasks.rados.rados.0.smithi076.stdout:1723: writing smithi07685715-47 from 1930738 to 2547304 tid 2 2024-09-21T09:45:16.729 INFO:tasks.rados.rados.0.smithi076.stdout:1723: writing smithi07685715-47 from 3040503 to 3143041 tid 3 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1724: read oid 43 snap -1 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1724: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1725: read oid 12 snap -1 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1725: expect (ObjNum 575 snap 182 seq_num 575) 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1726: write oid 3 current snap is 212 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: finishing write tid 4 to smithi07685715-17 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: finishing write tid 5 to smithi07685715-17 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: oid 17 updating version 0 to 947 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: oid 17 updating version 947 to 948 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: oid 17 updating version 948 to 949 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: oid 17 updating version 949 to 950 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 950 (ObjNum 661 snap 212 seq_num 661) dirty exists 2024-09-21T09:45:16.730 INFO:tasks.rados.rados.0.smithi076.stdout:1721: left oid 17 (ObjNum 661 snap 212 seq_num 661) 2024-09-21T09:45:16.731 INFO:tasks.rados.rados.0.smithi076.stdout:1722: finishing write tid 1 to smithi07685715-23 2024-09-21T09:45:16.731 INFO:tasks.rados.rados.0.smithi076.stdout:1726: seq_num 664 ranges {759848=488047,1989220=760224} 2024-09-21T09:45:16.736 INFO:tasks.rados.rados.0.smithi076.stdout:1726: writing smithi07685715-3 from 759848 to 1247895 tid 1 2024-09-21T09:45:16.744 INFO:tasks.rados.rados.0.smithi076.stdout:1726: writing smithi07685715-3 from 1989220 to 2749444 tid 2 2024-09-21T09:45:16.744 INFO:tasks.rados.rados.0.smithi076.stdout:1721: done (5 left) 2024-09-21T09:45:16.744 INFO:tasks.rados.rados.0.smithi076.stdout:1727: snap_remove snap 211 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: Reconfiguring mds.foo.smithi076.jymebx (monmap changed)... 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[106482]: Reconfiguring daemon mds.foo.smithi076.jymebx on smithi076 2024-09-21T09:45:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T09:45:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: Reconfiguring mds.foo.smithi076.jymebx (monmap changed)... 2024-09-21T09:45:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:45:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:16.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:16 smithi076 ceph-mon[110963]: Reconfiguring daemon mds.foo.smithi076.jymebx on smithi076 2024-09-21T09:45:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T09:45:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:17.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:17.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: Reconfiguring mds.foo.smithi076.jymebx (monmap changed)... 2024-09-21T09:45:17.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi076.jymebx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:45:17.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:17.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:16 smithi191 ceph-mon[102206]: Reconfiguring daemon mds.foo.smithi076.jymebx on smithi076 2024-09-21T09:45:17.657 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:45:17.686 INFO:tasks.rados.rados.0.smithi076.stdout:1727: done (5 left) 2024-09-21T09:45:17.686 INFO:tasks.rados.rados.0.smithi076.stdout:1728: snap_remove snap 209 2024-09-21T09:45:17.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: pgmap v224: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:45:17.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T09:45:17.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:17.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:17.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:17.999 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:18.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: pgmap v224: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: pgmap v224: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T09:45:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:18.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:18.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:45:18.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:45:18.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:18.222 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:45:18.222 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (3m) 66s ago 9m 19.6M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:45:18.222 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (3m) 16s ago 9m 79.7M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (8m) 66s ago 8m 19.7M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (8m) 16s ago 8m 17.2M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (4m) 16s ago 13m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (5m) 66s ago 15m 510M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (2m) 66s ago 15m 51.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (28s) 16s ago 14m 20.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 bb6de71151ea 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (2m) 66s ago 13m 38.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (4m) 66s ago 9m 12.2M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (4m) 16s ago 9m 11.0M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (13m) 66s ago 13m 106M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (12m) 66s ago 12m 83.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (12m) 66s ago 12m 104M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (12m) 66s ago 12m 79.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (11m) 16s ago 11m 98.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:45:18.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (11m) 16s ago 11m 105M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:45:18.224 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (10m) 16s ago 10m 78.0M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:45:18.224 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (10m) 16s ago 10m 78.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:45:18.224 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (4m) 16s ago 10m 45.8M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:45:18.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 10, 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:45:18.563 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1728: done (5 left) 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1729: rollback oid 13 current snap is 212 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1729: done (5 left) 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1730: read oid 44 snap -1 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1730: expect deleted 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1731: delete oid 41 current snap is 212 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1722: finishing write tid 2 to smithi07685715-23 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1722: finishing write tid 3 to smithi07685715-23 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1722: finishing write tid 4 to smithi07685715-23 2024-09-21T09:45:18.664 INFO:tasks.rados.rados.0.smithi076.stdout:1722: oid 23 updating version 0 to 1379 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1722: oid 23 version 1379 is already newer than 1377 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1722: oid 23 version 1379 is already newer than 1378 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1379 (ObjNum 662 snap 212 seq_num 662) dirty exists 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1722: left oid 23 (ObjNum 662 snap 212 seq_num 662) 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: finishing write tid 1 to smithi07685715-47 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: finishing write tid 2 to smithi07685715-47 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: finishing write tid 3 to smithi07685715-47 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: finishing write tid 4 to smithi07685715-47 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: finishing write tid 5 to smithi07685715-47 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: oid 47 updating version 0 to 1384 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: oid 47 version 1384 is already newer than 1381 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: oid 47 version 1384 is already newer than 1382 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:1723: oid 47 version 1384 is already newer than 1383 2024-09-21T09:45:18.665 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1384 (ObjNum 663 snap 212 seq_num 663) dirty exists 2024-09-21T09:45:18.666 INFO:tasks.rados.rados.0.smithi076.stdout:1723: left oid 47 (ObjNum 663 snap 212 seq_num 663) 2024-09-21T09:45:18.666 INFO:tasks.rados.rados.0.smithi076.stdout:1725: expect (ObjNum 575 snap 182 seq_num 575) 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: Reconfiguring daemon mon.b on smithi191 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/234041947' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:18.737 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:18 smithi191 ceph-mon[102206]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T09:45:18.838 INFO:tasks.rados.rados.0.smithi076.stdout:1724: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:45:18.865 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:45:18.865 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:45:18.865 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:45:18.865 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-21T09:45:18.865 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [ 2024-09-21T09:45:18.865 INFO:teuthology.orchestra.run.smithi076.stdout: "mon" 2024-09-21T09:45:18.866 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:45:18.866 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "3/3 daemons upgraded", 2024-09-21T09:45:18.866 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:45:18.866 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:45:18.866 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:45:18.866 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:45:18.866 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: Reconfiguring daemon mon.b on smithi191 2024-09-21T09:45:18.866 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.866 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T09:45:18.866 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/234041947' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[106482]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: Reconfiguring daemon mon.b on smithi191 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/234041947' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:18.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:18 smithi076 ceph-mon[110963]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T09:45:18.941 INFO:tasks.rados.rados.0.smithi076.stdout:1726: finishing write tid 1 to smithi07685715-3 2024-09-21T09:45:18.941 INFO:tasks.rados.rados.0.smithi076.stdout:1726: finishing write tid 2 to smithi07685715-3 2024-09-21T09:45:18.941 INFO:tasks.rados.rados.0.smithi076.stdout:1726: finishing write tid 3 to smithi07685715-3 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1726: finishing write tid 4 to smithi07685715-3 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1726: oid 3 updating version 0 to 84 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1726: oid 3 version 84 is already newer than 82 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1726: oid 3 version 84 is already newer than 83 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 84 (ObjNum 664 snap 212 seq_num 664) dirty exists 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1726: left oid 3 (ObjNum 664 snap 212 seq_num 664) 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1722: done (6 left) 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1723: done (5 left) 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1724: done (4 left) 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1725: done (3 left) 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1726: done (2 left) 2024-09-21T09:45:18.942 INFO:tasks.rados.rados.0.smithi076.stdout:1731: done (1 left) 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1732: read oid 23 snap -1 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1732: expect (ObjNum 662 snap 212 seq_num 662) 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1733: read oid 42 snap -1 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1733: expect deleted 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1734: read oid 45 snap -1 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1734: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1735: write oid 35 current snap is 212 2024-09-21T09:45:18.943 INFO:tasks.rados.rados.0.smithi076.stdout:1735: seq_num 666 ranges {793462=638545,2106184=763893} 2024-09-21T09:45:18.949 INFO:tasks.rados.rados.0.smithi076.stdout:1735: writing smithi07685715-35 from 793462 to 1432007 tid 1 2024-09-21T09:45:18.958 INFO:tasks.rados.rados.0.smithi076.stdout:1735: writing smithi07685715-35 from 2106184 to 2870077 tid 2 2024-09-21T09:45:18.958 INFO:tasks.rados.rados.0.smithi076.stdout:1730: done (4 left) 2024-09-21T09:45:18.958 INFO:tasks.rados.rados.0.smithi076.stdout:1736: read oid 40 snap -1 2024-09-21T09:45:18.958 INFO:tasks.rados.rados.0.smithi076.stdout:1736: expect deleted 2024-09-21T09:45:18.958 INFO:tasks.rados.rados.0.smithi076.stdout:1737: delete oid 30 current snap is 212 2024-09-21T09:45:18.958 INFO:tasks.rados.rados.0.smithi076.stdout:1734: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:45:19.093 INFO:tasks.rados.rados.0.smithi076.stdout:1732: expect (ObjNum 662 snap 212 seq_num 662) 2024-09-21T09:45:19.231 INFO:tasks.rados.rados.0.smithi076.stdout:1735: finishing write tid 1 to smithi07685715-35 2024-09-21T09:45:19.231 INFO:tasks.rados.rados.0.smithi076.stdout:1732: done (5 left) 2024-09-21T09:45:19.231 INFO:tasks.rados.rados.0.smithi076.stdout:1733: done (4 left) 2024-09-21T09:45:19.231 INFO:tasks.rados.rados.0.smithi076.stdout:1734: done (3 left) 2024-09-21T09:45:19.231 INFO:tasks.rados.rados.0.smithi076.stdout:1737: done (2 left) 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1738: delete oid 41 current snap is 212 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: finishing write tid 2 to smithi07685715-35 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: finishing write tid 3 to smithi07685715-35 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: finishing write tid 4 to smithi07685715-35 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: oid 35 updating version 0 to 1330 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: oid 35 updating version 1330 to 1331 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: oid 35 version 1331 is already newer than 1329 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1331 (ObjNum 666 snap 212 seq_num 666) dirty exists 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: left oid 35 (ObjNum 666 snap 212 seq_num 666) 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1735: done (2 left) 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1736: done (1 left) 2024-09-21T09:45:19.232 INFO:tasks.rados.rados.0.smithi076.stdout:1738: done (0 left) 2024-09-21T09:45:19.233 INFO:tasks.rados.rados.0.smithi076.stdout:1739: delete oid 27 current snap is 212 2024-09-21T09:45:19.234 INFO:tasks.rados.rados.0.smithi076.stdout:1739: done (0 left) 2024-09-21T09:45:19.234 INFO:tasks.rados.rados.0.smithi076.stdout:1740: read oid 13 snap -1 2024-09-21T09:45:19.234 INFO:tasks.rados.rados.0.smithi076.stdout:1740: expect (ObjNum 631 snap 203 seq_num 631) 2024-09-21T09:45:19.234 INFO:tasks.rados.rados.0.smithi076.stdout:1741: snap_create 2024-09-21T09:45:19.236 INFO:tasks.rados.rados.0.smithi076.stdout:1740: expect (ObjNum 631 snap 203 seq_num 631) 2024-09-21T09:45:19.899 INFO:tasks.rados.rados.0.smithi076.stdout:1740: done (1 left) 2024-09-21T09:45:19.899 INFO:tasks.rados.rados.0.smithi076.stdout:1741: done (0 left) 2024-09-21T09:45:19.899 INFO:tasks.rados.rados.0.smithi076.stdout:1742: snap_create 2024-09-21T09:45:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: pgmap v227: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:45:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:20.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:45:20.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:45:20.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:45:20.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:20.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:19 smithi191 ceph-mon[102206]: Reconfiguring daemon mgr.x on smithi191 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: pgmap v227: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:45:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[110963]: Reconfiguring daemon mgr.x on smithi191 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: pgmap v227: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:20.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:19 smithi076 ceph-mon[106482]: Reconfiguring daemon mgr.x on smithi191 2024-09-21T09:45:20.915 INFO:tasks.rados.rados.0.smithi076.stdout:1742: done (0 left) 2024-09-21T09:45:20.915 INFO:tasks.rados.rados.0.smithi076.stdout:1743: write oid 50 current snap is 214 2024-09-21T09:45:20.915 INFO:tasks.rados.rados.0.smithi076.stdout:1743: seq_num 670 ranges {460690=655894,1540112=771231,2716111=539778} 2024-09-21T09:45:20.921 INFO:tasks.rados.rados.0.smithi076.stdout:1743: writing smithi07685715-50 from 460690 to 1116584 tid 1 2024-09-21T09:45:20.927 INFO:tasks.rados.rados.0.smithi076.stdout:1743: writing smithi07685715-50 from 1540112 to 2311343 tid 2 2024-09-21T09:45:20.932 INFO:tasks.rados.rados.0.smithi076.stdout:1743: writing smithi07685715-50 from 2716111 to 3255889 tid 3 2024-09-21T09:45:20.932 INFO:tasks.rados.rados.0.smithi076.stdout:1744: read oid 40 snap -1 2024-09-21T09:45:20.933 INFO:tasks.rados.rados.0.smithi076.stdout:1744: expect deleted 2024-09-21T09:45:20.933 INFO:tasks.rados.rados.0.smithi076.stdout:1745: snap_create 2024-09-21T09:45:20.933 INFO:tasks.rados.rados.0.smithi076.stdout:1743: finishing write tid 1 to smithi07685715-50 2024-09-21T09:45:20.933 INFO:tasks.rados.rados.0.smithi076.stdout:1743: finishing write tid 2 to smithi07685715-50 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: finishing write tid 3 to smithi07685715-50 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: finishing write tid 4 to smithi07685715-50 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: finishing write tid 5 to smithi07685715-50 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: oid 50 updating version 0 to 531 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: oid 50 updating version 531 to 534 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: oid 50 version 534 is already newer than 533 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: oid 50 version 534 is already newer than 532 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 534 (ObjNum 670 snap 214 seq_num 670) dirty exists 2024-09-21T09:45:20.936 INFO:tasks.rados.rados.0.smithi076.stdout:1743: left oid 50 (ObjNum 670 snap 214 seq_num 670) 2024-09-21T09:45:21.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:20 smithi191 ceph-mon[102206]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T09:45:21.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:21.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:21.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:45:21.062 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[110963]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[106482]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:45:21.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1743: done (2 left) 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1744: done (1 left) 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1745: done (0 left) 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1746: read oid 25 snap -1 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1746: expect (ObjNum 643 snap 207 seq_num 643) 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1747: read oid 1 snap -1 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1747: expect (ObjNum 651 snap 208 seq_num 651) 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1748: write oid 29 current snap is 215 2024-09-21T09:45:21.915 INFO:tasks.rados.rados.0.smithi076.stdout:1748: seq_num 671 ranges {477497=731143,1998594=514889,3143534=133963} 2024-09-21T09:45:21.921 INFO:tasks.rados.rados.0.smithi076.stdout:1748: writing smithi07685715-29 from 477497 to 1208640 tid 1 2024-09-21T09:45:21.929 INFO:tasks.rados.rados.0.smithi076.stdout:1748: writing smithi07685715-29 from 1998594 to 2513483 tid 2 2024-09-21T09:45:21.933 INFO:tasks.rados.rados.0.smithi076.stdout:1748: writing smithi07685715-29 from 3143534 to 3277497 tid 3 2024-09-21T09:45:21.933 INFO:tasks.rados.rados.0.smithi076.stdout:1749: delete oid 3 current snap is 215 2024-09-21T09:45:21.934 INFO:tasks.rados.rados.0.smithi076.stdout:1746: expect (ObjNum 643 snap 207 seq_num 643) 2024-09-21T09:45:22.091 INFO:tasks.rados.rados.0.smithi076.stdout:1747: expect (ObjNum 651 snap 208 seq_num 651) 2024-09-21T09:45:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: pgmap v230: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:45:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.4 on smithi191 2024-09-21T09:45:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T09:45:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:45:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:21 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: finishing write tid 1 to smithi07685715-29 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: finishing write tid 2 to smithi07685715-29 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: finishing write tid 3 to smithi07685715-29 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: finishing write tid 4 to smithi07685715-29 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: finishing write tid 5 to smithi07685715-29 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: oid 29 updating version 0 to 470 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: oid 29 updating version 470 to 471 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: oid 29 version 471 is already newer than 469 2024-09-21T09:45:22.260 INFO:tasks.rados.rados.0.smithi076.stdout:1748: oid 29 version 471 is already newer than 468 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 471 (ObjNum 671 snap 215 seq_num 671) dirty exists 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1748: left oid 29 (ObjNum 671 snap 215 seq_num 671) 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1746: done (3 left) 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1747: done (2 left) 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1748: done (1 left) 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1749: done (0 left) 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1750: delete oid 44 current snap is 215 2024-09-21T09:45:22.261 INFO:tasks.rados.rados.0.smithi076.stdout:1750: done (0 left) 2024-09-21T09:45:22.262 INFO:tasks.rados.rados.0.smithi076.stdout:1751: write oid 25 current snap is 215 2024-09-21T09:45:22.262 INFO:tasks.rados.rados.0.smithi076.stdout:1751: seq_num 674 ranges {527918=556890,1657687=778569,3225803=102115} 2024-09-21T09:45:22.267 INFO:tasks.rados.rados.0.smithi076.stdout:1751: writing smithi07685715-25 from 527918 to 1084808 tid 1 2024-09-21T09:45:22.275 INFO:tasks.rados.rados.0.smithi076.stdout:1751: writing smithi07685715-25 from 1657687 to 2436256 tid 2 2024-09-21T09:45:22.280 INFO:tasks.rados.rados.0.smithi076.stdout:1751: writing smithi07685715-25 from 3225803 to 3327918 tid 3 2024-09-21T09:45:22.280 INFO:tasks.rados.rados.0.smithi076.stdout:1752: snap_remove snap 214 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: pgmap v230: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.4 on smithi191 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: pgmap v230: 89 pgs: 89 active+clean; 93 MiB data, 529 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:45:22.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.4 on smithi191 2024-09-21T09:45:22.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T09:45:22.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:22.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:22.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:45:22.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:21 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:22.925 INFO:tasks.rados.rados.0.smithi076.stdout:1752: done (1 left) 2024-09-21T09:45:22.925 INFO:tasks.rados.rados.0.smithi076.stdout:1753: snap_remove snap 213 2024-09-21T09:45:23.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:22 smithi076 ceph-mon[110963]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:45:23.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:22 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.5 on smithi191 2024-09-21T09:45:23.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:22 smithi076 ceph-mon[110963]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T09:45:23.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:22 smithi076 ceph-mon[106482]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:45:23.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:22 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.5 on smithi191 2024-09-21T09:45:23.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:22 smithi076 ceph-mon[106482]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T09:45:23.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:22 smithi191 ceph-mon[102206]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:45:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:22 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.5 on smithi191 2024-09-21T09:45:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:22 smithi191 ceph-mon[102206]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T09:45:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:45:23] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.51.0" 2024-09-21T09:45:23.664 INFO:tasks.rados.rados.0.smithi076.stdout:1753: done (1 left) 2024-09-21T09:45:23.664 INFO:tasks.rados.rados.0.smithi076.stdout:1754: snap_remove snap 212 2024-09-21T09:45:24.045 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: pgmap v233: 89 pgs: 89 active+clean; 88 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T09:45:24.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T09:45:24.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:24.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:24.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:45:24.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:24.046 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:23 smithi191 ceph-mon[102206]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: pgmap v233: 89 pgs: 89 active+clean; 88 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[110963]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T09:45:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: pgmap v233: 89 pgs: 89 active+clean; 88 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T09:45:24.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T09:45:24.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:24.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:24.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:45:24.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:24.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:23 smithi076 ceph-mon[106482]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T09:45:24.667 INFO:tasks.rados.rados.0.smithi076.stdout:1754: done (1 left) 2024-09-21T09:45:24.667 INFO:tasks.rados.rados.0.smithi076.stdout:1755: write oid 38 current snap is 215 2024-09-21T09:45:24.667 INFO:tasks.rados.rados.0.smithi076.stdout:1755: seq_num 675 ranges {544725=632139,1716169=522227,2736873=512491} 2024-09-21T09:45:24.673 INFO:tasks.rados.rados.0.smithi076.stdout:1755: writing smithi07685715-38 from 544725 to 1176864 tid 1 2024-09-21T09:45:24.679 INFO:tasks.rados.rados.0.smithi076.stdout:1755: writing smithi07685715-38 from 1716169 to 2238396 tid 2 2024-09-21T09:45:24.685 INFO:tasks.rados.rados.0.smithi076.stdout:1755: writing smithi07685715-38 from 2736873 to 3249364 tid 3 2024-09-21T09:45:24.685 INFO:tasks.rados.rados.0.smithi076.stdout:1756: rollback oid 29 current snap is 215 2024-09-21T09:45:24.685 INFO:tasks.rados.rados.0.smithi076.stdout:1756: done (2 left) 2024-09-21T09:45:24.685 INFO:tasks.rados.rados.0.smithi076.stdout:1757: write oid 34 current snap is 215 2024-09-21T09:45:24.685 INFO:tasks.rados.rados.0.smithi076.stdout:1757: seq_num 676 ranges {561532=707388,2058298=782238} 2024-09-21T09:45:24.692 INFO:tasks.rados.rados.0.smithi076.stdout:1757: writing smithi07685715-34 from 561532 to 1268920 tid 1 2024-09-21T09:45:24.701 INFO:tasks.rados.rados.0.smithi076.stdout:1757: writing smithi07685715-34 from 2058298 to 2840536 tid 2 2024-09-21T09:45:24.701 INFO:tasks.rados.rados.0.smithi076.stdout:1758: read oid 35 snap -1 2024-09-21T09:45:24.701 INFO:tasks.rados.rados.0.smithi076.stdout:1758: expect (ObjNum 666 snap 212 seq_num 666) 2024-09-21T09:45:24.701 INFO:tasks.rados.rados.0.smithi076.stdout:1759: write oid 15 current snap is 215 2024-09-21T09:45:24.701 INFO:tasks.rados.rados.0.smithi076.stdout:1759: seq_num 677 ranges {578339=498990,1833133=525896,2791719=586620} 2024-09-21T09:45:24.707 INFO:tasks.rados.rados.0.smithi076.stdout:1759: writing smithi07685715-15 from 578339 to 1077329 tid 1 2024-09-21T09:45:24.714 INFO:tasks.rados.rados.0.smithi076.stdout:1759: writing smithi07685715-15 from 1833133 to 2359029 tid 2 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1759: writing smithi07685715-15 from 2791719 to 3378339 tid 3 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1760: write oid 37 current snap is 215 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1751: finishing write tid 1 to smithi07685715-25 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1751: finishing write tid 2 to smithi07685715-25 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1751: finishing write tid 3 to smithi07685715-25 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1751: finishing write tid 4 to smithi07685715-25 2024-09-21T09:45:24.721 INFO:tasks.rados.rados.0.smithi076.stdout:1751: finishing write tid 5 to smithi07685715-25 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1751: oid 25 updating version 0 to 360 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1751: oid 25 version 360 is already newer than 358 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1751: oid 25 version 360 is already newer than 357 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1751: oid 25 version 360 is already newer than 359 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 360 (ObjNum 674 snap 215 seq_num 674) dirty exists 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1751: left oid 25 (ObjNum 674 snap 215 seq_num 674) 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: finishing write tid 1 to smithi07685715-38 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: finishing write tid 2 to smithi07685715-38 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: finishing write tid 3 to smithi07685715-38 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: finishing write tid 4 to smithi07685715-38 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: finishing write tid 5 to smithi07685715-38 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: oid 38 updating version 0 to 959 2024-09-21T09:45:24.722 INFO:tasks.rados.rados.0.smithi076.stdout:1755: oid 38 version 959 is already newer than 958 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1755: oid 38 updating version 959 to 960 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1755: oid 38 version 960 is already newer than 957 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 960 (ObjNum 675 snap 215 seq_num 675) dirty exists 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1755: left oid 38 (ObjNum 675 snap 215 seq_num 675) 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: finishing write tid 1 to smithi07685715-34 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: finishing write tid 2 to smithi07685715-34 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: finishing write tid 3 to smithi07685715-34 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: finishing write tid 4 to smithi07685715-34 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: oid 34 updating version 0 to 308 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: oid 34 updating version 308 to 309 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: oid 34 version 309 is already newer than 307 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 309 (ObjNum 676 snap 215 seq_num 676) dirty exists 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1757: left oid 34 (ObjNum 676 snap 215 seq_num 676) 2024-09-21T09:45:24.723 INFO:tasks.rados.rados.0.smithi076.stdout:1760: seq_num 678 ranges {595146=574239,1891615=669554,3219142=176004} 2024-09-21T09:45:24.726 INFO:tasks.rados.rados.0.smithi076.stdout:1760: writing smithi07685715-37 from 595146 to 1169385 tid 1 2024-09-21T09:45:24.734 INFO:tasks.rados.rados.0.smithi076.stdout:1760: writing smithi07685715-37 from 1891615 to 2561169 tid 2 2024-09-21T09:45:24.738 INFO:tasks.rados.rados.0.smithi076.stdout:1760: writing smithi07685715-37 from 3219142 to 3395146 tid 3 2024-09-21T09:45:24.738 INFO:tasks.rados.rados.0.smithi076.stdout:1751: done (5 left) 2024-09-21T09:45:24.739 INFO:tasks.rados.rados.0.smithi076.stdout:1755: done (4 left) 2024-09-21T09:45:24.739 INFO:tasks.rados.rados.0.smithi076.stdout:1757: done (3 left) 2024-09-21T09:45:24.739 INFO:tasks.rados.rados.0.smithi076.stdout:1761: snap_create 2024-09-21T09:45:24.739 INFO:tasks.rados.rados.0.smithi076.stdout:1758: expect (ObjNum 666 snap 212 seq_num 666) 2024-09-21T09:45:24.940 INFO:tasks.rados.rados.0.smithi076.stdout:1759: finishing write tid 1 to smithi07685715-15 2024-09-21T09:45:24.940 INFO:tasks.rados.rados.0.smithi076.stdout:1759: finishing write tid 2 to smithi07685715-15 2024-09-21T09:45:24.940 INFO:tasks.rados.rados.0.smithi076.stdout:1759: finishing write tid 3 to smithi07685715-15 2024-09-21T09:45:24.940 INFO:tasks.rados.rados.0.smithi076.stdout:1759: finishing write tid 4 to smithi07685715-15 2024-09-21T09:45:24.940 INFO:tasks.rados.rados.0.smithi076.stdout:1759: finishing write tid 5 to smithi07685715-15 2024-09-21T09:45:24.940 INFO:tasks.rados.rados.0.smithi076.stdout:1759: oid 15 updating version 0 to 345 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1759: oid 15 updating version 345 to 348 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1759: oid 15 version 348 is already newer than 346 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1759: oid 15 version 348 is already newer than 347 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 348 (ObjNum 677 snap 215 seq_num 677) dirty exists 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1759: left oid 15 (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: finishing write tid 1 to smithi07685715-37 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: finishing write tid 2 to smithi07685715-37 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: finishing write tid 3 to smithi07685715-37 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: finishing write tid 4 to smithi07685715-37 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: finishing write tid 5 to smithi07685715-37 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: oid 37 updating version 0 to 475 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: oid 37 version 475 is already newer than 472 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: oid 37 version 475 is already newer than 474 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: oid 37 version 475 is already newer than 473 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 475 (ObjNum 678 snap 215 seq_num 678) dirty exists 2024-09-21T09:45:24.941 INFO:tasks.rados.rados.0.smithi076.stdout:1760: left oid 37 (ObjNum 678 snap 215 seq_num 678) 2024-09-21T09:45:25.320 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:24 smithi191 ceph-mon[102206]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:45:25.320 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:24 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.6 on smithi191 2024-09-21T09:45:25.320 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:24 smithi191 ceph-mon[102206]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T09:45:25.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:24 smithi076 ceph-mon[106482]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:45:25.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:24 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.6 on smithi191 2024-09-21T09:45:25.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:24 smithi076 ceph-mon[106482]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T09:45:25.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:24 smithi076 ceph-mon[110963]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:45:25.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:24 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.6 on smithi191 2024-09-21T09:45:25.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:24 smithi076 ceph-mon[110963]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T09:45:25.666 INFO:tasks.rados.rados.0.smithi076.stdout:1758: done (3 left) 2024-09-21T09:45:25.666 INFO:tasks.rados.rados.0.smithi076.stdout:1759: done (2 left) 2024-09-21T09:45:25.666 INFO:tasks.rados.rados.0.smithi076.stdout:1760: done (1 left) 2024-09-21T09:45:25.667 INFO:tasks.rados.rados.0.smithi076.stdout:1761: done (0 left) 2024-09-21T09:45:25.667 INFO:tasks.rados.rados.0.smithi076.stdout:1762: snap_remove snap 215 2024-09-21T09:45:26.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: pgmap v236: 89 pgs: 89 active+clean; 88 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: Reconfiguring daemon osd.7 on smithi191 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:25 smithi191 ceph-mon[102206]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: pgmap v236: 89 pgs: 89 active+clean; 88 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: Reconfiguring daemon osd.7 on smithi191 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[106482]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: pgmap v236: 89 pgs: 89 active+clean; 88 MiB data, 524 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: Reconfiguring daemon osd.7 on smithi191 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:25 smithi076 ceph-mon[110963]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:45:26.671 INFO:tasks.rados.rados.0.smithi076.stdout:1762: done (0 left) 2024-09-21T09:45:26.671 INFO:tasks.rados.rados.0.smithi076.stdout:1763: snap_create 2024-09-21T09:45:27.672 INFO:tasks.rados.rados.0.smithi076.stdout:1763: done (0 left) 2024-09-21T09:45:27.673 INFO:tasks.rados.rados.0.smithi076.stdout:1764: read oid 19 snap -1 2024-09-21T09:45:27.673 INFO:tasks.rados.rados.0.smithi076.stdout:1764: expect deleted 2024-09-21T09:45:27.673 INFO:tasks.rados.rados.0.smithi076.stdout:1765: snap_create 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: pgmap v239: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: Reconfiguring mds.foo.smithi191.dhrpzx (monmap changed)... 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: Reconfiguring daemon mds.foo.smithi191.dhrpzx on smithi191 2024-09-21T09:45:27.823 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:27 smithi191 ceph-mon[102206]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: pgmap v239: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: Reconfiguring mds.foo.smithi191.dhrpzx (monmap changed)... 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: Reconfiguring daemon mds.foo.smithi191.dhrpzx on smithi191 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[110963]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:45:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: pgmap v239: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: Reconfiguring mds.foo.smithi191.dhrpzx (monmap changed)... 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi191.dhrpzx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: Reconfiguring daemon mds.foo.smithi191.dhrpzx on smithi191 2024-09-21T09:45:27.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:27 smithi076 ceph-mon[106482]: osdmap e513: 8 total, 8 up, 8 in 2024-09-21T09:45:28.663 INFO:tasks.rados.rados.0.smithi076.stdout:1764: done (1 left) 2024-09-21T09:45:28.664 INFO:tasks.rados.rados.0.smithi076.stdout:1765: done (0 left) 2024-09-21T09:45:28.664 INFO:tasks.rados.rados.0.smithi076.stdout:1766: delete oid 42 current snap is 218 2024-09-21T09:45:28.667 INFO:tasks.rados.rados.0.smithi076.stdout:1766: done (0 left) 2024-09-21T09:45:28.667 INFO:tasks.rados.rados.0.smithi076.stdout:1767: rollback oid 48 current snap is 218 2024-09-21T09:45:28.668 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 217 2024-09-21T09:45:28.668 INFO:tasks.rados.rados.0.smithi076.stdout:1768: write oid 24 current snap is 218 2024-09-21T09:45:28.668 INFO:tasks.rados.rados.0.smithi076.stdout:1768: seq_num 680 ranges {628760=724737,1892226=673223,3157635=271125} 2024-09-21T09:45:28.677 INFO:tasks.rados.rados.0.smithi076.stdout:1768: writing smithi07685715-24 from 628760 to 1353497 tid 1 2024-09-21T09:45:28.683 INFO:tasks.rados.rados.0.smithi076.stdout:1768: writing smithi07685715-24 from 1892226 to 2565449 tid 2 2024-09-21T09:45:28.688 INFO:tasks.rados.rados.0.smithi076.stdout:1768: writing smithi07685715-24 from 3157635 to 3428760 tid 3 2024-09-21T09:45:28.689 INFO:tasks.rados.rados.0.smithi076.stdout:1769: delete oid 11 current snap is 218 2024-09-21T09:45:28.689 INFO:tasks.rados.rados.0.smithi076.stdout:1767: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:45:28.689 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 0 (ObjNum 145 snap 0 seq_num 864206512) dirty dne 2024-09-21T09:45:28.689 INFO:tasks.rados.rados.0.smithi076.stdout:1768: finishing write tid 1 to smithi07685715-24 2024-09-21T09:45:28.689 INFO:tasks.rados.rados.0.smithi076.stdout:1768: finishing write tid 2 to smithi07685715-24 2024-09-21T09:45:28.690 INFO:tasks.rados.rados.0.smithi076.stdout:1768: finishing write tid 3 to smithi07685715-24 2024-09-21T09:45:28.691 INFO:tasks.rados.rados.0.smithi076.stdout:1767: done (2 left) 2024-09-21T09:45:28.691 INFO:tasks.rados.rados.0.smithi076.stdout:1769: done (1 left) 2024-09-21T09:45:28.691 INFO:tasks.rados.rados.0.smithi076.stdout:1770: read oid 33 snap 217 2024-09-21T09:45:28.691 INFO:tasks.rados.rados.0.smithi076.stdout:1770: expect (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:45:28.691 INFO:tasks.rados.rados.0.smithi076.stdout:1771: write oid 31 current snap is 218 2024-09-21T09:45:28.691 INFO:tasks.rados.rados.0.smithi076.stdout:1771: seq_num 682 ranges {662374=475235,1609190=676892,2812481=624101} 2024-09-21T09:45:28.697 INFO:tasks.rados.rados.0.smithi076.stdout:1771: writing smithi07685715-31 from 662374 to 1137609 tid 1 2024-09-21T09:45:28.703 INFO:tasks.rados.rados.0.smithi076.stdout:1771: writing smithi07685715-31 from 1609190 to 2286082 tid 2 2024-09-21T09:45:28.710 INFO:tasks.rados.rados.0.smithi076.stdout:1771: writing smithi07685715-31 from 2812481 to 3436582 tid 3 2024-09-21T09:45:28.710 INFO:tasks.rados.rados.0.smithi076.stdout:1772: snap_create 2024-09-21T09:45:28.710 INFO:tasks.rados.rados.0.smithi076.stdout:1768: finishing write tid 4 to smithi07685715-24 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1768: finishing write tid 5 to smithi07685715-24 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1768: oid 24 updating version 0 to 181 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1768: oid 24 updating version 181 to 183 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1768: oid 24 version 183 is already newer than 182 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1768: oid 24 version 183 is already newer than 180 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 183 (ObjNum 680 snap 218 seq_num 680) dirty exists 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1768: left oid 24 (ObjNum 680 snap 218 seq_num 680) 2024-09-21T09:45:28.711 INFO:tasks.rados.rados.0.smithi076.stdout:1770: expect (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: finishing write tid 1 to smithi07685715-31 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: finishing write tid 2 to smithi07685715-31 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: finishing write tid 3 to smithi07685715-31 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: finishing write tid 4 to smithi07685715-31 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: finishing write tid 5 to smithi07685715-31 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: oid 31 updating version 0 to 654 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: oid 31 version 654 is already newer than 651 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: oid 31 version 654 is already newer than 653 2024-09-21T09:45:28.837 INFO:tasks.rados.rados.0.smithi076.stdout:1771: oid 31 version 654 is already newer than 652 2024-09-21T09:45:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 654 (ObjNum 682 snap 218 seq_num 682) dirty exists 2024-09-21T09:45:28.838 INFO:tasks.rados.rados.0.smithi076.stdout:1771: left oid 31 (ObjNum 682 snap 218 seq_num 682) 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:45:28.944 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.945 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:45:28.947 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:45:28.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[106482]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:28.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:28 smithi076 ceph-mon[110963]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:45:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: osdmap e514: 8 total, 8 up, 8 in 2024-09-21T09:45:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:29.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:45:29.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:45:29.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:45:29.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:45:29.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:45:29.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:45:29.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:29.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:28 smithi191 ceph-mon[102206]: osdmap e515: 8 total, 8 up, 8 in 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1768: done (3 left) 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1770: done (2 left) 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1771: done (1 left) 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1772: done (0 left) 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1773: rollback oid 26 current snap is 219 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 216 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1774: read oid 24 snap -1 2024-09-21T09:45:29.689 INFO:tasks.rados.rados.0.smithi076.stdout:1774: expect (ObjNum 680 snap 218 seq_num 680) 2024-09-21T09:45:29.690 INFO:tasks.rados.rados.0.smithi076.stdout:1775: delete oid 31 current snap is 219 2024-09-21T09:45:29.696 INFO:tasks.rados.rados.0.smithi076.stdout:1774: expect (ObjNum 680 snap 218 seq_num 680) 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1773: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1773: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1773: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 618 (ObjNum 597 snap 190 seq_num 597) dirty exists 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1773: done (2 left) 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1774: done (1 left) 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1775: done (0 left) 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:1776: rollback oid 40 current snap is 219 2024-09-21T09:45:29.895 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 216 2024-09-21T09:45:29.896 INFO:tasks.rados.rados.0.smithi076.stdout:1777: read oid 30 snap -1 2024-09-21T09:45:29.896 INFO:tasks.rados.rados.0.smithi076.stdout:1777: expect deleted 2024-09-21T09:45:29.896 INFO:tasks.rados.rados.0.smithi076.stdout:1778: snap_create 2024-09-21T09:45:29.897 INFO:tasks.rados.rados.0.smithi076.stdout:1776: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:45:29.897 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 0 (ObjNum 609 snap 0 seq_num 1682331778) dirty dne 2024-09-21T09:45:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:45:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all grafana 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all loki 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all promtail 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Finalizing container_image settings 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: Upgrade: Complete! 2024-09-21T09:45:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:29 smithi191 ceph-mon[102206]: pgmap v242: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all grafana 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all loki 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all promtail 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Finalizing container_image settings 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: Upgrade: Complete! 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[106482]: pgmap v242: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:45:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all grafana 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all loki 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all promtail 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Finalizing container_image settings 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: Upgrade: Complete! 2024-09-21T09:45:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:29 smithi076 ceph-mon[110963]: pgmap v242: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:45:30.691 INFO:tasks.rados.rados.0.smithi076.stdout:1776: done (2 left) 2024-09-21T09:45:30.691 INFO:tasks.rados.rados.0.smithi076.stdout:1777: done (1 left) 2024-09-21T09:45:30.691 INFO:tasks.rados.rados.0.smithi076.stdout:1778: done (0 left) 2024-09-21T09:45:30.691 INFO:tasks.rados.rados.0.smithi076.stdout:1779: write oid 35 current snap is 220 2024-09-21T09:45:30.691 INFO:tasks.rados.rados.0.smithi076.stdout:1779: seq_num 684 ranges {695988=625733,2009801=680561,3150974=345014} 2024-09-21T09:45:30.698 INFO:tasks.rados.rados.0.smithi076.stdout:1779: writing smithi07685715-35 from 695988 to 1321721 tid 1 2024-09-21T09:45:30.705 INFO:tasks.rados.rados.0.smithi076.stdout:1779: writing smithi07685715-35 from 2009801 to 2690362 tid 2 2024-09-21T09:45:30.710 INFO:tasks.rados.rados.0.smithi076.stdout:1779: writing smithi07685715-35 from 3150974 to 3495988 tid 3 2024-09-21T09:45:30.710 INFO:tasks.rados.rados.0.smithi076.stdout:1780: write oid 48 current snap is 220 2024-09-21T09:45:30.710 INFO:tasks.rados.rados.0.smithi076.stdout:1780: seq_num 685 ranges {712795=417335,1784636=540572,2894750=457917} 2024-09-21T09:45:30.716 INFO:tasks.rados.rados.0.smithi076.stdout:1780: writing smithi07685715-48 from 712795 to 1130130 tid 1 2024-09-21T09:45:30.723 INFO:tasks.rados.rados.0.smithi076.stdout:1780: writing smithi07685715-48 from 1784636 to 2325208 tid 2 2024-09-21T09:45:30.729 INFO:tasks.rados.rados.0.smithi076.stdout:1780: writing smithi07685715-48 from 2894750 to 3352667 tid 3 2024-09-21T09:45:30.729 INFO:tasks.rados.rados.0.smithi076.stdout:1781: rollback oid 20 current snap is 220 2024-09-21T09:45:30.729 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 216 2024-09-21T09:45:30.729 INFO:tasks.rados.rados.0.smithi076.stdout:1779: finishing write tid 1 to smithi07685715-35 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: finishing write tid 2 to smithi07685715-35 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: finishing write tid 3 to smithi07685715-35 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: finishing write tid 4 to smithi07685715-35 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1782: read oid 14 snap -1 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1782: expect deleted 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1783: snap_create 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: finishing write tid 5 to smithi07685715-35 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: oid 35 updating version 0 to 1336 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: oid 35 version 1336 is already newer than 1334 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: oid 35 updating version 1336 to 1337 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: oid 35 version 1337 is already newer than 1335 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1337 (ObjNum 684 snap 220 seq_num 684) dirty exists 2024-09-21T09:45:30.730 INFO:tasks.rados.rados.0.smithi076.stdout:1779: left oid 35 (ObjNum 684 snap 220 seq_num 684) 2024-09-21T09:45:30.731 INFO:tasks.rados.rados.0.smithi076.stdout:1780: finishing write tid 1 to smithi07685715-48 2024-09-21T09:45:30.731 INFO:tasks.rados.rados.0.smithi076.stdout:1780: finishing write tid 2 to smithi07685715-48 2024-09-21T09:45:30.731 INFO:tasks.rados.rados.0.smithi076.stdout:1781: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:45:30.731 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:30.732 INFO:tasks.rados.rados.0.smithi076.stdout:1780: finishing write tid 3 to smithi07685715-48 2024-09-21T09:45:30.732 INFO:tasks.rados.rados.0.smithi076.stdout:1780: finishing write tid 4 to smithi07685715-48 2024-09-21T09:45:30.732 INFO:tasks.rados.rados.0.smithi076.stdout:1780: finishing write tid 5 to smithi07685715-48 2024-09-21T09:45:30.732 INFO:tasks.rados.rados.0.smithi076.stdout:1780: oid 48 updating version 0 to 1398 2024-09-21T09:45:30.732 INFO:tasks.rados.rados.0.smithi076.stdout:1780: oid 48 version 1398 is already newer than 1397 2024-09-21T09:45:30.732 INFO:tasks.rados.rados.0.smithi076.stdout:1780: oid 48 version 1398 is already newer than 1396 2024-09-21T09:45:30.733 INFO:tasks.rados.rados.0.smithi076.stdout:1780: oid 48 version 1398 is already newer than 1395 2024-09-21T09:45:30.733 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1398 (ObjNum 685 snap 220 seq_num 685) dirty exists 2024-09-21T09:45:30.733 INFO:tasks.rados.rados.0.smithi076.stdout:1780: left oid 48 (ObjNum 685 snap 220 seq_num 685) 2024-09-21T09:45:30.779 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:30 smithi076 ceph-mon[106482]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:45:30.779 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:30 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:30.779 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:30 smithi076 ceph-mon[110963]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:45:30.779 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:30 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:30 smithi191 ceph-mon[102206]: osdmap e516: 8 total, 8 up, 8 in 2024-09-21T09:45:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:30 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:31.700 INFO:tasks.rados.rados.0.smithi076.stdout:1779: done (4 left) 2024-09-21T09:45:31.700 INFO:tasks.rados.rados.0.smithi076.stdout:1780: done (3 left) 2024-09-21T09:45:31.700 INFO:tasks.rados.rados.0.smithi076.stdout:1781: done (2 left) 2024-09-21T09:45:31.700 INFO:tasks.rados.rados.0.smithi076.stdout:1782: done (1 left) 2024-09-21T09:45:31.700 INFO:tasks.rados.rados.0.smithi076.stdout:1783: done (0 left) 2024-09-21T09:45:31.700 INFO:tasks.rados.rados.0.smithi076.stdout:1784: delete oid 48 current snap is 221 2024-09-21T09:45:31.705 INFO:tasks.rados.rados.0.smithi076.stdout:1784: done (0 left) 2024-09-21T09:45:31.705 INFO:tasks.rados.rados.0.smithi076.stdout:1785: write oid 37 current snap is 221 2024-09-21T09:45:31.705 INFO:tasks.rados.rados.0.smithi076.stdout:1785: seq_num 687 ranges {746409=567833,1901600=544241,3065949=480460} 2024-09-21T09:45:31.712 INFO:tasks.rados.rados.0.smithi076.stdout:1785: writing smithi07685715-37 from 746409 to 1314242 tid 1 2024-09-21T09:45:31.718 INFO:tasks.rados.rados.0.smithi076.stdout:1785: writing smithi07685715-37 from 1901600 to 2445841 tid 2 2024-09-21T09:45:31.725 INFO:tasks.rados.rados.0.smithi076.stdout:1785: writing smithi07685715-37 from 3065949 to 3546409 tid 3 2024-09-21T09:45:31.725 INFO:tasks.rados.rados.0.smithi076.stdout:1786: snap_create 2024-09-21T09:45:31.725 INFO:tasks.rados.rados.0.smithi076.stdout:1785: finishing write tid 1 to smithi07685715-37 2024-09-21T09:45:31.725 INFO:tasks.rados.rados.0.smithi076.stdout:1785: finishing write tid 2 to smithi07685715-37 2024-09-21T09:45:31.727 INFO:tasks.rados.rados.0.smithi076.stdout:1785: finishing write tid 3 to smithi07685715-37 2024-09-21T09:45:31.727 INFO:tasks.rados.rados.0.smithi076.stdout:1785: finishing write tid 4 to smithi07685715-37 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:1785: finishing write tid 5 to smithi07685715-37 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:1785: oid 37 updating version 0 to 478 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:1785: oid 37 updating version 478 to 480 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:1785: oid 37 version 480 is already newer than 477 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:1785: oid 37 version 480 is already newer than 479 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 480 (ObjNum 687 snap 221 seq_num 687) dirty exists 2024-09-21T09:45:31.728 INFO:tasks.rados.rados.0.smithi076.stdout:1785: left oid 37 (ObjNum 687 snap 221 seq_num 687) 2024-09-21T09:45:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:31 smithi191 ceph-mon[102206]: pgmap v245: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:31 smithi191 ceph-mon[102206]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:45:32.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:31 smithi076 ceph-mon[106482]: pgmap v245: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:32.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:31 smithi076 ceph-mon[106482]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:45:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:31 smithi076 ceph-mon[110963]: pgmap v245: 89 pgs: 89 active+clean; 91 MiB data, 538 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:31 smithi076 ceph-mon[110963]: osdmap e517: 8 total, 8 up, 8 in 2024-09-21T09:45:32.706 INFO:tasks.rados.rados.0.smithi076.stdout:1785: done (1 left) 2024-09-21T09:45:32.707 INFO:tasks.rados.rados.0.smithi076.stdout:1786: done (0 left) 2024-09-21T09:45:32.707 INFO:tasks.rados.rados.0.smithi076.stdout:1787: delete oid 20 current snap is 222 2024-09-21T09:45:32.711 INFO:tasks.rados.rados.0.smithi076.stdout:1787: done (0 left) 2024-09-21T09:45:32.711 INFO:tasks.rados.rados.0.smithi076.stdout:1788: write oid 16 current snap is 222 2024-09-21T09:45:32.711 INFO:tasks.rados.rados.0.smithi076.stdout:1788: seq_num 689 ranges {780023=718331,1902211=547910,3004442=575581} 2024-09-21T09:45:32.719 INFO:tasks.rados.rados.0.smithi076.stdout:1788: writing smithi07685715-16 from 780023 to 1498354 tid 1 2024-09-21T09:45:32.724 INFO:tasks.rados.rados.0.smithi076.stdout:1788: writing smithi07685715-16 from 1902211 to 2450121 tid 2 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1788: writing smithi07685715-16 from 3004442 to 3580023 tid 3 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1789: read oid 11 snap -1 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1789: expect deleted 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1790: read oid 3 snap -1 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1790: expect deleted 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1791: rollback oid 28 current snap is 222 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 28 to 220 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1788: finishing write tid 1 to smithi07685715-16 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1788: finishing write tid 2 to smithi07685715-16 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1792: write oid 26 current snap is 222 2024-09-21T09:45:32.731 INFO:tasks.rados.rados.0.smithi076.stdout:1792: seq_num 690 ranges {796830=793580,2360693=691568} 2024-09-21T09:45:32.739 INFO:tasks.rados.rados.0.smithi076.stdout:1792: writing smithi07685715-26 from 796830 to 1590410 tid 1 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1792: writing smithi07685715-26 from 2360693 to 3052261 tid 2 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1793: rollback oid 45 current snap is 222 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 218 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1788: finishing write tid 3 to smithi07685715-16 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1788: finishing write tid 4 to smithi07685715-16 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1789: done (5 left) 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1790: done (4 left) 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1794: write oid 23 current snap is 222 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1788: finishing write tid 5 to smithi07685715-16 2024-09-21T09:45:32.748 INFO:tasks.rados.rados.0.smithi076.stdout:1788: oid 16 updating version 0 to 829 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:1788: oid 16 updating version 829 to 832 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:1788: oid 16 version 832 is already newer than 830 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:1788: oid 16 version 832 is already newer than 831 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 832 (ObjNum 689 snap 222 seq_num 689) dirty exists 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:1788: left oid 16 (ObjNum 689 snap 222 seq_num 689) 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:1791: finishing rollback tid 0 to smithi07685715-28 2024-09-21T09:45:32.749 INFO:tasks.rados.rados.0.smithi076.stdout:1794: seq_num 691 ranges {413637=468829,1619175=435226,2542935=758255} 2024-09-21T09:45:32.753 INFO:tasks.rados.rados.0.smithi076.stdout:1794: writing smithi07685715-23 from 413637 to 882466 tid 1 2024-09-21T09:45:32.759 INFO:tasks.rados.rados.0.smithi076.stdout:1794: writing smithi07685715-23 from 1619175 to 2054401 tid 2 2024-09-21T09:45:32.767 INFO:tasks.rados.rados.0.smithi076.stdout:1794: writing smithi07685715-23 from 2542935 to 3301190 tid 3 2024-09-21T09:45:32.767 INFO:tasks.rados.rados.0.smithi076.stdout:1788: done (4 left) 2024-09-21T09:45:32.767 INFO:tasks.rados.rados.0.smithi076.stdout:1795: read oid 25 snap -1 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1795: expect (ObjNum 674 snap 215 seq_num 674) 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1796: rollback oid 16 current snap is 222 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 218 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1791: finishing rollback tid 1 to smithi07685715-28 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1791: finishing rollback tid 2 to smithi07685715-28 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 1404 (ObjNum 529 snap 168 seq_num 529) dirty exists 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1792: finishing write tid 1 to smithi07685715-26 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1791: done (5 left) 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1797: read oid 2 snap -1 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1797: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:45:32.768 INFO:tasks.rados.rados.0.smithi076.stdout:1798: snap_remove snap 216 2024-09-21T09:45:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:45:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:33.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:33.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:33.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:33.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:33.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: osdmap e518: 8 total, 8 up, 8 in 2024-09-21T09:45:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:45:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:45:33.292 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:45:33] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.51.0" 2024-09-21T09:45:33.670 INFO:tasks.rados.rados.0.smithi076.stdout:1798: done (6 left) 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1799: snap_create 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1793: finishing rollback tid 0 to smithi07685715-45 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1793: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1793: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 352 (ObjNum 540 snap 171 seq_num 540) dirty exists 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: finishing write tid 2 to smithi07685715-26 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: finishing write tid 3 to smithi07685715-26 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: finishing write tid 4 to smithi07685715-26 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: oid 26 updating version 0 to 620 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: oid 26 updating version 620 to 621 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: oid 26 updating version 621 to 622 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 622 (ObjNum 690 snap 222 seq_num 690) dirty exists 2024-09-21T09:45:33.671 INFO:tasks.rados.rados.0.smithi076.stdout:1792: left oid 26 (ObjNum 690 snap 222 seq_num 690) 2024-09-21T09:45:33.672 INFO:tasks.rados.rados.0.smithi076.stdout:1794: finishing write tid 1 to smithi07685715-23 2024-09-21T09:45:33.672 INFO:tasks.rados.rados.0.smithi076.stdout:1794: finishing write tid 2 to smithi07685715-23 2024-09-21T09:45:33.672 INFO:tasks.rados.rados.0.smithi076.stdout:1797: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:45:33.786 INFO:tasks.rados.rados.0.smithi076.stdout:1796: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:45:33.786 INFO:tasks.rados.rados.0.smithi076.stdout:1796: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:45:33.786 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 834 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:33.786 INFO:tasks.rados.rados.0.smithi076.stdout:1794: finishing write tid 3 to smithi07685715-23 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: finishing write tid 4 to smithi07685715-23 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: finishing write tid 5 to smithi07685715-23 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: oid 23 updating version 0 to 1407 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: oid 23 version 1407 is already newer than 1406 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: oid 23 updating version 1407 to 1409 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: oid 23 version 1409 is already newer than 1408 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1409 (ObjNum 691 snap 222 seq_num 691) dirty exists 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1794: left oid 23 (ObjNum 691 snap 222 seq_num 691) 2024-09-21T09:45:33.787 INFO:tasks.rados.rados.0.smithi076.stdout:1795: expect (ObjNum 674 snap 215 seq_num 674) 2024-09-21T09:45:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:33 smithi191 ceph-mon[102206]: pgmap v248: 89 pgs: 89 active+clean; 99 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:45:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:33 smithi191 ceph-mon[102206]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:45:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:33 smithi191 ceph-mon[102206]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:45:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-mon[106482]: pgmap v248: 89 pgs: 89 active+clean; 99 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:45:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-mon[106482]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:45:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-mon[106482]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:45:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-mon[110963]: pgmap v248: 89 pgs: 89 active+clean; 99 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:45:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-mon[110963]: osdmap e519: 8 total, 8 up, 8 in 2024-09-21T09:45:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:33 smithi076 ceph-mon[110963]: osdmap e520: 8 total, 8 up, 8 in 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1792: done (6 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1793: done (5 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1794: done (4 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1795: done (3 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1796: done (2 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1797: done (1 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1799: done (0 left) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1800: read oid 43 snap 222 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1800: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1801: write oid 48 current snap is 223 2024-09-21T09:45:34.670 INFO:tasks.rados.rados.0.smithi076.stdout:1801: seq_num 692 ranges {430444=660431,1677657=695237,2970358=569527} 2024-09-21T09:45:34.676 INFO:tasks.rados.rados.0.smithi076.stdout:1801: writing smithi07685715-48 from 430444 to 1090875 tid 1 2024-09-21T09:45:34.683 INFO:tasks.rados.rados.0.smithi076.stdout:1801: writing smithi07685715-48 from 1677657 to 2372894 tid 2 2024-09-21T09:45:34.690 INFO:tasks.rados.rados.0.smithi076.stdout:1801: writing smithi07685715-48 from 2970358 to 3539885 tid 3 2024-09-21T09:45:34.690 INFO:tasks.rados.rados.0.smithi076.stdout:1802: write oid 7 current snap is 223 2024-09-21T09:45:34.690 INFO:tasks.rados.rados.0.smithi076.stdout:1802: seq_num 693 ranges {447251=735680,1736139=438895,2597781=780799} 2024-09-21T09:45:34.696 INFO:tasks.rados.rados.0.smithi076.stdout:1802: writing smithi07685715-7 from 447251 to 1182931 tid 1 2024-09-21T09:45:34.702 INFO:tasks.rados.rados.0.smithi076.stdout:1802: writing smithi07685715-7 from 1736139 to 2175034 tid 2 2024-09-21T09:45:34.709 INFO:tasks.rados.rados.0.smithi076.stdout:1802: writing smithi07685715-7 from 2597781 to 3378580 tid 3 2024-09-21T09:45:34.709 INFO:tasks.rados.rados.0.smithi076.stdout:1803: snap_create 2024-09-21T09:45:34.709 INFO:tasks.rados.rados.0.smithi076.stdout:1800: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: finishing write tid 1 to smithi07685715-48 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: finishing write tid 2 to smithi07685715-48 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: finishing write tid 3 to smithi07685715-48 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: finishing write tid 4 to smithi07685715-48 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: finishing write tid 5 to smithi07685715-48 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: oid 48 updating version 0 to 1416 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: oid 48 updating version 1416 to 1417 2024-09-21T09:45:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:1801: oid 48 version 1417 is already newer than 1415 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1801: oid 48 version 1417 is already newer than 1414 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1417 (ObjNum 692 snap 223 seq_num 692) dirty exists 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1801: left oid 48 (ObjNum 692 snap 223 seq_num 692) 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: finishing write tid 1 to smithi07685715-7 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: finishing write tid 2 to smithi07685715-7 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: finishing write tid 3 to smithi07685715-7 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: finishing write tid 4 to smithi07685715-7 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: finishing write tid 5 to smithi07685715-7 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: oid 7 updating version 0 to 961 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: oid 7 updating version 961 to 964 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: oid 7 version 964 is already newer than 962 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: oid 7 version 964 is already newer than 963 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 964 (ObjNum 693 snap 223 seq_num 693) dirty exists 2024-09-21T09:45:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:1802: left oid 7 (ObjNum 693 snap 223 seq_num 693) 2024-09-21T09:45:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:1800: done (3 left) 2024-09-21T09:45:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:1801: done (2 left) 2024-09-21T09:45:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:1802: done (1 left) 2024-09-21T09:45:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:1803: done (0 left) 2024-09-21T09:45:35.710 INFO:tasks.rados.rados.0.smithi076.stdout:1804: rollback oid 29 current snap is 224 2024-09-21T09:45:35.710 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 223 2024-09-21T09:45:35.710 INFO:tasks.rados.rados.0.smithi076.stdout:1805: write oid 18 current snap is 224 2024-09-21T09:45:35.710 INFO:tasks.rados.rados.0.smithi076.stdout:1805: seq_num 694 ranges {464058=410929,1394621=698906,2625204=592071,3656107=7951} 2024-09-21T09:45:35.714 INFO:tasks.rados.rados.0.smithi076.stdout:1805: writing smithi07685715-18 from 464058 to 874987 tid 1 2024-09-21T09:45:35.721 INFO:tasks.rados.rados.0.smithi076.stdout:1805: writing smithi07685715-18 from 1394621 to 2093527 tid 2 2024-09-21T09:45:35.727 INFO:tasks.rados.rados.0.smithi076.stdout:1805: writing smithi07685715-18 from 2625204 to 3217275 tid 3 2024-09-21T09:45:35.730 INFO:tasks.rados.rados.0.smithi076.stdout:1805: writing smithi07685715-18 from 3656107 to 3664058 tid 4 2024-09-21T09:45:35.730 INFO:tasks.rados.rados.0.smithi076.stdout:1806: write oid 34 current snap is 224 2024-09-21T09:45:35.730 INFO:tasks.rados.rados.0.smithi076.stdout:1804: finishing rollback tid 0 to smithi07685715-29 2024-09-21T09:45:35.730 INFO:tasks.rados.rados.0.smithi076.stdout:1806: seq_num 695 ranges {480865=486178,1453103=442564,2652627=519696} 2024-09-21T09:45:35.735 INFO:tasks.rados.rados.0.smithi076.stdout:1806: writing smithi07685715-34 from 480865 to 967043 tid 1 2024-09-21T09:45:35.740 INFO:tasks.rados.rados.0.smithi076.stdout:1806: writing smithi07685715-34 from 1453103 to 1895667 tid 2 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1806: writing smithi07685715-34 from 2652627 to 3172323 tid 3 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1807: write oid 38 current snap is 224 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1804: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1804: finishing rollback tid 2 to smithi07685715-29 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 487 (ObjNum 671 snap 215 seq_num 671) dirty exists 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1805: finishing write tid 1 to smithi07685715-18 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1805: finishing write tid 2 to smithi07685715-18 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1805: finishing write tid 3 to smithi07685715-18 2024-09-21T09:45:35.748 INFO:tasks.rados.rados.0.smithi076.stdout:1807: seq_num 696 ranges {497672=561427,1795232=702575,2963697=730968} 2024-09-21T09:45:35.753 INFO:tasks.rados.rados.0.smithi076.stdout:1807: writing smithi07685715-38 from 497672 to 1059099 tid 1 2024-09-21T09:45:35.761 INFO:tasks.rados.rados.0.smithi076.stdout:1807: writing smithi07685715-38 from 1795232 to 2497807 tid 2 2024-09-21T09:45:35.768 INFO:tasks.rados.rados.0.smithi076.stdout:1807: writing smithi07685715-38 from 2963697 to 3694665 tid 3 2024-09-21T09:45:35.768 INFO:tasks.rados.rados.0.smithi076.stdout:1804: done (3 left) 2024-09-21T09:45:35.768 INFO:tasks.rados.rados.0.smithi076.stdout:1808: write oid 37 current snap is 224 2024-09-21T09:45:35.768 INFO:tasks.rados.rados.0.smithi076.stdout:1805: finishing write tid 4 to smithi07685715-18 2024-09-21T09:45:35.768 INFO:tasks.rados.rados.0.smithi076.stdout:1805: finishing write tid 5 to smithi07685715-18 2024-09-21T09:45:35.768 INFO:tasks.rados.rados.0.smithi076.stdout:1805: finishing write tid 6 to smithi07685715-18 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1805: oid 18 updating version 0 to 552 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1805: oid 18 version 552 is already newer than 551 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1805: oid 18 version 552 is already newer than 549 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1805: oid 18 version 552 is already newer than 550 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1805: oid 18 updating version 552 to 553 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 553 (ObjNum 694 snap 224 seq_num 694) dirty exists 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1805: left oid 18 (ObjNum 694 snap 224 seq_num 694) 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1806: finishing write tid 1 to smithi07685715-34 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1806: finishing write tid 2 to smithi07685715-34 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1806: finishing write tid 3 to smithi07685715-34 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1806: finishing write tid 4 to smithi07685715-34 2024-09-21T09:45:35.769 INFO:tasks.rados.rados.0.smithi076.stdout:1808: seq_num 697 ranges {514479=636676,1853714=446233,2991120=542240} 2024-09-21T09:45:35.774 INFO:tasks.rados.rados.0.smithi076.stdout:1808: writing smithi07685715-37 from 514479 to 1151155 tid 1 2024-09-21T09:45:35.780 INFO:tasks.rados.rados.0.smithi076.stdout:1808: writing smithi07685715-37 from 1853714 to 2299947 tid 2 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1808: writing smithi07685715-37 from 2991120 to 3533360 tid 3 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1805: done (3 left) 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1809: snap_create 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1806: finishing write tid 5 to smithi07685715-34 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1806: oid 34 updating version 0 to 311 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1806: oid 34 updating version 311 to 312 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1806: oid 34 updating version 312 to 314 2024-09-21T09:45:35.787 INFO:tasks.rados.rados.0.smithi076.stdout:1806: oid 34 version 314 is already newer than 313 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 314 (ObjNum 695 snap 224 seq_num 695) dirty exists 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1806: left oid 34 (ObjNum 695 snap 224 seq_num 695) 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: finishing write tid 1 to smithi07685715-38 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: finishing write tid 2 to smithi07685715-38 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: finishing write tid 3 to smithi07685715-38 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: finishing write tid 4 to smithi07685715-38 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: finishing write tid 5 to smithi07685715-38 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: oid 38 updating version 0 to 966 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: oid 38 updating version 966 to 969 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: oid 38 version 969 is already newer than 968 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: oid 38 version 969 is already newer than 967 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 969 (ObjNum 696 snap 224 seq_num 696) dirty exists 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1807: left oid 38 (ObjNum 696 snap 224 seq_num 696) 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1808: finishing write tid 1 to smithi07685715-37 2024-09-21T09:45:35.788 INFO:tasks.rados.rados.0.smithi076.stdout:1808: finishing write tid 2 to smithi07685715-37 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: finishing write tid 3 to smithi07685715-37 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: finishing write tid 4 to smithi07685715-37 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: finishing write tid 5 to smithi07685715-37 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: oid 37 updating version 0 to 489 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: oid 37 updating version 489 to 490 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: oid 37 updating version 490 to 491 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: oid 37 updating version 491 to 492 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 492 (ObjNum 697 snap 224 seq_num 697) dirty exists 2024-09-21T09:45:35.790 INFO:tasks.rados.rados.0.smithi076.stdout:1808: left oid 37 (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:45:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:35 smithi191 ceph-mon[102206]: pgmap v251: 89 pgs: 89 active+clean; 99 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:45:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:35 smithi191 ceph-mon[102206]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:45:36.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:35 smithi076 ceph-mon[106482]: pgmap v251: 89 pgs: 89 active+clean; 99 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:45:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:35 smithi076 ceph-mon[106482]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:45:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:35 smithi076 ceph-mon[110963]: pgmap v251: 89 pgs: 89 active+clean; 99 MiB data, 544 MiB used, 715 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:45:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:35 smithi076 ceph-mon[110963]: osdmap e521: 8 total, 8 up, 8 in 2024-09-21T09:45:36.715 INFO:tasks.rados.rados.0.smithi076.stdout:1806: done (3 left) 2024-09-21T09:45:36.716 INFO:tasks.rados.rados.0.smithi076.stdout:1807: done (2 left) 2024-09-21T09:45:36.716 INFO:tasks.rados.rados.0.smithi076.stdout:1808: done (1 left) 2024-09-21T09:45:36.716 INFO:tasks.rados.rados.0.smithi076.stdout:1809: done (0 left) 2024-09-21T09:45:36.716 INFO:tasks.rados.rados.0.smithi076.stdout:1810: snap_create 2024-09-21T09:45:37.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:36 smithi191 ceph-mon[102206]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:45:37.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:36 smithi076 ceph-mon[106482]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:45:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:36 smithi076 ceph-mon[110963]: osdmap e522: 8 total, 8 up, 8 in 2024-09-21T09:45:37.717 INFO:tasks.rados.rados.0.smithi076.stdout:1810: done (0 left) 2024-09-21T09:45:37.717 INFO:tasks.rados.rados.0.smithi076.stdout:1811: read oid 48 snap 224 2024-09-21T09:45:37.717 INFO:tasks.rados.rados.0.smithi076.stdout:1811: expect (ObjNum 692 snap 223 seq_num 692) 2024-09-21T09:45:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:1812: read oid 5 snap 225 2024-09-21T09:45:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:1812: expect deleted 2024-09-21T09:45:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:1813: snap_create 2024-09-21T09:45:37.730 INFO:tasks.rados.rados.0.smithi076.stdout:1811: expect (ObjNum 692 snap 223 seq_num 692) 2024-09-21T09:45:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:37 smithi191 ceph-mon[102206]: pgmap v254: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.1 MiB/s wr, 22 op/s 2024-09-21T09:45:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:37 smithi191 ceph-mon[102206]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:45:38.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:37 smithi076 ceph-mon[110963]: pgmap v254: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.1 MiB/s wr, 22 op/s 2024-09-21T09:45:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:37 smithi076 ceph-mon[110963]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:45:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:37 smithi076 ceph-mon[106482]: pgmap v254: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.1 MiB/s wr, 22 op/s 2024-09-21T09:45:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:37 smithi076 ceph-mon[106482]: osdmap e523: 8 total, 8 up, 8 in 2024-09-21T09:45:38.671 INFO:tasks.rados.rados.0.smithi076.stdout:1811: done (2 left) 2024-09-21T09:45:38.671 INFO:tasks.rados.rados.0.smithi076.stdout:1812: done (1 left) 2024-09-21T09:45:38.671 INFO:tasks.rados.rados.0.smithi076.stdout:1813: done (0 left) 2024-09-21T09:45:38.671 INFO:tasks.rados.rados.0.smithi076.stdout:1814: snap_create 2024-09-21T09:45:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:38 smithi191 ceph-mon[102206]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:45:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:38 smithi191 ceph-mon[102206]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:45:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:38 smithi076 ceph-mon[106482]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:45:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:38 smithi076 ceph-mon[106482]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:45:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:38 smithi076 ceph-mon[110963]: osdmap e524: 8 total, 8 up, 8 in 2024-09-21T09:45:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:38 smithi076 ceph-mon[110963]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:45:39.670 INFO:tasks.rados.rados.0.smithi076.stdout:1814: done (0 left) 2024-09-21T09:45:39.670 INFO:tasks.rados.rados.0.smithi076.stdout:1815: read oid 31 snap 222 2024-09-21T09:45:39.670 INFO:tasks.rados.rados.0.smithi076.stdout:1815: expect deleted 2024-09-21T09:45:39.670 INFO:tasks.rados.rados.0.smithi076.stdout:1816: rollback oid 3 current snap is 228 2024-09-21T09:45:39.670 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 218 2024-09-21T09:45:39.671 INFO:tasks.rados.rados.0.smithi076.stdout:1817: rollback oid 37 current snap is 228 2024-09-21T09:45:39.671 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 226 2024-09-21T09:45:39.671 INFO:tasks.rados.rados.0.smithi076.stdout:1818: write oid 34 current snap is 228 2024-09-21T09:45:39.671 INFO:tasks.rados.rados.0.smithi076.stdout:1818: seq_num 698 ranges {531286=711925,1912196=706244,3018543=712743} 2024-09-21T09:45:39.677 INFO:tasks.rados.rados.0.smithi076.stdout:1818: writing smithi07685715-34 from 531286 to 1243211 tid 1 2024-09-21T09:45:39.685 INFO:tasks.rados.rados.0.smithi076.stdout:1818: writing smithi07685715-34 from 1912196 to 2618440 tid 2 2024-09-21T09:45:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:1818: writing smithi07685715-34 from 3018543 to 3731286 tid 3 2024-09-21T09:45:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:1819: snap_create 2024-09-21T09:45:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:1816: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:45:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 81 snap 0 seq_num 0) dirty dne 2024-09-21T09:45:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:1817: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:45:39.692 INFO:tasks.rados.rados.0.smithi076.stdout:1817: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:45:39.692 INFO:tasks.rados.rados.0.smithi076.stdout:1817: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:45:39.692 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 496 (ObjNum 697 snap 224 seq_num 697) dirty exists 2024-09-21T09:45:39.692 INFO:tasks.rados.rados.0.smithi076.stdout:1818: finishing write tid 1 to smithi07685715-34 2024-09-21T09:45:39.692 INFO:tasks.rados.rados.0.smithi076.stdout:1818: finishing write tid 2 to smithi07685715-34 2024-09-21T09:45:39.694 INFO:tasks.rados.rados.0.smithi076.stdout:1818: finishing write tid 3 to smithi07685715-34 2024-09-21T09:45:39.694 INFO:tasks.rados.rados.0.smithi076.stdout:1818: finishing write tid 4 to smithi07685715-34 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:1818: finishing write tid 5 to smithi07685715-34 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:1818: oid 34 updating version 0 to 319 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:1818: oid 34 version 319 is already newer than 317 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:1818: oid 34 version 319 is already newer than 318 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:1818: oid 34 version 319 is already newer than 316 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 319 (ObjNum 698 snap 228 seq_num 698) dirty exists 2024-09-21T09:45:39.695 INFO:tasks.rados.rados.0.smithi076.stdout:1818: left oid 34 (ObjNum 698 snap 228 seq_num 698) 2024-09-21T09:45:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:39 smithi191 ceph-mon[102206]: pgmap v257: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.1 MiB/s wr, 22 op/s 2024-09-21T09:45:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:39 smithi191 ceph-mon[102206]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:45:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:39 smithi076 ceph-mon[110963]: pgmap v257: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.1 MiB/s wr, 22 op/s 2024-09-21T09:45:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:39 smithi076 ceph-mon[110963]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:45:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:39 smithi076 ceph-mon[106482]: pgmap v257: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.1 MiB/s wr, 22 op/s 2024-09-21T09:45:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:39 smithi076 ceph-mon[106482]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1815: done (4 left) 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1816: done (3 left) 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1817: done (2 left) 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1818: done (1 left) 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1819: done (0 left) 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1820: rollback oid 49 current snap is 229 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 222 2024-09-21T09:45:40.672 INFO:tasks.rados.rados.0.smithi076.stdout:1821: snap_create 2024-09-21T09:45:40.676 INFO:tasks.rados.rados.0.smithi076.stdout:1820: finishing rollback tid 0 to smithi07685715-49 2024-09-21T09:45:40.676 INFO:tasks.rados.rados.0.smithi076.stdout:1820: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:45:40.676 INFO:tasks.rados.rados.0.smithi076.stdout:1820: finishing rollback tid 2 to smithi07685715-49 2024-09-21T09:45:40.677 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 1421 (ObjNum 639 snap 205 seq_num 639) dirty exists 2024-09-21T09:45:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:40 smithi191 ceph-mon[102206]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:45:41.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:40 smithi076 ceph-mon[106482]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:45:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:40 smithi076 ceph-mon[110963]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:45:41.676 INFO:tasks.rados.rados.0.smithi076.stdout:1820: done (1 left) 2024-09-21T09:45:41.676 INFO:tasks.rados.rados.0.smithi076.stdout:1821: done (0 left) 2024-09-21T09:45:41.676 INFO:tasks.rados.rados.0.smithi076.stdout:1822: delete oid 18 current snap is 230 2024-09-21T09:45:41.680 INFO:tasks.rados.rados.0.smithi076.stdout:1822: done (0 left) 2024-09-21T09:45:41.680 INFO:tasks.rados.rados.0.smithi076.stdout:1823: write oid 38 current snap is 230 2024-09-21T09:45:41.680 INFO:tasks.rados.rados.0.smithi076.stdout:1823: seq_num 700 ranges {564900=578776,1629160=709913,3073389=492409} 2024-09-21T09:45:41.686 INFO:tasks.rados.rados.0.smithi076.stdout:1823: writing smithi07685715-38 from 564900 to 1143676 tid 1 2024-09-21T09:45:41.692 INFO:tasks.rados.rados.0.smithi076.stdout:1823: writing smithi07685715-38 from 1629160 to 2339073 tid 2 2024-09-21T09:45:41.699 INFO:tasks.rados.rados.0.smithi076.stdout:1823: writing smithi07685715-38 from 3073389 to 3565798 tid 3 2024-09-21T09:45:41.699 INFO:tasks.rados.rados.0.smithi076.stdout:1824: snap_create 2024-09-21T09:45:41.699 INFO:tasks.rados.rados.0.smithi076.stdout:1823: finishing write tid 1 to smithi07685715-38 2024-09-21T09:45:41.700 INFO:tasks.rados.rados.0.smithi076.stdout:1823: finishing write tid 2 to smithi07685715-38 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: finishing write tid 3 to smithi07685715-38 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: finishing write tid 4 to smithi07685715-38 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: finishing write tid 5 to smithi07685715-38 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: oid 38 updating version 0 to 971 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: oid 38 updating version 971 to 972 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: oid 38 updating version 972 to 974 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: oid 38 version 974 is already newer than 973 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 974 (ObjNum 700 snap 230 seq_num 700) dirty exists 2024-09-21T09:45:41.703 INFO:tasks.rados.rados.0.smithi076.stdout:1823: left oid 38 (ObjNum 700 snap 230 seq_num 700) 2024-09-21T09:45:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:41 smithi191 ceph-mon[102206]: pgmap v260: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:41 smithi191 ceph-mon[102206]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:45:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:41 smithi076 ceph-mon[106482]: pgmap v260: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:41 smithi076 ceph-mon[106482]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:45:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:41 smithi076 ceph-mon[110963]: pgmap v260: 89 pgs: 89 active+clean; 119 MiB data, 585 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:41 smithi076 ceph-mon[110963]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:45:42.677 INFO:tasks.rados.rados.0.smithi076.stdout:1823: done (1 left) 2024-09-21T09:45:42.677 INFO:tasks.rados.rados.0.smithi076.stdout:1824: done (0 left) 2024-09-21T09:45:42.677 INFO:tasks.rados.rados.0.smithi076.stdout:1825: snap_remove snap 225 2024-09-21T09:45:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:45:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:45:43] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:45:43.676 INFO:tasks.rados.rados.0.smithi076.stdout:1825: done (0 left) 2024-09-21T09:45:43.676 INFO:tasks.rados.rados.0.smithi076.stdout:1826: read oid 4 snap -1 2024-09-21T09:45:43.676 INFO:tasks.rados.rados.0.smithi076.stdout:1826: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:45:43.676 INFO:tasks.rados.rados.0.smithi076.stdout:1827: snap_remove snap 221 2024-09-21T09:45:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:43 smithi191 ceph-mon[102206]: pgmap v263: 89 pgs: 89 active+clean; 121 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 521 KiB/s wr, 4 op/s 2024-09-21T09:45:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:43 smithi191 ceph-mon[102206]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:45:44.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:43 smithi076 ceph-mon[106482]: pgmap v263: 89 pgs: 89 active+clean; 121 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 521 KiB/s wr, 4 op/s 2024-09-21T09:45:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:43 smithi076 ceph-mon[106482]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:45:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:43 smithi076 ceph-mon[110963]: pgmap v263: 89 pgs: 89 active+clean; 121 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 521 KiB/s wr, 4 op/s 2024-09-21T09:45:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:43 smithi076 ceph-mon[110963]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:45:44.677 INFO:tasks.rados.rados.0.smithi076.stdout:1827: done (1 left) 2024-09-21T09:45:44.677 INFO:tasks.rados.rados.0.smithi076.stdout:1828: delete oid 17 current snap is 231 2024-09-21T09:45:44.677 INFO:tasks.rados.rados.0.smithi076.stdout:1826: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:45:44.770 INFO:tasks.rados.rados.0.smithi076.stdout:1826: done (1 left) 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:1828: done (0 left) 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:1829: read oid 11 snap -1 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:1829: expect deleted 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:1830: rollback oid 39 current snap is 231 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 217 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:1831: write oid 36 current snap is 231 2024-09-21T09:45:44.771 INFO:tasks.rados.rados.0.smithi076.stdout:1831: seq_num 702 ranges {598514=729274,1746124=597229,3128235=514953} 2024-09-21T09:45:44.777 INFO:tasks.rados.rados.0.smithi076.stdout:1831: writing smithi07685715-36 from 598514 to 1327788 tid 1 2024-09-21T09:45:44.783 INFO:tasks.rados.rados.0.smithi076.stdout:1831: writing smithi07685715-36 from 1746124 to 2343353 tid 2 2024-09-21T09:45:44.790 INFO:tasks.rados.rados.0.smithi076.stdout:1831: writing smithi07685715-36 from 3128235 to 3643188 tid 3 2024-09-21T09:45:44.790 INFO:tasks.rados.rados.0.smithi076.stdout:1832: write oid 9 current snap is 231 2024-09-21T09:45:44.790 INFO:tasks.rados.rados.0.smithi076.stdout:1832: seq_num 703 ranges {615321=404523,1804606=457240,2755658=726225} 2024-09-21T09:45:44.795 INFO:tasks.rados.rados.0.smithi076.stdout:1832: writing smithi07685715-9 from 615321 to 1019844 tid 1 2024-09-21T09:45:44.802 INFO:tasks.rados.rados.0.smithi076.stdout:1832: writing smithi07685715-9 from 1804606 to 2261846 tid 2 2024-09-21T09:45:44.808 INFO:tasks.rados.rados.0.smithi076.stdout:1832: writing smithi07685715-9 from 2755658 to 3481883 tid 3 2024-09-21T09:45:44.808 INFO:tasks.rados.rados.0.smithi076.stdout:1833: read oid 20 snap -1 2024-09-21T09:45:44.809 INFO:tasks.rados.rados.0.smithi076.stdout:1833: expect deleted 2024-09-21T09:45:44.809 INFO:tasks.rados.rados.0.smithi076.stdout:1834: snap_remove snap 227 2024-09-21T09:45:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:44 smithi191 ceph-mon[102206]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:45:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:44 smithi191 ceph-mon[102206]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:45:45.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:44 smithi076 ceph-mon[106482]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:45:45.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:44 smithi076 ceph-mon[106482]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:45:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:44 smithi076 ceph-mon[110963]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:45:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:44 smithi076 ceph-mon[110963]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:45:45.706 INFO:tasks.rados.rados.0.smithi076.stdout:1834: done (5 left) 2024-09-21T09:45:45.706 INFO:tasks.rados.rados.0.smithi076.stdout:1835: write oid 44 current snap is 231 2024-09-21T09:45:45.706 INFO:tasks.rados.rados.0.smithi076.stdout:1835: seq_num 704 ranges {632128=479772,1746735=600898,3066728=653850} 2024-09-21T09:45:45.711 INFO:tasks.rados.rados.0.smithi076.stdout:1835: writing smithi07685715-44 from 632128 to 1111900 tid 1 2024-09-21T09:45:45.718 INFO:tasks.rados.rados.0.smithi076.stdout:1835: writing smithi07685715-44 from 1746735 to 2347633 tid 2 2024-09-21T09:45:45.726 INFO:tasks.rados.rados.0.smithi076.stdout:1835: writing smithi07685715-44 from 3066728 to 3720578 tid 3 2024-09-21T09:45:45.726 INFO:tasks.rados.rados.0.smithi076.stdout:1836: rollback oid 33 current snap is 231 2024-09-21T09:45:45.726 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 218 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1830: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1830: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1830: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1343 (ObjNum 625 snap 200 seq_num 625) dirty exists 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1831: finishing write tid 1 to smithi07685715-36 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1831: finishing write tid 2 to smithi07685715-36 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1829: done (6 left) 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1830: done (5 left) 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1837: delete oid 8 current snap is 231 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1831: finishing write tid 3 to smithi07685715-36 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1831: finishing write tid 4 to smithi07685715-36 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1831: finishing write tid 5 to smithi07685715-36 2024-09-21T09:45:45.727 INFO:tasks.rados.rados.0.smithi076.stdout:1831: oid 36 updating version 0 to 540 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1831: oid 36 updating version 540 to 541 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1831: oid 36 version 541 is already newer than 539 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1831: oid 36 updating version 541 to 542 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 542 (ObjNum 702 snap 231 seq_num 702) dirty exists 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1831: left oid 36 (ObjNum 702 snap 231 seq_num 702) 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1832: finishing write tid 1 to smithi07685715-9 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1832: finishing write tid 2 to smithi07685715-9 2024-09-21T09:45:45.728 INFO:tasks.rados.rados.0.smithi076.stdout:1832: finishing write tid 3 to smithi07685715-9 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: finishing write tid 4 to smithi07685715-9 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: finishing write tid 5 to smithi07685715-9 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: oid 9 updating version 0 to 629 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: oid 9 version 629 is already newer than 628 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: oid 9 updating version 629 to 630 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: oid 9 version 630 is already newer than 627 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 630 (ObjNum 703 snap 231 seq_num 703) dirty exists 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1832: left oid 9 (ObjNum 703 snap 231 seq_num 703) 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1835: finishing write tid 1 to smithi07685715-44 2024-09-21T09:45:45.729 INFO:tasks.rados.rados.0.smithi076.stdout:1835: finishing write tid 2 to smithi07685715-44 2024-09-21T09:45:45.730 INFO:tasks.rados.rados.0.smithi076.stdout:1836: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:45:45.730 INFO:tasks.rados.rados.0.smithi076.stdout:1836: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:45:45.730 INFO:tasks.rados.rados.0.smithi076.stdout:1831: done (5 left) 2024-09-21T09:45:45.730 INFO:tasks.rados.rados.0.smithi076.stdout:1832: done (4 left) 2024-09-21T09:45:45.730 INFO:tasks.rados.rados.0.smithi076.stdout:1833: done (3 left) 2024-09-21T09:45:45.731 INFO:tasks.rados.rados.0.smithi076.stdout:1837: done (2 left) 2024-09-21T09:45:45.731 INFO:tasks.rados.rados.0.smithi076.stdout:1838: write oid 40 current snap is 231 2024-09-21T09:45:45.731 INFO:tasks.rados.rados.0.smithi076.stdout:1836: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:45:45.731 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 370 (ObjNum 637 snap 205 seq_num 637) dirty exists 2024-09-21T09:45:45.731 INFO:tasks.rados.rados.0.smithi076.stdout:1838: seq_num 706 ranges {665742=630270,1863699=604567,3121574=676394} 2024-09-21T09:45:45.736 INFO:tasks.rados.rados.0.smithi076.stdout:1838: writing smithi07685715-40 from 665742 to 1296012 tid 1 2024-09-21T09:45:45.742 INFO:tasks.rados.rados.0.smithi076.stdout:1838: writing smithi07685715-40 from 1863699 to 2468266 tid 2 2024-09-21T09:45:45.749 INFO:tasks.rados.rados.0.smithi076.stdout:1838: writing smithi07685715-40 from 3121574 to 3797968 tid 3 2024-09-21T09:45:45.750 INFO:tasks.rados.rados.0.smithi076.stdout:1836: done (2 left) 2024-09-21T09:45:45.750 INFO:tasks.rados.rados.0.smithi076.stdout:1839: write oid 10 current snap is 231 2024-09-21T09:45:45.750 INFO:tasks.rados.rados.0.smithi076.stdout:1839: seq_num 707 ranges {682549=705519,1922181=464578,3148997=487666} 2024-09-21T09:45:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:1839: writing smithi07685715-10 from 682549 to 1388068 tid 1 2024-09-21T09:45:45.762 INFO:tasks.rados.rados.0.smithi076.stdout:1839: writing smithi07685715-10 from 1922181 to 2386759 tid 2 2024-09-21T09:45:45.770 INFO:tasks.rados.rados.0.smithi076.stdout:1839: writing smithi07685715-10 from 3148997 to 3636663 tid 3 2024-09-21T09:45:45.770 INFO:tasks.rados.rados.0.smithi076.stdout:1840: rollback oid 22 current snap is 231 2024-09-21T09:45:45.770 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 220 2024-09-21T09:45:45.770 INFO:tasks.rados.rados.0.smithi076.stdout:1835: finishing write tid 3 to smithi07685715-44 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: finishing write tid 4 to smithi07685715-44 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: finishing write tid 5 to smithi07685715-44 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: oid 44 updating version 0 to 366 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: oid 44 updating version 366 to 367 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: oid 44 version 367 is already newer than 365 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: oid 44 version 367 is already newer than 364 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 367 (ObjNum 704 snap 231 seq_num 704) dirty exists 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: left oid 44 (ObjNum 704 snap 231 seq_num 704) 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1838: finishing write tid 1 to smithi07685715-40 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1835: done (3 left) 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1841: rollback oid 21 current snap is 231 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 223 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1838: finishing write tid 2 to smithi07685715-40 2024-09-21T09:45:45.771 INFO:tasks.rados.rados.0.smithi076.stdout:1838: finishing write tid 3 to smithi07685715-40 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: finishing write tid 4 to smithi07685715-40 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: finishing write tid 5 to smithi07685715-40 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: oid 40 updating version 0 to 837 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: oid 40 updating version 837 to 838 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: oid 40 version 838 is already newer than 835 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: oid 40 version 838 is already newer than 836 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 838 (ObjNum 706 snap 231 seq_num 706) dirty exists 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: left oid 40 (ObjNum 706 snap 231 seq_num 706) 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1839: finishing write tid 1 to smithi07685715-10 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1839: finishing write tid 2 to smithi07685715-10 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1838: done (3 left) 2024-09-21T09:45:45.772 INFO:tasks.rados.rados.0.smithi076.stdout:1842: snap_create 2024-09-21T09:45:45.773 INFO:tasks.rados.rados.0.smithi076.stdout:1839: finishing write tid 3 to smithi07685715-10 2024-09-21T09:45:45.773 INFO:tasks.rados.rados.0.smithi076.stdout:1839: finishing write tid 4 to smithi07685715-10 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1840: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1840: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1839: finishing write tid 5 to smithi07685715-10 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1839: oid 10 updating version 0 to 988 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1839: oid 10 version 988 is already newer than 986 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1839: oid 10 updating version 988 to 989 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1839: oid 10 version 989 is already newer than 987 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 989 (ObjNum 707 snap 231 seq_num 707) dirty exists 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1839: left oid 10 (ObjNum 707 snap 231 seq_num 707) 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1840: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 842 (ObjNum 627 snap 201 seq_num 627) dirty exists 2024-09-21T09:45:45.774 INFO:tasks.rados.rados.0.smithi076.stdout:1841: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:45:45.775 INFO:tasks.rados.rados.0.smithi076.stdout:1841: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:45:45.775 INFO:tasks.rados.rados.0.smithi076.stdout:1841: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:45:45.775 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1349 (ObjNum 579 snap 184 seq_num 579) dirty exists 2024-09-21T09:45:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:45 smithi191 ceph-mon[102206]: pgmap v266: 89 pgs: 89 active+clean; 121 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 521 KiB/s wr, 4 op/s 2024-09-21T09:45:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:45 smithi076 ceph-mon[106482]: pgmap v266: 89 pgs: 89 active+clean; 121 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 521 KiB/s wr, 4 op/s 2024-09-21T09:45:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:45 smithi076 ceph-mon[110963]: pgmap v266: 89 pgs: 89 active+clean; 121 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 3.9 MiB/s rd, 521 KiB/s wr, 4 op/s 2024-09-21T09:45:46.715 INFO:tasks.rados.rados.0.smithi076.stdout:1839: done (3 left) 2024-09-21T09:45:46.715 INFO:tasks.rados.rados.0.smithi076.stdout:1840: done (2 left) 2024-09-21T09:45:46.715 INFO:tasks.rados.rados.0.smithi076.stdout:1841: done (1 left) 2024-09-21T09:45:46.715 INFO:tasks.rados.rados.0.smithi076.stdout:1842: done (0 left) 2024-09-21T09:45:46.716 INFO:tasks.rados.rados.0.smithi076.stdout:1843: snap_create 2024-09-21T09:45:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:46 smithi191 ceph-mon[102206]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:45:47.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:46 smithi076 ceph-mon[106482]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:45:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:46 smithi076 ceph-mon[110963]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:45:47.715 INFO:tasks.rados.rados.0.smithi076.stdout:1843: done (0 left) 2024-09-21T09:45:47.715 INFO:tasks.rados.rados.0.smithi076.stdout:1844: snap_create 2024-09-21T09:45:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:47 smithi191 ceph-mon[102206]: pgmap v269: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:45:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:47 smithi191 ceph-mon[102206]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:45:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:47 smithi076 ceph-mon[106482]: pgmap v269: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:45:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:47 smithi076 ceph-mon[106482]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:45:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:47 smithi076 ceph-mon[110963]: pgmap v269: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:45:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:47 smithi076 ceph-mon[110963]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:45:48.671 INFO:tasks.rados.rados.0.smithi076.stdout:1844: done (0 left) 2024-09-21T09:45:48.671 INFO:tasks.rados.rados.0.smithi076.stdout:1845: read oid 26 snap 232 2024-09-21T09:45:48.671 INFO:tasks.rados.rados.0.smithi076.stdout:1845: expect (ObjNum 690 snap 222 seq_num 690) 2024-09-21T09:45:48.671 INFO:tasks.rados.rados.0.smithi076.stdout:1846: write oid 30 current snap is 234 2024-09-21T09:45:48.671 INFO:tasks.rados.rados.0.smithi076.stdout:1846: seq_num 708 ranges {699356=497121,1980663=608236,3176420=415291} 2024-09-21T09:45:48.678 INFO:tasks.rados.rados.0.smithi076.stdout:1846: writing smithi07685715-30 from 699356 to 1196477 tid 1 2024-09-21T09:45:48.686 INFO:tasks.rados.rados.0.smithi076.stdout:1846: writing smithi07685715-30 from 1980663 to 2588899 tid 2 2024-09-21T09:45:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:1846: writing smithi07685715-30 from 3176420 to 3591711 tid 3 2024-09-21T09:45:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:1847: snap_create 2024-09-21T09:45:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:1845: expect (ObjNum 690 snap 222 seq_num 690) 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: finishing write tid 1 to smithi07685715-30 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: finishing write tid 2 to smithi07685715-30 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: finishing write tid 3 to smithi07685715-30 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: finishing write tid 4 to smithi07685715-30 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: finishing write tid 5 to smithi07685715-30 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: oid 30 updating version 0 to 1435 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: oid 30 version 1435 is already newer than 1433 2024-09-21T09:45:48.885 INFO:tasks.rados.rados.0.smithi076.stdout:1846: oid 30 version 1435 is already newer than 1434 2024-09-21T09:45:48.886 INFO:tasks.rados.rados.0.smithi076.stdout:1846: oid 30 updating version 1435 to 1436 2024-09-21T09:45:48.886 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1436 (ObjNum 708 snap 234 seq_num 708) dirty exists 2024-09-21T09:45:48.886 INFO:tasks.rados.rados.0.smithi076.stdout:1846: left oid 30 (ObjNum 708 snap 234 seq_num 708) 2024-09-21T09:45:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:48 smithi191 ceph-mon[102206]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:45:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:48 smithi191 ceph-mon[102206]: osdmap e535: 8 total, 8 up, 8 in 2024-09-21T09:45:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:48 smithi076 ceph-mon[106482]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:45:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:48 smithi076 ceph-mon[106482]: osdmap e535: 8 total, 8 up, 8 in 2024-09-21T09:45:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:48 smithi076 ceph-mon[110963]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:45:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:48 smithi076 ceph-mon[110963]: osdmap e535: 8 total, 8 up, 8 in 2024-09-21T09:45:49.672 INFO:tasks.rados.rados.0.smithi076.stdout:1845: done (2 left) 2024-09-21T09:45:49.672 INFO:tasks.rados.rados.0.smithi076.stdout:1846: done (1 left) 2024-09-21T09:45:49.672 INFO:tasks.rados.rados.0.smithi076.stdout:1847: done (0 left) 2024-09-21T09:45:49.672 INFO:tasks.rados.rados.0.smithi076.stdout:1848: delete oid 30 current snap is 235 2024-09-21T09:45:49.677 INFO:tasks.rados.rados.0.smithi076.stdout:1848: done (0 left) 2024-09-21T09:45:49.677 INFO:tasks.rados.rados.0.smithi076.stdout:1849: read oid 24 snap -1 2024-09-21T09:45:49.677 INFO:tasks.rados.rados.0.smithi076.stdout:1849: expect (ObjNum 680 snap 218 seq_num 680) 2024-09-21T09:45:49.677 INFO:tasks.rados.rados.0.smithi076.stdout:1850: read oid 19 snap 228 2024-09-21T09:45:49.677 INFO:tasks.rados.rados.0.smithi076.stdout:1850: expect deleted 2024-09-21T09:45:49.677 INFO:tasks.rados.rados.0.smithi076.stdout:1851: snap_create 2024-09-21T09:45:49.682 INFO:tasks.rados.rados.0.smithi076.stdout:1849: expect (ObjNum 680 snap 218 seq_num 680) 2024-09-21T09:45:49.867 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:45:50.066 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:45:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:49 smithi191 ceph-mon[102206]: pgmap v272: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:45:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:49 smithi191 ceph-mon[102206]: osdmap e536: 8 total, 8 up, 8 in 2024-09-21T09:45:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:49 smithi076 ceph-mon[106482]: pgmap v272: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:45:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:49 smithi076 ceph-mon[106482]: osdmap e536: 8 total, 8 up, 8 in 2024-09-21T09:45:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:49 smithi076 ceph-mon[110963]: pgmap v272: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T09:45:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:49 smithi076 ceph-mon[110963]: osdmap e536: 8 total, 8 up, 8 in 2024-09-21T09:45:50.676 INFO:tasks.rados.rados.0.smithi076.stdout:1849: done (2 left) 2024-09-21T09:45:50.676 INFO:tasks.rados.rados.0.smithi076.stdout:1850: done (1 left) 2024-09-21T09:45:50.676 INFO:tasks.rados.rados.0.smithi076.stdout:1851: done (0 left) 2024-09-21T09:45:50.676 INFO:tasks.rados.rados.0.smithi076.stdout:1852: write oid 21 current snap is 236 2024-09-21T09:45:50.676 INFO:tasks.rados.rados.0.smithi076.stdout:1852: seq_num 710 ranges {732970=647619,2097627=611905,3231266=437835} 2024-09-21T09:45:50.683 INFO:tasks.rados.rados.0.smithi076.stdout:1852: writing smithi07685715-21 from 732970 to 1380589 tid 1 2024-09-21T09:45:50.691 INFO:tasks.rados.rados.0.smithi076.stdout:1852: writing smithi07685715-21 from 2097627 to 2709532 tid 2 2024-09-21T09:45:50.696 INFO:tasks.rados.rados.0.smithi076.stdout:1852: writing smithi07685715-21 from 3231266 to 3669101 tid 3 2024-09-21T09:45:50.696 INFO:tasks.rados.rados.0.smithi076.stdout:1853: write oid 30 current snap is 236 2024-09-21T09:45:50.696 INFO:tasks.rados.rados.0.smithi076.stdout:1853: seq_num 711 ranges {749777=722868,2156109=471916,3258689=649107} 2024-09-21T09:45:50.703 INFO:tasks.rados.rados.0.smithi076.stdout:1853: writing smithi07685715-30 from 749777 to 1472645 tid 1 2024-09-21T09:45:50.710 INFO:tasks.rados.rados.0.smithi076.stdout:1853: writing smithi07685715-30 from 2156109 to 2628025 tid 2 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1853: writing smithi07685715-30 from 3258689 to 3907796 tid 3 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1854: delete oid 4 current snap is 236 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1852: finishing write tid 1 to smithi07685715-21 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1852: finishing write tid 2 to smithi07685715-21 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1852: finishing write tid 3 to smithi07685715-21 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1852: finishing write tid 4 to smithi07685715-21 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1852: finishing write tid 5 to smithi07685715-21 2024-09-21T09:45:50.718 INFO:tasks.rados.rados.0.smithi076.stdout:1852: oid 21 updating version 0 to 1353 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:1852: oid 21 version 1353 is already newer than 1352 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:1852: oid 21 updating version 1353 to 1354 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:1852: oid 21 version 1354 is already newer than 1351 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1354 (ObjNum 710 snap 236 seq_num 710) dirty exists 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:1852: left oid 21 (ObjNum 710 snap 236 seq_num 710) 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:1853: finishing write tid 1 to smithi07685715-30 2024-09-21T09:45:50.719 INFO:tasks.rados.rados.0.smithi076.stdout:1853: finishing write tid 2 to smithi07685715-30 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: finishing write tid 3 to smithi07685715-30 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: finishing write tid 4 to smithi07685715-30 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: finishing write tid 5 to smithi07685715-30 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: oid 30 updating version 0 to 1442 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: oid 30 version 1442 is already newer than 1440 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: oid 30 version 1442 is already newer than 1441 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: oid 30 version 1442 is already newer than 1439 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1442 (ObjNum 711 snap 236 seq_num 711) dirty exists 2024-09-21T09:45:50.722 INFO:tasks.rados.rados.0.smithi076.stdout:1853: left oid 30 (ObjNum 711 snap 236 seq_num 711) 2024-09-21T09:45:50.723 INFO:tasks.rados.rados.0.smithi076.stdout:1852: done (2 left) 2024-09-21T09:45:50.723 INFO:tasks.rados.rados.0.smithi076.stdout:1853: done (1 left) 2024-09-21T09:45:50.723 INFO:tasks.rados.rados.0.smithi076.stdout:1854: done (0 left) 2024-09-21T09:45:50.723 INFO:tasks.rados.rados.0.smithi076.stdout:1855: snap_create 2024-09-21T09:45:51.070 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:50 smithi076 ceph-mon[106482]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:51.070 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:50 smithi076 ceph-mon[106482]: osdmap e537: 8 total, 8 up, 8 in 2024-09-21T09:45:51.071 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:50 smithi076 ceph-mon[110963]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:51.071 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:50 smithi076 ceph-mon[110963]: osdmap e537: 8 total, 8 up, 8 in 2024-09-21T09:45:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:50 smithi191 ceph-mon[102206]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:50 smithi191 ceph-mon[102206]: osdmap e537: 8 total, 8 up, 8 in 2024-09-21T09:45:51.727 INFO:tasks.rados.rados.0.smithi076.stdout:1855: done (0 left) 2024-09-21T09:45:51.727 INFO:tasks.rados.rados.0.smithi076.stdout:1856: read oid 7 snap -1 2024-09-21T09:45:51.728 INFO:tasks.rados.rados.0.smithi076.stdout:1856: expect (ObjNum 693 snap 223 seq_num 693) 2024-09-21T09:45:51.728 INFO:tasks.rados.rados.0.smithi076.stdout:1857: snap_remove snap 219 2024-09-21T09:45:51.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:51 smithi076 ceph-mon[106482]: pgmap v275: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:51 smithi076 ceph-mon[110963]: pgmap v275: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:51 smithi191 ceph-mon[102206]: pgmap v275: 89 pgs: 89 active+clean; 139 MiB data, 627 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:45:52.730 INFO:tasks.rados.rados.0.smithi076.stdout:1857: done (1 left) 2024-09-21T09:45:52.730 INFO:tasks.rados.rados.0.smithi076.stdout:1858: snap_create 2024-09-21T09:45:52.730 INFO:tasks.rados.rados.0.smithi076.stdout:1856: expect (ObjNum 693 snap 223 seq_num 693) 2024-09-21T09:45:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:52 smithi191 ceph-mon[102206]: osdmap e538: 8 total, 8 up, 8 in 2024-09-21T09:45:53.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:52 smithi076 ceph-mon[110963]: osdmap e538: 8 total, 8 up, 8 in 2024-09-21T09:45:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:52 smithi076 ceph-mon[106482]: osdmap e538: 8 total, 8 up, 8 in 2024-09-21T09:45:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:45:53] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:45:53.730 INFO:tasks.rados.rados.0.smithi076.stdout:1856: done (1 left) 2024-09-21T09:45:53.730 INFO:tasks.rados.rados.0.smithi076.stdout:1858: done (0 left) 2024-09-21T09:45:53.730 INFO:tasks.rados.rados.0.smithi076.stdout:1859: read oid 38 snap -1 2024-09-21T09:45:53.730 INFO:tasks.rados.rados.0.smithi076.stdout:1859: expect (ObjNum 700 snap 230 seq_num 700) 2024-09-21T09:45:53.730 INFO:tasks.rados.rados.0.smithi076.stdout:1860: snap_create 2024-09-21T09:45:53.736 INFO:tasks.rados.rados.0.smithi076.stdout:1859: expect (ObjNum 700 snap 230 seq_num 700) 2024-09-21T09:45:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:53 smithi191 ceph-mon[102206]: pgmap v278: 89 pgs: 89 active+clean; 148 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:45:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:53 smithi191 ceph-mon[102206]: osdmap e539: 8 total, 8 up, 8 in 2024-09-21T09:45:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:53 smithi191 ceph-mon[102206]: osdmap e540: 8 total, 8 up, 8 in 2024-09-21T09:45:54.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-mon[106482]: pgmap v278: 89 pgs: 89 active+clean; 148 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:45:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-mon[106482]: osdmap e539: 8 total, 8 up, 8 in 2024-09-21T09:45:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-mon[106482]: osdmap e540: 8 total, 8 up, 8 in 2024-09-21T09:45:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-mon[110963]: pgmap v278: 89 pgs: 89 active+clean; 148 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:45:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-mon[110963]: osdmap e539: 8 total, 8 up, 8 in 2024-09-21T09:45:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:53 smithi076 ceph-mon[110963]: osdmap e540: 8 total, 8 up, 8 in 2024-09-21T09:45:54.732 INFO:tasks.rados.rados.0.smithi076.stdout:1859: done (1 left) 2024-09-21T09:45:54.732 INFO:tasks.rados.rados.0.smithi076.stdout:1860: done (0 left) 2024-09-21T09:45:54.732 INFO:tasks.rados.rados.0.smithi076.stdout:1861: snap_create 2024-09-21T09:45:55.735 INFO:tasks.rados.rados.0.smithi076.stdout:1861: done (0 left) 2024-09-21T09:45:55.735 INFO:tasks.rados.rados.0.smithi076.stdout:1862: write oid 5 current snap is 240 2024-09-21T09:45:55.735 INFO:tasks.rados.rados.0.smithi076.stdout:1862: seq_num 713 ranges {783391=473366,1756720=475585,2797182=788004} 2024-09-21T09:45:55.742 INFO:tasks.rados.rados.0.smithi076.stdout:1862: writing smithi07685715-5 from 783391 to 1256757 tid 1 2024-09-21T09:45:55.747 INFO:tasks.rados.rados.0.smithi076.stdout:1862: writing smithi07685715-5 from 1756720 to 2232305 tid 2 2024-09-21T09:45:55.755 INFO:tasks.rados.rados.0.smithi076.stdout:1862: writing smithi07685715-5 from 2797182 to 3585186 tid 3 2024-09-21T09:45:55.755 INFO:tasks.rados.rados.0.smithi076.stdout:1863: rollback oid 4 current snap is 240 2024-09-21T09:45:55.755 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 218 2024-09-21T09:45:55.755 INFO:tasks.rados.rados.0.smithi076.stdout:1862: finishing write tid 1 to smithi07685715-5 2024-09-21T09:45:55.755 INFO:tasks.rados.rados.0.smithi076.stdout:1862: finishing write tid 2 to smithi07685715-5 2024-09-21T09:45:55.755 INFO:tasks.rados.rados.0.smithi076.stdout:1864: snap_create 2024-09-21T09:45:55.760 INFO:tasks.rados.rados.0.smithi076.stdout:1863: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: finishing write tid 3 to smithi07685715-5 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: finishing write tid 4 to smithi07685715-5 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: finishing write tid 5 to smithi07685715-5 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: oid 5 updating version 0 to 1361 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: oid 5 updating version 1361 to 1362 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: oid 5 updating version 1362 to 1363 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: oid 5 updating version 1363 to 1364 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1364 (ObjNum 713 snap 240 seq_num 713) dirty exists 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1862: left oid 5 (ObjNum 713 snap 240 seq_num 713) 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:1863: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:45:55.761 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 548 (ObjNum 607 snap 194 seq_num 607) dirty exists 2024-09-21T09:45:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:55 smithi191 ceph-mon[102206]: pgmap v281: 89 pgs: 89 active+clean; 148 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:45:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:55 smithi191 ceph-mon[102206]: osdmap e541: 8 total, 8 up, 8 in 2024-09-21T09:45:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:55 smithi076 ceph-mon[106482]: pgmap v281: 89 pgs: 89 active+clean; 148 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:45:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:55 smithi076 ceph-mon[106482]: osdmap e541: 8 total, 8 up, 8 in 2024-09-21T09:45:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:55 smithi076 ceph-mon[110963]: pgmap v281: 89 pgs: 89 active+clean; 148 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:45:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:55 smithi076 ceph-mon[110963]: osdmap e541: 8 total, 8 up, 8 in 2024-09-21T09:45:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:45:56.747 INFO:tasks.rados.rados.0.smithi076.stdout:1862: done (2 left) 2024-09-21T09:45:56.747 INFO:tasks.rados.rados.0.smithi076.stdout:1863: done (1 left) 2024-09-21T09:45:56.747 INFO:tasks.rados.rados.0.smithi076.stdout:1864: done (0 left) 2024-09-21T09:45:56.747 INFO:tasks.rados.rados.0.smithi076.stdout:1865: read oid 32 snap -1 2024-09-21T09:45:56.747 INFO:tasks.rados.rados.0.smithi076.stdout:1865: expect (ObjNum 605 snap 193 seq_num 605) 2024-09-21T09:45:56.749 INFO:tasks.rados.rados.0.smithi076.stdout:1866: snap_create 2024-09-21T09:45:56.752 INFO:tasks.rados.rados.0.smithi076.stdout:1865: expect (ObjNum 605 snap 193 seq_num 605) 2024-09-21T09:45:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:56 smithi191 ceph-mon[102206]: osdmap e542: 8 total, 8 up, 8 in 2024-09-21T09:45:57.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:56 smithi076 ceph-mon[106482]: osdmap e542: 8 total, 8 up, 8 in 2024-09-21T09:45:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:56 smithi076 ceph-mon[110963]: osdmap e542: 8 total, 8 up, 8 in 2024-09-21T09:45:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:1865: done (1 left) 2024-09-21T09:45:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:1866: done (0 left) 2024-09-21T09:45:57.747 INFO:tasks.rados.rados.0.smithi076.stdout:1867: snap_create 2024-09-21T09:45:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:57 smithi191 ceph-mon[102206]: pgmap v284: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 424 KiB/s wr, 5 op/s 2024-09-21T09:45:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:57 smithi191 ceph-mon[102206]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:45:58.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:57 smithi076 ceph-mon[106482]: pgmap v284: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 424 KiB/s wr, 5 op/s 2024-09-21T09:45:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:57 smithi076 ceph-mon[106482]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:45:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:57 smithi076 ceph-mon[110963]: pgmap v284: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 424 KiB/s wr, 5 op/s 2024-09-21T09:45:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:57 smithi076 ceph-mon[110963]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:45:58.673 INFO:tasks.rados.rados.0.smithi076.stdout:1867: done (0 left) 2024-09-21T09:45:58.674 INFO:tasks.rados.rados.0.smithi076.stdout:1868: write oid 39 current snap is 243 2024-09-21T09:45:58.674 INFO:tasks.rados.rados.0.smithi076.stdout:1868: seq_num 714 ranges {400198=548615,1415202=584996} 2024-09-21T09:45:58.678 INFO:tasks.rados.rados.0.smithi076.stdout:1868: writing smithi07685715-39 from 400198 to 948813 tid 1 2024-09-21T09:45:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:1868: writing smithi07685715-39 from 1415202 to 2000198 tid 2 2024-09-21T09:45:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:1869: read oid 32 snap -1 2024-09-21T09:45:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:1869: expect (ObjNum 605 snap 193 seq_num 605) 2024-09-21T09:45:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:1870: snap_create 2024-09-21T09:45:58.685 INFO:tasks.rados.rados.0.smithi076.stdout:1868: finishing write tid 1 to smithi07685715-39 2024-09-21T09:45:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:1868: finishing write tid 2 to smithi07685715-39 2024-09-21T09:45:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:1868: finishing write tid 3 to smithi07685715-39 2024-09-21T09:45:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:1868: finishing write tid 4 to smithi07685715-39 2024-09-21T09:45:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:1868: oid 39 updating version 0 to 1368 2024-09-21T09:45:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:1868: oid 39 version 1368 is already newer than 1366 2024-09-21T09:45:58.689 INFO:tasks.rados.rados.0.smithi076.stdout:1868: oid 39 version 1368 is already newer than 1367 2024-09-21T09:45:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1368 (ObjNum 714 snap 243 seq_num 714) dirty exists 2024-09-21T09:45:58.690 INFO:tasks.rados.rados.0.smithi076.stdout:1868: left oid 39 (ObjNum 714 snap 243 seq_num 714) 2024-09-21T09:45:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:1869: expect (ObjNum 605 snap 193 seq_num 605) 2024-09-21T09:45:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:58 smithi191 ceph-mon[102206]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:45:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:58 smithi191 ceph-mon[102206]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:45:59.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:58 smithi076 ceph-mon[106482]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:45:59.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:58 smithi076 ceph-mon[106482]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:45:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:58 smithi076 ceph-mon[110963]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:45:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:58 smithi076 ceph-mon[110963]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:45:59.675 INFO:tasks.rados.rados.0.smithi076.stdout:1868: done (2 left) 2024-09-21T09:45:59.676 INFO:tasks.rados.rados.0.smithi076.stdout:1869: done (1 left) 2024-09-21T09:45:59.676 INFO:tasks.rados.rados.0.smithi076.stdout:1870: done (0 left) 2024-09-21T09:45:59.676 INFO:tasks.rados.rados.0.smithi076.stdout:1871: snap_create 2024-09-21T09:46:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:59 smithi191 ceph-mon[102206]: pgmap v287: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 424 KiB/s wr, 5 op/s 2024-09-21T09:46:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:45:59 smithi191 ceph-mon[102206]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:46:00.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:59 smithi076 ceph-mon[106482]: pgmap v287: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 424 KiB/s wr, 5 op/s 2024-09-21T09:46:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:45:59 smithi076 ceph-mon[106482]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:46:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:59 smithi076 ceph-mon[110963]: pgmap v287: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 424 KiB/s wr, 5 op/s 2024-09-21T09:46:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:45:59 smithi076 ceph-mon[110963]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:46:00.678 INFO:tasks.rados.rados.0.smithi076.stdout:1871: done (0 left) 2024-09-21T09:46:00.678 INFO:tasks.rados.rados.0.smithi076.stdout:1872: read oid 35 snap -1 2024-09-21T09:46:00.678 INFO:tasks.rados.rados.0.smithi076.stdout:1872: expect (ObjNum 684 snap 220 seq_num 684) 2024-09-21T09:46:00.678 INFO:tasks.rados.rados.0.smithi076.stdout:1873: snap_remove snap 244 2024-09-21T09:46:01.682 INFO:tasks.rados.rados.0.smithi076.stdout:1873: done (1 left) 2024-09-21T09:46:01.682 INFO:tasks.rados.rados.0.smithi076.stdout:1874: write oid 25 current snap is 245 2024-09-21T09:46:01.682 INFO:tasks.rados.rados.0.smithi076.stdout:1872: expect (ObjNum 684 snap 220 seq_num 684) 2024-09-21T09:46:01.932 INFO:tasks.rados.rados.0.smithi076.stdout:1874: seq_num 715 ranges {417005=740217,1590037=426968} 2024-09-21T09:46:01.938 INFO:tasks.rados.rados.0.smithi076.stdout:1874: writing smithi07685715-25 from 417005 to 1157222 tid 1 2024-09-21T09:46:01.943 INFO:tasks.rados.rados.0.smithi076.stdout:1874: writing smithi07685715-25 from 1590037 to 2017005 tid 2 2024-09-21T09:46:01.943 INFO:tasks.rados.rados.0.smithi076.stdout:1872: done (1 left) 2024-09-21T09:46:01.943 INFO:tasks.rados.rados.0.smithi076.stdout:1875: snap_remove snap 217 2024-09-21T09:46:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:01 smithi191 ceph-mon[102206]: pgmap v290: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:01 smithi191 ceph-mon[102206]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:46:02.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:01 smithi076 ceph-mon[106482]: pgmap v290: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:01 smithi076 ceph-mon[106482]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:46:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:01 smithi076 ceph-mon[110963]: pgmap v290: 89 pgs: 89 active+clean; 154 MiB data, 648 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:01 smithi076 ceph-mon[110963]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: finishing write tid 1 to smithi07685715-25 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: finishing write tid 2 to smithi07685715-25 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: finishing write tid 3 to smithi07685715-25 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: finishing write tid 4 to smithi07685715-25 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: oid 25 updating version 0 to 376 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: oid 25 version 376 is already newer than 375 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: oid 25 version 376 is already newer than 374 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 376 (ObjNum 715 snap 245 seq_num 715) dirty exists 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: left oid 25 (ObjNum 715 snap 245 seq_num 715) 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1874: done (1 left) 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1875: done (0 left) 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1876: write oid 7 current snap is 245 2024-09-21T09:46:02.696 INFO:tasks.rados.rados.0.smithi076.stdout:1876: seq_num 716 ranges {433812=415466,1648519=385293} 2024-09-21T09:46:02.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: writing smithi07685715-7 from 433812 to 849278 tid 1 2024-09-21T09:46:02.708 INFO:tasks.rados.rados.0.smithi076.stdout:1876: writing smithi07685715-7 from 1648519 to 2033812 tid 2 2024-09-21T09:46:02.708 INFO:tasks.rados.rados.0.smithi076.stdout:1877: read oid 10 snap -1 2024-09-21T09:46:02.708 INFO:tasks.rados.rados.0.smithi076.stdout:1877: expect (ObjNum 707 snap 231 seq_num 707) 2024-09-21T09:46:02.708 INFO:tasks.rados.rados.0.smithi076.stdout:1878: snap_remove snap 226 2024-09-21T09:46:03.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:02 smithi076 ceph-mon[110963]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:46:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:02 smithi076 ceph-mon[106482]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:46:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:02 smithi191 ceph-mon[102206]: osdmap e548: 8 total, 8 up, 8 in 2024-09-21T09:46:03.290 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:46:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:46:03] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.51.0" 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1878: done (2 left) 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1879: snap_create 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: finishing write tid 1 to smithi07685715-7 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: finishing write tid 2 to smithi07685715-7 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: finishing write tid 3 to smithi07685715-7 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: finishing write tid 4 to smithi07685715-7 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: oid 7 updating version 0 to 1004 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: oid 7 updating version 1004 to 1005 2024-09-21T09:46:03.701 INFO:tasks.rados.rados.0.smithi076.stdout:1876: oid 7 version 1005 is already newer than 1001 2024-09-21T09:46:03.702 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1005 (ObjNum 716 snap 245 seq_num 716) dirty exists 2024-09-21T09:46:03.702 INFO:tasks.rados.rados.0.smithi076.stdout:1876: left oid 7 (ObjNum 716 snap 245 seq_num 716) 2024-09-21T09:46:03.702 INFO:tasks.rados.rados.0.smithi076.stdout:1877: expect (ObjNum 707 snap 231 seq_num 707) 2024-09-21T09:46:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:03 smithi191 ceph-mon[102206]: pgmap v293: 89 pgs: 89 active+clean; 155 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 277 KiB/s wr, 7 op/s 2024-09-21T09:46:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:03 smithi191 ceph-mon[102206]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:46:04.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:03 smithi076 ceph-mon[106482]: pgmap v293: 89 pgs: 89 active+clean; 155 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 277 KiB/s wr, 7 op/s 2024-09-21T09:46:04.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:03 smithi076 ceph-mon[106482]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:46:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:03 smithi076 ceph-mon[110963]: pgmap v293: 89 pgs: 89 active+clean; 155 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 277 KiB/s wr, 7 op/s 2024-09-21T09:46:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:03 smithi076 ceph-mon[110963]: osdmap e549: 8 total, 8 up, 8 in 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1876: done (2 left) 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1877: done (1 left) 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1879: done (0 left) 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1880: rollback oid 36 current snap is 246 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 238 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1881: read oid 34 snap -1 2024-09-21T09:46:04.706 INFO:tasks.rados.rados.0.smithi076.stdout:1881: expect (ObjNum 698 snap 228 seq_num 698) 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1882: rollback oid 46 current snap is 246 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 220 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1883: rollback oid 8 current snap is 246 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 218 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1884: read oid 14 snap -1 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1884: expect deleted 2024-09-21T09:46:04.707 INFO:tasks.rados.rados.0.smithi076.stdout:1885: snap_remove snap 228 2024-09-21T09:46:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:04 smithi191 ceph-mon[102206]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:46:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:04 smithi076 ceph-mon[106482]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:46:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:04 smithi076 ceph-mon[110963]: osdmap e550: 8 total, 8 up, 8 in 2024-09-21T09:46:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:1885: done (5 left) 2024-09-21T09:46:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:1886: write oid 6 current snap is 246 2024-09-21T09:46:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:1886: seq_num 717 ranges {450619=490715,1590648=459971} 2024-09-21T09:46:05.713 INFO:tasks.rados.rados.0.smithi076.stdout:1886: writing smithi07685715-6 from 450619 to 941334 tid 1 2024-09-21T09:46:05.720 INFO:tasks.rados.rados.0.smithi076.stdout:1886: writing smithi07685715-6 from 1590648 to 2050619 tid 2 2024-09-21T09:46:05.720 INFO:tasks.rados.rados.0.smithi076.stdout:1887: write oid 21 current snap is 246 2024-09-21T09:46:05.720 INFO:tasks.rados.rados.0.smithi076.stdout:1887: seq_num 718 ranges {467426=565964,1649130=418296} 2024-09-21T09:46:05.726 INFO:tasks.rados.rados.0.smithi076.stdout:1887: writing smithi07685715-21 from 467426 to 1033390 tid 1 2024-09-21T09:46:05.732 INFO:tasks.rados.rados.0.smithi076.stdout:1887: writing smithi07685715-21 from 1649130 to 2067426 tid 2 2024-09-21T09:46:05.732 INFO:tasks.rados.rados.0.smithi076.stdout:1888: read oid 27 snap -1 2024-09-21T09:46:05.732 INFO:tasks.rados.rados.0.smithi076.stdout:1888: expect deleted 2024-09-21T09:46:05.732 INFO:tasks.rados.rados.0.smithi076.stdout:1889: write oid 49 current snap is 246 2024-09-21T09:46:05.732 INFO:tasks.rados.rados.0.smithi076.stdout:1889: seq_num 719 ranges {484233=641213,1707612=376621} 2024-09-21T09:46:05.738 INFO:tasks.rados.rados.0.smithi076.stdout:1889: writing smithi07685715-49 from 484233 to 1125446 tid 1 2024-09-21T09:46:05.743 INFO:tasks.rados.rados.0.smithi076.stdout:1889: writing smithi07685715-49 from 1707612 to 2084233 tid 2 2024-09-21T09:46:05.743 INFO:tasks.rados.rados.0.smithi076.stdout:1883: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:46:05.743 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 0 (ObjNum 977 snap 0 seq_num 1283432608) dirty dne 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1880: finishing rollback tid 0 to smithi07685715-36 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1880: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1890: read oid 2 snap -1 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1890: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1883: done (9 left) 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1884: done (8 left) 2024-09-21T09:46:05.744 INFO:tasks.rados.rados.0.smithi076.stdout:1891: snap_remove snap 224 2024-09-21T09:46:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:05 smithi191 ceph-mon[102206]: pgmap v296: 89 pgs: 89 active+clean; 155 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 277 KiB/s wr, 7 op/s 2024-09-21T09:46:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:05 smithi191 ceph-mon[102206]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:46:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:05 smithi076 ceph-mon[106482]: pgmap v296: 89 pgs: 89 active+clean; 155 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 277 KiB/s wr, 7 op/s 2024-09-21T09:46:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:05 smithi076 ceph-mon[106482]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:46:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:05 smithi076 ceph-mon[110963]: pgmap v296: 89 pgs: 89 active+clean; 155 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 277 KiB/s wr, 7 op/s 2024-09-21T09:46:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:05 smithi076 ceph-mon[110963]: osdmap e551: 8 total, 8 up, 8 in 2024-09-21T09:46:06.718 INFO:tasks.rados.rados.0.smithi076.stdout:1891: done (8 left) 2024-09-21T09:46:06.718 INFO:tasks.rados.rados.0.smithi076.stdout:1892: snap_remove snap 243 2024-09-21T09:46:07.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:06 smithi191 ceph-mon[102206]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:46:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:06 smithi076 ceph-mon[110963]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:46:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:06 smithi076 ceph-mon[106482]: osdmap e552: 8 total, 8 up, 8 in 2024-09-21T09:46:07.719 INFO:tasks.rados.rados.0.smithi076.stdout:1892: done (8 left) 2024-09-21T09:46:07.719 INFO:tasks.rados.rados.0.smithi076.stdout:1893: snap_remove snap 230 2024-09-21T09:46:08.029 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:07 smithi191 ceph-mon[102206]: pgmap v299: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:46:08.030 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:07 smithi191 ceph-mon[102206]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:46:08.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:07 smithi076 ceph-mon[106482]: pgmap v299: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:46:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:07 smithi076 ceph-mon[106482]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:46:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:07 smithi076 ceph-mon[110963]: pgmap v299: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:46:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:07 smithi076 ceph-mon[110963]: osdmap e553: 8 total, 8 up, 8 in 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:1893: done (8 left) 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:1894: rollback oid 25 current snap is 246 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 234 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:1880: finishing rollback tid 2 to smithi07685715-36 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 556 (ObjNum 702 snap 231 seq_num 702) dirty exists 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:1882: finishing rollback tid 0 to smithi07685715-46 2024-09-21T09:46:08.680 INFO:tasks.rados.rados.0.smithi076.stdout:1882: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:46:08.681 INFO:tasks.rados.rados.0.smithi076.stdout:1882: finishing rollback tid 2 to smithi07685715-46 2024-09-21T09:46:08.681 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1382 (ObjNum 482 snap 155 seq_num 482) dirty exists 2024-09-21T09:46:08.681 INFO:tasks.rados.rados.0.smithi076.stdout:1881: expect (ObjNum 698 snap 228 seq_num 698) 2024-09-21T09:46:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:08 smithi191 ceph-mon[102206]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:46:08.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:08 smithi191 ceph-mon[102206]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: finishing write tid 1 to smithi07685715-6 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: finishing write tid 2 to smithi07685715-6 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: finishing write tid 3 to smithi07685715-6 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: finishing write tid 4 to smithi07685715-6 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: oid 6 updating version 0 to 369 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: oid 6 updating version 369 to 371 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: oid 6 version 371 is already newer than 370 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 371 (ObjNum 717 snap 246 seq_num 717) dirty exists 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1886: left oid 6 (ObjNum 717 snap 246 seq_num 717) 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1880: done (8 left) 2024-09-21T09:46:08.932 INFO:tasks.rados.rados.0.smithi076.stdout:1881: done (7 left) 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1882: done (6 left) 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1886: done (5 left) 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1895: rollback oid 26 current snap is 246 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 239 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: finishing write tid 1 to smithi07685715-21 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: finishing write tid 2 to smithi07685715-21 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: finishing write tid 3 to smithi07685715-21 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: finishing write tid 4 to smithi07685715-21 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: oid 21 updating version 0 to 1392 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: oid 21 version 1392 is already newer than 1391 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: oid 21 version 1392 is already newer than 1390 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1392 (ObjNum 718 snap 246 seq_num 718) dirty exists 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1887: left oid 21 (ObjNum 718 snap 246 seq_num 718) 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1889: finishing write tid 1 to smithi07685715-49 2024-09-21T09:46:08.933 INFO:tasks.rados.rados.0.smithi076.stdout:1890: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:46:09.044 INFO:tasks.rados.rados.0.smithi076.stdout:1889: finishing write tid 2 to smithi07685715-49 2024-09-21T09:46:09.044 INFO:tasks.rados.rados.0.smithi076.stdout:1889: finishing write tid 3 to smithi07685715-49 2024-09-21T09:46:09.044 INFO:tasks.rados.rados.0.smithi076.stdout:1889: finishing write tid 4 to smithi07685715-49 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1889: oid 49 updating version 0 to 1462 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1889: oid 49 updating version 1462 to 1463 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1889: oid 49 version 1463 is already newer than 1461 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 1463 (ObjNum 719 snap 246 seq_num 719) dirty exists 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1889: left oid 49 (ObjNum 719 snap 246 seq_num 719) 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1894: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1894: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1894: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 398 (ObjNum 674 snap 215 seq_num 674) dirty exists 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1895: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1895: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1895: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 638 (ObjNum 690 snap 222 seq_num 690) dirty exists 2024-09-21T09:46:09.045 INFO:tasks.rados.rados.0.smithi076.stdout:1887: done (5 left) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1888: done (4 left) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1889: done (3 left) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1890: done (2 left) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1894: done (1 left) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1895: done (0 left) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1896: rollback oid 25 current snap is 246 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 220 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1897: read oid 16 snap -1 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1897: expect deleted 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1898: read oid 33 snap -1 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1898: expect (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1899: read oid 38 snap -1 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1899: expect (ObjNum 700 snap 230 seq_num 700) 2024-09-21T09:46:09.046 INFO:tasks.rados.rados.0.smithi076.stdout:1900: snap_create 2024-09-21T09:46:09.047 INFO:tasks.rados.rados.0.smithi076.stdout:1896: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:46:09.047 INFO:tasks.rados.rados.0.smithi076.stdout:1896: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:46:09.047 INFO:tasks.rados.rados.0.smithi076.stdout:1896: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:46:09.047 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 405 (ObjNum 674 snap 215 seq_num 674) dirty exists 2024-09-21T09:46:09.049 INFO:tasks.rados.rados.0.smithi076.stdout:1898: expect (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:46:09.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:08 smithi076 ceph-mon[106482]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:46:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:08 smithi076 ceph-mon[106482]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:46:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:08 smithi076 ceph-mon[110963]: osdmap e554: 8 total, 8 up, 8 in 2024-09-21T09:46:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:08 smithi076 ceph-mon[110963]: osdmap e555: 8 total, 8 up, 8 in 2024-09-21T09:46:09.173 INFO:tasks.rados.rados.0.smithi076.stdout:1899: expect (ObjNum 700 snap 230 seq_num 700) 2024-09-21T09:46:09.744 INFO:tasks.rados.rados.0.smithi076.stdout:1896: done (4 left) 2024-09-21T09:46:09.744 INFO:tasks.rados.rados.0.smithi076.stdout:1897: done (3 left) 2024-09-21T09:46:09.744 INFO:tasks.rados.rados.0.smithi076.stdout:1898: done (2 left) 2024-09-21T09:46:09.744 INFO:tasks.rados.rados.0.smithi076.stdout:1899: done (1 left) 2024-09-21T09:46:09.744 INFO:tasks.rados.rados.0.smithi076.stdout:1900: done (0 left) 2024-09-21T09:46:09.745 INFO:tasks.rados.rados.0.smithi076.stdout:1901: rollback oid 11 current snap is 247 2024-09-21T09:46:09.745 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 246 2024-09-21T09:46:09.745 INFO:tasks.rados.rados.0.smithi076.stdout:1902: snap_remove snap 236 2024-09-21T09:46:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:09 smithi191 ceph-mon[102206]: pgmap v302: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:46:10.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:09 smithi076 ceph-mon[106482]: pgmap v302: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:46:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:09 smithi076 ceph-mon[110963]: pgmap v302: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:46:10.751 INFO:tasks.rados.rados.0.smithi076.stdout:1902: done (1 left) 2024-09-21T09:46:10.751 INFO:tasks.rados.rados.0.smithi076.stdout:1903: write oid 41 current snap is 247 2024-09-21T09:46:10.751 INFO:tasks.rados.rados.0.smithi076.stdout:1903: seq_num 720 ranges {501040=716462,1766094=334946} 2024-09-21T09:46:10.758 INFO:tasks.rados.rados.0.smithi076.stdout:1903: writing smithi07685715-41 from 501040 to 1217502 tid 1 2024-09-21T09:46:10.763 INFO:tasks.rados.rados.0.smithi076.stdout:1903: writing smithi07685715-41 from 1766094 to 2101040 tid 2 2024-09-21T09:46:10.763 INFO:tasks.rados.rados.0.smithi076.stdout:1904: snap_remove snap 238 2024-09-21T09:46:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:10 smithi191 ceph-mon[102206]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:46:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:11.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:10 smithi076 ceph-mon[106482]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:46:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:10 smithi076 ceph-mon[110963]: osdmap e556: 8 total, 8 up, 8 in 2024-09-21T09:46:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1904: done (2 left) 2024-09-21T09:46:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1905: write oid 14 current snap is 247 2024-09-21T09:46:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1901: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:46:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 0 (ObjNum 177 snap 0 seq_num 1283431808) dirty dne 2024-09-21T09:46:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1903: finishing write tid 1 to smithi07685715-41 2024-09-21T09:46:11.753 INFO:tasks.rados.rados.0.smithi076.stdout:1903: finishing write tid 2 to smithi07685715-41 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1903: finishing write tid 3 to smithi07685715-41 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1903: finishing write tid 4 to smithi07685715-41 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1903: oid 41 updating version 0 to 855 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1903: oid 41 updating version 855 to 857 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1903: oid 41 version 857 is already newer than 856 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 857 (ObjNum 720 snap 247 seq_num 720) dirty exists 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1903: left oid 41 (ObjNum 720 snap 247 seq_num 720) 2024-09-21T09:46:11.754 INFO:tasks.rados.rados.0.smithi076.stdout:1905: seq_num 721 ranges {517847=791711,2108223=9624} 2024-09-21T09:46:11.760 INFO:tasks.rados.rados.0.smithi076.stdout:1905: writing smithi07685715-14 from 517847 to 1309558 tid 1 2024-09-21T09:46:11.765 INFO:tasks.rados.rados.0.smithi076.stdout:1905: writing smithi07685715-14 from 2108223 to 2117847 tid 2 2024-09-21T09:46:11.765 INFO:tasks.rados.rados.0.smithi076.stdout:1901: done (2 left) 2024-09-21T09:46:11.765 INFO:tasks.rados.rados.0.smithi076.stdout:1903: done (1 left) 2024-09-21T09:46:11.765 INFO:tasks.rados.rados.0.smithi076.stdout:1906: snap_create 2024-09-21T09:46:11.767 INFO:tasks.rados.rados.0.smithi076.stdout:1905: finishing write tid 1 to smithi07685715-14 2024-09-21T09:46:11.767 INFO:tasks.rados.rados.0.smithi076.stdout:1905: finishing write tid 2 to smithi07685715-14 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:1905: finishing write tid 3 to smithi07685715-14 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:1905: finishing write tid 4 to smithi07685715-14 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:1905: oid 14 updating version 0 to 1043 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:1905: oid 14 version 1043 is already newer than 1042 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:1905: oid 14 updating version 1043 to 1044 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1044 (ObjNum 721 snap 247 seq_num 721) dirty exists 2024-09-21T09:46:11.768 INFO:tasks.rados.rados.0.smithi076.stdout:1905: left oid 14 (ObjNum 721 snap 247 seq_num 721) 2024-09-21T09:46:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:11 smithi191 ceph-mon[102206]: pgmap v305: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:11 smithi191 ceph-mon[102206]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:46:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:11 smithi076 ceph-mon[106482]: pgmap v305: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:11 smithi076 ceph-mon[106482]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:46:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:11 smithi076 ceph-mon[110963]: pgmap v305: 89 pgs: 89 active+clean; 157 MiB data, 657 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:11 smithi076 ceph-mon[110963]: osdmap e557: 8 total, 8 up, 8 in 2024-09-21T09:46:12.758 INFO:tasks.rados.rados.0.smithi076.stdout:1905: done (1 left) 2024-09-21T09:46:12.759 INFO:tasks.rados.rados.0.smithi076.stdout:1906: done (0 left) 2024-09-21T09:46:12.759 INFO:tasks.rados.rados.0.smithi076.stdout:1907: write oid 7 current snap is 248 2024-09-21T09:46:12.759 INFO:tasks.rados.rados.0.smithi076.stdout:1907: seq_num 722 ranges {534654=466960,1766705=367949} 2024-09-21T09:46:12.764 INFO:tasks.rados.rados.0.smithi076.stdout:1907: writing smithi07685715-7 from 534654 to 1001614 tid 1 2024-09-21T09:46:12.770 INFO:tasks.rados.rados.0.smithi076.stdout:1907: writing smithi07685715-7 from 1766705 to 2134654 tid 2 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:1908: rollback oid 45 current snap is 248 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 223 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:1907: finishing write tid 1 to smithi07685715-7 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:1909: read oid 44 snap -1 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:1909: expect (ObjNum 704 snap 231 seq_num 704) 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:1910: write oid 9 current snap is 248 2024-09-21T09:46:12.771 INFO:tasks.rados.rados.0.smithi076.stdout:1910: seq_num 723 ranges {551461=658562,1941540=209921} 2024-09-21T09:46:12.777 INFO:tasks.rados.rados.0.smithi076.stdout:1910: writing smithi07685715-9 from 551461 to 1210023 tid 1 2024-09-21T09:46:12.783 INFO:tasks.rados.rados.0.smithi076.stdout:1910: writing smithi07685715-9 from 1941540 to 2151461 tid 2 2024-09-21T09:46:12.783 INFO:tasks.rados.rados.0.smithi076.stdout:1911: write oid 3 current snap is 248 2024-09-21T09:46:12.783 INFO:tasks.rados.rados.0.smithi076.stdout:1907: finishing write tid 2 to smithi07685715-7 2024-09-21T09:46:12.783 INFO:tasks.rados.rados.0.smithi076.stdout:1911: seq_num 724 ranges {568268=733811,2000022=168246} 2024-09-21T09:46:12.790 INFO:tasks.rados.rados.0.smithi076.stdout:1911: writing smithi07685715-3 from 568268 to 1302079 tid 1 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1911: writing smithi07685715-3 from 2000022 to 2168268 tid 2 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1912: read oid 15 snap -1 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1912: expect (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1913: rollback oid 47 current snap is 248 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 245 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1907: finishing write tid 3 to smithi07685715-7 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1907: finishing write tid 4 to smithi07685715-7 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1907: oid 7 updating version 0 to 1048 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1907: oid 7 version 1048 is already newer than 1047 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1907: oid 7 version 1048 is already newer than 1046 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1048 (ObjNum 722 snap 248 seq_num 722) dirty exists 2024-09-21T09:46:12.796 INFO:tasks.rados.rados.0.smithi076.stdout:1907: left oid 7 (ObjNum 722 snap 248 seq_num 722) 2024-09-21T09:46:12.797 INFO:tasks.rados.rados.0.smithi076.stdout:1908: finishing rollback tid 0 to smithi07685715-45 2024-09-21T09:46:12.797 INFO:tasks.rados.rados.0.smithi076.stdout:1907: done (6 left) 2024-09-21T09:46:12.797 INFO:tasks.rados.rados.0.smithi076.stdout:1914: write oid 34 current snap is 248 2024-09-21T09:46:12.797 INFO:tasks.rados.rados.0.smithi076.stdout:1908: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:46:12.797 INFO:tasks.rados.rados.0.smithi076.stdout:1914: seq_num 725 ranges {585075=409060,1542151=642924} 2024-09-21T09:46:12.801 INFO:tasks.rados.rados.0.smithi076.stdout:1914: writing smithi07685715-34 from 585075 to 994135 tid 1 2024-09-21T09:46:12.808 INFO:tasks.rados.rados.0.smithi076.stdout:1914: writing smithi07685715-34 from 1542151 to 2185075 tid 2 2024-09-21T09:46:12.808 INFO:tasks.rados.rados.0.smithi076.stdout:1915: write oid 13 current snap is 248 2024-09-21T09:46:12.808 INFO:tasks.rados.rados.0.smithi076.stdout:1908: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:46:12.808 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 364 (ObjNum 540 snap 171 seq_num 540) dirty exists 2024-09-21T09:46:12.808 INFO:tasks.rados.rados.0.smithi076.stdout:1910: finishing write tid 1 to smithi07685715-9 2024-09-21T09:46:12.808 INFO:tasks.rados.rados.0.smithi076.stdout:1909: expect (ObjNum 704 snap 231 seq_num 704) 2024-09-21T09:46:12.930 INFO:tasks.rados.rados.0.smithi076.stdout:1910: finishing write tid 2 to smithi07685715-9 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1910: finishing write tid 3 to smithi07685715-9 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1910: finishing write tid 4 to smithi07685715-9 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1910: oid 9 updating version 0 to 645 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1910: oid 9 updating version 645 to 646 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1910: oid 9 version 646 is already newer than 644 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 646 (ObjNum 723 snap 248 seq_num 723) dirty exists 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1910: left oid 9 (ObjNum 723 snap 248 seq_num 723) 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1911: finishing write tid 1 to smithi07685715-3 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1911: finishing write tid 2 to smithi07685715-3 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1911: finishing write tid 3 to smithi07685715-3 2024-09-21T09:46:12.931 INFO:tasks.rados.rados.0.smithi076.stdout:1915: seq_num 726 ranges {601882=484309,1600633=524904} 2024-09-21T09:46:12.936 INFO:tasks.rados.rados.0.smithi076.stdout:1915: writing smithi07685715-13 from 601882 to 1086191 tid 1 2024-09-21T09:46:12.942 INFO:tasks.rados.rados.0.smithi076.stdout:1915: writing smithi07685715-13 from 1600633 to 2125537 tid 2 2024-09-21T09:46:12.942 INFO:tasks.rados.rados.0.smithi076.stdout:1908: done (7 left) 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1909: done (6 left) 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1910: done (5 left) 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1916: delete oid 1 current snap is 248 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1911: finishing write tid 4 to smithi07685715-3 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1911: oid 3 updating version 0 to 93 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1911: oid 3 updating version 93 to 95 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1911: oid 3 version 95 is already newer than 94 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 95 (ObjNum 724 snap 248 seq_num 724) dirty exists 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1911: left oid 3 (ObjNum 724 snap 248 seq_num 724) 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1913: finishing rollback tid 0 to smithi07685715-47 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1913: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1913: finishing rollback tid 2 to smithi07685715-47 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1477 (ObjNum 663 snap 212 seq_num 663) dirty exists 2024-09-21T09:46:12.943 INFO:tasks.rados.rados.0.smithi076.stdout:1912: expect (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: finishing write tid 1 to smithi07685715-34 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: finishing write tid 2 to smithi07685715-34 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: finishing write tid 3 to smithi07685715-34 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: finishing write tid 4 to smithi07685715-34 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: oid 34 updating version 0 to 337 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: oid 34 version 337 is already newer than 335 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:1914: oid 34 version 337 is already newer than 336 2024-09-21T09:46:13.051 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 337 (ObjNum 725 snap 248 seq_num 725) dirty exists 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1914: left oid 34 (ObjNum 725 snap 248 seq_num 725) 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1911: done (5 left) 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1912: done (4 left) 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1913: done (3 left) 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1914: done (2 left) 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1916: done (1 left) 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1917: rollback oid 6 current snap is 248 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 232 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: finishing write tid 1 to smithi07685715-13 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: finishing write tid 2 to smithi07685715-13 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: finishing write tid 3 to smithi07685715-13 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: finishing write tid 4 to smithi07685715-13 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: oid 13 updating version 0 to 276 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: oid 13 version 276 is already newer than 274 2024-09-21T09:46:13.052 INFO:tasks.rados.rados.0.smithi076.stdout:1915: oid 13 version 276 is already newer than 275 2024-09-21T09:46:13.053 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 276 (ObjNum 726 snap 248 seq_num 726) dirty exists 2024-09-21T09:46:13.053 INFO:tasks.rados.rados.0.smithi076.stdout:1915: left oid 13 (ObjNum 726 snap 248 seq_num 726) 2024-09-21T09:46:13.053 INFO:tasks.rados.rados.0.smithi076.stdout:1915: done (1 left) 2024-09-21T09:46:13.053 INFO:tasks.rados.rados.0.smithi076.stdout:1918: snap_remove snap 246 2024-09-21T09:46:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:12 smithi191 ceph-mon[102206]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:46:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:12 smithi076 ceph-mon[106482]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:46:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:12 smithi076 ceph-mon[110963]: osdmap e558: 8 total, 8 up, 8 in 2024-09-21T09:46:13.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:46:13] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:46:13.680 INFO:tasks.rados.rados.0.smithi076.stdout:1917: finishing rollback tid 0 to smithi07685715-6 2024-09-21T09:46:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:1917: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:46:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:1917: finishing rollback tid 2 to smithi07685715-6 2024-09-21T09:46:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 385 (ObjNum 492 snap 157 seq_num 492) dirty exists 2024-09-21T09:46:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:1917: done (1 left) 2024-09-21T09:46:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:1918: done (0 left) 2024-09-21T09:46:13.681 INFO:tasks.rados.rados.0.smithi076.stdout:1919: delete oid 37 current snap is 248 2024-09-21T09:46:13.686 INFO:tasks.rados.rados.0.smithi076.stdout:1919: done (0 left) 2024-09-21T09:46:13.686 INFO:tasks.rados.rados.0.smithi076.stdout:1920: snap_remove snap 231 2024-09-21T09:46:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:13 smithi191 ceph-mon[102206]: pgmap v308: 89 pgs: 89 active+clean; 161 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 257 KiB/s wr, 7 op/s 2024-09-21T09:46:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:13 smithi191 ceph-mon[102206]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:46:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:13 smithi191 ceph-mon[102206]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:46:14.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-mon[106482]: pgmap v308: 89 pgs: 89 active+clean; 161 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 257 KiB/s wr, 7 op/s 2024-09-21T09:46:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-mon[106482]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:46:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-mon[106482]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:46:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-mon[110963]: pgmap v308: 89 pgs: 89 active+clean; 161 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 257 KiB/s wr, 7 op/s 2024-09-21T09:46:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-mon[110963]: osdmap e559: 8 total, 8 up, 8 in 2024-09-21T09:46:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:13 smithi076 ceph-mon[110963]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:46:14.683 INFO:tasks.rados.rados.0.smithi076.stdout:1920: done (0 left) 2024-09-21T09:46:14.684 INFO:tasks.rados.rados.0.smithi076.stdout:1921: rollback oid 38 current snap is 248 2024-09-21T09:46:14.684 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 242 2024-09-21T09:46:14.684 INFO:tasks.rados.rados.0.smithi076.stdout:1922: read oid 36 snap -1 2024-09-21T09:46:14.684 INFO:tasks.rados.rados.0.smithi076.stdout:1922: expect (ObjNum 702 snap 231 seq_num 702) 2024-09-21T09:46:14.684 INFO:tasks.rados.rados.0.smithi076.stdout:1923: delete oid 20 current snap is 248 2024-09-21T09:46:14.688 INFO:tasks.rados.rados.0.smithi076.stdout:1921: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:46:14.689 INFO:tasks.rados.rados.0.smithi076.stdout:1921: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:46:14.690 INFO:tasks.rados.rados.0.smithi076.stdout:1921: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:46:14.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1056 (ObjNum 700 snap 230 seq_num 700) dirty exists 2024-09-21T09:46:14.691 INFO:tasks.rados.rados.0.smithi076.stdout:1921: done (2 left) 2024-09-21T09:46:14.691 INFO:tasks.rados.rados.0.smithi076.stdout:1923: done (1 left) 2024-09-21T09:46:14.691 INFO:tasks.rados.rados.0.smithi076.stdout:1924: read oid 5 snap -1 2024-09-21T09:46:14.691 INFO:tasks.rados.rados.0.smithi076.stdout:1924: expect (ObjNum 713 snap 240 seq_num 713) 2024-09-21T09:46:14.691 INFO:tasks.rados.rados.0.smithi076.stdout:1925: snap_remove snap 232 2024-09-21T09:46:15.693 INFO:tasks.rados.rados.0.smithi076.stdout:1925: done (2 left) 2024-09-21T09:46:15.693 INFO:tasks.rados.rados.0.smithi076.stdout:1926: delete oid 9 current snap is 248 2024-09-21T09:46:15.693 INFO:tasks.rados.rados.0.smithi076.stdout:1922: expect (ObjNum 702 snap 231 seq_num 702) 2024-09-21T09:46:15.816 INFO:tasks.rados.rados.0.smithi076.stdout:1924: expect (ObjNum 713 snap 240 seq_num 713) 2024-09-21T09:46:15.945 INFO:tasks.rados.rados.0.smithi076.stdout:1922: done (2 left) 2024-09-21T09:46:15.946 INFO:tasks.rados.rados.0.smithi076.stdout:1924: done (1 left) 2024-09-21T09:46:15.946 INFO:tasks.rados.rados.0.smithi076.stdout:1926: done (0 left) 2024-09-21T09:46:15.946 INFO:tasks.rados.rados.0.smithi076.stdout:1927: rollback oid 47 current snap is 248 2024-09-21T09:46:15.946 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 235 2024-09-21T09:46:15.946 INFO:tasks.rados.rados.0.smithi076.stdout:1928: write oid 48 current snap is 248 2024-09-21T09:46:15.946 INFO:tasks.rados.rados.0.smithi076.stdout:1928: seq_num 731 ranges {685917=576907,1893043=392874} 2024-09-21T09:46:15.954 INFO:tasks.rados.rados.0.smithi076.stdout:1928: writing smithi07685715-48 from 685917 to 1262824 tid 1 2024-09-21T09:46:15.958 INFO:tasks.rados.rados.0.smithi076.stdout:1928: writing smithi07685715-48 from 1893043 to 2285917 tid 2 2024-09-21T09:46:15.958 INFO:tasks.rados.rados.0.smithi076.stdout:1929: delete oid 35 current snap is 248 2024-09-21T09:46:15.959 INFO:tasks.rados.rados.0.smithi076.stdout:1927: finishing rollback tid 0 to smithi07685715-47 2024-09-21T09:46:15.959 INFO:tasks.rados.rados.0.smithi076.stdout:1927: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:46:15.959 INFO:tasks.rados.rados.0.smithi076.stdout:1927: finishing rollback tid 2 to smithi07685715-47 2024-09-21T09:46:15.959 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 1490 (ObjNum 663 snap 212 seq_num 663) dirty exists 2024-09-21T09:46:15.959 INFO:tasks.rados.rados.0.smithi076.stdout:1928: finishing write tid 1 to smithi07685715-48 2024-09-21T09:46:15.961 INFO:tasks.rados.rados.0.smithi076.stdout:1927: done (2 left) 2024-09-21T09:46:15.961 INFO:tasks.rados.rados.0.smithi076.stdout:1929: done (1 left) 2024-09-21T09:46:15.961 INFO:tasks.rados.rados.0.smithi076.stdout:1930: delete oid 1 current snap is 248 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: finishing write tid 2 to smithi07685715-48 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: finishing write tid 3 to smithi07685715-48 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: finishing write tid 4 to smithi07685715-48 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: oid 48 updating version 0 to 1493 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: oid 48 updating version 1493 to 1494 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: oid 48 version 1494 is already newer than 1492 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1494 (ObjNum 731 snap 248 seq_num 731) dirty exists 2024-09-21T09:46:15.962 INFO:tasks.rados.rados.0.smithi076.stdout:1928: left oid 48 (ObjNum 731 snap 248 seq_num 731) 2024-09-21T09:46:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:1928: done (1 left) 2024-09-21T09:46:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:1930: done (0 left) 2024-09-21T09:46:15.963 INFO:tasks.rados.rados.0.smithi076.stdout:1931: snap_create 2024-09-21T09:46:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:15 smithi191 ceph-mon[102206]: pgmap v311: 89 pgs: 89 active+clean; 161 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 257 KiB/s wr, 7 op/s 2024-09-21T09:46:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:15 smithi191 ceph-mon[102206]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:46:16.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:15 smithi076 ceph-mon[106482]: pgmap v311: 89 pgs: 89 active+clean; 161 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 257 KiB/s wr, 7 op/s 2024-09-21T09:46:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:15 smithi076 ceph-mon[106482]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:46:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:15 smithi076 ceph-mon[110963]: pgmap v311: 89 pgs: 89 active+clean; 161 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 257 KiB/s wr, 7 op/s 2024-09-21T09:46:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:15 smithi076 ceph-mon[110963]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1931: done (0 left) 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1932: read oid 34 snap -1 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1932: expect (ObjNum 725 snap 248 seq_num 725) 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1933: read oid 33 snap -1 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1933: expect (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:1934: rollback oid 12 current snap is 249 2024-09-21T09:46:16.696 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 223 2024-09-21T09:46:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:1935: write oid 26 current snap is 249 2024-09-21T09:46:16.697 INFO:tasks.rados.rados.0.smithi076.stdout:1935: seq_num 734 ranges {736338=402654,1552136=539580} 2024-09-21T09:46:16.702 INFO:tasks.rados.rados.0.smithi076.stdout:1935: writing smithi07685715-26 from 736338 to 1138992 tid 1 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1935: writing smithi07685715-26 from 1552136 to 2091716 tid 2 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1936: read oid 48 snap -1 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1936: expect (ObjNum 731 snap 248 seq_num 731) 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1937: read oid 9 snap 241 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1937: expect (ObjNum 703 snap 231 seq_num 703) 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1938: read oid 20 snap 240 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1938: expect deleted 2024-09-21T09:46:16.708 INFO:tasks.rados.rados.0.smithi076.stdout:1939: snap_remove snap 229 2024-09-21T09:46:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:16 smithi191 ceph-mon[102206]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:46:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:16 smithi076 ceph-mon[106482]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:46:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:16 smithi076 ceph-mon[110963]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:46:17.702 INFO:tasks.rados.rados.0.smithi076.stdout:1939: done (7 left) 2024-09-21T09:46:17.702 INFO:tasks.rados.rados.0.smithi076.stdout:1940: delete oid 44 current snap is 249 2024-09-21T09:46:17.703 INFO:tasks.rados.rados.0.smithi076.stdout:1933: expect (ObjNum 637 snap 205 seq_num 637) 2024-09-21T09:46:17.829 INFO:tasks.rados.rados.0.smithi076.stdout:1934: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:46:17.829 INFO:tasks.rados.rados.0.smithi076.stdout:1932: expect (ObjNum 725 snap 248 seq_num 725) 2024-09-21T09:46:17.981 INFO:tasks.rados.rados.0.smithi076.stdout:1934: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:46:17.981 INFO:tasks.rados.rados.0.smithi076.stdout:1934: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:46:17.981 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 661 (ObjNum 575 snap 182 seq_num 575) dirty exists 2024-09-21T09:46:17.981 INFO:tasks.rados.rados.0.smithi076.stdout:1935: finishing write tid 1 to smithi07685715-26 2024-09-21T09:46:17.981 INFO:tasks.rados.rados.0.smithi076.stdout:1936: expect (ObjNum 731 snap 248 seq_num 731) 2024-09-21T09:46:18.078 INFO:tasks.rados.rados.0.smithi076.stdout:1935: finishing write tid 2 to smithi07685715-26 2024-09-21T09:46:18.078 INFO:tasks.rados.rados.0.smithi076.stdout:1935: finishing write tid 3 to smithi07685715-26 2024-09-21T09:46:18.078 INFO:tasks.rados.rados.0.smithi076.stdout:1935: finishing write tid 4 to smithi07685715-26 2024-09-21T09:46:18.078 INFO:tasks.rados.rados.0.smithi076.stdout:1935: oid 26 updating version 0 to 663 2024-09-21T09:46:18.078 INFO:tasks.rados.rados.0.smithi076.stdout:1935: oid 26 updating version 663 to 664 2024-09-21T09:46:18.079 INFO:tasks.rados.rados.0.smithi076.stdout:1935: oid 26 updating version 664 to 665 2024-09-21T09:46:18.079 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 665 (ObjNum 734 snap 249 seq_num 734) dirty exists 2024-09-21T09:46:18.079 INFO:tasks.rados.rados.0.smithi076.stdout:1935: left oid 26 (ObjNum 734 snap 249 seq_num 734) 2024-09-21T09:46:18.079 INFO:tasks.rados.rados.0.smithi076.stdout:1937: expect (ObjNum 703 snap 231 seq_num 703) 2024-09-21T09:46:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:17 smithi191 ceph-mon[102206]: pgmap v314: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-21T09:46:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:17 smithi191 ceph-mon[102206]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:46:18.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:17 smithi076 ceph-mon[106482]: pgmap v314: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-21T09:46:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:17 smithi076 ceph-mon[106482]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:46:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:17 smithi076 ceph-mon[110963]: pgmap v314: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-21T09:46:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:17 smithi076 ceph-mon[110963]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1932: done (7 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1933: done (6 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1934: done (5 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1935: done (4 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1936: done (3 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1937: done (2 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1938: done (1 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1940: done (0 left) 2024-09-21T09:46:18.201 INFO:tasks.rados.rados.0.smithi076.stdout:1941: write oid 41 current snap is 249 2024-09-21T09:46:18.202 INFO:tasks.rados.rados.0.smithi076.stdout:1941: seq_num 736 ranges {769952=553152,2069100=300852} 2024-09-21T09:46:18.207 INFO:tasks.rados.rados.0.smithi076.stdout:1941: writing smithi07685715-41 from 769952 to 1323104 tid 1 2024-09-21T09:46:18.213 INFO:tasks.rados.rados.0.smithi076.stdout:1941: writing smithi07685715-41 from 2069100 to 2369952 tid 2 2024-09-21T09:46:18.213 INFO:tasks.rados.rados.0.smithi076.stdout:1942: read oid 39 snap -1 2024-09-21T09:46:18.213 INFO:tasks.rados.rados.0.smithi076.stdout:1942: expect (ObjNum 714 snap 243 seq_num 714) 2024-09-21T09:46:18.214 INFO:tasks.rados.rados.0.smithi076.stdout:1943: snap_create 2024-09-21T09:46:18.214 INFO:tasks.rados.rados.0.smithi076.stdout:1941: finishing write tid 1 to smithi07685715-41 2024-09-21T09:46:18.217 INFO:tasks.rados.rados.0.smithi076.stdout:1941: finishing write tid 2 to smithi07685715-41 2024-09-21T09:46:18.217 INFO:tasks.rados.rados.0.smithi076.stdout:1941: finishing write tid 3 to smithi07685715-41 2024-09-21T09:46:18.217 INFO:tasks.rados.rados.0.smithi076.stdout:1941: finishing write tid 4 to smithi07685715-41 2024-09-21T09:46:18.217 INFO:tasks.rados.rados.0.smithi076.stdout:1941: oid 41 updating version 0 to 862 2024-09-21T09:46:18.217 INFO:tasks.rados.rados.0.smithi076.stdout:1941: oid 41 version 862 is already newer than 861 2024-09-21T09:46:18.217 INFO:tasks.rados.rados.0.smithi076.stdout:1941: oid 41 updating version 862 to 863 2024-09-21T09:46:18.218 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 863 (ObjNum 736 snap 249 seq_num 736) dirty exists 2024-09-21T09:46:18.218 INFO:tasks.rados.rados.0.smithi076.stdout:1941: left oid 41 (ObjNum 736 snap 249 seq_num 736) 2024-09-21T09:46:18.220 INFO:tasks.rados.rados.0.smithi076.stdout:1942: expect (ObjNum 714 snap 243 seq_num 714) 2024-09-21T09:46:18.681 INFO:tasks.rados.rados.0.smithi076.stdout:1941: done (2 left) 2024-09-21T09:46:18.681 INFO:tasks.rados.rados.0.smithi076.stdout:1942: done (1 left) 2024-09-21T09:46:18.681 INFO:tasks.rados.rados.0.smithi076.stdout:1943: done (0 left) 2024-09-21T09:46:18.681 INFO:tasks.rados.rados.0.smithi076.stdout:1944: read oid 49 snap -1 2024-09-21T09:46:18.681 INFO:tasks.rados.rados.0.smithi076.stdout:1944: expect (ObjNum 719 snap 246 seq_num 719) 2024-09-21T09:46:18.681 INFO:tasks.rados.rados.0.smithi076.stdout:1945: snap_create 2024-09-21T09:46:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:1944: expect (ObjNum 719 snap 246 seq_num 719) 2024-09-21T09:46:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:18 smithi191 ceph-mon[102206]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:46:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:18 smithi191 ceph-mon[102206]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:46:19.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:18 smithi076 ceph-mon[106482]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:46:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:18 smithi076 ceph-mon[106482]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:46:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:18 smithi076 ceph-mon[110963]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:46:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:18 smithi076 ceph-mon[110963]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:46:19.684 INFO:tasks.rados.rados.0.smithi076.stdout:1944: done (1 left) 2024-09-21T09:46:19.684 INFO:tasks.rados.rados.0.smithi076.stdout:1945: done (0 left) 2024-09-21T09:46:19.684 INFO:tasks.rados.rados.0.smithi076.stdout:1946: rollback oid 34 current snap is 251 2024-09-21T09:46:19.684 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 223 2024-09-21T09:46:19.684 INFO:tasks.rados.rados.0.smithi076.stdout:1947: snap_create 2024-09-21T09:46:19.689 INFO:tasks.rados.rados.0.smithi076.stdout:1946: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:46:19.689 INFO:tasks.rados.rados.0.smithi076.stdout:1946: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:46:19.689 INFO:tasks.rados.rados.0.smithi076.stdout:1946: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:46:19.690 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 349 (ObjNum 676 snap 215 seq_num 676) dirty exists 2024-09-21T09:46:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:19 smithi191 ceph-mon[102206]: pgmap v317: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-21T09:46:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:19 smithi191 ceph-mon[102206]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:46:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:19 smithi076 ceph-mon[106482]: pgmap v317: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-21T09:46:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:19 smithi076 ceph-mon[106482]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:46:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:19 smithi076 ceph-mon[110963]: pgmap v317: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 20 op/s 2024-09-21T09:46:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:19 smithi076 ceph-mon[110963]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:46:20.688 INFO:tasks.rados.rados.0.smithi076.stdout:1946: done (1 left) 2024-09-21T09:46:20.688 INFO:tasks.rados.rados.0.smithi076.stdout:1947: done (0 left) 2024-09-21T09:46:20.688 INFO:tasks.rados.rados.0.smithi076.stdout:1948: read oid 37 snap -1 2024-09-21T09:46:20.688 INFO:tasks.rados.rados.0.smithi076.stdout:1948: expect deleted 2024-09-21T09:46:20.689 INFO:tasks.rados.rados.0.smithi076.stdout:1949: write oid 18 current snap is 252 2024-09-21T09:46:20.689 INFO:tasks.rados.rados.0.smithi076.stdout:1949: seq_num 737 ranges {786759=628401,2011229=375530} 2024-09-21T09:46:20.695 INFO:tasks.rados.rados.0.smithi076.stdout:1949: writing smithi07685715-18 from 786759 to 1415160 tid 1 2024-09-21T09:46:20.701 INFO:tasks.rados.rados.0.smithi076.stdout:1949: writing smithi07685715-18 from 2011229 to 2386759 tid 2 2024-09-21T09:46:20.701 INFO:tasks.rados.rados.0.smithi076.stdout:1950: delete oid 29 current snap is 252 2024-09-21T09:46:20.701 INFO:tasks.rados.rados.0.smithi076.stdout:1949: finishing write tid 1 to smithi07685715-18 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1948: done (2 left) 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1950: done (1 left) 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1951: read oid 45 snap -1 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1951: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1952: read oid 25 snap 241 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1952: expect (ObjNum 674 snap 215 seq_num 674) 2024-09-21T09:46:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:1953: read oid 15 snap -1 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1953: expect (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1954: snap_create 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: finishing write tid 2 to smithi07685715-18 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: finishing write tid 3 to smithi07685715-18 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: finishing write tid 4 to smithi07685715-18 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: oid 18 updating version 0 to 575 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: oid 18 version 575 is already newer than 574 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: oid 18 updating version 575 to 576 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 576 (ObjNum 737 snap 252 seq_num 737) dirty exists 2024-09-21T09:46:20.705 INFO:tasks.rados.rados.0.smithi076.stdout:1949: left oid 18 (ObjNum 737 snap 252 seq_num 737) 2024-09-21T09:46:20.709 INFO:tasks.rados.rados.0.smithi076.stdout:1951: expect (ObjNum 540 snap 171 seq_num 540) 2024-09-21T09:46:20.845 INFO:tasks.rados.rados.0.smithi076.stdout:1952: expect (ObjNum 674 snap 215 seq_num 674) 2024-09-21T09:46:21.064 INFO:tasks.rados.rados.0.smithi076.stdout:1953: expect (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:46:21.692 INFO:tasks.rados.rados.0.smithi076.stdout:1949: done (4 left) 2024-09-21T09:46:21.693 INFO:tasks.rados.rados.0.smithi076.stdout:1951: done (3 left) 2024-09-21T09:46:21.693 INFO:tasks.rados.rados.0.smithi076.stdout:1952: done (2 left) 2024-09-21T09:46:21.693 INFO:tasks.rados.rados.0.smithi076.stdout:1953: done (1 left) 2024-09-21T09:46:21.693 INFO:tasks.rados.rados.0.smithi076.stdout:1954: done (0 left) 2024-09-21T09:46:21.693 INFO:tasks.rados.rados.0.smithi076.stdout:1955: snap_remove snap 233 2024-09-21T09:46:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:21 smithi191 ceph-mon[102206]: pgmap v320: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:21 smithi191 ceph-mon[102206]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:46:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:21 smithi076 ceph-mon[106482]: pgmap v320: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:21 smithi076 ceph-mon[106482]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:46:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:21 smithi076 ceph-mon[110963]: pgmap v320: 89 pgs: 89 active+clean; 169 MiB data, 693 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:21 smithi076 ceph-mon[110963]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:46:22.702 INFO:tasks.rados.rados.0.smithi076.stdout:1955: done (0 left) 2024-09-21T09:46:22.702 INFO:tasks.rados.rados.0.smithi076.stdout:1956: read oid 43 snap 248 2024-09-21T09:46:22.702 INFO:tasks.rados.rados.0.smithi076.stdout:1956: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:46:22.703 INFO:tasks.rados.rados.0.smithi076.stdout:1957: delete oid 11 current snap is 253 2024-09-21T09:46:22.709 INFO:tasks.rados.rados.0.smithi076.stdout:1957: done (1 left) 2024-09-21T09:46:22.710 INFO:tasks.rados.rados.0.smithi076.stdout:1958: read oid 21 snap 239 2024-09-21T09:46:22.710 INFO:tasks.rados.rados.0.smithi076.stdout:1958: expect (ObjNum 710 snap 236 seq_num 710) 2024-09-21T09:46:22.710 INFO:tasks.rados.rados.0.smithi076.stdout:1959: snap_remove snap 250 2024-09-21T09:46:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:22 smithi191 ceph-mon[102206]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:46:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:22 smithi076 ceph-mon[106482]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:46:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:22 smithi076 ceph-mon[110963]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:46:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:46:23] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:46:23.684 INFO:tasks.rados.rados.0.smithi076.stdout:1959: done (2 left) 2024-09-21T09:46:23.684 INFO:tasks.rados.rados.0.smithi076.stdout:1960: snap_remove snap 242 2024-09-21T09:46:24.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:23 smithi191 ceph-mon[102206]: pgmap v323: 89 pgs: 89 active+clean; 172 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 684 KiB/s wr, 23 op/s 2024-09-21T09:46:24.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:23 smithi191 ceph-mon[102206]: osdmap e569: 8 total, 8 up, 8 in 2024-09-21T09:46:24.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:23 smithi191 ceph-mon[102206]: osdmap e570: 8 total, 8 up, 8 in 2024-09-21T09:46:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-mon[106482]: pgmap v323: 89 pgs: 89 active+clean; 172 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 684 KiB/s wr, 23 op/s 2024-09-21T09:46:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-mon[106482]: osdmap e569: 8 total, 8 up, 8 in 2024-09-21T09:46:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-mon[106482]: osdmap e570: 8 total, 8 up, 8 in 2024-09-21T09:46:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-mon[110963]: pgmap v323: 89 pgs: 89 active+clean; 172 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 684 KiB/s wr, 23 op/s 2024-09-21T09:46:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-mon[110963]: osdmap e569: 8 total, 8 up, 8 in 2024-09-21T09:46:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:23 smithi076 ceph-mon[110963]: osdmap e570: 8 total, 8 up, 8 in 2024-09-21T09:46:24.684 INFO:tasks.rados.rados.0.smithi076.stdout:1960: done (2 left) 2024-09-21T09:46:24.684 INFO:tasks.rados.rados.0.smithi076.stdout:1961: write oid 24 current snap is 253 2024-09-21T09:46:24.684 INFO:tasks.rados.rados.0.smithi076.stdout:1961: seq_num 740 ranges {437180=570501,1503028=550587} 2024-09-21T09:46:24.690 INFO:tasks.rados.rados.0.smithi076.stdout:1961: writing smithi07685715-24 from 437180 to 1007681 tid 1 2024-09-21T09:46:24.697 INFO:tasks.rados.rados.0.smithi076.stdout:1961: writing smithi07685715-24 from 1503028 to 2053615 tid 2 2024-09-21T09:46:24.697 INFO:tasks.rados.rados.0.smithi076.stdout:1962: write oid 29 current snap is 253 2024-09-21T09:46:24.697 INFO:tasks.rados.rados.0.smithi076.stdout:1962: seq_num 741 ranges {453987=645750,1845157=608830} 2024-09-21T09:46:24.704 INFO:tasks.rados.rados.0.smithi076.stdout:1962: writing smithi07685715-29 from 453987 to 1099737 tid 1 2024-09-21T09:46:24.711 INFO:tasks.rados.rados.0.smithi076.stdout:1962: writing smithi07685715-29 from 1845157 to 2453987 tid 2 2024-09-21T09:46:24.712 INFO:tasks.rados.rados.0.smithi076.stdout:1963: snap_remove snap 234 2024-09-21T09:46:25.704 INFO:tasks.rados.rados.0.smithi076.stdout:1963: done (4 left) 2024-09-21T09:46:25.705 INFO:tasks.rados.rados.0.smithi076.stdout:1964: read oid 40 snap 222 2024-09-21T09:46:25.705 INFO:tasks.rados.rados.0.smithi076.stdout:1964: expect deleted 2024-09-21T09:46:25.705 INFO:tasks.rados.rados.0.smithi076.stdout:1956: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:46:25.808 INFO:tasks.rados.rados.0.smithi076.stdout:1958: expect (ObjNum 710 snap 236 seq_num 710) 2024-09-21T09:46:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:25 smithi191 ceph-mon[102206]: pgmap v326: 89 pgs: 89 active+clean; 172 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 684 KiB/s wr, 23 op/s 2024-09-21T09:46:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:25 smithi191 ceph-mon[102206]: osdmap e571: 8 total, 8 up, 8 in 2024-09-21T09:46:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:26.118 INFO:tasks.rados.rados.0.smithi076.stdout:1961: finishing write tid 1 to smithi07685715-24 2024-09-21T09:46:26.118 INFO:tasks.rados.rados.0.smithi076.stdout:1961: finishing write tid 2 to smithi07685715-24 2024-09-21T09:46:26.118 INFO:tasks.rados.rados.0.smithi076.stdout:1961: finishing write tid 3 to smithi07685715-24 2024-09-21T09:46:26.118 INFO:tasks.rados.rados.0.smithi076.stdout:1961: finishing write tid 4 to smithi07685715-24 2024-09-21T09:46:26.118 INFO:tasks.rados.rados.0.smithi076.stdout:1961: oid 24 updating version 0 to 190 2024-09-21T09:46:26.118 INFO:tasks.rados.rados.0.smithi076.stdout:1961: oid 24 version 190 is already newer than 189 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1961: oid 24 updating version 190 to 191 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 191 (ObjNum 740 snap 253 seq_num 740) dirty exists 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1961: left oid 24 (ObjNum 740 snap 253 seq_num 740) 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: finishing write tid 1 to smithi07685715-29 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: finishing write tid 2 to smithi07685715-29 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: finishing write tid 3 to smithi07685715-29 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: finishing write tid 4 to smithi07685715-29 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: oid 29 updating version 0 to 538 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: oid 29 updating version 538 to 539 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: oid 29 updating version 539 to 540 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 540 (ObjNum 741 snap 253 seq_num 741) dirty exists 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1962: left oid 29 (ObjNum 741 snap 253 seq_num 741) 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1956: done (4 left) 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1958: done (3 left) 2024-09-21T09:46:26.119 INFO:tasks.rados.rados.0.smithi076.stdout:1961: done (2 left) 2024-09-21T09:46:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1962: done (1 left) 2024-09-21T09:46:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1964: done (0 left) 2024-09-21T09:46:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1965: write oid 38 current snap is 253 2024-09-21T09:46:26.120 INFO:tasks.rados.rados.0.smithi076.stdout:1965: seq_num 742 ranges {470794=720999,1903639=554256} 2024-09-21T09:46:26.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:25 smithi076 ceph-mon[110963]: pgmap v326: 89 pgs: 89 active+clean; 172 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 684 KiB/s wr, 23 op/s 2024-09-21T09:46:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:25 smithi076 ceph-mon[110963]: osdmap e571: 8 total, 8 up, 8 in 2024-09-21T09:46:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:25 smithi076 ceph-mon[106482]: pgmap v326: 89 pgs: 89 active+clean; 172 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 684 KiB/s wr, 23 op/s 2024-09-21T09:46:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:25 smithi076 ceph-mon[106482]: osdmap e571: 8 total, 8 up, 8 in 2024-09-21T09:46:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:26.124 INFO:tasks.rados.rados.0.smithi076.stdout:1965: writing smithi07685715-38 from 470794 to 1191793 tid 1 2024-09-21T09:46:26.131 INFO:tasks.rados.rados.0.smithi076.stdout:1965: writing smithi07685715-38 from 1903639 to 2457895 tid 2 2024-09-21T09:46:26.132 INFO:tasks.rados.rados.0.smithi076.stdout:1966: write oid 31 current snap is 253 2024-09-21T09:46:26.132 INFO:tasks.rados.rados.0.smithi076.stdout:1966: seq_num 743 ranges {487601=796248,1962121=525480} 2024-09-21T09:46:26.138 INFO:tasks.rados.rados.0.smithi076.stdout:1966: writing smithi07685715-31 from 487601 to 1283849 tid 1 2024-09-21T09:46:26.145 INFO:tasks.rados.rados.0.smithi076.stdout:1966: writing smithi07685715-31 from 1962121 to 2487601 tid 2 2024-09-21T09:46:26.145 INFO:tasks.rados.rados.0.smithi076.stdout:1967: snap_create 2024-09-21T09:46:26.145 INFO:tasks.rados.rados.0.smithi076.stdout:1965: finishing write tid 1 to smithi07685715-38 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: finishing write tid 2 to smithi07685715-38 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: finishing write tid 3 to smithi07685715-38 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: finishing write tid 4 to smithi07685715-38 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: oid 38 updating version 0 to 1082 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: oid 38 updating version 1082 to 1084 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: oid 38 version 1084 is already newer than 1083 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1084 (ObjNum 742 snap 253 seq_num 742) dirty exists 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1965: left oid 38 (ObjNum 742 snap 253 seq_num 742) 2024-09-21T09:46:26.146 INFO:tasks.rados.rados.0.smithi076.stdout:1966: finishing write tid 1 to smithi07685715-31 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: finishing write tid 2 to smithi07685715-31 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: finishing write tid 3 to smithi07685715-31 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: finishing write tid 4 to smithi07685715-31 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: oid 31 updating version 0 to 663 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: oid 31 version 663 is already newer than 662 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: oid 31 version 663 is already newer than 661 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 663 (ObjNum 743 snap 253 seq_num 743) dirty exists 2024-09-21T09:46:26.148 INFO:tasks.rados.rados.0.smithi076.stdout:1966: left oid 31 (ObjNum 743 snap 253 seq_num 743) 2024-09-21T09:46:26.708 INFO:tasks.rados.rados.0.smithi076.stdout:1965: done (2 left) 2024-09-21T09:46:26.708 INFO:tasks.rados.rados.0.smithi076.stdout:1966: done (1 left) 2024-09-21T09:46:26.709 INFO:tasks.rados.rados.0.smithi076.stdout:1967: done (0 left) 2024-09-21T09:46:26.709 INFO:tasks.rados.rados.0.smithi076.stdout:1968: snap_remove snap 223 2024-09-21T09:46:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:26 smithi191 ceph-mon[102206]: osdmap e572: 8 total, 8 up, 8 in 2024-09-21T09:46:27.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:26 smithi076 ceph-mon[106482]: osdmap e572: 8 total, 8 up, 8 in 2024-09-21T09:46:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:26 smithi076 ceph-mon[110963]: osdmap e572: 8 total, 8 up, 8 in 2024-09-21T09:46:27.710 INFO:tasks.rados.rados.0.smithi076.stdout:1968: done (0 left) 2024-09-21T09:46:27.711 INFO:tasks.rados.rados.0.smithi076.stdout:1969: snap_create 2024-09-21T09:46:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:27 smithi191 ceph-mon[102206]: pgmap v329: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:46:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:27 smithi191 ceph-mon[102206]: osdmap e573: 8 total, 8 up, 8 in 2024-09-21T09:46:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:27 smithi076 ceph-mon[106482]: pgmap v329: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:46:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:27 smithi076 ceph-mon[106482]: osdmap e573: 8 total, 8 up, 8 in 2024-09-21T09:46:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:27 smithi076 ceph-mon[110963]: pgmap v329: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:46:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:27 smithi076 ceph-mon[110963]: osdmap e573: 8 total, 8 up, 8 in 2024-09-21T09:46:28.685 INFO:tasks.rados.rados.0.smithi076.stdout:1969: done (0 left) 2024-09-21T09:46:28.685 INFO:tasks.rados.rados.0.smithi076.stdout:1970: read oid 49 snap -1 2024-09-21T09:46:28.685 INFO:tasks.rados.rados.0.smithi076.stdout:1970: expect (ObjNum 719 snap 246 seq_num 719) 2024-09-21T09:46:28.686 INFO:tasks.rados.rados.0.smithi076.stdout:1971: read oid 35 snap 237 2024-09-21T09:46:28.686 INFO:tasks.rados.rados.0.smithi076.stdout:1971: expect (ObjNum 684 snap 220 seq_num 684) 2024-09-21T09:46:28.686 INFO:tasks.rados.rados.0.smithi076.stdout:1972: snap_remove snap 253 2024-09-21T09:46:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:28 smithi191 ceph-mon[102206]: osdmap e574: 8 total, 8 up, 8 in 2024-09-21T09:46:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:28 smithi191 ceph-mon[102206]: osdmap e575: 8 total, 8 up, 8 in 2024-09-21T09:46:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:28 smithi076 ceph-mon[106482]: osdmap e574: 8 total, 8 up, 8 in 2024-09-21T09:46:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:28 smithi076 ceph-mon[106482]: osdmap e575: 8 total, 8 up, 8 in 2024-09-21T09:46:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:28 smithi076 ceph-mon[110963]: osdmap e574: 8 total, 8 up, 8 in 2024-09-21T09:46:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:28 smithi076 ceph-mon[110963]: osdmap e575: 8 total, 8 up, 8 in 2024-09-21T09:46:29.687 INFO:tasks.rados.rados.0.smithi076.stdout:1972: done (2 left) 2024-09-21T09:46:29.687 INFO:tasks.rados.rados.0.smithi076.stdout:1973: read oid 28 snap -1 2024-09-21T09:46:29.687 INFO:tasks.rados.rados.0.smithi076.stdout:1973: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:46:29.687 INFO:tasks.rados.rados.0.smithi076.stdout:1974: write oid 45 current snap is 255 2024-09-21T09:46:29.687 INFO:tasks.rados.rados.0.smithi076.stdout:1970: expect (ObjNum 719 snap 246 seq_num 719) 2024-09-21T09:46:29.828 INFO:tasks.rados.rados.0.smithi076.stdout:1971: expect (ObjNum 684 snap 220 seq_num 684) 2024-09-21T09:46:30.077 INFO:tasks.rados.rados.0.smithi076.stdout:1973: expect (ObjNum 529 snap 168 seq_num 529) 2024-09-21T09:46:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:29 smithi191 ceph-mon[102206]: pgmap v332: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:46:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:29 smithi191 ceph-mon[102206]: osdmap e576: 8 total, 8 up, 8 in 2024-09-21T09:46:30.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:29 smithi076 ceph-mon[106482]: pgmap v332: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:46:30.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:29 smithi076 ceph-mon[106482]: osdmap e576: 8 total, 8 up, 8 in 2024-09-21T09:46:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:29 smithi076 ceph-mon[110963]: pgmap v332: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-09-21T09:46:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:29 smithi076 ceph-mon[110963]: osdmap e576: 8 total, 8 up, 8 in 2024-09-21T09:46:30.172 INFO:tasks.rados.rados.0.smithi076.stdout:1974: seq_num 744 ranges {504408=471497,1620603=557925} 2024-09-21T09:46:30.177 INFO:tasks.rados.rados.0.smithi076.stdout:1974: writing smithi07685715-45 from 504408 to 975905 tid 1 2024-09-21T09:46:30.184 INFO:tasks.rados.rados.0.smithi076.stdout:1974: writing smithi07685715-45 from 1620603 to 2178528 tid 2 2024-09-21T09:46:30.184 INFO:tasks.rados.rados.0.smithi076.stdout:1970: done (3 left) 2024-09-21T09:46:30.184 INFO:tasks.rados.rados.0.smithi076.stdout:1971: done (2 left) 2024-09-21T09:46:30.184 INFO:tasks.rados.rados.0.smithi076.stdout:1973: done (1 left) 2024-09-21T09:46:30.184 INFO:tasks.rados.rados.0.smithi076.stdout:1975: snap_remove snap 240 2024-09-21T09:46:30.727 INFO:tasks.rados.rados.0.smithi076.stdout:1975: done (1 left) 2024-09-21T09:46:30.727 INFO:tasks.rados.rados.0.smithi076.stdout:1976: write oid 40 current snap is 255 2024-09-21T09:46:30.727 INFO:tasks.rados.rados.0.smithi076.stdout:1974: finishing write tid 1 to smithi07685715-45 2024-09-21T09:46:30.727 INFO:tasks.rados.rados.0.smithi076.stdout:1976: seq_num 745 ranges {521215=546746,1562732=701583} 2024-09-21T09:46:30.733 INFO:tasks.rados.rados.0.smithi076.stdout:1976: writing smithi07685715-40 from 521215 to 1067961 tid 1 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1976: writing smithi07685715-40 from 1562732 to 2264315 tid 2 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1977: rollback oid 20 current snap is 255 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 239 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: finishing write tid 2 to smithi07685715-45 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: finishing write tid 3 to smithi07685715-45 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: finishing write tid 4 to smithi07685715-45 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: oid 45 updating version 0 to 382 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: oid 45 updating version 382 to 383 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: oid 45 updating version 383 to 384 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 384 (ObjNum 744 snap 255 seq_num 744) dirty exists 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: left oid 45 (ObjNum 744 snap 255 seq_num 744) 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1976: finishing write tid 1 to smithi07685715-40 2024-09-21T09:46:30.742 INFO:tasks.rados.rados.0.smithi076.stdout:1974: done (2 left) 2024-09-21T09:46:30.743 INFO:tasks.rados.rados.0.smithi076.stdout:1978: snap_create 2024-09-21T09:46:30.743 INFO:tasks.rados.rados.0.smithi076.stdout:1977: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:46:30.744 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 0 (ObjNum 193 snap 0 seq_num 1283431824) dirty dne 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: finishing write tid 2 to smithi07685715-40 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: finishing write tid 3 to smithi07685715-40 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: finishing write tid 4 to smithi07685715-40 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: oid 40 updating version 0 to 867 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: oid 40 updating version 867 to 869 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: oid 40 version 869 is already newer than 868 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 869 (ObjNum 745 snap 255 seq_num 745) dirty exists 2024-09-21T09:46:30.746 INFO:tasks.rados.rados.0.smithi076.stdout:1976: left oid 40 (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:46:31.735 INFO:tasks.rados.rados.0.smithi076.stdout:1976: done (2 left) 2024-09-21T09:46:31.735 INFO:tasks.rados.rados.0.smithi076.stdout:1977: done (1 left) 2024-09-21T09:46:31.736 INFO:tasks.rados.rados.0.smithi076.stdout:1978: done (0 left) 2024-09-21T09:46:31.736 INFO:tasks.rados.rados.0.smithi076.stdout:1979: read oid 22 snap -1 2024-09-21T09:46:31.736 INFO:tasks.rados.rados.0.smithi076.stdout:1979: expect (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:46:31.736 INFO:tasks.rados.rados.0.smithi076.stdout:1980: delete oid 21 current snap is 256 2024-09-21T09:46:31.740 INFO:tasks.rados.rados.0.smithi076.stdout:1980: done (1 left) 2024-09-21T09:46:31.740 INFO:tasks.rados.rados.0.smithi076.stdout:1981: delete oid 28 current snap is 256 2024-09-21T09:46:31.743 INFO:tasks.rados.rados.0.smithi076.stdout:1979: expect (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:46:31.823 INFO:tasks.rados.rados.0.smithi076.stdout:1979: done (1 left) 2024-09-21T09:46:31.823 INFO:tasks.rados.rados.0.smithi076.stdout:1981: done (0 left) 2024-09-21T09:46:31.823 INFO:tasks.rados.rados.0.smithi076.stdout:1982: snap_remove snap 218 2024-09-21T09:46:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:31 smithi191 ceph-mon[102206]: pgmap v335: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:31 smithi191 ceph-mon[102206]: osdmap e577: 8 total, 8 up, 8 in 2024-09-21T09:46:32.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:31 smithi076 ceph-mon[106482]: pgmap v335: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:31 smithi076 ceph-mon[106482]: osdmap e577: 8 total, 8 up, 8 in 2024-09-21T09:46:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:31 smithi076 ceph-mon[110963]: pgmap v335: 89 pgs: 89 active+clean; 176 MiB data, 699 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:31 smithi076 ceph-mon[110963]: osdmap e577: 8 total, 8 up, 8 in 2024-09-21T09:46:32.744 INFO:tasks.rados.rados.0.smithi076.stdout:1982: done (0 left) 2024-09-21T09:46:32.744 INFO:tasks.rados.rados.0.smithi076.stdout:1983: write oid 6 current snap is 256 2024-09-21T09:46:32.744 INFO:tasks.rados.rados.0.smithi076.stdout:1983: seq_num 748 ranges {571636=488846,1854531=565263} 2024-09-21T09:46:32.747 INFO:tasks.rados.rados.0.smithi076.stdout:1983: writing smithi07685715-6 from 571636 to 1060482 tid 1 2024-09-21T09:46:32.755 INFO:tasks.rados.rados.0.smithi076.stdout:1983: writing smithi07685715-6 from 1854531 to 2419794 tid 2 2024-09-21T09:46:32.755 INFO:tasks.rados.rados.0.smithi076.stdout:1984: read oid 41 snap -1 2024-09-21T09:46:32.755 INFO:tasks.rados.rados.0.smithi076.stdout:1984: expect (ObjNum 736 snap 249 seq_num 736) 2024-09-21T09:46:32.756 INFO:tasks.rados.rados.0.smithi076.stdout:1985: read oid 27 snap 248 2024-09-21T09:46:32.756 INFO:tasks.rados.rados.0.smithi076.stdout:1985: expect deleted 2024-09-21T09:46:32.756 INFO:tasks.rados.rados.0.smithi076.stdout:1986: read oid 8 snap -1 2024-09-21T09:46:32.756 INFO:tasks.rados.rados.0.smithi076.stdout:1986: expect deleted 2024-09-21T09:46:32.756 INFO:tasks.rados.rados.0.smithi076.stdout:1987: delete oid 15 current snap is 256 2024-09-21T09:46:32.756 INFO:tasks.rados.rados.0.smithi076.stdout:1983: finishing write tid 1 to smithi07685715-6 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1986: done (4 left) 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1987: done (3 left) 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1988: read oid 4 snap -1 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1988: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1989: read oid 15 snap 248 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1989: expect (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:46:32.757 INFO:tasks.rados.rados.0.smithi076.stdout:1990: snap_remove snap 252 2024-09-21T09:46:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:32 smithi076 ceph-mon[106482]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T09:46:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:32 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:33.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:32 smithi076 ceph-mon[110963]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T09:46:33.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:32 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:33.043 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:32 smithi191 ceph-mon[102206]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T09:46:33.043 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:32 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:33.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:46:33] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.51.0" 2024-09-21T09:46:33.685 INFO:tasks.rados.rados.0.smithi076.stdout:1990: done (5 left) 2024-09-21T09:46:33.685 INFO:tasks.rados.rados.0.smithi076.stdout:1991: rollback oid 17 current snap is 256 2024-09-21T09:46:33.685 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 248 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: finishing write tid 2 to smithi07685715-6 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: finishing write tid 3 to smithi07685715-6 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: finishing write tid 4 to smithi07685715-6 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: oid 6 updating version 0 to 419 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: oid 6 version 419 is already newer than 418 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: oid 6 version 419 is already newer than 417 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 419 (ObjNum 748 snap 256 seq_num 748) dirty exists 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1983: left oid 6 (ObjNum 748 snap 256 seq_num 748) 2024-09-21T09:46:33.686 INFO:tasks.rados.rados.0.smithi076.stdout:1989: expect (ObjNum 677 snap 215 seq_num 677) 2024-09-21T09:46:33.806 INFO:tasks.rados.rados.0.smithi076.stdout:1984: expect (ObjNum 736 snap 249 seq_num 736) 2024-09-21T09:46:33.912 INFO:tasks.rados.rados.0.smithi076.stdout:1988: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:46:33.961 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-mon[110963]: pgmap v338: 89 pgs: 89 active+clean; 178 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 556 KiB/s wr, 8 op/s 2024-09-21T09:46:33.961 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-mon[110963]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T09:46:33.961 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-mon[110963]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T09:46:33.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-mon[106482]: pgmap v338: 89 pgs: 89 active+clean; 178 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 556 KiB/s wr, 8 op/s 2024-09-21T09:46:33.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-mon[106482]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T09:46:33.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:33 smithi076 ceph-mon[106482]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T09:46:34.004 INFO:tasks.rados.rados.0.smithi076.stdout:1991: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:46:34.004 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 978 (ObjNum 289 snap 0 seq_num 1683088819) dirty dne 2024-09-21T09:46:34.004 INFO:tasks.rados.rados.0.smithi076.stdout:1983: done (5 left) 2024-09-21T09:46:34.004 INFO:tasks.rados.rados.0.smithi076.stdout:1984: done (4 left) 2024-09-21T09:46:34.004 INFO:tasks.rados.rados.0.smithi076.stdout:1985: done (3 left) 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1988: done (2 left) 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1989: done (1 left) 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1991: done (0 left) 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1992: read oid 14 snap -1 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1992: expect (ObjNum 721 snap 247 seq_num 721) 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1993: write oid 33 current snap is 256 2024-09-21T09:46:34.005 INFO:tasks.rados.rados.0.smithi076.stdout:1993: seq_num 750 ranges {605250=639344,1855142=452579} 2024-09-21T09:46:34.011 INFO:tasks.rados.rados.0.smithi076.stdout:1993: writing smithi07685715-33 from 605250 to 1244594 tid 1 2024-09-21T09:46:34.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:33 smithi191 ceph-mon[102206]: pgmap v338: 89 pgs: 89 active+clean; 178 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 556 KiB/s wr, 8 op/s 2024-09-21T09:46:34.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:33 smithi191 ceph-mon[102206]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T09:46:34.012 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:33 smithi191 ceph-mon[102206]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T09:46:34.017 INFO:tasks.rados.rados.0.smithi076.stdout:1993: writing smithi07685715-33 from 1855142 to 2307721 tid 2 2024-09-21T09:46:34.017 INFO:tasks.rados.rados.0.smithi076.stdout:1994: snap_create 2024-09-21T09:46:34.017 INFO:tasks.rados.rados.0.smithi076.stdout:1992: expect (ObjNum 721 snap 247 seq_num 721) 2024-09-21T09:46:34.088 INFO:tasks.rados.rados.0.smithi076.stdout:1993: finishing write tid 1 to smithi07685715-33 2024-09-21T09:46:34.088 INFO:tasks.rados.rados.0.smithi076.stdout:1993: finishing write tid 2 to smithi07685715-33 2024-09-21T09:46:34.088 INFO:tasks.rados.rados.0.smithi076.stdout:1993: finishing write tid 3 to smithi07685715-33 2024-09-21T09:46:34.088 INFO:tasks.rados.rados.0.smithi076.stdout:1993: finishing write tid 4 to smithi07685715-33 2024-09-21T09:46:34.088 INFO:tasks.rados.rados.0.smithi076.stdout:1993: oid 33 updating version 0 to 435 2024-09-21T09:46:34.089 INFO:tasks.rados.rados.0.smithi076.stdout:1993: oid 33 updating version 435 to 436 2024-09-21T09:46:34.089 INFO:tasks.rados.rados.0.smithi076.stdout:1993: oid 33 updating version 436 to 437 2024-09-21T09:46:34.089 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 437 (ObjNum 750 snap 256 seq_num 750) dirty exists 2024-09-21T09:46:34.089 INFO:tasks.rados.rados.0.smithi076.stdout:1993: left oid 33 (ObjNum 750 snap 256 seq_num 750) 2024-09-21T09:46:34.743 INFO:tasks.rados.rados.0.smithi076.stdout:1992: done (2 left) 2024-09-21T09:46:34.744 INFO:tasks.rados.rados.0.smithi076.stdout:1993: done (1 left) 2024-09-21T09:46:34.744 INFO:tasks.rados.rados.0.smithi076.stdout:1994: done (0 left) 2024-09-21T09:46:34.744 INFO:tasks.rados.rados.0.smithi076.stdout:1995: snap_remove snap 220 2024-09-21T09:46:35.751 INFO:tasks.rados.rados.0.smithi076.stdout:1995: done (0 left) 2024-09-21T09:46:35.751 INFO:tasks.rados.rados.0.smithi076.stdout:1996: read oid 40 snap 222 2024-09-21T09:46:35.751 INFO:tasks.rados.rados.0.smithi076.stdout:1996: expect deleted 2024-09-21T09:46:35.751 INFO:tasks.rados.rados.0.smithi076.stdout:1997: delete oid 42 current snap is 257 2024-09-21T09:46:35.758 INFO:tasks.rados.rados.0.smithi076.stdout:1996: done (1 left) 2024-09-21T09:46:35.758 INFO:tasks.rados.rados.0.smithi076.stdout:1997: done (0 left) 2024-09-21T09:46:35.758 INFO:tasks.rados.rados.0.smithi076.stdout:1998: snap_create 2024-09-21T09:46:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:35 smithi191 ceph-mon[102206]: pgmap v341: 89 pgs: 89 active+clean; 178 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 556 KiB/s wr, 8 op/s 2024-09-21T09:46:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:35 smithi191 ceph-mon[102206]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:46:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:35 smithi076 ceph-mon[106482]: pgmap v341: 89 pgs: 89 active+clean; 178 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 556 KiB/s wr, 8 op/s 2024-09-21T09:46:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:35 smithi076 ceph-mon[106482]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:46:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:35 smithi076 ceph-mon[110963]: pgmap v341: 89 pgs: 89 active+clean; 178 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 556 KiB/s wr, 8 op/s 2024-09-21T09:46:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:35 smithi076 ceph-mon[110963]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T09:46:36.755 INFO:tasks.rados.rados.0.smithi076.stdout:1998: done (0 left) 2024-09-21T09:46:36.755 INFO:tasks.rados.rados.0.smithi076.stdout:1999: rollback oid 31 current snap is 258 2024-09-21T09:46:36.755 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 241 2024-09-21T09:46:36.755 INFO:tasks.rados.rados.0.smithi076.stdout:2000: delete oid 14 current snap is 258 2024-09-21T09:46:36.758 INFO:tasks.rados.rados.0.smithi076.stdout:1999: finishing rollback tid 0 to smithi07685715-31 2024-09-21T09:46:36.758 INFO:tasks.rados.rados.0.smithi076.stdout:2000: done (1 left) 2024-09-21T09:46:36.758 INFO:tasks.rados.rados.0.smithi076.stdout:2001: write oid 32 current snap is 258 2024-09-21T09:46:36.758 INFO:tasks.rados.rados.0.smithi076.stdout:1999: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:46:36.758 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 668 (ObjNum 129 snap 0 seq_num 1472411688) dirty dne 2024-09-21T09:46:36.758 INFO:tasks.rados.rados.0.smithi076.stdout:2001: seq_num 753 ranges {655671=581444,2030588=625083} 2024-09-21T09:46:36.764 INFO:tasks.rados.rados.0.smithi076.stdout:2001: writing smithi07685715-32 from 655671 to 1237115 tid 1 2024-09-21T09:46:36.772 INFO:tasks.rados.rados.0.smithi076.stdout:2001: writing smithi07685715-32 from 2030588 to 2655671 tid 2 2024-09-21T09:46:36.773 INFO:tasks.rados.rados.0.smithi076.stdout:1999: done (1 left) 2024-09-21T09:46:36.773 INFO:tasks.rados.rados.0.smithi076.stdout:2002: write oid 48 current snap is 258 2024-09-21T09:46:36.773 INFO:tasks.rados.rados.0.smithi076.stdout:2001: finishing write tid 1 to smithi07685715-32 2024-09-21T09:46:36.773 INFO:tasks.rados.rados.0.smithi076.stdout:2002: seq_num 754 ranges {672478=656693,2089070=459917} 2024-09-21T09:46:36.780 INFO:tasks.rados.rados.0.smithi076.stdout:2002: writing smithi07685715-48 from 672478 to 1329171 tid 1 2024-09-21T09:46:36.787 INFO:tasks.rados.rados.0.smithi076.stdout:2002: writing smithi07685715-48 from 2089070 to 2548987 tid 2 2024-09-21T09:46:36.787 INFO:tasks.rados.rados.0.smithi076.stdout:2003: snap_remove snap 256 2024-09-21T09:46:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:36 smithi191 ceph-mon[102206]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:46:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:36 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:36 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:37.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:36 smithi076 ceph-mon[106482]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:46:37.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:36 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:36 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:36 smithi076 ceph-mon[110963]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T09:46:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:36 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:36 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2003: done (2 left) 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2004: delete oid 49 current snap is 258 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2001: finishing write tid 2 to smithi07685715-32 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2001: finishing write tid 3 to smithi07685715-32 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2001: finishing write tid 4 to smithi07685715-32 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2001: oid 32 updating version 0 to 670 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2001: oid 32 updating version 670 to 672 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:2001: oid 32 version 672 is already newer than 671 2024-09-21T09:46:37.765 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 672 (ObjNum 753 snap 258 seq_num 753) dirty exists 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2001: left oid 32 (ObjNum 753 snap 258 seq_num 753) 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: finishing write tid 1 to smithi07685715-48 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: finishing write tid 2 to smithi07685715-48 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: finishing write tid 3 to smithi07685715-48 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: finishing write tid 4 to smithi07685715-48 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: oid 48 updating version 0 to 1556 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: oid 48 updating version 1556 to 1557 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: oid 48 updating version 1557 to 1558 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1558 (ObjNum 754 snap 258 seq_num 754) dirty exists 2024-09-21T09:46:37.766 INFO:tasks.rados.rados.0.smithi076.stdout:2002: left oid 48 (ObjNum 754 snap 258 seq_num 754) 2024-09-21T09:46:37.769 INFO:tasks.rados.rados.0.smithi076.stdout:2001: done (2 left) 2024-09-21T09:46:37.769 INFO:tasks.rados.rados.0.smithi076.stdout:2002: done (1 left) 2024-09-21T09:46:37.769 INFO:tasks.rados.rados.0.smithi076.stdout:2004: done (0 left) 2024-09-21T09:46:37.769 INFO:tasks.rados.rados.0.smithi076.stdout:2005: rollback oid 25 current snap is 258 2024-09-21T09:46:37.769 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 25 to 248 2024-09-21T09:46:37.769 INFO:tasks.rados.rados.0.smithi076.stdout:2006: snap_create 2024-09-21T09:46:37.773 INFO:tasks.rados.rados.0.smithi076.stdout:2005: finishing rollback tid 0 to smithi07685715-25 2024-09-21T09:46:37.774 INFO:tasks.rados.rados.0.smithi076.stdout:2005: finishing rollback tid 1 to smithi07685715-25 2024-09-21T09:46:37.774 INFO:tasks.rados.rados.0.smithi076.stdout:2005: finishing rollback tid 2 to smithi07685715-25 2024-09-21T09:46:37.774 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 445 (ObjNum 674 snap 215 seq_num 674) dirty exists 2024-09-21T09:46:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: pgmap v344: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 524 KiB/s wr, 14 op/s 2024-09-21T09:46:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:46:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: pgmap v344: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 524 KiB/s wr, 14 op/s 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: pgmap v344: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 524 KiB/s wr, 14 op/s 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:38.687 INFO:tasks.rados.rados.0.smithi076.stdout:2005: done (1 left) 2024-09-21T09:46:38.688 INFO:tasks.rados.rados.0.smithi076.stdout:2006: done (0 left) 2024-09-21T09:46:38.688 INFO:tasks.rados.rados.0.smithi076.stdout:2007: write oid 42 current snap is 259 2024-09-21T09:46:38.688 INFO:tasks.rados.rados.0.smithi076.stdout:2007: seq_num 756 ranges {706092=407191,1806034=463586} 2024-09-21T09:46:38.693 INFO:tasks.rados.rados.0.smithi076.stdout:2007: writing smithi07685715-42 from 706092 to 1113283 tid 1 2024-09-21T09:46:38.700 INFO:tasks.rados.rados.0.smithi076.stdout:2007: writing smithi07685715-42 from 1806034 to 2269620 tid 2 2024-09-21T09:46:38.700 INFO:tasks.rados.rados.0.smithi076.stdout:2008: delete oid 14 current snap is 259 2024-09-21T09:46:38.700 INFO:tasks.rados.rados.0.smithi076.stdout:2007: finishing write tid 1 to smithi07685715-42 2024-09-21T09:46:38.703 INFO:tasks.rados.rados.0.smithi076.stdout:2007: finishing write tid 2 to smithi07685715-42 2024-09-21T09:46:38.703 INFO:tasks.rados.rados.0.smithi076.stdout:2007: finishing write tid 3 to smithi07685715-42 2024-09-21T09:46:38.703 INFO:tasks.rados.rados.0.smithi076.stdout:2007: finishing write tid 4 to smithi07685715-42 2024-09-21T09:46:38.704 INFO:tasks.rados.rados.0.smithi076.stdout:2007: oid 42 updating version 0 to 586 2024-09-21T09:46:38.704 INFO:tasks.rados.rados.0.smithi076.stdout:2007: oid 42 version 586 is already newer than 585 2024-09-21T09:46:38.704 INFO:tasks.rados.rados.0.smithi076.stdout:2007: oid 42 version 586 is already newer than 584 2024-09-21T09:46:38.704 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 586 (ObjNum 756 snap 259 seq_num 756) dirty exists 2024-09-21T09:46:38.704 INFO:tasks.rados.rados.0.smithi076.stdout:2007: left oid 42 (ObjNum 756 snap 259 seq_num 756) 2024-09-21T09:46:38.704 INFO:tasks.rados.rados.0.smithi076.stdout:2007: done (1 left) 2024-09-21T09:46:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2008: done (0 left) 2024-09-21T09:46:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2009: read oid 11 snap -1 2024-09-21T09:46:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2009: expect deleted 2024-09-21T09:46:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2010: write oid 24 current snap is 259 2024-09-21T09:46:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2010: seq_num 758 ranges {739706=557689,1806645=467255} 2024-09-21T09:46:38.711 INFO:tasks.rados.rados.0.smithi076.stdout:2010: writing smithi07685715-24 from 739706 to 1297395 tid 1 2024-09-21T09:46:38.717 INFO:tasks.rados.rados.0.smithi076.stdout:2010: writing smithi07685715-24 from 1806645 to 2273900 tid 2 2024-09-21T09:46:38.717 INFO:tasks.rados.rados.0.smithi076.stdout:2011: write oid 3 current snap is 259 2024-09-21T09:46:38.717 INFO:tasks.rados.rados.0.smithi076.stdout:2011: seq_num 759 ranges {756513=632938,1865127=727266} 2024-09-21T09:46:38.725 INFO:tasks.rados.rados.0.smithi076.stdout:2011: writing smithi07685715-3 from 756513 to 1389451 tid 1 2024-09-21T09:46:38.731 INFO:tasks.rados.rados.0.smithi076.stdout:2011: writing smithi07685715-3 from 1865127 to 2592393 tid 2 2024-09-21T09:46:38.731 INFO:tasks.rados.rados.0.smithi076.stdout:2012: write oid 42 current snap is 259 2024-09-21T09:46:38.731 INFO:tasks.rados.rados.0.smithi076.stdout:2012: seq_num 760 ranges {773320=708187,1923609=470924} 2024-09-21T09:46:38.739 INFO:tasks.rados.rados.0.smithi076.stdout:2012: writing smithi07685715-42 from 773320 to 1481507 tid 1 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2012: writing smithi07685715-42 from 1923609 to 2394533 tid 2 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: finishing write tid 1 to smithi07685715-24 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: finishing write tid 2 to smithi07685715-24 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: finishing write tid 3 to smithi07685715-24 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: finishing write tid 4 to smithi07685715-24 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: oid 24 updating version 0 to 206 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: oid 24 version 206 is already newer than 205 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: oid 24 updating version 206 to 207 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 207 (ObjNum 758 snap 259 seq_num 758) dirty exists 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2010: left oid 24 (ObjNum 758 snap 259 seq_num 758) 2024-09-21T09:46:38.745 INFO:tasks.rados.rados.0.smithi076.stdout:2011: finishing write tid 1 to smithi07685715-3 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: finishing write tid 2 to smithi07685715-3 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: finishing write tid 3 to smithi07685715-3 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2013: read oid 26 snap -1 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2013: expect (ObjNum 734 snap 249 seq_num 734) 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2009: done (4 left) 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2010: done (3 left) 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2014: snap_create 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: finishing write tid 4 to smithi07685715-3 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: oid 3 updating version 0 to 99 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: oid 3 version 99 is already newer than 97 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: oid 3 version 99 is already newer than 98 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 99 (ObjNum 759 snap 259 seq_num 759) dirty exists 2024-09-21T09:46:38.746 INFO:tasks.rados.rados.0.smithi076.stdout:2011: left oid 3 (ObjNum 759 snap 259 seq_num 759) 2024-09-21T09:46:38.747 INFO:tasks.rados.rados.0.smithi076.stdout:2012: finishing write tid 1 to smithi07685715-42 2024-09-21T09:46:38.747 INFO:tasks.rados.rados.0.smithi076.stdout:2012: finishing write tid 2 to smithi07685715-42 2024-09-21T09:46:38.747 INFO:tasks.rados.rados.0.smithi076.stdout:2012: finishing write tid 3 to smithi07685715-42 2024-09-21T09:46:38.748 INFO:tasks.rados.rados.0.smithi076.stdout:2012: finishing write tid 4 to smithi07685715-42 2024-09-21T09:46:38.748 INFO:tasks.rados.rados.0.smithi076.stdout:2012: oid 42 updating version 0 to 589 2024-09-21T09:46:38.748 INFO:tasks.rados.rados.0.smithi076.stdout:2012: oid 42 version 589 is already newer than 587 2024-09-21T09:46:38.748 INFO:tasks.rados.rados.0.smithi076.stdout:2012: oid 42 version 589 is already newer than 588 2024-09-21T09:46:38.748 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 589 (ObjNum 760 snap 259 seq_num 760) dirty exists 2024-09-21T09:46:38.749 INFO:tasks.rados.rados.0.smithi076.stdout:2012: left oid 42 (ObjNum 760 snap 259 seq_num 760) 2024-09-21T09:46:38.749 INFO:tasks.rados.rados.0.smithi076.stdout:2013: expect (ObjNum 734 snap 249 seq_num 734) 2024-09-21T09:46:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:38 smithi191 ceph-mon[102206]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:46:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:38 smithi191 ceph-mon[102206]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:46:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:38 smithi076 ceph-mon[106482]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:46:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:38 smithi076 ceph-mon[106482]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:46:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:38 smithi076 ceph-mon[110963]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T09:46:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:38 smithi076 ceph-mon[110963]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T09:46:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:2011: done (3 left) 2024-09-21T09:46:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:2012: done (2 left) 2024-09-21T09:46:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:2013: done (1 left) 2024-09-21T09:46:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:2014: done (0 left) 2024-09-21T09:46:39.691 INFO:tasks.rados.rados.0.smithi076.stdout:2015: snap_remove snap 247 2024-09-21T09:46:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:39 smithi191 ceph-mon[102206]: pgmap v347: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 524 KiB/s wr, 14 op/s 2024-09-21T09:46:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:39 smithi191 ceph-mon[102206]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:46:40.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:39 smithi076 ceph-mon[106482]: pgmap v347: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 524 KiB/s wr, 14 op/s 2024-09-21T09:46:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:39 smithi076 ceph-mon[106482]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:46:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:39 smithi076 ceph-mon[110963]: pgmap v347: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 524 KiB/s wr, 14 op/s 2024-09-21T09:46:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:39 smithi076 ceph-mon[110963]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T09:46:40.693 INFO:tasks.rados.rados.0.smithi076.stdout:2015: done (0 left) 2024-09-21T09:46:40.693 INFO:tasks.rados.rados.0.smithi076.stdout:2016: delete oid 10 current snap is 260 2024-09-21T09:46:40.703 INFO:tasks.rados.rados.0.smithi076.stdout:2016: done (0 left) 2024-09-21T09:46:40.703 INFO:tasks.rados.rados.0.smithi076.stdout:2017: read oid 18 snap -1 2024-09-21T09:46:40.703 INFO:tasks.rados.rados.0.smithi076.stdout:2017: expect (ObjNum 737 snap 252 seq_num 737) 2024-09-21T09:46:40.703 INFO:tasks.rados.rados.0.smithi076.stdout:2018: read oid 28 snap -1 2024-09-21T09:46:40.703 INFO:tasks.rados.rados.0.smithi076.stdout:2018: expect deleted 2024-09-21T09:46:40.703 INFO:tasks.rados.rados.0.smithi076.stdout:2019: snap_remove snap 235 2024-09-21T09:46:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:40 smithi191 ceph-mon[102206]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:46:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:40 smithi076 ceph-mon[106482]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:46:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:40 smithi076 ceph-mon[110963]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T09:46:41.695 INFO:tasks.rados.rados.0.smithi076.stdout:2017: expect (ObjNum 737 snap 252 seq_num 737) 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2017: done (2 left) 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2018: done (1 left) 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2019: done (0 left) 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2020: read oid 9 snap -1 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2020: expect deleted 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2021: write oid 43 current snap is 260 2024-09-21T09:46:41.778 INFO:tasks.rados.rados.0.smithi076.stdout:2021: seq_num 762 ranges {406934=575038,1640573=474593,2642791=164143} 2024-09-21T09:46:41.783 INFO:tasks.rados.rados.0.smithi076.stdout:2021: writing smithi07685715-43 from 406934 to 981972 tid 1 2024-09-21T09:46:41.789 INFO:tasks.rados.rados.0.smithi076.stdout:2021: writing smithi07685715-43 from 1640573 to 2115166 tid 2 2024-09-21T09:46:41.793 INFO:tasks.rados.rados.0.smithi076.stdout:2021: writing smithi07685715-43 from 2642791 to 2806934 tid 3 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2022: read oid 40 snap -1 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2022: expect (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2023: read oid 38 snap -1 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2023: expect (ObjNum 742 snap 253 seq_num 742) 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2024: snap_create 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2021: finishing write tid 1 to smithi07685715-43 2024-09-21T09:46:41.794 INFO:tasks.rados.rados.0.smithi076.stdout:2021: finishing write tid 2 to smithi07685715-43 2024-09-21T09:46:41.795 INFO:tasks.rados.rados.0.smithi076.stdout:2021: finishing write tid 3 to smithi07685715-43 2024-09-21T09:46:41.795 INFO:tasks.rados.rados.0.smithi076.stdout:2021: finishing write tid 4 to smithi07685715-43 2024-09-21T09:46:41.795 INFO:tasks.rados.rados.0.smithi076.stdout:2021: finishing write tid 5 to smithi07685715-43 2024-09-21T09:46:41.795 INFO:tasks.rados.rados.0.smithi076.stdout:2021: oid 43 updating version 0 to 371 2024-09-21T09:46:41.795 INFO:tasks.rados.rados.0.smithi076.stdout:2021: oid 43 updating version 371 to 374 2024-09-21T09:46:41.796 INFO:tasks.rados.rados.0.smithi076.stdout:2021: oid 43 version 374 is already newer than 373 2024-09-21T09:46:41.796 INFO:tasks.rados.rados.0.smithi076.stdout:2021: oid 43 version 374 is already newer than 372 2024-09-21T09:46:41.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 374 (ObjNum 762 snap 260 seq_num 762) dirty exists 2024-09-21T09:46:41.796 INFO:tasks.rados.rados.0.smithi076.stdout:2021: left oid 43 (ObjNum 762 snap 260 seq_num 762) 2024-09-21T09:46:41.797 INFO:tasks.rados.rados.0.smithi076.stdout:2023: expect (ObjNum 742 snap 253 seq_num 742) 2024-09-21T09:46:42.001 INFO:tasks.rados.rados.0.smithi076.stdout:2022: expect (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:46:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:41 smithi191 ceph-mon[102206]: pgmap v350: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:41 smithi191 ceph-mon[102206]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:46:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:41 smithi076 ceph-mon[106482]: pgmap v350: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:41 smithi076 ceph-mon[106482]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:46:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:41 smithi076 ceph-mon[110963]: pgmap v350: 89 pgs: 89 active+clean; 169 MiB data, 688 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:41 smithi076 ceph-mon[110963]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T09:46:42.787 INFO:tasks.rados.rados.0.smithi076.stdout:2020: done (4 left) 2024-09-21T09:46:42.787 INFO:tasks.rados.rados.0.smithi076.stdout:2021: done (3 left) 2024-09-21T09:46:42.787 INFO:tasks.rados.rados.0.smithi076.stdout:2022: done (2 left) 2024-09-21T09:46:42.787 INFO:tasks.rados.rados.0.smithi076.stdout:2023: done (1 left) 2024-09-21T09:46:42.787 INFO:tasks.rados.rados.0.smithi076.stdout:2024: done (0 left) 2024-09-21T09:46:42.788 INFO:tasks.rados.rados.0.smithi076.stdout:2025: rollback oid 21 current snap is 261 2024-09-21T09:46:42.788 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 241 2024-09-21T09:46:42.788 INFO:tasks.rados.rados.0.smithi076.stdout:2026: snap_create 2024-09-21T09:46:42.792 INFO:tasks.rados.rados.0.smithi076.stdout:2025: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:46:42.792 INFO:tasks.rados.rados.0.smithi076.stdout:2025: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:46:42.792 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1546 (ObjNum 710 snap 236 seq_num 710) dirty exists 2024-09-21T09:46:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:46:43] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:46:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:2025: done (1 left) 2024-09-21T09:46:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:2026: done (0 left) 2024-09-21T09:46:43.690 INFO:tasks.rados.rados.0.smithi076.stdout:2027: read oid 43 snap -1 2024-09-21T09:46:43.691 INFO:tasks.rados.rados.0.smithi076.stdout:2027: expect (ObjNum 762 snap 260 seq_num 762) 2024-09-21T09:46:43.691 INFO:tasks.rados.rados.0.smithi076.stdout:2028: write oid 8 current snap is 262 2024-09-21T09:46:43.691 INFO:tasks.rados.rados.0.smithi076.stdout:2028: seq_num 763 ranges {423741=650287,1699055=618251} 2024-09-21T09:46:43.696 INFO:tasks.rados.rados.0.smithi076.stdout:2028: writing smithi07685715-8 from 423741 to 1074028 tid 1 2024-09-21T09:46:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:2028: writing smithi07685715-8 from 1699055 to 2317306 tid 2 2024-09-21T09:46:43.703 INFO:tasks.rados.rados.0.smithi076.stdout:2029: snap_remove snap 241 2024-09-21T09:46:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:43 smithi191 ceph-mon[102206]: pgmap v353: 89 pgs: 89 active+clean; 175 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:46:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:43 smithi191 ceph-mon[102206]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:46:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:43 smithi191 ceph-mon[102206]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:46:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-mon[106482]: pgmap v353: 89 pgs: 89 active+clean; 175 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:46:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-mon[106482]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:46:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-mon[106482]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:46:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-mon[110963]: pgmap v353: 89 pgs: 89 active+clean; 175 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:46:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-mon[110963]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T09:46:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:43 smithi076 ceph-mon[110963]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T09:46:44.694 INFO:tasks.rados.rados.0.smithi076.stdout:2029: done (2 left) 2024-09-21T09:46:44.694 INFO:tasks.rados.rados.0.smithi076.stdout:2030: write oid 25 current snap is 262 2024-09-21T09:46:44.694 INFO:tasks.rados.rados.0.smithi076.stdout:2030: seq_num 764 ranges {440548=725536,1757537=478262,2813990=26558} 2024-09-21T09:46:44.701 INFO:tasks.rados.rados.0.smithi076.stdout:2030: writing smithi07685715-25 from 440548 to 1166084 tid 1 2024-09-21T09:46:44.708 INFO:tasks.rados.rados.0.smithi076.stdout:2030: writing smithi07685715-25 from 1757537 to 2235799 tid 2 2024-09-21T09:46:44.712 INFO:tasks.rados.rados.0.smithi076.stdout:2030: writing smithi07685715-25 from 2813990 to 2840548 tid 3 2024-09-21T09:46:44.712 INFO:tasks.rados.rados.0.smithi076.stdout:2031: rollback oid 14 current snap is 262 2024-09-21T09:46:44.712 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 255 2024-09-21T09:46:44.712 INFO:tasks.rados.rados.0.smithi076.stdout:2027: expect (ObjNum 762 snap 260 seq_num 762) 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: finishing write tid 1 to smithi07685715-8 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: finishing write tid 2 to smithi07685715-8 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: finishing write tid 3 to smithi07685715-8 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: finishing write tid 4 to smithi07685715-8 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: oid 8 updating version 0 to 1579 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: oid 8 updating version 1579 to 1581 2024-09-21T09:46:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:2028: oid 8 version 1581 is already newer than 1580 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1581 (ObjNum 763 snap 262 seq_num 763) dirty exists 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2028: left oid 8 (ObjNum 763 snap 262 seq_num 763) 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2030: finishing write tid 1 to smithi07685715-25 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2030: finishing write tid 2 to smithi07685715-25 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2030: finishing write tid 3 to smithi07685715-25 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2030: finishing write tid 4 to smithi07685715-25 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2027: done (3 left) 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2028: done (2 left) 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2032: delete oid 29 current snap is 262 2024-09-21T09:46:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:2031: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2030: finishing write tid 5 to smithi07685715-25 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2030: oid 25 updating version 0 to 461 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2030: oid 25 updating version 461 to 462 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2030: oid 25 version 462 is already newer than 460 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2030: oid 25 version 462 is already newer than 459 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 462 (ObjNum 764 snap 262 seq_num 764) dirty exists 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2030: left oid 25 (ObjNum 764 snap 262 seq_num 764) 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:2031: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:46:44.838 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1138 (ObjNum 721 snap 247 seq_num 721) dirty exists 2024-09-21T09:46:44.839 INFO:tasks.rados.rados.0.smithi076.stdout:2030: done (2 left) 2024-09-21T09:46:44.839 INFO:tasks.rados.rados.0.smithi076.stdout:2031: done (1 left) 2024-09-21T09:46:44.839 INFO:tasks.rados.rados.0.smithi076.stdout:2032: done (0 left) 2024-09-21T09:46:44.839 INFO:tasks.rados.rados.0.smithi076.stdout:2033: snap_remove snap 260 2024-09-21T09:46:45.791 INFO:tasks.rados.rados.0.smithi076.stdout:2033: done (0 left) 2024-09-21T09:46:45.791 INFO:tasks.rados.rados.0.smithi076.stdout:2034: write oid 7 current snap is 262 2024-09-21T09:46:45.791 INFO:tasks.rados.rados.0.smithi076.stdout:2034: seq_num 766 ranges {474162=476034,1358148=481931,2352483=465303} 2024-09-21T09:46:45.796 INFO:tasks.rados.rados.0.smithi076.stdout:2034: writing smithi07685715-7 from 474162 to 950196 tid 1 2024-09-21T09:46:45.801 INFO:tasks.rados.rados.0.smithi076.stdout:2034: writing smithi07685715-7 from 1358148 to 1840079 tid 2 2024-09-21T09:46:45.807 INFO:tasks.rados.rados.0.smithi076.stdout:2034: writing smithi07685715-7 from 2352483 to 2817786 tid 3 2024-09-21T09:46:45.807 INFO:tasks.rados.rados.0.smithi076.stdout:2035: rollback oid 45 current snap is 262 2024-09-21T09:46:45.807 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 245 2024-09-21T09:46:45.807 INFO:tasks.rados.rados.0.smithi076.stdout:2034: finishing write tid 1 to smithi07685715-7 2024-09-21T09:46:45.807 INFO:tasks.rados.rados.0.smithi076.stdout:2034: finishing write tid 2 to smithi07685715-7 2024-09-21T09:46:45.807 INFO:tasks.rados.rados.0.smithi076.stdout:2036: read oid 17 snap -1 2024-09-21T09:46:45.808 INFO:tasks.rados.rados.0.smithi076.stdout:2036: expect deleted 2024-09-21T09:46:45.808 INFO:tasks.rados.rados.0.smithi076.stdout:2037: read oid 32 snap -1 2024-09-21T09:46:45.808 INFO:tasks.rados.rados.0.smithi076.stdout:2037: expect (ObjNum 753 snap 258 seq_num 753) 2024-09-21T09:46:45.808 INFO:tasks.rados.rados.0.smithi076.stdout:2038: snap_create 2024-09-21T09:46:45.810 INFO:tasks.rados.rados.0.smithi076.stdout:2034: finishing write tid 3 to smithi07685715-7 2024-09-21T09:46:45.810 INFO:tasks.rados.rados.0.smithi076.stdout:2034: finishing write tid 4 to smithi07685715-7 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:2034: finishing write tid 5 to smithi07685715-7 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:2034: oid 7 updating version 0 to 1140 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:2034: oid 7 updating version 1140 to 1142 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:2034: oid 7 updating version 1142 to 1143 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:2034: oid 7 version 1143 is already newer than 1141 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1143 (ObjNum 766 snap 262 seq_num 766) dirty exists 2024-09-21T09:46:45.811 INFO:tasks.rados.rados.0.smithi076.stdout:2034: left oid 7 (ObjNum 766 snap 262 seq_num 766) 2024-09-21T09:46:45.813 INFO:tasks.rados.rados.0.smithi076.stdout:2035: finishing rollback tid 0 to smithi07685715-45 2024-09-21T09:46:45.814 INFO:tasks.rados.rados.0.smithi076.stdout:2035: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:46:45.814 INFO:tasks.rados.rados.0.smithi076.stdout:2037: expect (ObjNum 753 snap 258 seq_num 753) 2024-09-21T09:46:45.933 INFO:tasks.rados.rados.0.smithi076.stdout:2035: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:46:45.933 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 414 (ObjNum 540 snap 171 seq_num 540) dirty exists 2024-09-21T09:46:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:45 smithi191 ceph-mon[102206]: pgmap v356: 89 pgs: 89 active+clean; 175 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:46:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:45 smithi191 ceph-mon[102206]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:46:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:45 smithi076 ceph-mon[106482]: pgmap v356: 89 pgs: 89 active+clean; 175 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:46:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:45 smithi076 ceph-mon[106482]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:46:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:45 smithi076 ceph-mon[110963]: pgmap v356: 89 pgs: 89 active+clean; 175 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:46:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:45 smithi076 ceph-mon[110963]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2034: done (4 left) 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2035: done (3 left) 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2036: done (2 left) 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2037: done (1 left) 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2038: done (0 left) 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2039: write oid 20 current snap is 263 2024-09-21T09:46:46.798 INFO:tasks.rados.rados.0.smithi076.stdout:2039: seq_num 767 ranges {490969=551283,1816630=625589} 2024-09-21T09:46:46.803 INFO:tasks.rados.rados.0.smithi076.stdout:2039: writing smithi07685715-20 from 490969 to 1042252 tid 1 2024-09-21T09:46:46.811 INFO:tasks.rados.rados.0.smithi076.stdout:2039: writing smithi07685715-20 from 1816630 to 2442219 tid 2 2024-09-21T09:46:46.811 INFO:tasks.rados.rados.0.smithi076.stdout:2040: write oid 36 current snap is 263 2024-09-21T09:46:46.811 INFO:tasks.rados.rados.0.smithi076.stdout:2040: seq_num 768 ranges {507776=742885,1991465=485600} 2024-09-21T09:46:46.818 INFO:tasks.rados.rados.0.smithi076.stdout:2040: writing smithi07685715-36 from 507776 to 1250661 tid 1 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2040: writing smithi07685715-36 from 1991465 to 2477065 tid 2 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2041: delete oid 2 current snap is 263 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: finishing write tid 1 to smithi07685715-20 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: finishing write tid 2 to smithi07685715-20 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: finishing write tid 3 to smithi07685715-20 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: finishing write tid 4 to smithi07685715-20 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: oid 20 updating version 0 to 579 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: oid 20 updating version 579 to 580 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: oid 20 version 580 is already newer than 578 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 580 (ObjNum 767 snap 263 seq_num 767) dirty exists 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2039: left oid 20 (ObjNum 767 snap 263 seq_num 767) 2024-09-21T09:46:46.825 INFO:tasks.rados.rados.0.smithi076.stdout:2040: finishing write tid 1 to smithi07685715-36 2024-09-21T09:46:46.828 INFO:tasks.rados.rados.0.smithi076.stdout:2040: finishing write tid 2 to smithi07685715-36 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2040: finishing write tid 3 to smithi07685715-36 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2039: done (2 left) 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2041: done (1 left) 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2042: read oid 20 snap -1 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2042: expect (ObjNum 767 snap 263 seq_num 767) 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2043: snap_create 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2040: finishing write tid 4 to smithi07685715-36 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2040: oid 36 updating version 0 to 606 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2040: oid 36 version 606 is already newer than 604 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2040: oid 36 version 606 is already newer than 605 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 606 (ObjNum 768 snap 263 seq_num 768) dirty exists 2024-09-21T09:46:46.829 INFO:tasks.rados.rados.0.smithi076.stdout:2040: left oid 36 (ObjNum 768 snap 263 seq_num 768) 2024-09-21T09:46:46.840 INFO:tasks.rados.rados.0.smithi076.stdout:2042: expect (ObjNum 767 snap 263 seq_num 767) 2024-09-21T09:46:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:46 smithi191 ceph-mon[102206]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:46:47.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:46 smithi076 ceph-mon[106482]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:46:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:46 smithi076 ceph-mon[110963]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T09:46:47.800 INFO:tasks.rados.rados.0.smithi076.stdout:2040: done (2 left) 2024-09-21T09:46:47.800 INFO:tasks.rados.rados.0.smithi076.stdout:2042: done (1 left) 2024-09-21T09:46:47.800 INFO:tasks.rados.rados.0.smithi076.stdout:2043: done (0 left) 2024-09-21T09:46:47.800 INFO:tasks.rados.rados.0.smithi076.stdout:2044: read oid 22 snap -1 2024-09-21T09:46:47.800 INFO:tasks.rados.rados.0.smithi076.stdout:2044: expect (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:46:47.801 INFO:tasks.rados.rados.0.smithi076.stdout:2045: delete oid 41 current snap is 264 2024-09-21T09:46:47.806 INFO:tasks.rados.rados.0.smithi076.stdout:2044: expect (ObjNum 627 snap 201 seq_num 627) 2024-09-21T09:46:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:2044: done (1 left) 2024-09-21T09:46:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:2045: done (0 left) 2024-09-21T09:46:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:2046: snap_create 2024-09-21T09:46:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:47 smithi191 ceph-mon[102206]: pgmap v359: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:46:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:47 smithi191 ceph-mon[102206]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:46:48.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:47 smithi076 ceph-mon[106482]: pgmap v359: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:46:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:47 smithi076 ceph-mon[106482]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:46:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:47 smithi076 ceph-mon[110963]: pgmap v359: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:46:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:47 smithi076 ceph-mon[110963]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T09:46:48.691 INFO:tasks.rados.rados.0.smithi076.stdout:2046: done (0 left) 2024-09-21T09:46:48.691 INFO:tasks.rados.rados.0.smithi076.stdout:2047: read oid 25 snap -1 2024-09-21T09:46:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:2047: expect (ObjNum 764 snap 262 seq_num 764) 2024-09-21T09:46:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:2048: read oid 11 snap -1 2024-09-21T09:46:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:2048: expect deleted 2024-09-21T09:46:48.692 INFO:tasks.rados.rados.0.smithi076.stdout:2049: snap_remove snap 258 2024-09-21T09:46:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:48 smithi191 ceph-mon[102206]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:46:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:48 smithi191 ceph-mon[102206]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:46:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:48 smithi076 ceph-mon[106482]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:46:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:48 smithi076 ceph-mon[106482]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:46:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:48 smithi076 ceph-mon[110963]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T09:46:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:48 smithi076 ceph-mon[110963]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2049: done (2 left) 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2050: read oid 31 snap -1 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2050: expect deleted 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2051: read oid 21 snap -1 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2051: expect (ObjNum 710 snap 236 seq_num 710) 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2052: read oid 47 snap -1 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2052: expect (ObjNum 663 snap 212 seq_num 663) 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2053: delete oid 42 current snap is 265 2024-09-21T09:46:49.698 INFO:tasks.rados.rados.0.smithi076.stdout:2047: expect (ObjNum 764 snap 262 seq_num 764) 2024-09-21T09:46:49.943 INFO:tasks.rados.rados.0.smithi076.stdout:2051: expect (ObjNum 710 snap 236 seq_num 710) 2024-09-21T09:46:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:49 smithi191 ceph-mon[102206]: pgmap v362: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:46:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:49 smithi191 ceph-mon[102206]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:46:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:49 smithi076 ceph-mon[106482]: pgmap v362: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:46:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:49 smithi076 ceph-mon[106482]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:46:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:49 smithi076 ceph-mon[110963]: pgmap v362: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:46:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:49 smithi076 ceph-mon[110963]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T09:46:50.256 INFO:tasks.rados.rados.0.smithi076.stdout:2052: expect (ObjNum 663 snap 212 seq_num 663) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2047: done (5 left) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2048: done (4 left) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2050: done (3 left) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2051: done (2 left) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2052: done (1 left) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2053: done (0 left) 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2054: write oid 46 current snap is 265 2024-09-21T09:46:50.398 INFO:tasks.rados.rados.0.smithi076.stdout:2054: seq_num 772 ranges {575004=643881,1709040=492938,2917021=57983} 2024-09-21T09:46:50.403 INFO:tasks.rados.rados.0.smithi076.stdout:2054: writing smithi07685715-46 from 575004 to 1218885 tid 1 2024-09-21T09:46:50.409 INFO:tasks.rados.rados.0.smithi076.stdout:2054: writing smithi07685715-46 from 1709040 to 2201978 tid 2 2024-09-21T09:46:50.414 INFO:tasks.rados.rados.0.smithi076.stdout:2054: writing smithi07685715-46 from 2917021 to 2975004 tid 3 2024-09-21T09:46:50.414 INFO:tasks.rados.rados.0.smithi076.stdout:2055: read oid 2 snap 249 2024-09-21T09:46:50.414 INFO:tasks.rados.rados.0.smithi076.stdout:2055: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:46:50.414 INFO:tasks.rados.rados.0.smithi076.stdout:2056: snap_create 2024-09-21T09:46:50.414 INFO:tasks.rados.rados.0.smithi076.stdout:2054: finishing write tid 1 to smithi07685715-46 2024-09-21T09:46:50.414 INFO:tasks.rados.rados.0.smithi076.stdout:2054: finishing write tid 2 to smithi07685715-46 2024-09-21T09:46:50.416 INFO:tasks.rados.rados.0.smithi076.stdout:2055: expect (ObjNum 481 snap 155 seq_num 481) 2024-09-21T09:46:50.525 INFO:tasks.rados.rados.0.smithi076.stdout:2054: finishing write tid 3 to smithi07685715-46 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: finishing write tid 4 to smithi07685715-46 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: finishing write tid 5 to smithi07685715-46 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: oid 46 updating version 0 to 1561 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: oid 46 version 1561 is already newer than 1559 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: oid 46 version 1561 is already newer than 1558 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: oid 46 version 1561 is already newer than 1560 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1561 (ObjNum 772 snap 265 seq_num 772) dirty exists 2024-09-21T09:46:50.526 INFO:tasks.rados.rados.0.smithi076.stdout:2054: left oid 46 (ObjNum 772 snap 265 seq_num 772) 2024-09-21T09:46:50.814 INFO:tasks.rados.rados.0.smithi076.stdout:2054: done (2 left) 2024-09-21T09:46:50.814 INFO:tasks.rados.rados.0.smithi076.stdout:2055: done (1 left) 2024-09-21T09:46:50.814 INFO:tasks.rados.rados.0.smithi076.stdout:2056: done (0 left) 2024-09-21T09:46:50.814 INFO:tasks.rados.rados.0.smithi076.stdout:2057: snap_remove snap 262 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:2057: done (0 left) 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:2058: rollback oid 10 current snap is 266 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 259 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:2059: read oid 44 snap 222 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:2059: expect deleted 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:2060: rollback oid 43 current snap is 266 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 254 2024-09-21T09:46:51.817 INFO:tasks.rados.rados.0.smithi076.stdout:2061: snap_create 2024-09-21T09:46:51.824 INFO:tasks.rados.rados.0.smithi076.stdout:2060: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:46:51.824 INFO:tasks.rados.rados.0.smithi076.stdout:2060: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:46:51.824 INFO:tasks.rados.rados.0.smithi076.stdout:2058: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:46:51.824 INFO:tasks.rados.rados.0.smithi076.stdout:2060: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:46:51.824 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 384 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:46:51.826 INFO:tasks.rados.rados.0.smithi076.stdout:2058: finishing rollback tid 2 to smithi07685715-10 2024-09-21T09:46:51.826 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1149 (ObjNum 707 snap 231 seq_num 707) dirty exists 2024-09-21T09:46:52.041 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:51 smithi076 ceph-mon[106482]: pgmap v365: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:52.042 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:51 smithi076 ceph-mon[106482]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:46:52.042 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:51 smithi076 ceph-mon[110963]: pgmap v365: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:52.042 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:51 smithi076 ceph-mon[110963]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:46:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:51 smithi191 ceph-mon[102206]: pgmap v365: 89 pgs: 89 active+clean; 189 MiB data, 713 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:46:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:51 smithi191 ceph-mon[102206]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T09:46:52.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for all mons to upgrade' 2024-09-21T09:46:52.817 INFO:tasks.rados.rados.0.smithi076.stdout:2058: done (3 left) 2024-09-21T09:46:52.817 INFO:tasks.rados.rados.0.smithi076.stdout:2059: done (2 left) 2024-09-21T09:46:52.817 INFO:tasks.rados.rados.0.smithi076.stdout:2060: done (1 left) 2024-09-21T09:46:52.817 INFO:tasks.rados.rados.0.smithi076.stdout:2061: done (0 left) 2024-09-21T09:46:52.817 INFO:tasks.rados.rados.0.smithi076.stdout:2062: snap_remove snap 265 2024-09-21T09:46:52.821 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:52 smithi076 ceph-mon[106482]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:46:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:52 smithi076 ceph-mon[106482]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:46:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:52 smithi076 ceph-mon[110963]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:46:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:52 smithi076 ceph-mon[110963]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:46:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:52 smithi191 ceph-mon[102206]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T09:46:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:52 smithi191 ceph-mon[102206]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T09:46:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:46:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:46:53] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.51.0" 2024-09-21T09:46:53.691 INFO:tasks.rados.rados.0.smithi076.stdout:2062: done (0 left) 2024-09-21T09:46:53.691 INFO:tasks.rados.rados.0.smithi076.stdout:2063: write oid 3 current snap is 267 2024-09-21T09:46:53.691 INFO:tasks.rados.rados.0.smithi076.stdout:2063: seq_num 773 ranges {591811=719130,1767522=636596,2828091=163720} 2024-09-21T09:46:53.698 INFO:tasks.rados.rados.0.smithi076.stdout:2063: writing smithi07685715-3 from 591811 to 1310941 tid 1 2024-09-21T09:46:53.704 INFO:tasks.rados.rados.0.smithi076.stdout:2063: writing smithi07685715-3 from 1767522 to 2404118 tid 2 2024-09-21T09:46:53.707 INFO:tasks.rados.rados.0.smithi076.stdout:2063: writing smithi07685715-3 from 2828091 to 2991811 tid 3 2024-09-21T09:46:53.707 INFO:tasks.rados.rados.0.smithi076.stdout:2064: snap_create 2024-09-21T09:46:53.707 INFO:tasks.rados.rados.0.smithi076.stdout:2063: finishing write tid 1 to smithi07685715-3 2024-09-21T09:46:53.709 INFO:tasks.rados.rados.0.smithi076.stdout:2063: finishing write tid 2 to smithi07685715-3 2024-09-21T09:46:53.709 INFO:tasks.rados.rados.0.smithi076.stdout:2063: finishing write tid 3 to smithi07685715-3 2024-09-21T09:46:53.709 INFO:tasks.rados.rados.0.smithi076.stdout:2063: finishing write tid 4 to smithi07685715-3 2024-09-21T09:46:53.709 INFO:tasks.rados.rados.0.smithi076.stdout:2063: finishing write tid 5 to smithi07685715-3 2024-09-21T09:46:53.709 INFO:tasks.rados.rados.0.smithi076.stdout:2063: oid 3 updating version 0 to 106 2024-09-21T09:46:53.709 INFO:tasks.rados.rados.0.smithi076.stdout:2063: oid 3 version 106 is already newer than 105 2024-09-21T09:46:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2063: oid 3 version 106 is already newer than 104 2024-09-21T09:46:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2063: oid 3 version 106 is already newer than 103 2024-09-21T09:46:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 106 (ObjNum 773 snap 267 seq_num 773) dirty exists 2024-09-21T09:46:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2063: left oid 3 (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:46:54.006 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:53 smithi076 ceph-mon[106482]: pgmap v368: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 880 KiB/s wr, 17 op/s 2024-09-21T09:46:54.006 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:53 smithi076 ceph-mon[106482]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:46:54.007 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:53 smithi076 ceph-mon[110963]: pgmap v368: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 880 KiB/s wr, 17 op/s 2024-09-21T09:46:54.007 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:53 smithi076 ceph-mon[110963]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:46:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:53 smithi191 ceph-mon[102206]: pgmap v368: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 880 KiB/s wr, 17 op/s 2024-09-21T09:46:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:53 smithi191 ceph-mon[102206]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T09:46:54.518 INFO:teuthology.orchestra.run.smithi076.stdout:wait for all mons to upgrade 2024-09-21T09:46:54.698 INFO:tasks.rados.rados.0.smithi076.stdout:2063: done (1 left) 2024-09-21T09:46:54.698 INFO:tasks.rados.rados.0.smithi076.stdout:2064: done (0 left) 2024-09-21T09:46:54.698 INFO:tasks.rados.rados.0.smithi076.stdout:2065: delete oid 20 current snap is 268 2024-09-21T09:46:54.700 INFO:tasks.rados.rados.0.smithi076.stdout:2065: done (0 left) 2024-09-21T09:46:54.700 INFO:tasks.rados.rados.0.smithi076.stdout:2066: rollback oid 21 current snap is 268 2024-09-21T09:46:54.700 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 222 2024-09-21T09:46:54.700 INFO:tasks.rados.rados.0.smithi076.stdout:2067: snap_remove snap 248 2024-09-21T09:46:55.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-09-21T09:46:55.372 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:55.700 INFO:tasks.rados.rados.0.smithi076.stdout:2067: done (1 left) 2024-09-21T09:46:55.701 INFO:tasks.rados.rados.0.smithi076.stdout:2068: snap_create 2024-09-21T09:46:55.701 INFO:tasks.rados.rados.0.smithi076.stdout:2066: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:46:55.701 INFO:tasks.rados.rados.0.smithi076.stdout:2066: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:46:55.701 INFO:tasks.rados.rados.0.smithi076.stdout:2066: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:46:55.701 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1567 (ObjNum 579 snap 184 seq_num 579) dirty exists 2024-09-21T09:46:55.775 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:55 smithi076 ceph-mon[110963]: pgmap v371: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 880 KiB/s wr, 17 op/s 2024-09-21T09:46:55.775 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:55 smithi076 ceph-mon[110963]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:46:55.775 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:55.775 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:55 smithi076 ceph-mon[106482]: pgmap v371: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 880 KiB/s wr, 17 op/s 2024-09-21T09:46:55.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:55 smithi076 ceph-mon[106482]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:46:55.776 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:55 smithi191 ceph-mon[102206]: pgmap v371: 89 pgs: 89 active+clean; 195 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 880 KiB/s wr, 17 op/s 2024-09-21T09:46:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:55 smithi191 ceph-mon[102206]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T09:46:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:46:56.699 INFO:tasks.rados.rados.0.smithi076.stdout:2066: done (1 left) 2024-09-21T09:46:56.699 INFO:tasks.rados.rados.0.smithi076.stdout:2068: done (0 left) 2024-09-21T09:46:56.699 INFO:tasks.rados.rados.0.smithi076.stdout:2069: write oid 8 current snap is 269 2024-09-21T09:46:56.699 INFO:tasks.rados.rados.0.smithi076.stdout:2069: seq_num 775 ranges {625425=469628,1768133=640265} 2024-09-21T09:46:56.704 INFO:tasks.rados.rados.0.smithi076.stdout:2069: writing smithi07685715-8 from 625425 to 1095053 tid 1 2024-09-21T09:46:56.713 INFO:tasks.rados.rados.0.smithi076.stdout:2069: writing smithi07685715-8 from 1768133 to 2408398 tid 2 2024-09-21T09:46:56.714 INFO:tasks.rados.rados.0.smithi076.stdout:2070: snap_create 2024-09-21T09:46:56.714 INFO:tasks.rados.rados.0.smithi076.stdout:2069: finishing write tid 1 to smithi07685715-8 2024-09-21T09:46:56.719 INFO:tasks.rados.rados.0.smithi076.stdout:2069: finishing write tid 2 to smithi07685715-8 2024-09-21T09:46:56.719 INFO:tasks.rados.rados.0.smithi076.stdout:2069: finishing write tid 3 to smithi07685715-8 2024-09-21T09:46:56.719 INFO:tasks.rados.rados.0.smithi076.stdout:2069: finishing write tid 4 to smithi07685715-8 2024-09-21T09:46:56.719 INFO:tasks.rados.rados.0.smithi076.stdout:2069: oid 8 updating version 0 to 1598 2024-09-21T09:46:56.719 INFO:tasks.rados.rados.0.smithi076.stdout:2069: oid 8 version 1598 is already newer than 1597 2024-09-21T09:46:56.720 INFO:tasks.rados.rados.0.smithi076.stdout:2069: oid 8 updating version 1598 to 1599 2024-09-21T09:46:56.720 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1599 (ObjNum 775 snap 269 seq_num 775) dirty exists 2024-09-21T09:46:56.720 INFO:tasks.rados.rados.0.smithi076.stdout:2069: left oid 8 (ObjNum 775 snap 269 seq_num 775) 2024-09-21T09:46:56.931 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:56 smithi076 ceph-mon[106482]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:46:56.931 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:56 smithi076 ceph-mon[106482]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:46:56.931 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:56 smithi076 ceph-mon[110963]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:46:56.931 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:56 smithi076 ceph-mon[110963]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:46:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:56 smithi191 ceph-mon[102206]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T09:46:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:56 smithi191 ceph-mon[102206]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T09:46:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:2069: done (1 left) 2024-09-21T09:46:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:2070: done (0 left) 2024-09-21T09:46:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:2071: write oid 10 current snap is 270 2024-09-21T09:46:57.716 INFO:tasks.rados.rados.0.smithi076.stdout:2071: seq_num 776 ranges {642232=661230,1942968=783923} 2024-09-21T09:46:57.723 INFO:tasks.rados.rados.0.smithi076.stdout:2071: writing smithi07685715-10 from 642232 to 1303462 tid 1 2024-09-21T09:46:57.731 INFO:tasks.rados.rados.0.smithi076.stdout:2071: writing smithi07685715-10 from 1942968 to 2726891 tid 2 2024-09-21T09:46:57.731 INFO:tasks.rados.rados.0.smithi076.stdout:2072: delete oid 47 current snap is 270 2024-09-21T09:46:57.731 INFO:tasks.rados.rados.0.smithi076.stdout:2071: finishing write tid 1 to smithi07685715-10 2024-09-21T09:46:57.734 INFO:tasks.rados.rados.0.smithi076.stdout:2072: done (1 left) 2024-09-21T09:46:57.734 INFO:tasks.rados.rados.0.smithi076.stdout:2073: snap_create 2024-09-21T09:46:57.735 INFO:tasks.rados.rados.0.smithi076.stdout:2071: finishing write tid 2 to smithi07685715-10 2024-09-21T09:46:57.735 INFO:tasks.rados.rados.0.smithi076.stdout:2071: finishing write tid 3 to smithi07685715-10 2024-09-21T09:46:57.735 INFO:tasks.rados.rados.0.smithi076.stdout:2071: finishing write tid 4 to smithi07685715-10 2024-09-21T09:46:57.735 INFO:tasks.rados.rados.0.smithi076.stdout:2071: oid 10 updating version 0 to 1159 2024-09-21T09:46:57.735 INFO:tasks.rados.rados.0.smithi076.stdout:2071: oid 10 updating version 1159 to 1161 2024-09-21T09:46:57.736 INFO:tasks.rados.rados.0.smithi076.stdout:2071: oid 10 version 1161 is already newer than 1160 2024-09-21T09:46:57.736 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1161 (ObjNum 776 snap 270 seq_num 776) dirty exists 2024-09-21T09:46:57.736 INFO:tasks.rados.rados.0.smithi076.stdout:2071: left oid 10 (ObjNum 776 snap 270 seq_num 776) 2024-09-21T09:46:58.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:57 smithi076 ceph-mon[106482]: pgmap v374: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 371 KiB/s wr, 3 op/s 2024-09-21T09:46:58.112 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:57 smithi076 ceph-mon[110963]: pgmap v374: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 371 KiB/s wr, 3 op/s 2024-09-21T09:46:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:57 smithi191 ceph-mon[102206]: pgmap v374: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 371 KiB/s wr, 3 op/s 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2071: done (1 left) 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2073: done (0 left) 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2074: rollback oid 33 current snap is 271 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 270 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2075: read oid 44 snap -1 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2075: expect deleted 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2076: write oid 23 current snap is 271 2024-09-21T09:46:58.692 INFO:tasks.rados.rados.0.smithi076.stdout:2076: seq_num 778 ranges {675846=411728,1543579=787592,2848853=226993} 2024-09-21T09:46:58.698 INFO:tasks.rados.rados.0.smithi076.stdout:2076: writing smithi07685715-23 from 675846 to 1087574 tid 1 2024-09-21T09:46:58.705 INFO:tasks.rados.rados.0.smithi076.stdout:2076: writing smithi07685715-23 from 1543579 to 2331171 tid 2 2024-09-21T09:46:58.709 INFO:tasks.rados.rados.0.smithi076.stdout:2076: writing smithi07685715-23 from 2848853 to 3075846 tid 3 2024-09-21T09:46:58.709 INFO:tasks.rados.rados.0.smithi076.stdout:2077: snap_remove snap 237 2024-09-21T09:46:59.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:58 smithi076 ceph-mon[106482]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:59.017 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:58 smithi076 ceph-mon[106482]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:46:59.017 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:58 smithi076 ceph-mon[106482]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:46:59.017 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:58 smithi076 ceph-mon[110963]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:59.017 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:58 smithi076 ceph-mon[110963]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:46:59.017 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:58 smithi076 ceph-mon[110963]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:46:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:58 smithi191 ceph-mon[102206]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:58 smithi191 ceph-mon[102206]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T09:46:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:58 smithi191 ceph-mon[102206]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T09:46:59.341 INFO:teuthology.orchestra.run.smithi076.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:46:59.695 INFO:tasks.rados.rados.0.smithi076.stdout:2077: done (3 left) 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2078: read oid 7 snap 254 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2078: expect (ObjNum 722 snap 248 seq_num 722) 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2079: read oid 20 snap -1 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2079: expect deleted 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2080: delete oid 11 current snap is 271 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2074: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2074: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2074: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 474 (ObjNum 750 snap 256 seq_num 750) dirty exists 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2076: finishing write tid 1 to smithi07685715-23 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2076: finishing write tid 2 to smithi07685715-23 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2076: finishing write tid 3 to smithi07685715-23 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2076: finishing write tid 4 to smithi07685715-23 2024-09-21T09:46:59.696 INFO:tasks.rados.rados.0.smithi076.stdout:2076: finishing write tid 5 to smithi07685715-23 2024-09-21T09:46:59.697 INFO:tasks.rados.rados.0.smithi076.stdout:2076: oid 23 updating version 0 to 1604 2024-09-21T09:46:59.697 INFO:tasks.rados.rados.0.smithi076.stdout:2076: oid 23 updating version 1604 to 1606 2024-09-21T09:46:59.697 INFO:tasks.rados.rados.0.smithi076.stdout:2076: oid 23 version 1606 is already newer than 1605 2024-09-21T09:46:59.697 INFO:tasks.rados.rados.0.smithi076.stdout:2076: oid 23 version 1606 is already newer than 1603 2024-09-21T09:46:59.697 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1606 (ObjNum 778 snap 271 seq_num 778) dirty exists 2024-09-21T09:46:59.697 INFO:tasks.rados.rados.0.smithi076.stdout:2076: left oid 23 (ObjNum 778 snap 271 seq_num 778) 2024-09-21T09:46:59.701 INFO:tasks.rados.rados.0.smithi076.stdout:2078: expect (ObjNum 722 snap 248 seq_num 722) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2074: done (5 left) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2075: done (4 left) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2076: done (3 left) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2078: done (2 left) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2079: done (1 left) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2080: done (0 left) 2024-09-21T09:46:59.816 INFO:tasks.rados.rados.0.smithi076.stdout:2081: snap_remove snap 266 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: pgmap v377: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 371 KiB/s wr, 3 op/s 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[110963]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: pgmap v377: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 371 KiB/s wr, 3 op/s 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:59.921 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:46:59.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:46:59.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:46:59.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:46:59.922 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:46:59 smithi076 ceph-mon[106482]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:46:59.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:47:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: pgmap v377: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 371 KiB/s wr, 3 op/s 2024-09-21T09:47:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:46:59 smithi191 ceph-mon[102206]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T09:47:00.156 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:00.743 INFO:tasks.rados.rados.0.smithi076.stdout:2081: done (0 left) 2024-09-21T09:47:00.743 INFO:tasks.rados.rados.0.smithi076.stdout:2082: snap_remove snap 263 2024-09-21T09:47:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:00 smithi191 ceph-mon[102206]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:47:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:00 smithi191 ceph-mon[102206]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:47:01.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:00 smithi076 ceph-mon[110963]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:47:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:00 smithi076 ceph-mon[110963]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:47:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:00 smithi076 ceph-mon[106482]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:47:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:00 smithi076 ceph-mon[106482]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:47:01.742 INFO:tasks.rados.rados.0.smithi076.stdout:2082: done (0 left) 2024-09-21T09:47:01.742 INFO:tasks.rados.rados.0.smithi076.stdout:2083: write oid 20 current snap is 271 2024-09-21T09:47:01.742 INFO:tasks.rados.rados.0.smithi076.stdout:2083: seq_num 780 ranges {709460=562226,2060543=791261} 2024-09-21T09:47:01.750 INFO:tasks.rados.rados.0.smithi076.stdout:2083: writing smithi07685715-20 from 709460 to 1271686 tid 1 2024-09-21T09:47:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2083: writing smithi07685715-20 from 2060543 to 2851804 tid 2 2024-09-21T09:47:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2084: snap_create 2024-09-21T09:47:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2083: finishing write tid 1 to smithi07685715-20 2024-09-21T09:47:01.762 INFO:tasks.rados.rados.0.smithi076.stdout:2083: finishing write tid 2 to smithi07685715-20 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:2083: finishing write tid 3 to smithi07685715-20 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:2083: finishing write tid 4 to smithi07685715-20 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:2083: oid 20 updating version 0 to 596 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:2083: oid 20 updating version 596 to 597 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:2083: oid 20 version 597 is already newer than 595 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 597 (ObjNum 780 snap 271 seq_num 780) dirty exists 2024-09-21T09:47:01.763 INFO:tasks.rados.rados.0.smithi076.stdout:2083: left oid 20 (ObjNum 780 snap 271 seq_num 780) 2024-09-21T09:47:01.867 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:01 smithi076 ceph-mon[106482]: pgmap v380: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:01.867 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:01 smithi076 ceph-mon[106482]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:47:01.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:01 smithi076 ceph-mon[110963]: pgmap v380: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:01.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:01 smithi076 ceph-mon[110963]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:47:02.066 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:47:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:01 smithi191 ceph-mon[102206]: pgmap v380: 89 pgs: 89 active+clean; 205 MiB data, 737 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:01 smithi191 ceph-mon[102206]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T09:47:02.646 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (5m) 26s ago 11m 19.7M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (5m) 25s ago 11m 80.6M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (10m) 26s ago 10m 21.3M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (10m) 25s ago 10m 17.6M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (6m) 25s ago 15m 457M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (7m) 26s ago 17m 514M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (4m) 26s ago 17m 62.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:47:02.647 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (2m) 25s ago 15m 37.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 bb6de71151ea 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (4m) 26s ago 15m 48.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (6m) 26s ago 11m 11.0M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (6m) 25s ago 10m 12.0M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (15m) 26s ago 15m 116M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (14m) 26s ago 14m 91.3M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (14m) 26s ago 14m 107M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (13m) 26s ago 13m 86.7M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (13m) 25s ago 13m 101M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (13m) 25s ago 13m 116M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (12m) 25s ago 12m 80.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (12m) 25s ago 12m 83.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:47:02.648 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (5m) 25s ago 11m 48.0M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:47:02.747 INFO:tasks.rados.rados.0.smithi076.stdout:2083: done (1 left) 2024-09-21T09:47:02.748 INFO:tasks.rados.rados.0.smithi076.stdout:2084: done (0 left) 2024-09-21T09:47:02.748 INFO:tasks.rados.rados.0.smithi076.stdout:2085: snap_remove snap 222 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:47:02.992 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 10, 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:47:02.993 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:47:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:02 smithi191 ceph-mon[102206]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:47:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:02 smithi191 ceph-mon[102206]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:03.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:02 smithi076 ceph-mon[106482]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:47:03.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:02 smithi076 ceph-mon[106482]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:02 smithi076 ceph-mon[110963]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T09:47:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:02 smithi076 ceph-mon[110963]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:03.298 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:47:03.298 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:47:03.298 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:47:03.298 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) mon", 2024-09-21T09:47:03.298 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [ 2024-09-21T09:47:03.299 INFO:teuthology.orchestra.run.smithi076.stdout: "mon" 2024-09-21T09:47:03.299 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:47:03.299 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "3/3 daemons upgraded", 2024-09-21T09:47:03.299 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 image", 2024-09-21T09:47:03.299 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:47:03.299 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:47:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:47:03] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.51.0" 2024-09-21T09:47:03.697 INFO:tasks.rados.rados.0.smithi076.stdout:2085: done (0 left) 2024-09-21T09:47:03.698 INFO:tasks.rados.rados.0.smithi076.stdout:2086: write oid 8 current snap is 272 2024-09-21T09:47:03.698 INFO:tasks.rados.rados.0.smithi076.stdout:2086: seq_num 781 ranges {726267=637475,2119025=651272} 2024-09-21T09:47:03.706 INFO:tasks.rados.rados.0.smithi076.stdout:2086: writing smithi07685715-8 from 726267 to 1363742 tid 1 2024-09-21T09:47:03.715 INFO:tasks.rados.rados.0.smithi076.stdout:2086: writing smithi07685715-8 from 2119025 to 2770297 tid 2 2024-09-21T09:47:03.715 INFO:tasks.rados.rados.0.smithi076.stdout:2087: delete oid 15 current snap is 272 2024-09-21T09:47:03.718 INFO:tasks.rados.rados.0.smithi076.stdout:2087: done (1 left) 2024-09-21T09:47:03.719 INFO:tasks.rados.rados.0.smithi076.stdout:2088: delete oid 34 current snap is 272 2024-09-21T09:47:03.721 INFO:tasks.rados.rados.0.smithi076.stdout:2086: finishing write tid 1 to smithi07685715-8 2024-09-21T09:47:03.722 INFO:tasks.rados.rados.0.smithi076.stdout:2088: done (1 left) 2024-09-21T09:47:03.722 INFO:tasks.rados.rados.0.smithi076.stdout:2089: write oid 9 current snap is 272 2024-09-21T09:47:03.722 INFO:tasks.rados.rados.0.smithi076.stdout:2089: seq_num 784 ranges {776688=579575,1894471=798599} 2024-09-21T09:47:03.729 INFO:tasks.rados.rados.0.smithi076.stdout:2089: writing smithi07685715-9 from 776688 to 1356263 tid 1 2024-09-21T09:47:03.737 INFO:tasks.rados.rados.0.smithi076.stdout:2089: writing smithi07685715-9 from 1894471 to 2693070 tid 2 2024-09-21T09:47:03.737 INFO:tasks.rados.rados.0.smithi076.stdout:2090: snap_create 2024-09-21T09:47:03.737 INFO:tasks.rados.rados.0.smithi076.stdout:2086: finishing write tid 2 to smithi07685715-8 2024-09-21T09:47:03.737 INFO:tasks.rados.rados.0.smithi076.stdout:2086: finishing write tid 3 to smithi07685715-8 2024-09-21T09:47:03.737 INFO:tasks.rados.rados.0.smithi076.stdout:2086: finishing write tid 4 to smithi07685715-8 2024-09-21T09:47:03.738 INFO:tasks.rados.rados.0.smithi076.stdout:2086: oid 8 updating version 0 to 1634 2024-09-21T09:47:03.738 INFO:tasks.rados.rados.0.smithi076.stdout:2086: oid 8 updating version 1634 to 1639 2024-09-21T09:47:03.738 INFO:tasks.rados.rados.0.smithi076.stdout:2086: oid 8 updating version 1639 to 1640 2024-09-21T09:47:03.738 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1640 (ObjNum 781 snap 272 seq_num 781) dirty exists 2024-09-21T09:47:03.738 INFO:tasks.rados.rados.0.smithi076.stdout:2086: left oid 8 (ObjNum 781 snap 272 seq_num 781) 2024-09-21T09:47:03.738 INFO:tasks.rados.rados.0.smithi076.stdout:2089: finishing write tid 1 to smithi07685715-9 2024-09-21T09:47:03.762 INFO:tasks.rados.rados.0.smithi076.stdout:2089: finishing write tid 2 to smithi07685715-9 2024-09-21T09:47:03.762 INFO:tasks.rados.rados.0.smithi076.stdout:2089: finishing write tid 3 to smithi07685715-9 2024-09-21T09:47:03.763 INFO:tasks.rados.rados.0.smithi076.stdout:2089: finishing write tid 4 to smithi07685715-9 2024-09-21T09:47:03.763 INFO:tasks.rados.rados.0.smithi076.stdout:2089: oid 9 updating version 0 to 740 2024-09-21T09:47:03.763 INFO:tasks.rados.rados.0.smithi076.stdout:2089: oid 9 version 740 is already newer than 739 2024-09-21T09:47:03.763 INFO:tasks.rados.rados.0.smithi076.stdout:2089: oid 9 version 740 is already newer than 738 2024-09-21T09:47:03.763 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 740 (ObjNum 784 snap 272 seq_num 784) dirty exists 2024-09-21T09:47:03.763 INFO:tasks.rados.rados.0.smithi076.stdout:2089: left oid 9 (ObjNum 784 snap 272 seq_num 784) 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: pgmap v383: 89 pgs: 89 active+clean; 209 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 907 KiB/s rd, 973 KiB/s wr, 6 op/s 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/123651172' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:47:04.116 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:47:04.117 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:04.118 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:04.119 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.119 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:03 smithi191 ceph-mon[102206]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:47:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: pgmap v383: 89 pgs: 89 active+clean; 209 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 907 KiB/s rd, 973 KiB/s wr, 6 op/s 2024-09-21T09:47:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:47:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/123651172' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: pgmap v383: 89 pgs: 89 active+clean; 209 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 907 KiB/s rd, 973 KiB/s wr, 6 op/s 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/123651172' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:04.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.128 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:47:04.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:47:04.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.131 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[110963]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:47:04.132 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.133 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:04.134 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:03 smithi076 ceph-mon[106482]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T09:47:04.710 INFO:tasks.rados.rados.0.smithi076.stdout:2086: done (2 left) 2024-09-21T09:47:04.710 INFO:tasks.rados.rados.0.smithi076.stdout:2089: done (1 left) 2024-09-21T09:47:04.710 INFO:tasks.rados.rados.0.smithi076.stdout:2090: done (0 left) 2024-09-21T09:47:04.710 INFO:tasks.rados.rados.0.smithi076.stdout:2091: write oid 18 current snap is 273 2024-09-21T09:47:04.710 INFO:tasks.rados.rados.0.smithi076.stdout:2091: seq_num 785 ranges {793495=654824,1952953=658610,3157167=36328} 2024-09-21T09:47:04.717 INFO:tasks.rados.rados.0.smithi076.stdout:2091: writing smithi07685715-18 from 793495 to 1448319 tid 1 2024-09-21T09:47:04.724 INFO:tasks.rados.rados.0.smithi076.stdout:2091: writing smithi07685715-18 from 1952953 to 2611563 tid 2 2024-09-21T09:47:04.727 INFO:tasks.rados.rados.0.smithi076.stdout:2091: writing smithi07685715-18 from 3157167 to 3193495 tid 3 2024-09-21T09:47:04.727 INFO:tasks.rados.rados.0.smithi076.stdout:2092: snap_remove snap 267 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all mgr 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all mon 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all crash 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all rgw 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:47:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all nfs 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all grafana 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all loki 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all promtail 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Finalizing container_image settings 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: Upgrade: Complete! 2024-09-21T09:47:05.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:04 smithi191 ceph-mon[102206]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all mgr 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all mon 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all crash 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all rgw 2024-09-21T09:47:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all nfs 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all grafana 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all loki 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all promtail 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Finalizing container_image settings 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: Upgrade: Complete! 2024-09-21T09:47:05.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[106482]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all mgr 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all mon 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all crash 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all rgw 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all iscsi 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all nfs 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all node-exporter 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all prometheus 2024-09-21T09:47:05.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all alertmanager 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all grafana 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all loki 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all promtail 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Finalizing container_image settings 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: Upgrade: Complete! 2024-09-21T09:47:05.126 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:04 smithi076 ceph-mon[110963]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2092: done (1 left) 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2093: delete oid 38 current snap is 273 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: finishing write tid 1 to smithi07685715-18 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: finishing write tid 2 to smithi07685715-18 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: finishing write tid 3 to smithi07685715-18 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: finishing write tid 4 to smithi07685715-18 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: finishing write tid 5 to smithi07685715-18 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: oid 18 updating version 0 to 600 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: oid 18 updating version 600 to 601 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: oid 18 updating version 601 to 602 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: oid 18 version 602 is already newer than 599 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 602 (ObjNum 785 snap 273 seq_num 785) dirty exists 2024-09-21T09:47:05.708 INFO:tasks.rados.rados.0.smithi076.stdout:2091: left oid 18 (ObjNum 785 snap 273 seq_num 785) 2024-09-21T09:47:05.714 INFO:tasks.rados.rados.0.smithi076.stdout:2091: done (1 left) 2024-09-21T09:47:05.714 INFO:tasks.rados.rados.0.smithi076.stdout:2093: done (0 left) 2024-09-21T09:47:05.714 INFO:tasks.rados.rados.0.smithi076.stdout:2094: read oid 43 snap -1 2024-09-21T09:47:05.715 INFO:tasks.rados.rados.0.smithi076.stdout:2094: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:47:05.715 INFO:tasks.rados.rados.0.smithi076.stdout:2095: snap_remove snap 251 2024-09-21T09:47:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:05 smithi076 ceph-mon[106482]: pgmap v386: 89 pgs: 89 active+clean; 209 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 907 KiB/s rd, 973 KiB/s wr, 6 op/s 2024-09-21T09:47:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:05.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:05 smithi076 ceph-mon[106482]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:47:05.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:05 smithi076 ceph-mon[110963]: pgmap v386: 89 pgs: 89 active+clean; 209 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 907 KiB/s rd, 973 KiB/s wr, 6 op/s 2024-09-21T09:47:05.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:05.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:05 smithi076 ceph-mon[110963]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:47:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:05 smithi191 ceph-mon[102206]: pgmap v386: 89 pgs: 89 active+clean; 209 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 907 KiB/s rd, 973 KiB/s wr, 6 op/s 2024-09-21T09:47:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:47:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:05 smithi191 ceph-mon[102206]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T09:47:06.714 INFO:tasks.rados.rados.0.smithi076.stdout:2094: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:47:06.819 INFO:tasks.rados.rados.0.smithi076.stdout:2094: done (1 left) 2024-09-21T09:47:06.819 INFO:tasks.rados.rados.0.smithi076.stdout:2095: done (0 left) 2024-09-21T09:47:06.819 INFO:tasks.rados.rados.0.smithi076.stdout:2096: rollback oid 48 current snap is 273 2024-09-21T09:47:06.819 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 261 2024-09-21T09:47:06.819 INFO:tasks.rados.rados.0.smithi076.stdout:2097: snap_create 2024-09-21T09:47:06.822 INFO:tasks.rados.rados.0.smithi076.stdout:2096: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:47:06.822 INFO:tasks.rados.rados.0.smithi076.stdout:2096: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:47:06.822 INFO:tasks.rados.rados.0.smithi076.stdout:2096: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:47:06.823 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1660 (ObjNum 754 snap 258 seq_num 754) dirty exists 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2096: done (1 left) 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2097: done (0 left) 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2098: read oid 24 snap -1 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2098: expect (ObjNum 758 snap 259 seq_num 758) 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2099: read oid 23 snap -1 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2099: expect (ObjNum 778 snap 271 seq_num 778) 2024-09-21T09:47:07.779 INFO:tasks.rados.rados.0.smithi076.stdout:2100: delete oid 46 current snap is 274 2024-09-21T09:47:07.784 INFO:tasks.rados.rados.0.smithi076.stdout:2100: done (2 left) 2024-09-21T09:47:07.784 INFO:tasks.rados.rados.0.smithi076.stdout:2101: write oid 6 current snap is 274 2024-09-21T09:47:07.785 INFO:tasks.rados.rados.0.smithi076.stdout:2101: seq_num 788 ranges {443916=480571,1612046=405937,2606730=495052} 2024-09-21T09:47:07.789 INFO:tasks.rados.rados.0.smithi076.stdout:2101: writing smithi07685715-6 from 443916 to 924487 tid 1 2024-09-21T09:47:07.795 INFO:tasks.rados.rados.0.smithi076.stdout:2101: writing smithi07685715-6 from 1612046 to 2017983 tid 2 2024-09-21T09:47:07.802 INFO:tasks.rados.rados.0.smithi076.stdout:2101: writing smithi07685715-6 from 2606730 to 3101782 tid 3 2024-09-21T09:47:07.802 INFO:tasks.rados.rados.0.smithi076.stdout:2102: snap_remove snap 268 2024-09-21T09:47:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:07 smithi191 ceph-mon[102206]: pgmap v389: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:07 smithi191 ceph-mon[102206]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:47:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:07 smithi076 ceph-mon[106482]: pgmap v389: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:07 smithi076 ceph-mon[106482]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:47:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:07 smithi076 ceph-mon[110963]: pgmap v389: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:07 smithi076 ceph-mon[110963]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T09:47:08.698 INFO:tasks.rados.rados.0.smithi076.stdout:2102: done (3 left) 2024-09-21T09:47:08.698 INFO:tasks.rados.rados.0.smithi076.stdout:2103: snap_remove snap 264 2024-09-21T09:47:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:08 smithi191 ceph-mon[102206]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:47:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:08 smithi191 ceph-mon[102206]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:47:09.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:08 smithi076 ceph-mon[106482]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:47:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:08 smithi076 ceph-mon[106482]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:47:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:08 smithi076 ceph-mon[110963]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T09:47:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:08 smithi076 ceph-mon[110963]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T09:47:09.699 INFO:tasks.rados.rados.0.smithi076.stdout:2103: done (3 left) 2024-09-21T09:47:09.699 INFO:tasks.rados.rados.0.smithi076.stdout:2104: rollback oid 27 current snap is 274 2024-09-21T09:47:09.699 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 239 2024-09-21T09:47:09.699 INFO:tasks.rados.rados.0.smithi076.stdout:2098: expect (ObjNum 758 snap 259 seq_num 758) 2024-09-21T09:47:09.881 INFO:tasks.rados.rados.0.smithi076.stdout:2101: finishing write tid 1 to smithi07685715-6 2024-09-21T09:47:09.881 INFO:tasks.rados.rados.0.smithi076.stdout:2099: expect (ObjNum 778 snap 271 seq_num 778) 2024-09-21T09:47:10.049 INFO:tasks.rados.rados.0.smithi076.stdout:2101: finishing write tid 2 to smithi07685715-6 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: finishing write tid 3 to smithi07685715-6 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: finishing write tid 4 to smithi07685715-6 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: finishing write tid 5 to smithi07685715-6 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: oid 6 updating version 0 to 451 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: oid 6 version 451 is already newer than 449 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: oid 6 updating version 451 to 452 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:2101: oid 6 version 452 is already newer than 450 2024-09-21T09:47:10.050 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 452 (ObjNum 788 snap 274 seq_num 788) dirty exists 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:2101: left oid 6 (ObjNum 788 snap 274 seq_num 788) 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:2098: done (3 left) 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:2099: done (2 left) 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:2101: done (1 left) 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:2105: write oid 45 current snap is 274 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:2104: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:47:10.051 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 0 (ObjNum 129 snap 0 seq_num 864494816) dirty dne 2024-09-21T09:47:10.052 INFO:tasks.rados.rados.0.smithi076.stdout:2105: seq_num 789 ranges {460723=555820,1670528=549595,2634153=626570} 2024-09-21T09:47:10.054 INFO:tasks.rados.rados.0.smithi076.stdout:2105: writing smithi07685715-45 from 460723 to 1016543 tid 1 2024-09-21T09:47:10.060 INFO:tasks.rados.rados.0.smithi076.stdout:2105: writing smithi07685715-45 from 1670528 to 2220123 tid 2 2024-09-21T09:47:10.066 INFO:tasks.rados.rados.0.smithi076.stdout:2105: writing smithi07685715-45 from 2634153 to 3260723 tid 3 2024-09-21T09:47:10.067 INFO:tasks.rados.rados.0.smithi076.stdout:2104: done (1 left) 2024-09-21T09:47:10.067 INFO:tasks.rados.rados.0.smithi076.stdout:2106: snap_remove snap 270 2024-09-21T09:47:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:09 smithi191 ceph-mon[102206]: pgmap v392: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:09 smithi191 ceph-mon[102206]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:47:10.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:09 smithi076 ceph-mon[106482]: pgmap v392: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:09 smithi076 ceph-mon[106482]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:47:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:09 smithi076 ceph-mon[110963]: pgmap v392: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:47:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:09 smithi076 ceph-mon[110963]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T09:47:10.795 INFO:tasks.rados.rados.0.smithi076.stdout:2105: finishing write tid 1 to smithi07685715-45 2024-09-21T09:47:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2105: finishing write tid 2 to smithi07685715-45 2024-09-21T09:47:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2105: finishing write tid 3 to smithi07685715-45 2024-09-21T09:47:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2105: finishing write tid 4 to smithi07685715-45 2024-09-21T09:47:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2105: finishing write tid 5 to smithi07685715-45 2024-09-21T09:47:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2105: oid 45 updating version 0 to 435 2024-09-21T09:47:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2105: oid 45 updating version 435 to 438 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2105: oid 45 version 438 is already newer than 437 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2105: oid 45 version 438 is already newer than 436 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 438 (ObjNum 789 snap 274 seq_num 789) dirty exists 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2105: left oid 45 (ObjNum 789 snap 274 seq_num 789) 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2105: done (1 left) 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2106: done (0 left) 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2107: read oid 40 snap -1 2024-09-21T09:47:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2107: expect (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:47:10.798 INFO:tasks.rados.rados.0.smithi076.stdout:2108: snap_remove snap 272 2024-09-21T09:47:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:11.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:2108: done (1 left) 2024-09-21T09:47:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:2109: read oid 7 snap -1 2024-09-21T09:47:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:2109: expect (ObjNum 766 snap 262 seq_num 766) 2024-09-21T09:47:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:2110: write oid 44 current snap is 274 2024-09-21T09:47:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:2110: seq_num 790 ranges {477530=631069,1612657=409606,2545223=517596} 2024-09-21T09:47:11.807 INFO:tasks.rados.rados.0.smithi076.stdout:2110: writing smithi07685715-44 from 477530 to 1108599 tid 1 2024-09-21T09:47:11.813 INFO:tasks.rados.rados.0.smithi076.stdout:2110: writing smithi07685715-44 from 1612657 to 2022263 tid 2 2024-09-21T09:47:11.819 INFO:tasks.rados.rados.0.smithi076.stdout:2110: writing smithi07685715-44 from 2545223 to 3062819 tid 3 2024-09-21T09:47:11.819 INFO:tasks.rados.rados.0.smithi076.stdout:2111: read oid 6 snap -1 2024-09-21T09:47:11.819 INFO:tasks.rados.rados.0.smithi076.stdout:2111: expect (ObjNum 788 snap 274 seq_num 788) 2024-09-21T09:47:11.819 INFO:tasks.rados.rados.0.smithi076.stdout:2112: delete oid 37 current snap is 274 2024-09-21T09:47:11.819 INFO:tasks.rados.rados.0.smithi076.stdout:2107: expect (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:47:11.933 INFO:tasks.rados.rados.0.smithi076.stdout:2110: finishing write tid 1 to smithi07685715-44 2024-09-21T09:47:11.933 INFO:tasks.rados.rados.0.smithi076.stdout:2110: finishing write tid 2 to smithi07685715-44 2024-09-21T09:47:11.934 INFO:tasks.rados.rados.0.smithi076.stdout:2109: expect (ObjNum 766 snap 262 seq_num 766) 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: finishing write tid 3 to smithi07685715-44 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: finishing write tid 4 to smithi07685715-44 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: finishing write tid 5 to smithi07685715-44 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: oid 44 updating version 0 to 462 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: oid 44 updating version 462 to 463 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: oid 44 updating version 463 to 464 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: oid 44 version 464 is already newer than 461 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 464 (ObjNum 790 snap 274 seq_num 790) dirty exists 2024-09-21T09:47:12.087 INFO:tasks.rados.rados.0.smithi076.stdout:2110: left oid 44 (ObjNum 790 snap 274 seq_num 790) 2024-09-21T09:47:12.088 INFO:tasks.rados.rados.0.smithi076.stdout:2111: expect (ObjNum 788 snap 274 seq_num 788) 2024-09-21T09:47:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:11 smithi191 ceph-mon[102206]: pgmap v395: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:11 smithi191 ceph-mon[102206]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:47:12.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:11 smithi076 ceph-mon[106482]: pgmap v395: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:11 smithi076 ceph-mon[106482]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:47:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:11 smithi076 ceph-mon[110963]: pgmap v395: 89 pgs: 89 active+clean; 201 MiB data, 744 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:11 smithi076 ceph-mon[110963]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2107: done (4 left) 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2109: done (3 left) 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2110: done (2 left) 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2111: done (1 left) 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2112: done (0 left) 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2113: write oid 26 current snap is 274 2024-09-21T09:47:12.290 INFO:tasks.rados.rados.0.smithi076.stdout:2113: seq_num 792 ranges {511144=497920,1445974=413275,2316422=656493} 2024-09-21T09:47:12.295 INFO:tasks.rados.rados.0.smithi076.stdout:2113: writing smithi07685715-26 from 511144 to 1009064 tid 1 2024-09-21T09:47:12.299 INFO:tasks.rados.rados.0.smithi076.stdout:2113: writing smithi07685715-26 from 1445974 to 1859249 tid 2 2024-09-21T09:47:12.306 INFO:tasks.rados.rados.0.smithi076.stdout:2113: writing smithi07685715-26 from 2316422 to 2972915 tid 3 2024-09-21T09:47:12.306 INFO:tasks.rados.rados.0.smithi076.stdout:2114: snap_remove snap 271 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2114: done (1 left) 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2115: delete oid 10 current snap is 274 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: finishing write tid 1 to smithi07685715-26 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: finishing write tid 2 to smithi07685715-26 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: finishing write tid 3 to smithi07685715-26 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: finishing write tid 4 to smithi07685715-26 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: finishing write tid 5 to smithi07685715-26 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: oid 26 updating version 0 to 744 2024-09-21T09:47:12.811 INFO:tasks.rados.rados.0.smithi076.stdout:2113: oid 26 version 744 is already newer than 742 2024-09-21T09:47:12.812 INFO:tasks.rados.rados.0.smithi076.stdout:2113: oid 26 version 744 is already newer than 743 2024-09-21T09:47:12.812 INFO:tasks.rados.rados.0.smithi076.stdout:2113: oid 26 updating version 744 to 745 2024-09-21T09:47:12.812 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 745 (ObjNum 792 snap 274 seq_num 792) dirty exists 2024-09-21T09:47:12.812 INFO:tasks.rados.rados.0.smithi076.stdout:2113: left oid 26 (ObjNum 792 snap 274 seq_num 792) 2024-09-21T09:47:12.818 INFO:tasks.rados.rados.0.smithi076.stdout:2113: done (1 left) 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2115: done (0 left) 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2116: read oid 20 snap -1 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2116: expect (ObjNum 780 snap 271 seq_num 780) 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2117: read oid 33 snap -1 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2117: expect (ObjNum 750 snap 256 seq_num 750) 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2118: read oid 42 snap -1 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2118: expect deleted 2024-09-21T09:47:12.819 INFO:tasks.rados.rados.0.smithi076.stdout:2119: snap_remove snap 261 2024-09-21T09:47:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:12 smithi191 ceph-mon[102206]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:47:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:12 smithi076 ceph-mon[106482]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:47:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:12 smithi076 ceph-mon[110963]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T09:47:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:47:13] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.51.0" 2024-09-21T09:47:13.700 INFO:tasks.rados.rados.0.smithi076.stdout:2119: done (3 left) 2024-09-21T09:47:13.700 INFO:tasks.rados.rados.0.smithi076.stdout:2120: delete oid 17 current snap is 274 2024-09-21T09:47:13.700 INFO:tasks.rados.rados.0.smithi076.stdout:2117: expect (ObjNum 750 snap 256 seq_num 750) 2024-09-21T09:47:13.832 INFO:tasks.rados.rados.0.smithi076.stdout:2116: expect (ObjNum 780 snap 271 seq_num 780) 2024-09-21T09:47:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:2116: done (3 left) 2024-09-21T09:47:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:2117: done (2 left) 2024-09-21T09:47:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:2118: done (1 left) 2024-09-21T09:47:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:2120: done (0 left) 2024-09-21T09:47:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:2121: snap_remove snap 257 2024-09-21T09:47:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:13 smithi191 ceph-mon[102206]: pgmap v398: 89 pgs: 89 active+clean; 200 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 761 KiB/s wr, 10 op/s 2024-09-21T09:47:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:13 smithi191 ceph-mon[102206]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:47:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:13 smithi191 ceph-mon[102206]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:47:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-mon[106482]: pgmap v398: 89 pgs: 89 active+clean; 200 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 761 KiB/s wr, 10 op/s 2024-09-21T09:47:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-mon[106482]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:47:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-mon[106482]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:47:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-mon[110963]: pgmap v398: 89 pgs: 89 active+clean; 200 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 761 KiB/s wr, 10 op/s 2024-09-21T09:47:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-mon[110963]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T09:47:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:13 smithi076 ceph-mon[110963]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T09:47:14.821 INFO:tasks.rados.rados.0.smithi076.stdout:2121: done (0 left) 2024-09-21T09:47:14.822 INFO:tasks.rados.rados.0.smithi076.stdout:2122: snap_create 2024-09-21T09:47:15.819 INFO:tasks.rados.rados.0.smithi076.stdout:2122: done (0 left) 2024-09-21T09:47:15.819 INFO:tasks.rados.rados.0.smithi076.stdout:2123: write oid 17 current snap is 275 2024-09-21T09:47:15.820 INFO:tasks.rados.rados.0.smithi076.stdout:2123: seq_num 795 ranges {561565=723667,1905067=560602,3082338=279227} 2024-09-21T09:47:15.826 INFO:tasks.rados.rados.0.smithi076.stdout:2123: writing smithi07685715-17 from 561565 to 1285232 tid 1 2024-09-21T09:47:15.833 INFO:tasks.rados.rados.0.smithi076.stdout:2123: writing smithi07685715-17 from 1905067 to 2465669 tid 2 2024-09-21T09:47:15.838 INFO:tasks.rados.rados.0.smithi076.stdout:2123: writing smithi07685715-17 from 3082338 to 3361565 tid 3 2024-09-21T09:47:15.838 INFO:tasks.rados.rados.0.smithi076.stdout:2124: write oid 49 current snap is 275 2024-09-21T09:47:15.838 INFO:tasks.rados.rados.0.smithi076.stdout:2124: seq_num 796 ranges {578372=798916,1963549=420613,2826114=417934} 2024-09-21T09:47:15.845 INFO:tasks.rados.rados.0.smithi076.stdout:2124: writing smithi07685715-49 from 578372 to 1377288 tid 1 2024-09-21T09:47:15.851 INFO:tasks.rados.rados.0.smithi076.stdout:2124: writing smithi07685715-49 from 1963549 to 2384162 tid 2 2024-09-21T09:47:15.856 INFO:tasks.rados.rados.0.smithi076.stdout:2124: writing smithi07685715-49 from 2826114 to 3244048 tid 3 2024-09-21T09:47:15.856 INFO:tasks.rados.rados.0.smithi076.stdout:2125: delete oid 4 current snap is 275 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: finishing write tid 1 to smithi07685715-17 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: finishing write tid 2 to smithi07685715-17 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: finishing write tid 3 to smithi07685715-17 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: finishing write tid 4 to smithi07685715-17 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: finishing write tid 5 to smithi07685715-17 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: oid 17 updating version 0 to 1216 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: oid 17 version 1216 is already newer than 1215 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: oid 17 updating version 1216 to 1217 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: oid 17 updating version 1217 to 1218 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1218 (ObjNum 795 snap 275 seq_num 795) dirty exists 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2123: left oid 17 (ObjNum 795 snap 275 seq_num 795) 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2124: finishing write tid 1 to smithi07685715-49 2024-09-21T09:47:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2124: finishing write tid 2 to smithi07685715-49 2024-09-21T09:47:15.860 INFO:tasks.rados.rados.0.smithi076.stdout:2124: finishing write tid 3 to smithi07685715-49 2024-09-21T09:47:15.860 INFO:tasks.rados.rados.0.smithi076.stdout:2124: finishing write tid 4 to smithi07685715-49 2024-09-21T09:47:15.860 INFO:tasks.rados.rados.0.smithi076.stdout:2124: finishing write tid 5 to smithi07685715-49 2024-09-21T09:47:15.860 INFO:tasks.rados.rados.0.smithi076.stdout:2124: oid 49 updating version 0 to 1705 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2124: oid 49 version 1705 is already newer than 1703 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2124: oid 49 updating version 1705 to 1706 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2124: oid 49 version 1706 is already newer than 1704 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 1706 (ObjNum 796 snap 275 seq_num 796) dirty exists 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2124: left oid 49 (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2123: done (2 left) 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2124: done (1 left) 2024-09-21T09:47:15.861 INFO:tasks.rados.rados.0.smithi076.stdout:2125: done (0 left) 2024-09-21T09:47:15.862 INFO:tasks.rados.rados.0.smithi076.stdout:2126: snap_create 2024-09-21T09:47:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:15 smithi191 ceph-mon[102206]: pgmap v401: 89 pgs: 89 active+clean; 200 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 761 KiB/s wr, 10 op/s 2024-09-21T09:47:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:15 smithi191 ceph-mon[102206]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:47:16.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:15 smithi076 ceph-mon[106482]: pgmap v401: 89 pgs: 89 active+clean; 200 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 761 KiB/s wr, 10 op/s 2024-09-21T09:47:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:15 smithi076 ceph-mon[106482]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:47:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:15 smithi076 ceph-mon[110963]: pgmap v401: 89 pgs: 89 active+clean; 200 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 761 KiB/s wr, 10 op/s 2024-09-21T09:47:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:15 smithi076 ceph-mon[110963]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T09:47:16.830 INFO:tasks.rados.rados.0.smithi076.stdout:2126: done (0 left) 2024-09-21T09:47:16.831 INFO:tasks.rados.rados.0.smithi076.stdout:2127: read oid 4 snap 273 2024-09-21T09:47:16.831 INFO:tasks.rados.rados.0.smithi076.stdout:2127: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:47:16.831 INFO:tasks.rados.rados.0.smithi076.stdout:2128: snap_create 2024-09-21T09:47:16.837 INFO:tasks.rados.rados.0.smithi076.stdout:2127: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:47:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:16 smithi191 ceph-mon[102206]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:47:17.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:16 smithi076 ceph-mon[110963]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:47:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:16 smithi076 ceph-mon[106482]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T09:47:17.832 INFO:tasks.rados.rados.0.smithi076.stdout:2127: done (1 left) 2024-09-21T09:47:17.833 INFO:tasks.rados.rados.0.smithi076.stdout:2128: done (0 left) 2024-09-21T09:47:17.833 INFO:tasks.rados.rados.0.smithi076.stdout:2129: snap_remove snap 239 2024-09-21T09:47:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:17 smithi191 ceph-mon[102206]: pgmap v404: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:47:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:17 smithi191 ceph-mon[102206]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:47:18.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:17 smithi076 ceph-mon[106482]: pgmap v404: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:47:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:17 smithi076 ceph-mon[106482]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:47:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:17 smithi076 ceph-mon[110963]: pgmap v404: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:47:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:17 smithi076 ceph-mon[110963]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T09:47:18.702 INFO:tasks.rados.rados.0.smithi076.stdout:2129: done (0 left) 2024-09-21T09:47:18.702 INFO:tasks.rados.rados.0.smithi076.stdout:2130: snap_remove snap 273 2024-09-21T09:47:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:18 smithi191 ceph-mon[102206]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:47:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:18 smithi191 ceph-mon[102206]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:47:19.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:18 smithi076 ceph-mon[110963]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:47:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:18 smithi076 ceph-mon[110963]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:47:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:18 smithi076 ceph-mon[106482]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T09:47:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:18 smithi076 ceph-mon[106482]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T09:47:19.702 INFO:tasks.rados.rados.0.smithi076.stdout:2130: done (0 left) 2024-09-21T09:47:19.702 INFO:tasks.rados.rados.0.smithi076.stdout:2131: delete oid 34 current snap is 277 2024-09-21T09:47:19.708 INFO:tasks.rados.rados.0.smithi076.stdout:2131: done (0 left) 2024-09-21T09:47:19.708 INFO:tasks.rados.rados.0.smithi076.stdout:2132: write oid 31 current snap is 277 2024-09-21T09:47:19.709 INFO:tasks.rados.rados.0.smithi076.stdout:2132: seq_num 799 ranges {628793=741016,2138995=567940,3192030=236763} 2024-09-21T09:47:19.716 INFO:tasks.rados.rados.0.smithi076.stdout:2132: writing smithi07685715-31 from 628793 to 1369809 tid 1 2024-09-21T09:47:19.723 INFO:tasks.rados.rados.0.smithi076.stdout:2132: writing smithi07685715-31 from 2138995 to 2706935 tid 2 2024-09-21T09:47:19.727 INFO:tasks.rados.rados.0.smithi076.stdout:2132: writing smithi07685715-31 from 3192030 to 3428793 tid 3 2024-09-21T09:47:19.727 INFO:tasks.rados.rados.0.smithi076.stdout:2133: read oid 23 snap 259 2024-09-21T09:47:19.728 INFO:tasks.rados.rados.0.smithi076.stdout:2133: expect (ObjNum 691 snap 222 seq_num 691) 2024-09-21T09:47:19.728 INFO:tasks.rados.rados.0.smithi076.stdout:2134: snap_create 2024-09-21T09:47:19.728 INFO:tasks.rados.rados.0.smithi076.stdout:2132: finishing write tid 1 to smithi07685715-31 2024-09-21T09:47:19.728 INFO:tasks.rados.rados.0.smithi076.stdout:2132: finishing write tid 2 to smithi07685715-31 2024-09-21T09:47:19.729 INFO:tasks.rados.rados.0.smithi076.stdout:2132: finishing write tid 3 to smithi07685715-31 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:2132: finishing write tid 4 to smithi07685715-31 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:2132: finishing write tid 5 to smithi07685715-31 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:2132: oid 31 updating version 0 to 700 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:2132: oid 31 version 700 is already newer than 697 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:2132: oid 31 version 700 is already newer than 698 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:2132: oid 31 version 700 is already newer than 699 2024-09-21T09:47:19.730 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 700 (ObjNum 799 snap 277 seq_num 799) dirty exists 2024-09-21T09:47:19.731 INFO:tasks.rados.rados.0.smithi076.stdout:2132: left oid 31 (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:19.739 INFO:tasks.rados.rados.0.smithi076.stdout:2133: expect (ObjNum 691 snap 222 seq_num 691) 2024-09-21T09:47:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:19 smithi191 ceph-mon[102206]: pgmap v407: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:47:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:19 smithi191 ceph-mon[102206]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:47:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:19 smithi076 ceph-mon[106482]: pgmap v407: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:47:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:19 smithi076 ceph-mon[106482]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:47:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:19 smithi076 ceph-mon[110963]: pgmap v407: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-09-21T09:47:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:19 smithi076 ceph-mon[110963]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T09:47:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:2132: done (2 left) 2024-09-21T09:47:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:2133: done (1 left) 2024-09-21T09:47:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:2134: done (0 left) 2024-09-21T09:47:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:2135: write oid 9 current snap is 278 2024-09-21T09:47:20.704 INFO:tasks.rados.rados.0.smithi076.stdout:2135: seq_num 800 ranges {645600=416265,1797477=711598,3219453=226147} 2024-09-21T09:47:20.709 INFO:tasks.rados.rados.0.smithi076.stdout:2135: writing smithi07685715-9 from 645600 to 1061865 tid 1 2024-09-21T09:47:20.717 INFO:tasks.rados.rados.0.smithi076.stdout:2135: writing smithi07685715-9 from 1797477 to 2509075 tid 2 2024-09-21T09:47:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2135: writing smithi07685715-9 from 3219453 to 3445600 tid 3 2024-09-21T09:47:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2136: read oid 10 snap 269 2024-09-21T09:47:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2136: expect (ObjNum 707 snap 231 seq_num 707) 2024-09-21T09:47:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2137: snap_create 2024-09-21T09:47:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2135: finishing write tid 1 to smithi07685715-9 2024-09-21T09:47:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2135: finishing write tid 2 to smithi07685715-9 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: finishing write tid 3 to smithi07685715-9 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: finishing write tid 4 to smithi07685715-9 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: finishing write tid 5 to smithi07685715-9 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: oid 9 updating version 0 to 763 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: oid 9 updating version 763 to 764 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: oid 9 version 764 is already newer than 762 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: oid 9 version 764 is already newer than 761 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 764 (ObjNum 800 snap 278 seq_num 800) dirty exists 2024-09-21T09:47:20.726 INFO:tasks.rados.rados.0.smithi076.stdout:2135: left oid 9 (ObjNum 800 snap 278 seq_num 800) 2024-09-21T09:47:20.730 INFO:tasks.rados.rados.0.smithi076.stdout:2136: expect (ObjNum 707 snap 231 seq_num 707) 2024-09-21T09:47:21.710 INFO:tasks.rados.rados.0.smithi076.stdout:2135: done (2 left) 2024-09-21T09:47:21.711 INFO:tasks.rados.rados.0.smithi076.stdout:2136: done (1 left) 2024-09-21T09:47:21.711 INFO:tasks.rados.rados.0.smithi076.stdout:2137: done (0 left) 2024-09-21T09:47:21.711 INFO:tasks.rados.rados.0.smithi076.stdout:2138: read oid 3 snap -1 2024-09-21T09:47:21.711 INFO:tasks.rados.rados.0.smithi076.stdout:2138: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:47:21.711 INFO:tasks.rados.rados.0.smithi076.stdout:2139: write oid 39 current snap is 279 2024-09-21T09:47:21.711 INFO:tasks.rados.rados.0.smithi076.stdout:2139: seq_num 801 ranges {662407=491514,1855959=571609,2846876=615531} 2024-09-21T09:47:21.716 INFO:tasks.rados.rados.0.smithi076.stdout:2139: writing smithi07685715-39 from 662407 to 1153921 tid 1 2024-09-21T09:47:21.724 INFO:tasks.rados.rados.0.smithi076.stdout:2139: writing smithi07685715-39 from 1855959 to 2427568 tid 2 2024-09-21T09:47:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2139: writing smithi07685715-39 from 2846876 to 3462407 tid 3 2024-09-21T09:47:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2140: read oid 5 snap 275 2024-09-21T09:47:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2140: expect (ObjNum 713 snap 240 seq_num 713) 2024-09-21T09:47:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2141: snap_create 2024-09-21T09:47:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2138: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:47:21.903 INFO:tasks.rados.rados.0.smithi076.stdout:2139: finishing write tid 1 to smithi07685715-39 2024-09-21T09:47:21.903 INFO:tasks.rados.rados.0.smithi076.stdout:2139: finishing write tid 2 to smithi07685715-39 2024-09-21T09:47:21.903 INFO:tasks.rados.rados.0.smithi076.stdout:2139: finishing write tid 3 to smithi07685715-39 2024-09-21T09:47:21.903 INFO:tasks.rados.rados.0.smithi076.stdout:2139: finishing write tid 4 to smithi07685715-39 2024-09-21T09:47:21.903 INFO:tasks.rados.rados.0.smithi076.stdout:2139: finishing write tid 5 to smithi07685715-39 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:2139: oid 39 updating version 0 to 1640 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:2139: oid 39 version 1640 is already newer than 1638 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:2139: oid 39 version 1640 is already newer than 1639 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:2139: oid 39 updating version 1640 to 1641 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1641 (ObjNum 801 snap 279 seq_num 801) dirty exists 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:2139: left oid 39 (ObjNum 801 snap 279 seq_num 801) 2024-09-21T09:47:21.904 INFO:tasks.rados.rados.0.smithi076.stdout:2140: expect (ObjNum 713 snap 240 seq_num 713) 2024-09-21T09:47:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:21 smithi191 ceph-mon[102206]: pgmap v410: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:21 smithi191 ceph-mon[102206]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:47:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:21 smithi076 ceph-mon[106482]: pgmap v410: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:21 smithi076 ceph-mon[106482]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:47:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:21 smithi076 ceph-mon[110963]: pgmap v410: 89 pgs: 89 active+clean; 207 MiB data, 740 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:21 smithi076 ceph-mon[110963]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T09:47:22.718 INFO:tasks.rados.rados.0.smithi076.stdout:2138: done (3 left) 2024-09-21T09:47:22.718 INFO:tasks.rados.rados.0.smithi076.stdout:2139: done (2 left) 2024-09-21T09:47:22.718 INFO:tasks.rados.rados.0.smithi076.stdout:2140: done (1 left) 2024-09-21T09:47:22.719 INFO:tasks.rados.rados.0.smithi076.stdout:2141: done (0 left) 2024-09-21T09:47:22.719 INFO:tasks.rados.rados.0.smithi076.stdout:2142: delete oid 11 current snap is 280 2024-09-21T09:47:22.723 INFO:tasks.rados.rados.0.smithi076.stdout:2142: done (0 left) 2024-09-21T09:47:22.724 INFO:tasks.rados.rados.0.smithi076.stdout:2143: snap_create 2024-09-21T09:47:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:22 smithi191 ceph-mon[102206]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:47:23.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:22 smithi076 ceph-mon[106482]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:47:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:22 smithi076 ceph-mon[110963]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T09:47:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:47:23] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.51.0" 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2143: done (0 left) 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2144: read oid 43 snap -1 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2144: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2145: read oid 1 snap -1 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2145: expect deleted 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2146: write oid 42 current snap is 281 2024-09-21T09:47:23.707 INFO:tasks.rados.rados.0.smithi076.stdout:2146: seq_num 803 ranges {696021=642012,1856570=575278,3185369=310652} 2024-09-21T09:47:23.714 INFO:tasks.rados.rados.0.smithi076.stdout:2146: writing smithi07685715-42 from 696021 to 1338033 tid 1 2024-09-21T09:47:23.721 INFO:tasks.rados.rados.0.smithi076.stdout:2146: writing smithi07685715-42 from 1856570 to 2431848 tid 2 2024-09-21T09:47:23.727 INFO:tasks.rados.rados.0.smithi076.stdout:2146: writing smithi07685715-42 from 3185369 to 3496021 tid 3 2024-09-21T09:47:23.727 INFO:tasks.rados.rados.0.smithi076.stdout:2147: delete oid 34 current snap is 281 2024-09-21T09:47:23.727 INFO:tasks.rados.rados.0.smithi076.stdout:2144: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: finishing write tid 1 to smithi07685715-42 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: finishing write tid 2 to smithi07685715-42 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: finishing write tid 3 to smithi07685715-42 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: finishing write tid 4 to smithi07685715-42 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: finishing write tid 5 to smithi07685715-42 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: oid 42 updating version 0 to 625 2024-09-21T09:47:23.827 INFO:tasks.rados.rados.0.smithi076.stdout:2146: oid 42 version 625 is already newer than 623 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2146: oid 42 updating version 625 to 626 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2146: oid 42 version 626 is already newer than 624 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 626 (ObjNum 803 snap 281 seq_num 803) dirty exists 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2146: left oid 42 (ObjNum 803 snap 281 seq_num 803) 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2144: done (3 left) 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2145: done (2 left) 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2146: done (1 left) 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2147: done (0 left) 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2148: rollback oid 37 current snap is 281 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 245 2024-09-21T09:47:23.828 INFO:tasks.rados.rados.0.smithi076.stdout:2149: snap_remove snap 255 2024-09-21T09:47:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:23 smithi191 ceph-mon[102206]: pgmap v413: 89 pgs: 89 active+clean; 204 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 708 KiB/s wr, 8 op/s 2024-09-21T09:47:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:23 smithi191 ceph-mon[102206]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:47:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:23 smithi191 ceph-mon[102206]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:47:24.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-mon[110963]: pgmap v413: 89 pgs: 89 active+clean; 204 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 708 KiB/s wr, 8 op/s 2024-09-21T09:47:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-mon[110963]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:47:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-mon[110963]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:47:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-mon[106482]: pgmap v413: 89 pgs: 89 active+clean; 204 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 708 KiB/s wr, 8 op/s 2024-09-21T09:47:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-mon[106482]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T09:47:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:23 smithi076 ceph-mon[106482]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:2149: done (1 left) 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:2150: read oid 2 snap -1 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:2150: expect deleted 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:2151: snap_create 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:2148: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:2148: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:47:24.727 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 618 (ObjNum 697 snap 224 seq_num 697) dirty exists 2024-09-21T09:47:25.732 INFO:tasks.rados.rados.0.smithi076.stdout:2148: done (2 left) 2024-09-21T09:47:25.732 INFO:tasks.rados.rados.0.smithi076.stdout:2150: done (1 left) 2024-09-21T09:47:25.732 INFO:tasks.rados.rados.0.smithi076.stdout:2151: done (0 left) 2024-09-21T09:47:25.732 INFO:tasks.rados.rados.0.smithi076.stdout:2152: delete oid 13 current snap is 282 2024-09-21T09:47:25.737 INFO:tasks.rados.rados.0.smithi076.stdout:2152: done (0 left) 2024-09-21T09:47:25.737 INFO:tasks.rados.rados.0.smithi076.stdout:2153: write oid 20 current snap is 282 2024-09-21T09:47:25.737 INFO:tasks.rados.rados.0.smithi076.stdout:2153: seq_num 806 ranges {746442=584112,1748369=722605,2983991=562451} 2024-09-21T09:47:25.744 INFO:tasks.rados.rados.0.smithi076.stdout:2153: writing smithi07685715-20 from 746442 to 1330554 tid 1 2024-09-21T09:47:25.750 INFO:tasks.rados.rados.0.smithi076.stdout:2153: writing smithi07685715-20 from 1748369 to 2470974 tid 2 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:2153: writing smithi07685715-20 from 2983991 to 3546442 tid 3 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:2154: rollback oid 43 current snap is 282 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 280 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:2153: finishing write tid 1 to smithi07685715-20 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:2153: finishing write tid 2 to smithi07685715-20 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:2155: rollback oid 5 current snap is 282 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 269 2024-09-21T09:47:25.757 INFO:tasks.rados.rados.0.smithi076.stdout:2156: write oid 16 current snap is 282 2024-09-21T09:47:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2156: seq_num 807 ranges {763249=659361,2090498=582616,3295061=268188} 2024-09-21T09:47:25.765 INFO:tasks.rados.rados.0.smithi076.stdout:2156: writing smithi07685715-16 from 763249 to 1422610 tid 1 2024-09-21T09:47:25.772 INFO:tasks.rados.rados.0.smithi076.stdout:2156: writing smithi07685715-16 from 2090498 to 2673114 tid 2 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2156: writing smithi07685715-16 from 3295061 to 3563249 tid 3 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2157: read oid 42 snap 269 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2157: expect deleted 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2158: write oid 17 current snap is 282 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2155: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2153: finishing write tid 3 to smithi07685715-20 2024-09-21T09:47:25.777 INFO:tasks.rados.rados.0.smithi076.stdout:2158: seq_num 808 ranges {780056=734610,2148980=726274,3322484=257572} 2024-09-21T09:47:25.785 INFO:tasks.rados.rados.0.smithi076.stdout:2158: writing smithi07685715-17 from 780056 to 1514666 tid 1 2024-09-21T09:47:25.793 INFO:tasks.rados.rados.0.smithi076.stdout:2158: writing smithi07685715-17 from 2148980 to 2875254 tid 2 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2158: writing smithi07685715-17 from 3322484 to 3580056 tid 3 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2159: write oid 48 current snap is 282 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2153: finishing write tid 4 to smithi07685715-20 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2153: finishing write tid 5 to smithi07685715-20 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2153: oid 20 updating version 0 to 624 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2153: oid 20 updating version 624 to 625 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2153: oid 20 version 625 is already newer than 622 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:2153: oid 20 version 625 is already newer than 623 2024-09-21T09:47:25.797 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 625 (ObjNum 806 snap 282 seq_num 806) dirty exists 2024-09-21T09:47:25.798 INFO:tasks.rados.rados.0.smithi076.stdout:2153: left oid 20 (ObjNum 806 snap 282 seq_num 806) 2024-09-21T09:47:25.798 INFO:tasks.rados.rados.0.smithi076.stdout:2155: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:47:25.798 INFO:tasks.rados.rados.0.smithi076.stdout:2159: seq_num 809 ranges {796863=409859,1807462=586285,3066260=530603} 2024-09-21T09:47:25.803 INFO:tasks.rados.rados.0.smithi076.stdout:2159: writing smithi07685715-48 from 796863 to 1206722 tid 1 2024-09-21T09:47:25.810 INFO:tasks.rados.rados.0.smithi076.stdout:2159: writing smithi07685715-48 from 1807462 to 2393747 tid 2 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:2159: writing smithi07685715-48 from 3066260 to 3596863 tid 3 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:2153: done (6 left) 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:2160: write oid 1 current snap is 282 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:2155: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1652 (ObjNum 713 snap 240 seq_num 713) dirty exists 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:2154: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:47:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:2154: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:47:25.818 INFO:tasks.rados.rados.0.smithi076.stdout:2154: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:47:25.818 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 440 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:47:25.818 INFO:tasks.rados.rados.0.smithi076.stdout:2156: finishing write tid 1 to smithi07685715-16 2024-09-21T09:47:25.818 INFO:tasks.rados.rados.0.smithi076.stdout:2156: finishing write tid 2 to smithi07685715-16 2024-09-21T09:47:25.818 INFO:tasks.rados.rados.0.smithi076.stdout:2160: seq_num 810 ranges {413670=485108,1465944=729943,2977330=524801} 2024-09-21T09:47:25.822 INFO:tasks.rados.rados.0.smithi076.stdout:2160: writing smithi07685715-1 from 413670 to 898778 tid 1 2024-09-21T09:47:25.829 INFO:tasks.rados.rados.0.smithi076.stdout:2160: writing smithi07685715-1 from 1465944 to 2195887 tid 2 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2160: writing smithi07685715-1 from 2977330 to 3502131 tid 3 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2154: done (6 left) 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2155: done (5 left) 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2161: delete oid 5 current snap is 282 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: finishing write tid 3 to smithi07685715-16 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: finishing write tid 4 to smithi07685715-16 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: finishing write tid 5 to smithi07685715-16 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: oid 16 updating version 0 to 913 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: oid 16 version 913 is already newer than 912 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: oid 16 updating version 913 to 914 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: oid 16 updating version 914 to 915 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 915 (ObjNum 807 snap 282 seq_num 807) dirty exists 2024-09-21T09:47:25.837 INFO:tasks.rados.rados.0.smithi076.stdout:2156: left oid 16 (ObjNum 807 snap 282 seq_num 807) 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: finishing write tid 1 to smithi07685715-17 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: finishing write tid 2 to smithi07685715-17 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: finishing write tid 3 to smithi07685715-17 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: finishing write tid 4 to smithi07685715-17 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: finishing write tid 5 to smithi07685715-17 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: oid 17 updating version 0 to 1237 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: oid 17 updating version 1237 to 1239 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: oid 17 version 1239 is already newer than 1238 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: oid 17 version 1239 is already newer than 1236 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1239 (ObjNum 808 snap 282 seq_num 808) dirty exists 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2158: left oid 17 (ObjNum 808 snap 282 seq_num 808) 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2159: finishing write tid 1 to smithi07685715-48 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2159: finishing write tid 2 to smithi07685715-48 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2159: finishing write tid 3 to smithi07685715-48 2024-09-21T09:47:25.838 INFO:tasks.rados.rados.0.smithi076.stdout:2159: finishing write tid 4 to smithi07685715-48 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2159: finishing write tid 5 to smithi07685715-48 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2159: oid 48 updating version 0 to 1728 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2159: oid 48 updating version 1728 to 1729 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2159: oid 48 updating version 1729 to 1731 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2159: oid 48 version 1731 is already newer than 1730 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1731 (ObjNum 809 snap 282 seq_num 809) dirty exists 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2159: left oid 48 (ObjNum 809 snap 282 seq_num 809) 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2160: finishing write tid 1 to smithi07685715-1 2024-09-21T09:47:25.839 INFO:tasks.rados.rados.0.smithi076.stdout:2160: finishing write tid 2 to smithi07685715-1 2024-09-21T09:47:25.840 INFO:tasks.rados.rados.0.smithi076.stdout:2160: finishing write tid 3 to smithi07685715-1 2024-09-21T09:47:25.840 INFO:tasks.rados.rados.0.smithi076.stdout:2160: finishing write tid 4 to smithi07685715-1 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: finishing write tid 5 to smithi07685715-1 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: oid 1 updating version 0 to 1654 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: oid 1 version 1654 is already newer than 1653 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: oid 1 updating version 1654 to 1656 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: oid 1 version 1656 is already newer than 1655 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 1656 (ObjNum 810 snap 282 seq_num 810) dirty exists 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: left oid 1 (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2156: done (5 left) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2157: done (4 left) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2158: done (3 left) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2159: done (2 left) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2160: done (1 left) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2161: done (0 left) 2024-09-21T09:47:25.841 INFO:tasks.rados.rados.0.smithi076.stdout:2162: delete oid 18 current snap is 282 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2162: done (0 left) 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2163: read oid 43 snap -1 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2163: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2164: read oid 19 snap -1 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2164: expect deleted 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2165: read oid 35 snap -1 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2165: expect deleted 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2166: rollback oid 22 current snap is 282 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 269 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2167: read oid 31 snap -1 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2167: expect (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:25.843 INFO:tasks.rados.rados.0.smithi076.stdout:2168: snap_remove snap 278 2024-09-21T09:47:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:25 smithi191 ceph-mon[102206]: pgmap v416: 89 pgs: 89 active+clean; 204 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 708 KiB/s wr, 8 op/s 2024-09-21T09:47:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:25 smithi191 ceph-mon[102206]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:47:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:26.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:25 smithi076 ceph-mon[106482]: pgmap v416: 89 pgs: 89 active+clean; 204 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 708 KiB/s wr, 8 op/s 2024-09-21T09:47:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:25 smithi076 ceph-mon[106482]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:47:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:25 smithi076 ceph-mon[110963]: pgmap v416: 89 pgs: 89 active+clean; 204 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 708 KiB/s wr, 8 op/s 2024-09-21T09:47:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:25 smithi076 ceph-mon[110963]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T09:47:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:26.744 INFO:tasks.rados.rados.0.smithi076.stdout:2168: done (5 left) 2024-09-21T09:47:26.744 INFO:tasks.rados.rados.0.smithi076.stdout:2169: read oid 17 snap -1 2024-09-21T09:47:26.744 INFO:tasks.rados.rados.0.smithi076.stdout:2169: expect (ObjNum 808 snap 282 seq_num 808) 2024-09-21T09:47:26.744 INFO:tasks.rados.rados.0.smithi076.stdout:2170: write oid 2 current snap is 282 2024-09-21T09:47:26.744 INFO:tasks.rados.rados.0.smithi076.stdout:2170: seq_num 813 ranges {464091=710855,1925037=477270,2943246=474970} 2024-09-21T09:47:26.750 INFO:tasks.rados.rados.0.smithi076.stdout:2170: writing smithi07685715-2 from 464091 to 1174946 tid 1 2024-09-21T09:47:26.758 INFO:tasks.rados.rados.0.smithi076.stdout:2170: writing smithi07685715-2 from 1925037 to 2402307 tid 2 2024-09-21T09:47:26.764 INFO:tasks.rados.rados.0.smithi076.stdout:2170: writing smithi07685715-2 from 2943246 to 3418216 tid 3 2024-09-21T09:47:26.764 INFO:tasks.rados.rados.0.smithi076.stdout:2171: snap_remove snap 274 2024-09-21T09:47:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:26 smithi191 ceph-mon[102206]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:47:27.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:26 smithi076 ceph-mon[106482]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:47:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:26 smithi076 ceph-mon[110963]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T09:47:27.747 INFO:tasks.rados.rados.0.smithi076.stdout:2171: done (7 left) 2024-09-21T09:47:27.748 INFO:tasks.rados.rados.0.smithi076.stdout:2172: delete oid 24 current snap is 282 2024-09-21T09:47:27.748 INFO:tasks.rados.rados.0.smithi076.stdout:2166: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:47:27.748 INFO:tasks.rados.rados.0.smithi076.stdout:2166: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:47:27.748 INFO:tasks.rados.rados.0.smithi076.stdout:2166: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:47:27.748 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 919 (ObjNum 627 snap 201 seq_num 627) dirty exists 2024-09-21T09:47:27.748 INFO:tasks.rados.rados.0.smithi076.stdout:2163: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:47:27.848 INFO:tasks.rados.rados.0.smithi076.stdout:2167: expect (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:27.961 INFO:tasks.rados.rados.0.smithi076.stdout:2169: expect (ObjNum 808 snap 282 seq_num 808) 2024-09-21T09:47:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:27 smithi191 ceph-mon[102206]: pgmap v419: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:47:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:27 smithi191 ceph-mon[102206]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: finishing write tid 1 to smithi07685715-2 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: finishing write tid 2 to smithi07685715-2 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: finishing write tid 3 to smithi07685715-2 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: finishing write tid 4 to smithi07685715-2 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: finishing write tid 5 to smithi07685715-2 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: oid 2 updating version 0 to 340 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: oid 2 version 340 is already newer than 339 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: oid 2 version 340 is already newer than 335 2024-09-21T09:47:28.120 INFO:tasks.rados.rados.0.smithi076.stdout:2170: oid 2 version 340 is already newer than 338 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 340 (ObjNum 813 snap 282 seq_num 813) dirty exists 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2170: left oid 2 (ObjNum 813 snap 282 seq_num 813) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2163: done (7 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2164: done (6 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2165: done (5 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2166: done (4 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2167: done (3 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2169: done (2 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2170: done (1 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2172: done (0 left) 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2173: rollback oid 13 current snap is 282 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 245 2024-09-21T09:47:28.121 INFO:tasks.rados.rados.0.smithi076.stdout:2174: delete oid 33 current snap is 282 2024-09-21T09:47:28.122 INFO:tasks.rados.rados.0.smithi076.stdout:2173: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:47:28.122 INFO:tasks.rados.rados.0.smithi076.stdout:2173: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:47:28.122 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 344 (ObjNum 631 snap 203 seq_num 631) dirty exists 2024-09-21T09:47:28.122 INFO:tasks.rados.rados.0.smithi076.stdout:2173: done (1 left) 2024-09-21T09:47:28.122 INFO:tasks.rados.rados.0.smithi076.stdout:2174: done (0 left) 2024-09-21T09:47:28.122 INFO:tasks.rados.rados.0.smithi076.stdout:2175: snap_create 2024-09-21T09:47:28.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:27 smithi076 ceph-mon[106482]: pgmap v419: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:47:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:27 smithi076 ceph-mon[106482]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:47:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:27 smithi076 ceph-mon[110963]: pgmap v419: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:47:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:27 smithi076 ceph-mon[110963]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T09:47:28.701 INFO:tasks.rados.rados.0.smithi076.stdout:2175: done (0 left) 2024-09-21T09:47:28.702 INFO:tasks.rados.rados.0.smithi076.stdout:2176: write oid 4 current snap is 283 2024-09-21T09:47:28.702 INFO:tasks.rados.rados.0.smithi076.stdout:2176: seq_num 816 ranges {514512=652955,1700483=740950,3025515=688997} 2024-09-21T09:47:28.707 INFO:tasks.rados.rados.0.smithi076.stdout:2176: writing smithi07685715-4 from 514512 to 1167467 tid 1 2024-09-21T09:47:28.714 INFO:tasks.rados.rados.0.smithi076.stdout:2176: writing smithi07685715-4 from 1700483 to 2441433 tid 2 2024-09-21T09:47:28.721 INFO:tasks.rados.rados.0.smithi076.stdout:2176: writing smithi07685715-4 from 3025515 to 3714512 tid 3 2024-09-21T09:47:28.721 INFO:tasks.rados.rados.0.smithi076.stdout:2177: read oid 37 snap -1 2024-09-21T09:47:28.721 INFO:tasks.rados.rados.0.smithi076.stdout:2177: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:47:28.721 INFO:tasks.rados.rados.0.smithi076.stdout:2178: snap_remove snap 269 2024-09-21T09:47:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:28 smithi191 ceph-mon[102206]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:47:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:28 smithi191 ceph-mon[102206]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:47:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:28 smithi076 ceph-mon[106482]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:47:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:28 smithi076 ceph-mon[106482]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:47:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:28 smithi076 ceph-mon[110963]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T09:47:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:28 smithi076 ceph-mon[110963]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T09:47:29.708 INFO:tasks.rados.rados.0.smithi076.stdout:2178: done (2 left) 2024-09-21T09:47:29.709 INFO:tasks.rados.rados.0.smithi076.stdout:2179: write oid 33 current snap is 283 2024-09-21T09:47:29.709 INFO:tasks.rados.rados.0.smithi076.stdout:2179: seq_num 817 ranges {531319=728204,1758965=484608,2652938=520058} 2024-09-21T09:47:29.715 INFO:tasks.rados.rados.0.smithi076.stdout:2179: writing smithi07685715-33 from 531319 to 1259523 tid 1 2024-09-21T09:47:29.721 INFO:tasks.rados.rados.0.smithi076.stdout:2179: writing smithi07685715-33 from 1758965 to 2243573 tid 2 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2179: writing smithi07685715-33 from 2652938 to 3172996 tid 3 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2180: rollback oid 32 current snap is 283 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 282 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: finishing write tid 1 to smithi07685715-4 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: finishing write tid 2 to smithi07685715-4 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: finishing write tid 3 to smithi07685715-4 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: finishing write tid 4 to smithi07685715-4 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: finishing write tid 5 to smithi07685715-4 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: oid 4 updating version 0 to 638 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: oid 4 version 638 is already newer than 635 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: oid 4 version 638 is already newer than 637 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:2176: oid 4 version 638 is already newer than 636 2024-09-21T09:47:29.727 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 638 (ObjNum 816 snap 283 seq_num 816) dirty exists 2024-09-21T09:47:29.728 INFO:tasks.rados.rados.0.smithi076.stdout:2176: left oid 4 (ObjNum 816 snap 283 seq_num 816) 2024-09-21T09:47:29.728 INFO:tasks.rados.rados.0.smithi076.stdout:2177: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:47:29.979 INFO:tasks.rados.rados.0.smithi076.stdout:2179: finishing write tid 1 to smithi07685715-33 2024-09-21T09:47:29.979 INFO:tasks.rados.rados.0.smithi076.stdout:2179: finishing write tid 2 to smithi07685715-33 2024-09-21T09:47:29.979 INFO:tasks.rados.rados.0.smithi076.stdout:2179: finishing write tid 3 to smithi07685715-33 2024-09-21T09:47:29.979 INFO:tasks.rados.rados.0.smithi076.stdout:2179: finishing write tid 4 to smithi07685715-33 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: finishing write tid 5 to smithi07685715-33 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: oid 33 updating version 0 to 522 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: oid 33 version 522 is already newer than 520 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: oid 33 version 522 is already newer than 521 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: oid 33 version 522 is already newer than 519 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 522 (ObjNum 817 snap 283 seq_num 817) dirty exists 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: left oid 33 (ObjNum 817 snap 283 seq_num 817) 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2180: finishing rollback tid 0 to smithi07685715-32 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2176: done (3 left) 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2177: done (2 left) 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2179: done (1 left) 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2181: snap_create 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2180: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:2180: finishing rollback tid 2 to smithi07685715-32 2024-09-21T09:47:29.980 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 708 (ObjNum 753 snap 258 seq_num 753) dirty exists 2024-09-21T09:47:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:29 smithi191 ceph-mon[102206]: pgmap v422: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:47:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:29 smithi191 ceph-mon[102206]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:47:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:29 smithi076 ceph-mon[106482]: pgmap v422: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:47:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:29 smithi076 ceph-mon[106482]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:47:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:29 smithi076 ceph-mon[110963]: pgmap v422: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:47:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:29 smithi076 ceph-mon[110963]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T09:47:30.761 INFO:tasks.rados.rados.0.smithi076.stdout:2180: done (1 left) 2024-09-21T09:47:30.761 INFO:tasks.rados.rados.0.smithi076.stdout:2181: done (0 left) 2024-09-21T09:47:30.761 INFO:tasks.rados.rados.0.smithi076.stdout:2182: write oid 29 current snap is 284 2024-09-21T09:47:30.761 INFO:tasks.rados.rados.0.smithi076.stdout:2182: seq_num 818 ranges {548126=403453,1417447=744619,2680361=447683,3680450=67676} 2024-09-21T09:47:30.766 INFO:tasks.rados.rados.0.smithi076.stdout:2182: writing smithi07685715-29 from 548126 to 951579 tid 1 2024-09-21T09:47:30.773 INFO:tasks.rados.rados.0.smithi076.stdout:2182: writing smithi07685715-29 from 1417447 to 2162066 tid 2 2024-09-21T09:47:30.778 INFO:tasks.rados.rados.0.smithi076.stdout:2182: writing smithi07685715-29 from 2680361 to 3128044 tid 3 2024-09-21T09:47:30.782 INFO:tasks.rados.rados.0.smithi076.stdout:2182: writing smithi07685715-29 from 3680450 to 3748126 tid 4 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2183: read oid 21 snap 249 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2183: expect (ObjNum 718 snap 246 seq_num 718) 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2184: rollback oid 46 current snap is 284 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 275 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2182: finishing write tid 1 to smithi07685715-29 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2182: finishing write tid 2 to smithi07685715-29 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2182: finishing write tid 3 to smithi07685715-29 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2185: write oid 34 current snap is 284 2024-09-21T09:47:30.783 INFO:tasks.rados.rados.0.smithi076.stdout:2185: seq_num 819 ranges {564933=478702,1759576=488277,2991431=658955} 2024-09-21T09:47:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2185: writing smithi07685715-34 from 564933 to 1043635 tid 1 2024-09-21T09:47:30.794 INFO:tasks.rados.rados.0.smithi076.stdout:2185: writing smithi07685715-34 from 1759576 to 2247853 tid 2 2024-09-21T09:47:30.802 INFO:tasks.rados.rados.0.smithi076.stdout:2185: writing smithi07685715-34 from 2991431 to 3650386 tid 3 2024-09-21T09:47:30.802 INFO:tasks.rados.rados.0.smithi076.stdout:2186: delete oid 28 current snap is 284 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: finishing write tid 4 to smithi07685715-29 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: finishing write tid 5 to smithi07685715-29 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: finishing write tid 6 to smithi07685715-29 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: oid 29 updating version 0 to 632 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: oid 29 updating version 632 to 633 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: oid 29 version 633 is already newer than 630 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: oid 29 updating version 633 to 634 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: oid 29 version 634 is already newer than 631 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 634 (ObjNum 818 snap 284 seq_num 818) dirty exists 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2182: left oid 29 (ObjNum 818 snap 284 seq_num 818) 2024-09-21T09:47:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2183: expect (ObjNum 718 snap 246 seq_num 718) 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2184: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1606 (ObjNum 225 snap 0 seq_num 864514544) dirty dne 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2185: finishing write tid 1 to smithi07685715-34 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2185: finishing write tid 2 to smithi07685715-34 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2182: done (4 left) 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2183: done (3 left) 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2184: done (2 left) 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2186: done (1 left) 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2187: snap_create 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2185: finishing write tid 3 to smithi07685715-34 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2185: finishing write tid 4 to smithi07685715-34 2024-09-21T09:47:30.929 INFO:tasks.rados.rados.0.smithi076.stdout:2185: finishing write tid 5 to smithi07685715-34 2024-09-21T09:47:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2185: oid 34 updating version 0 to 451 2024-09-21T09:47:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2185: oid 34 updating version 451 to 452 2024-09-21T09:47:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2185: oid 34 version 452 is already newer than 449 2024-09-21T09:47:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2185: oid 34 version 452 is already newer than 450 2024-09-21T09:47:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 452 (ObjNum 819 snap 284 seq_num 819) dirty exists 2024-09-21T09:47:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2185: left oid 34 (ObjNum 819 snap 284 seq_num 819) 2024-09-21T09:47:31.764 INFO:tasks.rados.rados.0.smithi076.stdout:2185: done (1 left) 2024-09-21T09:47:31.764 INFO:tasks.rados.rados.0.smithi076.stdout:2187: done (0 left) 2024-09-21T09:47:31.764 INFO:tasks.rados.rados.0.smithi076.stdout:2188: read oid 40 snap -1 2024-09-21T09:47:31.764 INFO:tasks.rados.rados.0.smithi076.stdout:2188: expect (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:47:31.764 INFO:tasks.rados.rados.0.smithi076.stdout:2189: delete oid 33 current snap is 285 2024-09-21T09:47:31.767 INFO:tasks.rados.rados.0.smithi076.stdout:2189: done (1 left) 2024-09-21T09:47:31.768 INFO:tasks.rados.rados.0.smithi076.stdout:2190: write oid 32 current snap is 285 2024-09-21T09:47:31.768 INFO:tasks.rados.rados.0.smithi076.stdout:2190: seq_num 822 ranges {615354=420802,1651375=751957,3190053=492771} 2024-09-21T09:47:31.773 INFO:tasks.rados.rados.0.smithi076.stdout:2190: writing smithi07685715-32 from 615354 to 1036156 tid 1 2024-09-21T09:47:31.781 INFO:tasks.rados.rados.0.smithi076.stdout:2190: writing smithi07685715-32 from 1651375 to 2403332 tid 2 2024-09-21T09:47:31.788 INFO:tasks.rados.rados.0.smithi076.stdout:2190: writing smithi07685715-32 from 3190053 to 3682824 tid 3 2024-09-21T09:47:31.788 INFO:tasks.rados.rados.0.smithi076.stdout:2191: snap_remove snap 276 2024-09-21T09:47:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:31 smithi191 ceph-mon[102206]: pgmap v425: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:31 smithi191 ceph-mon[102206]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:47:32.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:31 smithi076 ceph-mon[106482]: pgmap v425: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:31 smithi076 ceph-mon[106482]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:47:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:31 smithi076 ceph-mon[110963]: pgmap v425: 89 pgs: 89 active+clean; 225 MiB data, 759 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:31 smithi076 ceph-mon[110963]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T09:47:32.774 INFO:tasks.rados.rados.0.smithi076.stdout:2191: done (2 left) 2024-09-21T09:47:32.774 INFO:tasks.rados.rados.0.smithi076.stdout:2192: read oid 31 snap 282 2024-09-21T09:47:32.774 INFO:tasks.rados.rados.0.smithi076.stdout:2192: expect (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:32.774 INFO:tasks.rados.rados.0.smithi076.stdout:2193: write oid 15 current snap is 285 2024-09-21T09:47:32.775 INFO:tasks.rados.rados.0.smithi076.stdout:2193: seq_num 823 ranges {632161=496051,1593504=495615,2701123=420396,3611645=220516} 2024-09-21T09:47:32.782 INFO:tasks.rados.rados.0.smithi076.stdout:2193: writing smithi07685715-15 from 632161 to 1128212 tid 1 2024-09-21T09:47:32.787 INFO:tasks.rados.rados.0.smithi076.stdout:2193: writing smithi07685715-15 from 1593504 to 2089119 tid 2 2024-09-21T09:47:32.793 INFO:tasks.rados.rados.0.smithi076.stdout:2193: writing smithi07685715-15 from 2701123 to 3121519 tid 3 2024-09-21T09:47:32.797 INFO:tasks.rados.rados.0.smithi076.stdout:2193: writing smithi07685715-15 from 3611645 to 3832161 tid 4 2024-09-21T09:47:32.797 INFO:tasks.rados.rados.0.smithi076.stdout:2194: read oid 35 snap -1 2024-09-21T09:47:32.798 INFO:tasks.rados.rados.0.smithi076.stdout:2194: expect deleted 2024-09-21T09:47:32.798 INFO:tasks.rados.rados.0.smithi076.stdout:2195: write oid 41 current snap is 285 2024-09-21T09:47:32.798 INFO:tasks.rados.rados.0.smithi076.stdout:2195: seq_num 824 ranges {648968=571300,1651986=639273,2728546=631668} 2024-09-21T09:47:32.804 INFO:tasks.rados.rados.0.smithi076.stdout:2195: writing smithi07685715-41 from 648968 to 1220268 tid 1 2024-09-21T09:47:32.810 INFO:tasks.rados.rados.0.smithi076.stdout:2195: writing smithi07685715-41 from 1651986 to 2291259 tid 2 2024-09-21T09:47:32.816 INFO:tasks.rados.rados.0.smithi076.stdout:2195: writing smithi07685715-41 from 2728546 to 3360214 tid 3 2024-09-21T09:47:32.816 INFO:tasks.rados.rados.0.smithi076.stdout:2196: delete oid 24 current snap is 285 2024-09-21T09:47:32.816 INFO:tasks.rados.rados.0.smithi076.stdout:2188: expect (ObjNum 745 snap 255 seq_num 745) 2024-09-21T09:47:32.929 INFO:tasks.rados.rados.0.smithi076.stdout:2190: finishing write tid 1 to smithi07685715-32 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: finishing write tid 2 to smithi07685715-32 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: finishing write tid 3 to smithi07685715-32 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: finishing write tid 4 to smithi07685715-32 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: finishing write tid 5 to smithi07685715-32 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: oid 32 updating version 0 to 711 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: oid 32 updating version 711 to 712 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: oid 32 updating version 712 to 713 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: oid 32 version 713 is already newer than 710 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 713 (ObjNum 822 snap 285 seq_num 822) dirty exists 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2190: left oid 32 (ObjNum 822 snap 285 seq_num 822) 2024-09-21T09:47:32.930 INFO:tasks.rados.rados.0.smithi076.stdout:2192: expect (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:33.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:32 smithi076 ceph-mon[106482]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:47:33.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:32 smithi076 ceph-mon[110963]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: finishing write tid 1 to smithi07685715-15 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: finishing write tid 2 to smithi07685715-15 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: finishing write tid 3 to smithi07685715-15 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: finishing write tid 4 to smithi07685715-15 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: finishing write tid 5 to smithi07685715-15 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: finishing write tid 6 to smithi07685715-15 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: oid 15 updating version 0 to 463 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: oid 15 updating version 463 to 464 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: oid 15 version 464 is already newer than 462 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: oid 15 version 464 is already newer than 461 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: oid 15 updating version 464 to 465 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 465 (ObjNum 823 snap 285 seq_num 823) dirty exists 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2193: left oid 15 (ObjNum 823 snap 285 seq_num 823) 2024-09-21T09:47:33.039 INFO:tasks.rados.rados.0.smithi076.stdout:2195: finishing write tid 1 to smithi07685715-41 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: finishing write tid 2 to smithi07685715-41 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: finishing write tid 3 to smithi07685715-41 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: finishing write tid 4 to smithi07685715-41 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: finishing write tid 5 to smithi07685715-41 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: oid 41 updating version 0 to 929 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: oid 41 updating version 929 to 930 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: oid 41 version 930 is already newer than 928 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: oid 41 updating version 930 to 931 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 931 (ObjNum 824 snap 285 seq_num 824) dirty exists 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: left oid 41 (ObjNum 824 snap 285 seq_num 824) 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2188: done (6 left) 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2190: done (5 left) 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2192: done (4 left) 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2193: done (3 left) 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2194: done (2 left) 2024-09-21T09:47:33.040 INFO:tasks.rados.rados.0.smithi076.stdout:2195: done (1 left) 2024-09-21T09:47:33.041 INFO:tasks.rados.rados.0.smithi076.stdout:2196: done (0 left) 2024-09-21T09:47:33.041 INFO:tasks.rados.rados.0.smithi076.stdout:2197: snap_remove snap 245 2024-09-21T09:47:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:32 smithi191 ceph-mon[102206]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T09:47:33.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:47:33] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.51.0" 2024-09-21T09:47:33.705 INFO:tasks.rados.rados.0.smithi076.stdout:2197: done (0 left) 2024-09-21T09:47:33.705 INFO:tasks.rados.rados.0.smithi076.stdout:2198: write oid 25 current snap is 285 2024-09-21T09:47:33.705 INFO:tasks.rados.rados.0.smithi076.stdout:2198: seq_num 826 ranges {682582=721798,2168950=642942,3583392=299190} 2024-09-21T09:47:33.712 INFO:tasks.rados.rados.0.smithi076.stdout:2198: writing smithi07685715-25 from 682582 to 1404380 tid 1 2024-09-21T09:47:33.721 INFO:tasks.rados.rados.0.smithi076.stdout:2198: writing smithi07685715-25 from 2168950 to 2811892 tid 2 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2198: writing smithi07685715-25 from 3583392 to 3882582 tid 3 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2199: read oid 33 snap -1 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2199: expect deleted 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2200: write oid 50 current snap is 285 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2198: finishing write tid 1 to smithi07685715-25 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2198: finishing write tid 2 to smithi07685715-25 2024-09-21T09:47:33.727 INFO:tasks.rados.rados.0.smithi076.stdout:2200: seq_num 827 ranges {699389=797047,2111079=502953,3094462=581837} 2024-09-21T09:47:33.735 INFO:tasks.rados.rados.0.smithi076.stdout:2200: writing smithi07685715-50 from 699389 to 1496436 tid 1 2024-09-21T09:47:33.741 INFO:tasks.rados.rados.0.smithi076.stdout:2200: writing smithi07685715-50 from 2111079 to 2614032 tid 2 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2200: writing smithi07685715-50 from 3094462 to 3676299 tid 3 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2201: delete oid 42 current snap is 285 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: finishing write tid 3 to smithi07685715-25 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: finishing write tid 4 to smithi07685715-25 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: finishing write tid 5 to smithi07685715-25 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: oid 25 updating version 0 to 533 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: oid 25 updating version 533 to 534 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: oid 25 updating version 534 to 535 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2198: oid 25 version 535 is already newer than 532 2024-09-21T09:47:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 535 (ObjNum 826 snap 285 seq_num 826) dirty exists 2024-09-21T09:47:33.748 INFO:tasks.rados.rados.0.smithi076.stdout:2198: left oid 25 (ObjNum 826 snap 285 seq_num 826) 2024-09-21T09:47:33.748 INFO:tasks.rados.rados.0.smithi076.stdout:2200: finishing write tid 1 to smithi07685715-50 2024-09-21T09:47:33.748 INFO:tasks.rados.rados.0.smithi076.stdout:2200: finishing write tid 2 to smithi07685715-50 2024-09-21T09:47:33.749 INFO:tasks.rados.rados.0.smithi076.stdout:2198: done (3 left) 2024-09-21T09:47:33.749 INFO:tasks.rados.rados.0.smithi076.stdout:2199: done (2 left) 2024-09-21T09:47:33.749 INFO:tasks.rados.rados.0.smithi076.stdout:2201: done (1 left) 2024-09-21T09:47:33.749 INFO:tasks.rados.rados.0.smithi076.stdout:2202: write oid 6 current snap is 285 2024-09-21T09:47:33.749 INFO:tasks.rados.rados.0.smithi076.stdout:2202: seq_num 829 ranges {733003=663898,1944396=506622,2865661=604381} 2024-09-21T09:47:33.756 INFO:tasks.rados.rados.0.smithi076.stdout:2202: writing smithi07685715-6 from 733003 to 1396901 tid 1 2024-09-21T09:47:33.762 INFO:tasks.rados.rados.0.smithi076.stdout:2202: writing smithi07685715-6 from 1944396 to 2451018 tid 2 2024-09-21T09:47:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:2202: writing smithi07685715-6 from 2865661 to 3470042 tid 3 2024-09-21T09:47:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:2203: rollback oid 12 current snap is 285 2024-09-21T09:47:33.768 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 284 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: finishing write tid 3 to smithi07685715-50 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: finishing write tid 4 to smithi07685715-50 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: finishing write tid 5 to smithi07685715-50 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: oid 50 updating version 0 to 649 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: oid 50 version 649 is already newer than 647 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: oid 50 version 649 is already newer than 648 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: oid 50 version 649 is already newer than 646 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 649 (ObjNum 827 snap 285 seq_num 827) dirty exists 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: left oid 50 (ObjNum 827 snap 285 seq_num 827) 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2202: finishing write tid 1 to smithi07685715-6 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2202: finishing write tid 2 to smithi07685715-6 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2200: done (2 left) 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2204: read oid 34 snap 280 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2204: expect deleted 2024-09-21T09:47:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2205: snap_create 2024-09-21T09:47:33.770 INFO:tasks.rados.rados.0.smithi076.stdout:2202: finishing write tid 3 to smithi07685715-6 2024-09-21T09:47:33.771 INFO:tasks.rados.rados.0.smithi076.stdout:2203: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:47:33.771 INFO:tasks.rados.rados.0.smithi076.stdout:2203: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:47:33.771 INFO:tasks.rados.rados.0.smithi076.stdout:2203: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:47:33.771 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 782 (ObjNum 575 snap 182 seq_num 575) dirty exists 2024-09-21T09:47:33.772 INFO:tasks.rados.rados.0.smithi076.stdout:2202: finishing write tid 4 to smithi07685715-6 2024-09-21T09:47:33.772 INFO:tasks.rados.rados.0.smithi076.stdout:2202: finishing write tid 5 to smithi07685715-6 2024-09-21T09:47:33.772 INFO:tasks.rados.rados.0.smithi076.stdout:2202: oid 6 updating version 0 to 487 2024-09-21T09:47:33.772 INFO:tasks.rados.rados.0.smithi076.stdout:2202: oid 6 version 487 is already newer than 486 2024-09-21T09:47:33.773 INFO:tasks.rados.rados.0.smithi076.stdout:2202: oid 6 updating version 487 to 489 2024-09-21T09:47:33.773 INFO:tasks.rados.rados.0.smithi076.stdout:2202: oid 6 version 489 is already newer than 488 2024-09-21T09:47:33.773 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 489 (ObjNum 829 snap 285 seq_num 829) dirty exists 2024-09-21T09:47:33.773 INFO:tasks.rados.rados.0.smithi076.stdout:2202: left oid 6 (ObjNum 829 snap 285 seq_num 829) 2024-09-21T09:47:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-mon[106482]: pgmap v428: 89 pgs: 89 active+clean; 237 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:47:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-mon[106482]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:47:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-mon[106482]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:47:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-mon[110963]: pgmap v428: 89 pgs: 89 active+clean; 237 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:47:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-mon[110963]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:47:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:33 smithi076 ceph-mon[110963]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:47:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:33 smithi191 ceph-mon[102206]: pgmap v428: 89 pgs: 89 active+clean; 237 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:47:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:33 smithi191 ceph-mon[102206]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T09:47:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:33 smithi191 ceph-mon[102206]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T09:47:34.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'sleep 60' 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:2202: done (3 left) 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:2203: done (2 left) 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:2204: done (1 left) 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:2205: done (0 left) 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:2206: rollback oid 48 current snap is 286 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 283 2024-09-21T09:47:34.706 INFO:tasks.rados.rados.0.smithi076.stdout:2207: snap_remove snap 277 2024-09-21T09:47:34.755 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:34 smithi076 ceph-mon[106482]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:34 smithi076 ceph-mon[106482]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:47:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:34 smithi076 ceph-mon[110963]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:34 smithi076 ceph-mon[110963]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:47:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:34 smithi191 ceph-mon[102206]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:34 smithi191 ceph-mon[102206]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T09:47:35.707 INFO:tasks.rados.rados.0.smithi076.stdout:2207: done (1 left) 2024-09-21T09:47:35.707 INFO:tasks.rados.rados.0.smithi076.stdout:2208: delete oid 33 current snap is 286 2024-09-21T09:47:35.707 INFO:tasks.rados.rados.0.smithi076.stdout:2206: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:47:35.707 INFO:tasks.rados.rados.0.smithi076.stdout:2206: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:47:35.707 INFO:tasks.rados.rados.0.smithi076.stdout:2206: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:47:35.707 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1760 (ObjNum 809 snap 282 seq_num 809) dirty exists 2024-09-21T09:47:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:2206: done (1 left) 2024-09-21T09:47:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:2208: done (0 left) 2024-09-21T09:47:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:2209: read oid 35 snap -1 2024-09-21T09:47:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:2209: expect deleted 2024-09-21T09:47:35.709 INFO:tasks.rados.rados.0.smithi076.stdout:2210: read oid 30 snap -1 2024-09-21T09:47:35.710 INFO:tasks.rados.rados.0.smithi076.stdout:2210: expect (ObjNum 711 snap 236 seq_num 711) 2024-09-21T09:47:35.710 INFO:tasks.rados.rados.0.smithi076.stdout:2211: snap_create 2024-09-21T09:47:35.728 INFO:tasks.rados.rados.0.smithi076.stdout:2210: expect (ObjNum 711 snap 236 seq_num 711) 2024-09-21T09:47:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:35 smithi076 ceph-mon[106482]: pgmap v431: 89 pgs: 89 active+clean; 237 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:47:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:35 smithi076 ceph-mon[106482]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:47:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:35 smithi076 ceph-mon[110963]: pgmap v431: 89 pgs: 89 active+clean; 237 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:47:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:35 smithi076 ceph-mon[110963]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:47:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:35 smithi191 ceph-mon[102206]: pgmap v431: 89 pgs: 89 active+clean; 237 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 2.1 MiB/s wr, 15 op/s 2024-09-21T09:47:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:35 smithi191 ceph-mon[102206]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T09:47:36.708 INFO:tasks.rados.rados.0.smithi076.stdout:2209: done (2 left) 2024-09-21T09:47:36.708 INFO:tasks.rados.rados.0.smithi076.stdout:2210: done (1 left) 2024-09-21T09:47:36.708 INFO:tasks.rados.rados.0.smithi076.stdout:2211: done (0 left) 2024-09-21T09:47:36.709 INFO:tasks.rados.rados.0.smithi076.stdout:2212: snap_create 2024-09-21T09:47:37.714 INFO:tasks.rados.rados.0.smithi076.stdout:2212: done (0 left) 2024-09-21T09:47:37.714 INFO:tasks.rados.rados.0.smithi076.stdout:2213: rollback oid 14 current snap is 288 2024-09-21T09:47:37.714 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 285 2024-09-21T09:47:37.714 INFO:tasks.rados.rados.0.smithi076.stdout:2214: snap_create 2024-09-21T09:47:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:2213: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:47:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:2213: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:47:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:2213: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:47:37.718 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1259 (ObjNum 721 snap 247 seq_num 721) dirty exists 2024-09-21T09:47:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:37 smithi191 ceph-mon[102206]: pgmap v434: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:47:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:37 smithi191 ceph-mon[102206]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:47:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:37 smithi076 ceph-mon[106482]: pgmap v434: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:47:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:37 smithi076 ceph-mon[106482]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:47:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:37 smithi076 ceph-mon[110963]: pgmap v434: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:47:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:37 smithi076 ceph-mon[110963]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T09:47:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2213: done (1 left) 2024-09-21T09:47:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2214: done (0 left) 2024-09-21T09:47:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2215: rollback oid 41 current snap is 289 2024-09-21T09:47:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 284 2024-09-21T09:47:38.705 INFO:tasks.rados.rados.0.smithi076.stdout:2216: snap_remove snap 279 2024-09-21T09:47:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:38 smithi191 ceph-mon[102206]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:47:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:38 smithi191 ceph-mon[102206]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:47:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:38 smithi076 ceph-mon[106482]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:47:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:38 smithi076 ceph-mon[106482]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:47:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:38 smithi076 ceph-mon[110963]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T09:47:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:38 smithi076 ceph-mon[110963]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T09:47:39.708 INFO:tasks.rados.rados.0.smithi076.stdout:2216: done (1 left) 2024-09-21T09:47:39.708 INFO:tasks.rados.rados.0.smithi076.stdout:2217: read oid 21 snap -1 2024-09-21T09:47:39.708 INFO:tasks.rados.rados.0.smithi076.stdout:2217: expect (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:47:39.708 INFO:tasks.rados.rados.0.smithi076.stdout:2218: write oid 44 current snap is 289 2024-09-21T09:47:39.708 INFO:tasks.rados.rados.0.smithi076.stdout:2218: seq_num 831 ranges {766617=414396,1945007=510291,3204154=626925} 2024-09-21T09:47:39.714 INFO:tasks.rados.rados.0.smithi076.stdout:2218: writing smithi07685715-44 from 766617 to 1181013 tid 1 2024-09-21T09:47:39.722 INFO:tasks.rados.rados.0.smithi076.stdout:2218: writing smithi07685715-44 from 1945007 to 2455298 tid 2 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:2218: writing smithi07685715-44 from 3204154 to 3831079 tid 3 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:2219: rollback oid 29 current snap is 289 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 286 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:2215: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:2215: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 940 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:2215: done (3 left) 2024-09-21T09:47:39.731 INFO:tasks.rados.rados.0.smithi076.stdout:2220: write oid 46 current snap is 289 2024-09-21T09:47:39.732 INFO:tasks.rados.rados.0.smithi076.stdout:2218: finishing write tid 1 to smithi07685715-44 2024-09-21T09:47:39.732 INFO:tasks.rados.rados.0.smithi076.stdout:2217: expect (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:47:40.002 INFO:tasks.rados.rados.0.smithi076.stdout:2218: finishing write tid 2 to smithi07685715-44 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2219: finishing rollback tid 0 to smithi07685715-29 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2219: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: finishing write tid 3 to smithi07685715-44 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: finishing write tid 4 to smithi07685715-44 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2219: finishing rollback tid 2 to smithi07685715-29 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 648 (ObjNum 818 snap 284 seq_num 818) dirty exists 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: finishing write tid 5 to smithi07685715-44 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: oid 44 updating version 0 to 497 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: oid 44 version 497 is already newer than 496 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: oid 44 version 497 is already newer than 495 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: oid 44 updating version 497 to 498 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 498 (ObjNum 831 snap 289 seq_num 831) dirty exists 2024-09-21T09:47:40.003 INFO:tasks.rados.rados.0.smithi076.stdout:2218: left oid 44 (ObjNum 831 snap 289 seq_num 831) 2024-09-21T09:47:40.004 INFO:tasks.rados.rados.0.smithi076.stdout:2220: seq_num 832 ranges {783424=489645,2003489=653949,3231577=554550} 2024-09-21T09:47:40.009 INFO:tasks.rados.rados.0.smithi076.stdout:2220: writing smithi07685715-46 from 783424 to 1273069 tid 1 2024-09-21T09:47:40.016 INFO:tasks.rados.rados.0.smithi076.stdout:2220: writing smithi07685715-46 from 2003489 to 2657438 tid 2 2024-09-21T09:47:40.022 INFO:tasks.rados.rados.0.smithi076.stdout:2220: writing smithi07685715-46 from 3231577 to 3786127 tid 3 2024-09-21T09:47:40.022 INFO:tasks.rados.rados.0.smithi076.stdout:2217: done (3 left) 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2218: done (2 left) 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2219: done (1 left) 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2221: read oid 31 snap -1 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2221: expect (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2222: snap_create 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2220: finishing write tid 1 to smithi07685715-46 2024-09-21T09:47:40.023 INFO:tasks.rados.rados.0.smithi076.stdout:2220: finishing write tid 2 to smithi07685715-46 2024-09-21T09:47:40.024 INFO:tasks.rados.rados.0.smithi076.stdout:2221: expect (ObjNum 799 snap 277 seq_num 799) 2024-09-21T09:47:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:39 smithi191 ceph-mon[102206]: pgmap v437: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:47:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:39 smithi191 ceph-mon[102206]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:47:40.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:39 smithi076 ceph-mon[110963]: pgmap v437: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:47:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:39 smithi076 ceph-mon[110963]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:47:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:39 smithi076 ceph-mon[106482]: pgmap v437: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:47:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:39 smithi076 ceph-mon[106482]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: finishing write tid 3 to smithi07685715-46 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: finishing write tid 4 to smithi07685715-46 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: finishing write tid 5 to smithi07685715-46 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: oid 46 updating version 0 to 1698 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: oid 46 version 1698 is already newer than 1696 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: oid 46 version 1698 is already newer than 1695 2024-09-21T09:47:40.135 INFO:tasks.rados.rados.0.smithi076.stdout:2220: oid 46 version 1698 is already newer than 1697 2024-09-21T09:47:40.136 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1698 (ObjNum 832 snap 289 seq_num 832) dirty exists 2024-09-21T09:47:40.136 INFO:tasks.rados.rados.0.smithi076.stdout:2220: left oid 46 (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:47:40.733 INFO:tasks.rados.rados.0.smithi076.stdout:2220: done (2 left) 2024-09-21T09:47:40.733 INFO:tasks.rados.rados.0.smithi076.stdout:2221: done (1 left) 2024-09-21T09:47:40.734 INFO:tasks.rados.rados.0.smithi076.stdout:2222: done (0 left) 2024-09-21T09:47:40.734 INFO:tasks.rados.rados.0.smithi076.stdout:2223: write oid 44 current snap is 290 2024-09-21T09:47:40.734 INFO:tasks.rados.rados.0.smithi076.stdout:2223: seq_num 833 ranges {400231=564894,1661971=338260} 2024-09-21T09:47:40.738 INFO:tasks.rados.rados.0.smithi076.stdout:2223: writing smithi07685715-44 from 400231 to 965125 tid 1 2024-09-21T09:47:40.744 INFO:tasks.rados.rados.0.smithi076.stdout:2223: writing smithi07685715-44 from 1661971 to 2000231 tid 2 2024-09-21T09:47:40.744 INFO:tasks.rados.rados.0.smithi076.stdout:2224: delete oid 13 current snap is 290 2024-09-21T09:47:40.744 INFO:tasks.rados.rados.0.smithi076.stdout:2223: finishing write tid 1 to smithi07685715-44 2024-09-21T09:47:40.746 INFO:tasks.rados.rados.0.smithi076.stdout:2224: done (1 left) 2024-09-21T09:47:40.746 INFO:tasks.rados.rados.0.smithi076.stdout:2225: read oid 46 snap -1 2024-09-21T09:47:40.746 INFO:tasks.rados.rados.0.smithi076.stdout:2225: expect (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:47:40.746 INFO:tasks.rados.rados.0.smithi076.stdout:2226: read oid 4 snap -1 2024-09-21T09:47:40.746 INFO:tasks.rados.rados.0.smithi076.stdout:2226: expect (ObjNum 816 snap 283 seq_num 816) 2024-09-21T09:47:40.746 INFO:tasks.rados.rados.0.smithi076.stdout:2227: snap_remove snap 282 2024-09-21T09:47:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: finishing write tid 2 to smithi07685715-44 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: finishing write tid 3 to smithi07685715-44 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: finishing write tid 4 to smithi07685715-44 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: oid 44 updating version 0 to 502 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: oid 44 version 502 is already newer than 501 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: oid 44 version 502 is already newer than 500 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 502 (ObjNum 833 snap 290 seq_num 833) dirty exists 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2223: left oid 44 (ObjNum 833 snap 290 seq_num 833) 2024-09-21T09:47:41.742 INFO:tasks.rados.rados.0.smithi076.stdout:2226: expect (ObjNum 816 snap 283 seq_num 816) 2024-09-21T09:47:41.859 INFO:tasks.rados.rados.0.smithi076.stdout:2225: expect (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:47:41.987 INFO:tasks.rados.rados.0.smithi076.stdout:2223: done (3 left) 2024-09-21T09:47:41.987 INFO:tasks.rados.rados.0.smithi076.stdout:2225: done (2 left) 2024-09-21T09:47:41.987 INFO:tasks.rados.rados.0.smithi076.stdout:2226: done (1 left) 2024-09-21T09:47:41.987 INFO:tasks.rados.rados.0.smithi076.stdout:2227: done (0 left) 2024-09-21T09:47:41.987 INFO:tasks.rados.rados.0.smithi076.stdout:2228: snap_remove snap 249 2024-09-21T09:47:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:41 smithi191 ceph-mon[102206]: pgmap v440: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:41 smithi191 ceph-mon[102206]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:47:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:41 smithi076 ceph-mon[106482]: pgmap v440: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:41 smithi076 ceph-mon[106482]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:47:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:41 smithi076 ceph-mon[110963]: pgmap v440: 89 pgs: 89 active+clean; 221 MiB data, 779 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:41 smithi076 ceph-mon[110963]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T09:47:42.747 INFO:tasks.rados.rados.0.smithi076.stdout:2228: done (0 left) 2024-09-21T09:47:42.747 INFO:tasks.rados.rados.0.smithi076.stdout:2229: write oid 18 current snap is 290 2024-09-21T09:47:42.747 INFO:tasks.rados.rados.0.smithi076.stdout:2229: seq_num 835 ranges {433845=715392,1662582=371263} 2024-09-21T09:47:42.754 INFO:tasks.rados.rados.0.smithi076.stdout:2229: writing smithi07685715-18 from 433845 to 1149237 tid 1 2024-09-21T09:47:42.760 INFO:tasks.rados.rados.0.smithi076.stdout:2229: writing smithi07685715-18 from 1662582 to 2033845 tid 2 2024-09-21T09:47:42.760 INFO:tasks.rados.rados.0.smithi076.stdout:2230: snap_remove snap 280 2024-09-21T09:47:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:42 smithi191 ceph-mon[102206]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:47:43.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:42 smithi076 ceph-mon[106482]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:47:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:42 smithi076 ceph-mon[110963]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T09:47:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:47:43] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.51.0" 2024-09-21T09:47:43.709 INFO:tasks.rados.rados.0.smithi076.stdout:2230: done (1 left) 2024-09-21T09:47:43.709 INFO:tasks.rados.rados.0.smithi076.stdout:2231: write oid 34 current snap is 290 2024-09-21T09:47:43.709 INFO:tasks.rados.rados.0.smithi076.stdout:2229: finishing write tid 1 to smithi07685715-18 2024-09-21T09:47:43.710 INFO:tasks.rados.rados.0.smithi076.stdout:2229: finishing write tid 2 to smithi07685715-18 2024-09-21T09:47:43.710 INFO:tasks.rados.rados.0.smithi076.stdout:2229: finishing write tid 3 to smithi07685715-18 2024-09-21T09:47:43.710 INFO:tasks.rados.rados.0.smithi076.stdout:2229: finishing write tid 4 to smithi07685715-18 2024-09-21T09:47:43.710 INFO:tasks.rados.rados.0.smithi076.stdout:2229: oid 18 updating version 0 to 649 2024-09-21T09:47:43.710 INFO:tasks.rados.rados.0.smithi076.stdout:2229: oid 18 version 649 is already newer than 648 2024-09-21T09:47:43.710 INFO:tasks.rados.rados.0.smithi076.stdout:2229: oid 18 version 649 is already newer than 647 2024-09-21T09:47:43.711 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 649 (ObjNum 835 snap 290 seq_num 835) dirty exists 2024-09-21T09:47:43.711 INFO:tasks.rados.rados.0.smithi076.stdout:2229: left oid 18 (ObjNum 835 snap 290 seq_num 835) 2024-09-21T09:47:43.711 INFO:tasks.rados.rados.0.smithi076.stdout:2231: seq_num 836 ranges {450652=790641,1721064=329588} 2024-09-21T09:47:43.716 INFO:tasks.rados.rados.0.smithi076.stdout:2231: writing smithi07685715-34 from 450652 to 1241293 tid 1 2024-09-21T09:47:43.721 INFO:tasks.rados.rados.0.smithi076.stdout:2231: writing smithi07685715-34 from 1721064 to 2050652 tid 2 2024-09-21T09:47:43.722 INFO:tasks.rados.rados.0.smithi076.stdout:2229: done (1 left) 2024-09-21T09:47:43.722 INFO:tasks.rados.rados.0.smithi076.stdout:2232: delete oid 28 current snap is 290 2024-09-21T09:47:43.722 INFO:tasks.rados.rados.0.smithi076.stdout:2231: finishing write tid 1 to smithi07685715-34 2024-09-21T09:47:43.725 INFO:tasks.rados.rados.0.smithi076.stdout:2231: finishing write tid 2 to smithi07685715-34 2024-09-21T09:47:43.725 INFO:tasks.rados.rados.0.smithi076.stdout:2231: finishing write tid 3 to smithi07685715-34 2024-09-21T09:47:43.725 INFO:tasks.rados.rados.0.smithi076.stdout:2231: finishing write tid 4 to smithi07685715-34 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2231: oid 34 updating version 0 to 471 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2231: oid 34 updating version 471 to 472 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2231: oid 34 version 472 is already newer than 470 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 472 (ObjNum 836 snap 290 seq_num 836) dirty exists 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2231: left oid 34 (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2231: done (1 left) 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2232: done (0 left) 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2233: read oid 16 snap -1 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2233: expect (ObjNum 807 snap 282 seq_num 807) 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2234: write oid 27 current snap is 290 2024-09-21T09:47:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2234: seq_num 838 ranges {484266=657492,1554381=529885} 2024-09-21T09:47:43.730 INFO:tasks.rados.rados.0.smithi076.stdout:2234: writing smithi07685715-27 from 484266 to 1141758 tid 1 2024-09-21T09:47:43.735 INFO:tasks.rados.rados.0.smithi076.stdout:2234: writing smithi07685715-27 from 1554381 to 2084266 tid 2 2024-09-21T09:47:43.736 INFO:tasks.rados.rados.0.smithi076.stdout:2235: snap_create 2024-09-21T09:47:43.738 INFO:tasks.rados.rados.0.smithi076.stdout:2233: expect (ObjNum 807 snap 282 seq_num 807) 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: finishing write tid 1 to smithi07685715-27 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: finishing write tid 2 to smithi07685715-27 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: finishing write tid 3 to smithi07685715-27 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: finishing write tid 4 to smithi07685715-27 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: oid 27 updating version 0 to 953 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: oid 27 version 953 is already newer than 952 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: oid 27 version 953 is already newer than 951 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 953 (ObjNum 838 snap 290 seq_num 838) dirty exists 2024-09-21T09:47:43.855 INFO:tasks.rados.rados.0.smithi076.stdout:2234: left oid 27 (ObjNum 838 snap 290 seq_num 838) 2024-09-21T09:47:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:43 smithi191 ceph-mon[102206]: pgmap v443: 89 pgs: 89 active+clean; 227 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1015 KiB/s wr, 13 op/s 2024-09-21T09:47:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:43 smithi191 ceph-mon[102206]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:47:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:43 smithi191 ceph-mon[102206]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:47:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-mon[106482]: pgmap v443: 89 pgs: 89 active+clean; 227 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1015 KiB/s wr, 13 op/s 2024-09-21T09:47:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-mon[106482]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:47:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-mon[106482]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:47:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-mon[110963]: pgmap v443: 89 pgs: 89 active+clean; 227 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1015 KiB/s wr, 13 op/s 2024-09-21T09:47:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-mon[110963]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T09:47:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:43 smithi076 ceph-mon[110963]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T09:47:44.708 INFO:tasks.rados.rados.0.smithi076.stdout:2233: done (2 left) 2024-09-21T09:47:44.708 INFO:tasks.rados.rados.0.smithi076.stdout:2234: done (1 left) 2024-09-21T09:47:44.708 INFO:tasks.rados.rados.0.smithi076.stdout:2235: done (0 left) 2024-09-21T09:47:44.708 INFO:tasks.rados.rados.0.smithi076.stdout:2236: write oid 12 current snap is 291 2024-09-21T09:47:44.708 INFO:tasks.rados.rados.0.smithi076.stdout:2236: seq_num 839 ranges {501073=732741,1896510=204563} 2024-09-21T09:47:44.714 INFO:tasks.rados.rados.0.smithi076.stdout:2236: writing smithi07685715-12 from 501073 to 1233814 tid 1 2024-09-21T09:47:44.719 INFO:tasks.rados.rados.0.smithi076.stdout:2236: writing smithi07685715-12 from 1896510 to 2101073 tid 2 2024-09-21T09:47:44.719 INFO:tasks.rados.rados.0.smithi076.stdout:2237: write oid 38 current snap is 291 2024-09-21T09:47:44.720 INFO:tasks.rados.rados.0.smithi076.stdout:2236: finishing write tid 1 to smithi07685715-12 2024-09-21T09:47:44.720 INFO:tasks.rados.rados.0.smithi076.stdout:2237: seq_num 840 ranges {517880=407990,1554992=562888} 2024-09-21T09:47:44.724 INFO:tasks.rados.rados.0.smithi076.stdout:2237: writing smithi07685715-38 from 517880 to 925870 tid 1 2024-09-21T09:47:44.731 INFO:tasks.rados.rados.0.smithi076.stdout:2237: writing smithi07685715-38 from 1554992 to 2117880 tid 2 2024-09-21T09:47:44.731 INFO:tasks.rados.rados.0.smithi076.stdout:2238: delete oid 39 current snap is 291 2024-09-21T09:47:44.731 INFO:tasks.rados.rados.0.smithi076.stdout:2236: finishing write tid 2 to smithi07685715-12 2024-09-21T09:47:44.731 INFO:tasks.rados.rados.0.smithi076.stdout:2236: finishing write tid 3 to smithi07685715-12 2024-09-21T09:47:44.731 INFO:tasks.rados.rados.0.smithi076.stdout:2236: finishing write tid 4 to smithi07685715-12 2024-09-21T09:47:44.732 INFO:tasks.rados.rados.0.smithi076.stdout:2236: oid 12 updating version 0 to 800 2024-09-21T09:47:44.732 INFO:tasks.rados.rados.0.smithi076.stdout:2236: oid 12 version 800 is already newer than 799 2024-09-21T09:47:44.732 INFO:tasks.rados.rados.0.smithi076.stdout:2236: oid 12 version 800 is already newer than 798 2024-09-21T09:47:44.732 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 800 (ObjNum 839 snap 291 seq_num 839) dirty exists 2024-09-21T09:47:44.732 INFO:tasks.rados.rados.0.smithi076.stdout:2236: left oid 12 (ObjNum 839 snap 291 seq_num 839) 2024-09-21T09:47:44.732 INFO:tasks.rados.rados.0.smithi076.stdout:2237: finishing write tid 1 to smithi07685715-38 2024-09-21T09:47:44.735 INFO:tasks.rados.rados.0.smithi076.stdout:2236: done (2 left) 2024-09-21T09:47:44.735 INFO:tasks.rados.rados.0.smithi076.stdout:2238: done (1 left) 2024-09-21T09:47:44.735 INFO:tasks.rados.rados.0.smithi076.stdout:2239: write oid 30 current snap is 291 2024-09-21T09:47:44.735 INFO:tasks.rados.rados.0.smithi076.stdout:2239: seq_num 842 ranges {551494=558488,1671956=479538} 2024-09-21T09:47:44.741 INFO:tasks.rados.rados.0.smithi076.stdout:2239: writing smithi07685715-30 from 551494 to 1109982 tid 1 2024-09-21T09:47:44.747 INFO:tasks.rados.rados.0.smithi076.stdout:2239: writing smithi07685715-30 from 1671956 to 2151494 tid 2 2024-09-21T09:47:44.747 INFO:tasks.rados.rados.0.smithi076.stdout:2240: snap_create 2024-09-21T09:47:44.747 INFO:tasks.rados.rados.0.smithi076.stdout:2237: finishing write tid 2 to smithi07685715-38 2024-09-21T09:47:44.747 INFO:tasks.rados.rados.0.smithi076.stdout:2237: finishing write tid 3 to smithi07685715-38 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:2237: finishing write tid 4 to smithi07685715-38 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:2237: oid 38 updating version 0 to 1279 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:2237: oid 38 version 1279 is already newer than 1278 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:2237: oid 38 updating version 1279 to 1280 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1280 (ObjNum 840 snap 291 seq_num 840) dirty exists 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:2237: left oid 38 (ObjNum 840 snap 291 seq_num 840) 2024-09-21T09:47:44.748 INFO:tasks.rados.rados.0.smithi076.stdout:2239: finishing write tid 1 to smithi07685715-30 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: finishing write tid 2 to smithi07685715-30 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: finishing write tid 3 to smithi07685715-30 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: finishing write tid 4 to smithi07685715-30 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: oid 30 updating version 0 to 1781 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: oid 30 updating version 1781 to 1782 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: oid 30 updating version 1782 to 1783 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1783 (ObjNum 842 snap 291 seq_num 842) dirty exists 2024-09-21T09:47:44.751 INFO:tasks.rados.rados.0.smithi076.stdout:2239: left oid 30 (ObjNum 842 snap 291 seq_num 842) 2024-09-21T09:47:45.756 INFO:tasks.rados.rados.0.smithi076.stdout:2237: done (2 left) 2024-09-21T09:47:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2239: done (1 left) 2024-09-21T09:47:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2240: done (0 left) 2024-09-21T09:47:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2241: read oid 46 snap -1 2024-09-21T09:47:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2241: expect (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:47:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2242: write oid 5 current snap is 292 2024-09-21T09:47:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2242: seq_num 843 ranges {568301=633737,1614085=415952} 2024-09-21T09:47:45.763 INFO:tasks.rados.rados.0.smithi076.stdout:2242: writing smithi07685715-5 from 568301 to 1202038 tid 1 2024-09-21T09:47:45.767 INFO:tasks.rados.rados.0.smithi076.stdout:2242: writing smithi07685715-5 from 1614085 to 2030037 tid 2 2024-09-21T09:47:45.767 INFO:tasks.rados.rados.0.smithi076.stdout:2243: write oid 37 current snap is 292 2024-09-21T09:47:45.767 INFO:tasks.rados.rados.0.smithi076.stdout:2243: seq_num 844 ranges {585108=425339,1788920=396188} 2024-09-21T09:47:45.773 INFO:tasks.rados.rados.0.smithi076.stdout:2243: writing smithi07685715-37 from 585108 to 1010447 tid 1 2024-09-21T09:47:45.779 INFO:tasks.rados.rados.0.smithi076.stdout:2243: writing smithi07685715-37 from 1788920 to 2185108 tid 2 2024-09-21T09:47:45.779 INFO:tasks.rados.rados.0.smithi076.stdout:2244: delete oid 16 current snap is 292 2024-09-21T09:47:45.779 INFO:tasks.rados.rados.0.smithi076.stdout:2241: expect (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:47:45.908 INFO:tasks.rados.rados.0.smithi076.stdout:2242: finishing write tid 1 to smithi07685715-5 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: finishing write tid 2 to smithi07685715-5 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: finishing write tid 3 to smithi07685715-5 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: finishing write tid 4 to smithi07685715-5 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: oid 5 updating version 0 to 1712 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: oid 5 version 1712 is already newer than 1711 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: oid 5 updating version 1712 to 1713 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1713 (ObjNum 843 snap 292 seq_num 843) dirty exists 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2242: left oid 5 (ObjNum 843 snap 292 seq_num 843) 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2243: finishing write tid 1 to smithi07685715-37 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2243: finishing write tid 2 to smithi07685715-37 2024-09-21T09:47:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:2243: finishing write tid 3 to smithi07685715-37 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2243: finishing write tid 4 to smithi07685715-37 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2243: oid 37 updating version 0 to 656 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2243: oid 37 version 656 is already newer than 654 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2243: oid 37 version 656 is already newer than 655 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 656 (ObjNum 844 snap 292 seq_num 844) dirty exists 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2243: left oid 37 (ObjNum 844 snap 292 seq_num 844) 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2241: done (3 left) 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2242: done (2 left) 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2243: done (1 left) 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2244: done (0 left) 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2245: read oid 4 snap 254 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2245: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:47:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:2246: snap_remove snap 259 2024-09-21T09:47:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:45 smithi191 ceph-mon[102206]: pgmap v446: 89 pgs: 89 active+clean; 227 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1015 KiB/s wr, 13 op/s 2024-09-21T09:47:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:45 smithi191 ceph-mon[102206]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:47:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:45 smithi076 ceph-mon[106482]: pgmap v446: 89 pgs: 89 active+clean; 227 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1015 KiB/s wr, 13 op/s 2024-09-21T09:47:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:45 smithi076 ceph-mon[106482]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:47:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:45 smithi076 ceph-mon[110963]: pgmap v446: 89 pgs: 89 active+clean; 227 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1015 KiB/s wr, 13 op/s 2024-09-21T09:47:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:45 smithi076 ceph-mon[110963]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T09:47:46.765 INFO:tasks.rados.rados.0.smithi076.stdout:2246: done (1 left) 2024-09-21T09:47:46.765 INFO:tasks.rados.rados.0.smithi076.stdout:2247: delete oid 45 current snap is 292 2024-09-21T09:47:46.765 INFO:tasks.rados.rados.0.smithi076.stdout:2245: expect (ObjNum 607 snap 194 seq_num 607) 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:2245: done (1 left) 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:2247: done (0 left) 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:2248: rollback oid 10 current snap is 292 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 287 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:2249: read oid 16 snap 254 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:2249: expect deleted 2024-09-21T09:47:46.858 INFO:tasks.rados.rados.0.smithi076.stdout:2250: snap_remove snap 291 2024-09-21T09:47:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:46 smithi191 ceph-mon[102206]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:47:47.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:46 smithi076 ceph-mon[106482]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:47:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:46 smithi076 ceph-mon[110963]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T09:47:47.764 INFO:tasks.rados.rados.0.smithi076.stdout:2250: done (2 left) 2024-09-21T09:47:47.764 INFO:tasks.rados.rados.0.smithi076.stdout:2251: delete oid 33 current snap is 292 2024-09-21T09:47:47.764 INFO:tasks.rados.rados.0.smithi076.stdout:2248: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:47:47.764 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1199 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:47:47.767 INFO:tasks.rados.rados.0.smithi076.stdout:2248: done (2 left) 2024-09-21T09:47:47.768 INFO:tasks.rados.rados.0.smithi076.stdout:2249: done (1 left) 2024-09-21T09:47:47.768 INFO:tasks.rados.rados.0.smithi076.stdout:2251: done (0 left) 2024-09-21T09:47:47.768 INFO:tasks.rados.rados.0.smithi076.stdout:2252: write oid 39 current snap is 292 2024-09-21T09:47:47.768 INFO:tasks.rados.rados.0.smithi076.stdout:2252: seq_num 848 ranges {652336=726335,1906495=345841} 2024-09-21T09:47:47.775 INFO:tasks.rados.rados.0.smithi076.stdout:2252: writing smithi07685715-39 from 652336 to 1378671 tid 1 2024-09-21T09:47:47.780 INFO:tasks.rados.rados.0.smithi076.stdout:2252: writing smithi07685715-39 from 1906495 to 2252336 tid 2 2024-09-21T09:47:47.780 INFO:tasks.rados.rados.0.smithi076.stdout:2253: rollback oid 17 current snap is 292 2024-09-21T09:47:47.780 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 290 2024-09-21T09:47:47.780 INFO:tasks.rados.rados.0.smithi076.stdout:2252: finishing write tid 1 to smithi07685715-39 2024-09-21T09:47:47.780 INFO:tasks.rados.rados.0.smithi076.stdout:2254: snap_remove snap 284 2024-09-21T09:47:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:47 smithi191 ceph-mon[102206]: pgmap v449: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-21T09:47:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:47 smithi191 ceph-mon[102206]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:47:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:47 smithi076 ceph-mon[106482]: pgmap v449: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-21T09:47:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:47 smithi076 ceph-mon[106482]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:47:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:47 smithi076 ceph-mon[110963]: pgmap v449: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-21T09:47:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:47 smithi076 ceph-mon[110963]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T09:47:48.710 INFO:tasks.rados.rados.0.smithi076.stdout:2254: done (2 left) 2024-09-21T09:47:48.711 INFO:tasks.rados.rados.0.smithi076.stdout:2255: read oid 2 snap -1 2024-09-21T09:47:48.711 INFO:tasks.rados.rados.0.smithi076.stdout:2255: expect (ObjNum 813 snap 282 seq_num 813) 2024-09-21T09:47:48.711 INFO:tasks.rados.rados.0.smithi076.stdout:2256: write oid 9 current snap is 292 2024-09-21T09:47:48.711 INFO:tasks.rados.rados.0.smithi076.stdout:2256: seq_num 849 ranges {669143=401584,1564977=426959} 2024-09-21T09:47:48.717 INFO:tasks.rados.rados.0.smithi076.stdout:2256: writing smithi07685715-9 from 669143 to 1070727 tid 1 2024-09-21T09:47:48.723 INFO:tasks.rados.rados.0.smithi076.stdout:2256: writing smithi07685715-9 from 1564977 to 1991936 tid 2 2024-09-21T09:47:48.723 INFO:tasks.rados.rados.0.smithi076.stdout:2257: write oid 35 current snap is 292 2024-09-21T09:47:48.723 INFO:tasks.rados.rados.0.smithi076.stdout:2257: seq_num 850 ranges {685950=476833,1623459=570617} 2024-09-21T09:47:48.729 INFO:tasks.rados.rados.0.smithi076.stdout:2257: writing smithi07685715-35 from 685950 to 1162783 tid 1 2024-09-21T09:47:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2257: writing smithi07685715-35 from 1623459 to 2194076 tid 2 2024-09-21T09:47:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2258: read oid 4 snap 287 2024-09-21T09:47:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2258: expect (ObjNum 816 snap 283 seq_num 816) 2024-09-21T09:47:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2259: rollback oid 36 current snap is 292 2024-09-21T09:47:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 290 2024-09-21T09:47:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2253: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2260: read oid 44 snap -1 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2260: expect (ObjNum 833 snap 290 seq_num 833) 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2261: read oid 3 snap -1 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2261: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2262: read oid 28 snap 275 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2262: expect deleted 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2263: rollback oid 5 current snap is 292 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 285 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2253: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2253: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1291 (ObjNum 808 snap 282 seq_num 808) dirty exists 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2252: finishing write tid 2 to smithi07685715-39 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2253: done (10 left) 2024-09-21T09:47:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:2264: write oid 45 current snap is 292 2024-09-21T09:47:48.738 INFO:tasks.rados.rados.0.smithi076.stdout:2252: finishing write tid 3 to smithi07685715-39 2024-09-21T09:47:48.738 INFO:tasks.rados.rados.0.smithi076.stdout:2264: seq_num 851 ranges {702757=552082,1681941=430628} 2024-09-21T09:47:48.742 INFO:tasks.rados.rados.0.smithi076.stdout:2264: writing smithi07685715-45 from 702757 to 1254839 tid 1 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2264: writing smithi07685715-45 from 1681941 to 2112569 tid 2 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2265: read oid 14 snap 287 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2265: expect (ObjNum 721 snap 247 seq_num 721) 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2266: read oid 50 snap -1 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2266: expect (ObjNum 827 snap 285 seq_num 827) 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2267: delete oid 12 current snap is 292 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2252: finishing write tid 4 to smithi07685715-39 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2252: oid 39 updating version 0 to 1722 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2252: oid 39 version 1722 is already newer than 1721 2024-09-21T09:47:48.748 INFO:tasks.rados.rados.0.smithi076.stdout:2252: oid 39 version 1722 is already newer than 1720 2024-09-21T09:47:48.749 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1722 (ObjNum 848 snap 292 seq_num 848) dirty exists 2024-09-21T09:47:48.749 INFO:tasks.rados.rados.0.smithi076.stdout:2252: left oid 39 (ObjNum 848 snap 292 seq_num 848) 2024-09-21T09:47:48.749 INFO:tasks.rados.rados.0.smithi076.stdout:2256: finishing write tid 1 to smithi07685715-9 2024-09-21T09:47:48.749 INFO:tasks.rados.rados.0.smithi076.stdout:2255: expect (ObjNum 813 snap 282 seq_num 813) 2024-09-21T09:47:48.868 INFO:tasks.rados.rados.0.smithi076.stdout:2256: finishing write tid 2 to smithi07685715-9 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: finishing write tid 3 to smithi07685715-9 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: finishing write tid 4 to smithi07685715-9 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: oid 9 updating version 0 to 808 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: oid 9 updating version 808 to 809 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: oid 9 updating version 809 to 810 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 810 (ObjNum 849 snap 292 seq_num 849) dirty exists 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: left oid 9 (ObjNum 849 snap 292 seq_num 849) 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2257: finishing write tid 1 to smithi07685715-35 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2252: done (13 left) 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2255: done (12 left) 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2256: done (11 left) 2024-09-21T09:47:48.869 INFO:tasks.rados.rados.0.smithi076.stdout:2267: done (10 left) 2024-09-21T09:47:48.870 INFO:tasks.rados.rados.0.smithi076.stdout:2268: write oid 15 current snap is 292 2024-09-21T09:47:48.870 INFO:tasks.rados.rados.0.smithi076.stdout:2260: expect (ObjNum 833 snap 290 seq_num 833) 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: finishing write tid 2 to smithi07685715-35 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: finishing write tid 3 to smithi07685715-35 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: finishing write tid 4 to smithi07685715-35 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: oid 35 updating version 0 to 1726 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: oid 35 version 1726 is already newer than 1725 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: oid 35 updating version 1726 to 1727 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1727 (ObjNum 850 snap 292 seq_num 850) dirty exists 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2257: left oid 35 (ObjNum 850 snap 292 seq_num 850) 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2263: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:47:48.973 INFO:tasks.rados.rados.0.smithi076.stdout:2268: seq_num 853 ranges {736371=418933,1798905=434297} 2024-09-21T09:47:48.979 INFO:tasks.rados.rados.0.smithi076.stdout:2268: writing smithi07685715-15 from 736371 to 1155304 tid 1 2024-09-21T09:47:48.985 INFO:tasks.rados.rados.0.smithi076.stdout:2268: writing smithi07685715-15 from 1798905 to 2233202 tid 2 2024-09-21T09:47:48.985 INFO:tasks.rados.rados.0.smithi076.stdout:2257: done (10 left) 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2260: done (9 left) 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2269: delete oid 32 current snap is 292 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2263: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1729 (ObjNum 129 snap 0 seq_num 864413840) dirty dne 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: finishing write tid 1 to smithi07685715-45 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: finishing write tid 2 to smithi07685715-45 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: finishing write tid 3 to smithi07685715-45 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: finishing write tid 4 to smithi07685715-45 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: oid 45 updating version 0 to 484 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: oid 45 version 484 is already newer than 483 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: oid 45 version 484 is already newer than 482 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 484 (ObjNum 851 snap 292 seq_num 851) dirty exists 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2264: left oid 45 (ObjNum 851 snap 292 seq_num 851) 2024-09-21T09:47:48.986 INFO:tasks.rados.rados.0.smithi076.stdout:2261: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:47:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:48 smithi191 ceph-mon[102206]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:47:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:48 smithi191 ceph-mon[102206]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:47:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:48 smithi076 ceph-mon[106482]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:47:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:48 smithi076 ceph-mon[106482]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:47:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:48 smithi076 ceph-mon[110963]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T09:47:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:48 smithi076 ceph-mon[110963]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T09:47:49.160 INFO:tasks.rados.rados.0.smithi076.stdout:2258: expect (ObjNum 816 snap 283 seq_num 816) 2024-09-21T09:47:49.273 INFO:tasks.rados.rados.0.smithi076.stdout:2259: finishing rollback tid 0 to smithi07685715-36 2024-09-21T09:47:49.273 INFO:tasks.rados.rados.0.smithi076.stdout:2259: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:47:49.273 INFO:tasks.rados.rados.0.smithi076.stdout:2259: finishing rollback tid 2 to smithi07685715-36 2024-09-21T09:47:49.273 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 675 (ObjNum 768 snap 263 seq_num 768) dirty exists 2024-09-21T09:47:49.274 INFO:tasks.rados.rados.0.smithi076.stdout:2265: expect (ObjNum 721 snap 247 seq_num 721) 2024-09-21T09:47:49.342 INFO:tasks.rados.rados.0.smithi076.stdout:2266: expect (ObjNum 827 snap 285 seq_num 827) 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: finishing write tid 1 to smithi07685715-15 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: finishing write tid 2 to smithi07685715-15 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: finishing write tid 3 to smithi07685715-15 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: finishing write tid 4 to smithi07685715-15 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: oid 15 updating version 0 to 488 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: oid 15 version 488 is already newer than 487 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: oid 15 version 488 is already newer than 486 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 488 (ObjNum 853 snap 292 seq_num 853) dirty exists 2024-09-21T09:47:49.508 INFO:tasks.rados.rados.0.smithi076.stdout:2268: left oid 15 (ObjNum 853 snap 292 seq_num 853) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2258: done (9 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2259: done (8 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2261: done (7 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2262: done (6 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2263: done (5 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2264: done (4 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2265: done (3 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2266: done (2 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2268: done (1 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2269: done (0 left) 2024-09-21T09:47:49.509 INFO:tasks.rados.rados.0.smithi076.stdout:2270: read oid 28 snap -1 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:2270: expect deleted 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:2271: rollback oid 4 current snap is 292 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 290 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:2272: read oid 1 snap 288 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:2272: expect (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:2273: write oid 40 current snap is 292 2024-09-21T09:47:49.510 INFO:tasks.rados.rados.0.smithi076.stdout:2273: seq_num 855 ranges {769985=569431,1915869=437966} 2024-09-21T09:47:49.514 INFO:tasks.rados.rados.0.smithi076.stdout:2273: writing smithi07685715-40 from 769985 to 1339416 tid 1 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:2273: writing smithi07685715-40 from 1915869 to 2353835 tid 2 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:2274: write oid 9 current snap is 292 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:2271: finishing rollback tid 0 to smithi07685715-4 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:2271: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:2271: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 679 (ObjNum 816 snap 283 seq_num 816) dirty exists 2024-09-21T09:47:49.520 INFO:tasks.rados.rados.0.smithi076.stdout:2274: seq_num 856 ranges {786792=644680,1857998=528794} 2024-09-21T09:47:49.528 INFO:tasks.rados.rados.0.smithi076.stdout:2274: writing smithi07685715-9 from 786792 to 1431472 tid 1 2024-09-21T09:47:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:2274: writing smithi07685715-9 from 1857998 to 2386792 tid 2 2024-09-21T09:47:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:2270: done (4 left) 2024-09-21T09:47:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:2271: done (3 left) 2024-09-21T09:47:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:2275: snap_create 2024-09-21T09:47:49.533 INFO:tasks.rados.rados.0.smithi076.stdout:2272: expect (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:47:49.649 INFO:tasks.rados.rados.0.smithi076.stdout:2273: finishing write tid 1 to smithi07685715-40 2024-09-21T09:47:49.649 INFO:tasks.rados.rados.0.smithi076.stdout:2273: finishing write tid 2 to smithi07685715-40 2024-09-21T09:47:49.649 INFO:tasks.rados.rados.0.smithi076.stdout:2273: finishing write tid 3 to smithi07685715-40 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2273: finishing write tid 4 to smithi07685715-40 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2273: oid 40 updating version 0 to 966 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2273: oid 40 updating version 966 to 967 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2273: oid 40 version 967 is already newer than 965 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 967 (ObjNum 855 snap 292 seq_num 855) dirty exists 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2273: left oid 40 (ObjNum 855 snap 292 seq_num 855) 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2274: finishing write tid 1 to smithi07685715-9 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2274: finishing write tid 2 to smithi07685715-9 2024-09-21T09:47:49.650 INFO:tasks.rados.rados.0.smithi076.stdout:2274: finishing write tid 3 to smithi07685715-9 2024-09-21T09:47:49.651 INFO:tasks.rados.rados.0.smithi076.stdout:2274: finishing write tid 4 to smithi07685715-9 2024-09-21T09:47:49.651 INFO:tasks.rados.rados.0.smithi076.stdout:2274: oid 9 updating version 0 to 812 2024-09-21T09:47:49.651 INFO:tasks.rados.rados.0.smithi076.stdout:2274: oid 9 updating version 812 to 813 2024-09-21T09:47:49.651 INFO:tasks.rados.rados.0.smithi076.stdout:2274: oid 9 updating version 813 to 814 2024-09-21T09:47:49.651 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 814 (ObjNum 856 snap 292 seq_num 856) dirty exists 2024-09-21T09:47:49.651 INFO:tasks.rados.rados.0.smithi076.stdout:2274: left oid 9 (ObjNum 856 snap 292 seq_num 856) 2024-09-21T09:47:49.772 INFO:tasks.rados.rados.0.smithi076.stdout:2272: done (3 left) 2024-09-21T09:47:49.772 INFO:tasks.rados.rados.0.smithi076.stdout:2273: done (2 left) 2024-09-21T09:47:49.773 INFO:tasks.rados.rados.0.smithi076.stdout:2274: done (1 left) 2024-09-21T09:47:49.773 INFO:tasks.rados.rados.0.smithi076.stdout:2275: done (0 left) 2024-09-21T09:47:49.773 INFO:tasks.rados.rados.0.smithi076.stdout:2276: snap_create 2024-09-21T09:47:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:49 smithi191 ceph-mon[102206]: pgmap v452: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-21T09:47:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:49 smithi076 ceph-mon[106482]: pgmap v452: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-21T09:47:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:49 smithi076 ceph-mon[110963]: pgmap v452: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-21T09:47:50.778 INFO:tasks.rados.rados.0.smithi076.stdout:2276: done (0 left) 2024-09-21T09:47:50.778 INFO:tasks.rados.rados.0.smithi076.stdout:2277: delete oid 9 current snap is 294 2024-09-21T09:47:50.784 INFO:tasks.rados.rados.0.smithi076.stdout:2277: done (0 left) 2024-09-21T09:47:50.784 INFO:tasks.rados.rados.0.smithi076.stdout:2278: write oid 48 current snap is 294 2024-09-21T09:47:50.784 INFO:tasks.rados.rados.0.smithi076.stdout:2278: seq_num 858 ranges {420406=795178,1974962=445444} 2024-09-21T09:47:50.790 INFO:tasks.rados.rados.0.smithi076.stdout:2278: writing smithi07685715-48 from 420406 to 1215584 tid 1 2024-09-21T09:47:50.797 INFO:tasks.rados.rados.0.smithi076.stdout:2278: writing smithi07685715-48 from 1974962 to 2420406 tid 2 2024-09-21T09:47:50.797 INFO:tasks.rados.rados.0.smithi076.stdout:2279: snap_remove snap 287 2024-09-21T09:47:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:50 smithi191 ceph-mon[102206]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:47:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:50 smithi076 ceph-mon[106482]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:47:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:50 smithi076 ceph-mon[110963]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T09:47:51.789 INFO:tasks.rados.rados.0.smithi076.stdout:2279: done (1 left) 2024-09-21T09:47:51.789 INFO:tasks.rados.rados.0.smithi076.stdout:2280: read oid 34 snap -1 2024-09-21T09:47:51.789 INFO:tasks.rados.rados.0.smithi076.stdout:2280: expect (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:47:51.789 INFO:tasks.rados.rados.0.smithi076.stdout:2281: snap_remove snap 281 2024-09-21T09:47:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:51 smithi191 ceph-mon[102206]: pgmap v455: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:51 smithi191 ceph-mon[102206]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:47:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:51 smithi076 ceph-mon[106482]: pgmap v455: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:51 smithi076 ceph-mon[106482]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:47:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:51 smithi076 ceph-mon[110963]: pgmap v455: 89 pgs: 89 active+clean; 230 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:47:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:51 smithi076 ceph-mon[110963]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T09:47:52.793 INFO:tasks.rados.rados.0.smithi076.stdout:2281: done (2 left) 2024-09-21T09:47:52.793 INFO:tasks.rados.rados.0.smithi076.stdout:2282: snap_remove snap 286 2024-09-21T09:47:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:52 smithi191 ceph-mon[102206]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:47:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:52 smithi076 ceph-mon[106482]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:47:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:52 smithi076 ceph-mon[110963]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T09:47:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:47:53] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.51.0" 2024-09-21T09:47:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2282: done (2 left) 2024-09-21T09:47:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2283: write oid 7 current snap is 294 2024-09-21T09:47:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2278: finishing write tid 1 to smithi07685715-48 2024-09-21T09:47:53.710 INFO:tasks.rados.rados.0.smithi076.stdout:2283: seq_num 859 ranges {437213=470427,1633444=728951} 2024-09-21T09:47:53.716 INFO:tasks.rados.rados.0.smithi076.stdout:2283: writing smithi07685715-7 from 437213 to 907640 tid 1 2024-09-21T09:47:53.726 INFO:tasks.rados.rados.0.smithi076.stdout:2283: writing smithi07685715-7 from 1633444 to 2362395 tid 2 2024-09-21T09:47:53.726 INFO:tasks.rados.rados.0.smithi076.stdout:2284: snap_remove snap 254 2024-09-21T09:47:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:53 smithi191 ceph-mon[102206]: pgmap v458: 89 pgs: 89 active+clean; 232 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-21T09:47:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:53 smithi191 ceph-mon[102206]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:47:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:53 smithi191 ceph-mon[102206]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:47:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-mon[106482]: pgmap v458: 89 pgs: 89 active+clean; 232 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-21T09:47:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-mon[106482]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:47:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-mon[106482]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:47:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-mon[110963]: pgmap v458: 89 pgs: 89 active+clean; 232 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-21T09:47:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-mon[110963]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T09:47:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:53 smithi076 ceph-mon[110963]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T09:47:54.712 INFO:tasks.rados.rados.0.smithi076.stdout:2284: done (3 left) 2024-09-21T09:47:54.712 INFO:tasks.rados.rados.0.smithi076.stdout:2285: write oid 19 current snap is 294 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: finishing write tid 2 to smithi07685715-48 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: finishing write tid 3 to smithi07685715-48 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: finishing write tid 4 to smithi07685715-48 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: oid 48 updating version 0 to 1797 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: oid 48 updating version 1797 to 1799 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: oid 48 version 1799 is already newer than 1798 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1799 (ObjNum 858 snap 294 seq_num 858) dirty exists 2024-09-21T09:47:54.714 INFO:tasks.rados.rados.0.smithi076.stdout:2278: left oid 48 (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:47:54.715 INFO:tasks.rados.rados.0.smithi076.stdout:2280: expect (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:47:54.782 INFO:tasks.rados.rados.0.smithi076.stdout:2283: finishing write tid 1 to smithi07685715-7 2024-09-21T09:47:54.782 INFO:tasks.rados.rados.0.smithi076.stdout:2283: finishing write tid 2 to smithi07685715-7 2024-09-21T09:47:54.782 INFO:tasks.rados.rados.0.smithi076.stdout:2283: finishing write tid 3 to smithi07685715-7 2024-09-21T09:47:54.782 INFO:tasks.rados.rados.0.smithi076.stdout:2283: finishing write tid 4 to smithi07685715-7 2024-09-21T09:47:54.783 INFO:tasks.rados.rados.0.smithi076.stdout:2283: oid 7 updating version 0 to 1309 2024-09-21T09:47:54.783 INFO:tasks.rados.rados.0.smithi076.stdout:2283: oid 7 updating version 1309 to 1310 2024-09-21T09:47:54.783 INFO:tasks.rados.rados.0.smithi076.stdout:2283: oid 7 updating version 1310 to 1311 2024-09-21T09:47:54.783 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1311 (ObjNum 859 snap 294 seq_num 859) dirty exists 2024-09-21T09:47:54.783 INFO:tasks.rados.rados.0.smithi076.stdout:2283: left oid 7 (ObjNum 859 snap 294 seq_num 859) 2024-09-21T09:47:54.783 INFO:tasks.rados.rados.0.smithi076.stdout:2285: seq_num 860 ranges {454020=662029,1691926=588962} 2024-09-21T09:47:54.788 INFO:tasks.rados.rados.0.smithi076.stdout:2285: writing smithi07685715-19 from 454020 to 1116049 tid 1 2024-09-21T09:47:54.794 INFO:tasks.rados.rados.0.smithi076.stdout:2285: writing smithi07685715-19 from 1691926 to 2280888 tid 2 2024-09-21T09:47:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:2278: done (3 left) 2024-09-21T09:47:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:2280: done (2 left) 2024-09-21T09:47:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:2283: done (1 left) 2024-09-21T09:47:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:2286: snap_create 2024-09-21T09:47:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:2285: finishing write tid 1 to smithi07685715-19 2024-09-21T09:47:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:2285: finishing write tid 2 to smithi07685715-19 2024-09-21T09:47:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:2285: finishing write tid 3 to smithi07685715-19 2024-09-21T09:47:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:2285: finishing write tid 4 to smithi07685715-19 2024-09-21T09:47:54.797 INFO:tasks.rados.rados.0.smithi076.stdout:2285: oid 19 updating version 0 to 746 2024-09-21T09:47:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:2285: oid 19 version 746 is already newer than 745 2024-09-21T09:47:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:2285: oid 19 version 746 is already newer than 744 2024-09-21T09:47:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 746 (ObjNum 860 snap 294 seq_num 860) dirty exists 2024-09-21T09:47:54.798 INFO:tasks.rados.rados.0.smithi076.stdout:2285: left oid 19 (ObjNum 860 snap 294 seq_num 860) 2024-09-21T09:47:55.799 INFO:tasks.rados.rados.0.smithi076.stdout:2285: done (1 left) 2024-09-21T09:47:55.800 INFO:tasks.rados.rados.0.smithi076.stdout:2286: done (0 left) 2024-09-21T09:47:55.800 INFO:tasks.rados.rados.0.smithi076.stdout:2287: delete oid 11 current snap is 295 2024-09-21T09:47:55.805 INFO:tasks.rados.rados.0.smithi076.stdout:2287: done (0 left) 2024-09-21T09:47:55.805 INFO:tasks.rados.rados.0.smithi076.stdout:2288: rollback oid 44 current snap is 295 2024-09-21T09:47:55.805 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 275 2024-09-21T09:47:55.805 INFO:tasks.rados.rados.0.smithi076.stdout:2289: write oid 45 current snap is 295 2024-09-21T09:47:55.805 INFO:tasks.rados.rados.0.smithi076.stdout:2289: seq_num 862 ranges {487634=412527,1408890=592631} 2024-09-21T09:47:55.810 INFO:tasks.rados.rados.0.smithi076.stdout:2289: writing smithi07685715-45 from 487634 to 900161 tid 1 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2289: writing smithi07685715-45 from 1408890 to 2001521 tid 2 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2290: rollback oid 41 current snap is 295 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 288 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2288: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2288: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2288: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 532 (ObjNum 790 snap 274 seq_num 790) dirty exists 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2289: finishing write tid 1 to smithi07685715-45 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2288: done (2 left) 2024-09-21T09:47:55.817 INFO:tasks.rados.rados.0.smithi076.stdout:2291: delete oid 17 current snap is 295 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:2289: finishing write tid 2 to smithi07685715-45 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:2289: finishing write tid 3 to smithi07685715-45 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:2289: finishing write tid 4 to smithi07685715-45 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:2289: oid 45 updating version 0 to 504 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:2289: oid 45 updating version 504 to 505 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:2289: oid 45 updating version 505 to 506 2024-09-21T09:47:55.820 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 506 (ObjNum 862 snap 295 seq_num 862) dirty exists 2024-09-21T09:47:55.821 INFO:tasks.rados.rados.0.smithi076.stdout:2289: left oid 45 (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:47:55.821 INFO:tasks.rados.rados.0.smithi076.stdout:2290: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:47:55.821 INFO:tasks.rados.rados.0.smithi076.stdout:2290: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:47:55.822 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 991 (ObjNum 824 snap 285 seq_num 824) dirty exists 2024-09-21T09:47:55.822 INFO:tasks.rados.rados.0.smithi076.stdout:2289: done (2 left) 2024-09-21T09:47:55.822 INFO:tasks.rados.rados.0.smithi076.stdout:2290: done (1 left) 2024-09-21T09:47:55.822 INFO:tasks.rados.rados.0.smithi076.stdout:2291: done (0 left) 2024-09-21T09:47:55.822 INFO:tasks.rados.rados.0.smithi076.stdout:2292: snap_create 2024-09-21T09:47:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:55 smithi191 ceph-mon[102206]: pgmap v461: 89 pgs: 89 active+clean; 232 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-21T09:47:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:55 smithi191 ceph-mon[102206]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:47:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:55 smithi076 ceph-mon[106482]: pgmap v461: 89 pgs: 89 active+clean; 232 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-21T09:47:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:55 smithi076 ceph-mon[106482]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:47:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:55 smithi076 ceph-mon[110963]: pgmap v461: 89 pgs: 89 active+clean; 232 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-21T09:47:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:55 smithi076 ceph-mon[110963]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T09:47:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:47:56.811 INFO:tasks.rados.rados.0.smithi076.stdout:2292: done (0 left) 2024-09-21T09:47:56.811 INFO:tasks.rados.rados.0.smithi076.stdout:2293: write oid 22 current snap is 296 2024-09-21T09:47:56.812 INFO:tasks.rados.rados.0.smithi076.stdout:2293: seq_num 864 ranges {521248=563025,1809501=596300} 2024-09-21T09:47:56.817 INFO:tasks.rados.rados.0.smithi076.stdout:2293: writing smithi07685715-22 from 521248 to 1084273 tid 1 2024-09-21T09:47:56.825 INFO:tasks.rados.rados.0.smithi076.stdout:2293: writing smithi07685715-22 from 1809501 to 2405801 tid 2 2024-09-21T09:47:56.825 INFO:tasks.rados.rados.0.smithi076.stdout:2294: rollback oid 4 current snap is 296 2024-09-21T09:47:56.825 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 275 2024-09-21T09:47:56.825 INFO:tasks.rados.rados.0.smithi076.stdout:2293: finishing write tid 1 to smithi07685715-22 2024-09-21T09:47:56.825 INFO:tasks.rados.rados.0.smithi076.stdout:2295: delete oid 42 current snap is 296 2024-09-21T09:47:56.828 INFO:tasks.rados.rados.0.smithi076.stdout:2294: finishing rollback tid 0 to smithi07685715-4 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:2294: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 700 (ObjNum 609 snap 0 seq_num 1682286601) dirty dne 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:2293: finishing write tid 2 to smithi07685715-22 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:2294: done (2 left) 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:2295: done (1 left) 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:2296: write oid 10 current snap is 296 2024-09-21T09:47:56.830 INFO:tasks.rados.rados.0.smithi076.stdout:2296: seq_num 866 ranges {554862=713523,1926465=599969} 2024-09-21T09:47:56.836 INFO:tasks.rados.rados.0.smithi076.stdout:2296: writing smithi07685715-10 from 554862 to 1268385 tid 1 2024-09-21T09:47:56.844 INFO:tasks.rados.rados.0.smithi076.stdout:2296: writing smithi07685715-10 from 1926465 to 2526434 tid 2 2024-09-21T09:47:56.844 INFO:tasks.rados.rados.0.smithi076.stdout:2297: snap_create 2024-09-21T09:47:56.844 INFO:tasks.rados.rados.0.smithi076.stdout:2293: finishing write tid 3 to smithi07685715-22 2024-09-21T09:47:56.844 INFO:tasks.rados.rados.0.smithi076.stdout:2293: finishing write tid 4 to smithi07685715-22 2024-09-21T09:47:56.844 INFO:tasks.rados.rados.0.smithi076.stdout:2293: oid 22 updating version 0 to 994 2024-09-21T09:47:56.844 INFO:tasks.rados.rados.0.smithi076.stdout:2293: oid 22 updating version 994 to 995 2024-09-21T09:47:56.845 INFO:tasks.rados.rados.0.smithi076.stdout:2293: oid 22 version 995 is already newer than 993 2024-09-21T09:47:56.845 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 995 (ObjNum 864 snap 296 seq_num 864) dirty exists 2024-09-21T09:47:56.845 INFO:tasks.rados.rados.0.smithi076.stdout:2293: left oid 22 (ObjNum 864 snap 296 seq_num 864) 2024-09-21T09:47:56.845 INFO:tasks.rados.rados.0.smithi076.stdout:2296: finishing write tid 1 to smithi07685715-10 2024-09-21T09:47:56.847 INFO:tasks.rados.rados.0.smithi076.stdout:2296: finishing write tid 2 to smithi07685715-10 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:2296: finishing write tid 3 to smithi07685715-10 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:2296: finishing write tid 4 to smithi07685715-10 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:2296: oid 10 updating version 0 to 1324 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:2296: oid 10 version 1324 is already newer than 1322 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:2296: oid 10 version 1324 is already newer than 1323 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1324 (ObjNum 866 snap 296 seq_num 866) dirty exists 2024-09-21T09:47:56.848 INFO:tasks.rados.rados.0.smithi076.stdout:2296: left oid 10 (ObjNum 866 snap 296 seq_num 866) 2024-09-21T09:47:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:56 smithi191 ceph-mon[102206]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:47:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:56 smithi076 ceph-mon[106482]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:47:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:56 smithi076 ceph-mon[110963]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T09:47:57.818 INFO:tasks.rados.rados.0.smithi076.stdout:2293: done (2 left) 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2296: done (1 left) 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2297: done (0 left) 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2298: read oid 3 snap -1 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2298: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2299: read oid 22 snap -1 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2299: expect (ObjNum 864 snap 296 seq_num 864) 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2300: rollback oid 35 current snap is 297 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 289 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2301: read oid 21 snap -1 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2301: expect (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2302: read oid 8 snap 283 2024-09-21T09:47:57.819 INFO:tasks.rados.rados.0.smithi076.stdout:2302: expect (ObjNum 781 snap 272 seq_num 781) 2024-09-21T09:47:57.820 INFO:tasks.rados.rados.0.smithi076.stdout:2303: snap_create 2024-09-21T09:47:57.836 INFO:tasks.rados.rados.0.smithi076.stdout:2299: expect (ObjNum 864 snap 296 seq_num 864) 2024-09-21T09:47:57.952 INFO:tasks.rados.rados.0.smithi076.stdout:2300: finishing rollback tid 0 to smithi07685715-35 2024-09-21T09:47:57.952 INFO:tasks.rados.rados.0.smithi076.stdout:2300: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:47:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1749 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:47:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:2298: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:47:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:57 smithi191 ceph-mon[102206]: pgmap v464: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 6 op/s 2024-09-21T09:47:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:57 smithi191 ceph-mon[102206]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:47:58.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:57 smithi076 ceph-mon[110963]: pgmap v464: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 6 op/s 2024-09-21T09:47:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:57 smithi076 ceph-mon[110963]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:47:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:57 smithi076 ceph-mon[106482]: pgmap v464: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 6 op/s 2024-09-21T09:47:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:57 smithi076 ceph-mon[106482]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T09:47:58.127 INFO:tasks.rados.rados.0.smithi076.stdout:2302: expect (ObjNum 781 snap 272 seq_num 781) 2024-09-21T09:47:58.311 INFO:tasks.rados.rados.0.smithi076.stdout:2301: expect (ObjNum 579 snap 184 seq_num 579) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2298: done (5 left) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2299: done (4 left) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2300: done (3 left) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2301: done (2 left) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2302: done (1 left) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2303: done (0 left) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2304: read oid 48 snap 288 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2304: expect (ObjNum 809 snap 282 seq_num 809) 2024-09-21T09:47:58.711 INFO:tasks.rados.rados.0.smithi076.stdout:2305: delete oid 12 current snap is 298 2024-09-21T09:47:58.714 INFO:tasks.rados.rados.0.smithi076.stdout:2305: done (1 left) 2024-09-21T09:47:58.714 INFO:tasks.rados.rados.0.smithi076.stdout:2306: snap_create 2024-09-21T09:47:58.726 INFO:tasks.rados.rados.0.smithi076.stdout:2304: expect (ObjNum 809 snap 282 seq_num 809) 2024-09-21T09:47:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:58 smithi191 ceph-mon[102206]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:47:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:58 smithi191 ceph-mon[102206]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:47:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:58 smithi076 ceph-mon[106482]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:47:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:58 smithi076 ceph-mon[106482]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:47:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:58 smithi076 ceph-mon[110963]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T09:47:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:58 smithi076 ceph-mon[110963]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T09:47:59.712 INFO:tasks.rados.rados.0.smithi076.stdout:2304: done (1 left) 2024-09-21T09:47:59.712 INFO:tasks.rados.rados.0.smithi076.stdout:2306: done (0 left) 2024-09-21T09:47:59.712 INFO:tasks.rados.rados.0.smithi076.stdout:2307: read oid 45 snap 283 2024-09-21T09:47:59.712 INFO:tasks.rados.rados.0.smithi076.stdout:2307: expect (ObjNum 789 snap 274 seq_num 789) 2024-09-21T09:47:59.712 INFO:tasks.rados.rados.0.smithi076.stdout:2308: delete oid 47 current snap is 299 2024-09-21T09:47:59.718 INFO:tasks.rados.rados.0.smithi076.stdout:2308: done (1 left) 2024-09-21T09:47:59.718 INFO:tasks.rados.rados.0.smithi076.stdout:2309: snap_create 2024-09-21T09:47:59.723 INFO:tasks.rados.rados.0.smithi076.stdout:2307: expect (ObjNum 789 snap 274 seq_num 789) 2024-09-21T09:48:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:59 smithi191 ceph-mon[102206]: pgmap v467: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 6 op/s 2024-09-21T09:48:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:47:59 smithi191 ceph-mon[102206]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:48:00.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:59 smithi076 ceph-mon[106482]: pgmap v467: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 6 op/s 2024-09-21T09:48:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:47:59 smithi076 ceph-mon[106482]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:48:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:59 smithi076 ceph-mon[110963]: pgmap v467: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 844 KiB/s wr, 6 op/s 2024-09-21T09:48:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:47:59 smithi076 ceph-mon[110963]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T09:48:00.719 INFO:tasks.rados.rados.0.smithi076.stdout:2307: done (1 left) 2024-09-21T09:48:00.721 INFO:tasks.rados.rados.0.smithi076.stdout:2309: done (0 left) 2024-09-21T09:48:00.721 INFO:tasks.rados.rados.0.smithi076.stdout:2310: snap_create 2024-09-21T09:48:01.722 INFO:tasks.rados.rados.0.smithi076.stdout:2310: done (0 left) 2024-09-21T09:48:01.722 INFO:tasks.rados.rados.0.smithi076.stdout:2311: read oid 34 snap -1 2024-09-21T09:48:01.722 INFO:tasks.rados.rados.0.smithi076.stdout:2311: expect (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:48:01.722 INFO:tasks.rados.rados.0.smithi076.stdout:2312: read oid 49 snap -1 2024-09-21T09:48:01.722 INFO:tasks.rados.rados.0.smithi076.stdout:2312: expect (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:48:01.722 INFO:tasks.rados.rados.0.smithi076.stdout:2313: snap_create 2024-09-21T09:48:01.730 INFO:tasks.rados.rados.0.smithi076.stdout:2311: expect (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:48:01.793 INFO:tasks.rados.rados.0.smithi076.stdout:2312: expect (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:48:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:01 smithi191 ceph-mon[102206]: pgmap v470: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:01 smithi191 ceph-mon[102206]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:48:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:01 smithi076 ceph-mon[106482]: pgmap v470: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:01 smithi076 ceph-mon[106482]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:48:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:01 smithi076 ceph-mon[110963]: pgmap v470: 89 pgs: 89 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:01 smithi076 ceph-mon[110963]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T09:48:02.722 INFO:tasks.rados.rados.0.smithi076.stdout:2311: done (2 left) 2024-09-21T09:48:02.722 INFO:tasks.rados.rados.0.smithi076.stdout:2312: done (1 left) 2024-09-21T09:48:02.723 INFO:tasks.rados.rados.0.smithi076.stdout:2313: done (0 left) 2024-09-21T09:48:02.723 INFO:tasks.rados.rados.0.smithi076.stdout:2314: read oid 40 snap 299 2024-09-21T09:48:02.723 INFO:tasks.rados.rados.0.smithi076.stdout:2314: expect (ObjNum 855 snap 292 seq_num 855) 2024-09-21T09:48:02.723 INFO:tasks.rados.rados.0.smithi076.stdout:2315: write oid 38 current snap is 302 2024-09-21T09:48:02.723 INFO:tasks.rados.rados.0.smithi076.stdout:2315: seq_num 869 ranges {605283=655623,1701911=747296} 2024-09-21T09:48:02.729 INFO:tasks.rados.rados.0.smithi076.stdout:2315: writing smithi07685715-38 from 605283 to 1260906 tid 1 2024-09-21T09:48:02.737 INFO:tasks.rados.rados.0.smithi076.stdout:2315: writing smithi07685715-38 from 1701911 to 2449207 tid 2 2024-09-21T09:48:02.737 INFO:tasks.rados.rados.0.smithi076.stdout:2316: read oid 44 snap 298 2024-09-21T09:48:02.737 INFO:tasks.rados.rados.0.smithi076.stdout:2316: expect (ObjNum 790 snap 274 seq_num 790) 2024-09-21T09:48:02.737 INFO:tasks.rados.rados.0.smithi076.stdout:2317: snap_create 2024-09-21T09:48:02.737 INFO:tasks.rados.rados.0.smithi076.stdout:2314: expect (ObjNum 855 snap 292 seq_num 855) 2024-09-21T09:48:02.873 INFO:tasks.rados.rados.0.smithi076.stdout:2315: finishing write tid 1 to smithi07685715-38 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: finishing write tid 2 to smithi07685715-38 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: finishing write tid 3 to smithi07685715-38 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: finishing write tid 4 to smithi07685715-38 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: oid 38 updating version 0 to 1327 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: oid 38 version 1327 is already newer than 1326 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: oid 38 updating version 1327 to 1328 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1328 (ObjNum 869 snap 302 seq_num 869) dirty exists 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2315: left oid 38 (ObjNum 869 snap 302 seq_num 869) 2024-09-21T09:48:02.874 INFO:tasks.rados.rados.0.smithi076.stdout:2316: expect (ObjNum 790 snap 274 seq_num 790) 2024-09-21T09:48:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:02 smithi076 ceph-mon[106482]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:48:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:02 smithi076 ceph-mon[106482]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:48:03.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:02 smithi076 ceph-mon[110963]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:48:03.024 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:02 smithi076 ceph-mon[110963]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:48:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:02 smithi191 ceph-mon[102206]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T09:48:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:02 smithi191 ceph-mon[102206]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T09:48:03.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:48:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:48:03] "GET /metrics HTTP/1.1" 200 36134 "" "Prometheus/2.51.0" 2024-09-21T09:48:03.746 INFO:tasks.rados.rados.0.smithi076.stdout:2314: done (3 left) 2024-09-21T09:48:03.746 INFO:tasks.rados.rados.0.smithi076.stdout:2315: done (2 left) 2024-09-21T09:48:03.746 INFO:tasks.rados.rados.0.smithi076.stdout:2316: done (1 left) 2024-09-21T09:48:03.747 INFO:tasks.rados.rados.0.smithi076.stdout:2317: done (0 left) 2024-09-21T09:48:03.747 INFO:tasks.rados.rados.0.smithi076.stdout:2318: snap_create 2024-09-21T09:48:04.048 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:03 smithi076 ceph-mon[106482]: pgmap v473: 89 pgs: 89 active+clean; 189 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 604 KiB/s wr, 15 op/s 2024-09-21T09:48:04.048 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:04.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:03 smithi076 ceph-mon[110963]: pgmap v473: 89 pgs: 89 active+clean; 189 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 604 KiB/s wr, 15 op/s 2024-09-21T09:48:04.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:04.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:03 smithi191 ceph-mon[102206]: pgmap v473: 89 pgs: 89 active+clean; 189 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 604 KiB/s wr, 15 op/s 2024-09-21T09:48:04.052 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:04.755 INFO:tasks.rados.rados.0.smithi076.stdout:2318: done (0 left) 2024-09-21T09:48:04.755 INFO:tasks.rados.rados.0.smithi076.stdout:2319: read oid 36 snap 297 2024-09-21T09:48:04.755 INFO:tasks.rados.rados.0.smithi076.stdout:2319: expect (ObjNum 768 snap 263 seq_num 768) 2024-09-21T09:48:04.755 INFO:tasks.rados.rados.0.smithi076.stdout:2320: write oid 21 current snap is 304 2024-09-21T09:48:04.755 INFO:tasks.rados.rados.0.smithi076.stdout:2320: seq_num 870 ranges {622090=730872,1760393=607307} 2024-09-21T09:48:04.762 INFO:tasks.rados.rados.0.smithi076.stdout:2320: writing smithi07685715-21 from 622090 to 1352962 tid 1 2024-09-21T09:48:04.767 INFO:tasks.rados.rados.0.smithi076.stdout:2320: writing smithi07685715-21 from 1760393 to 2367700 tid 2 2024-09-21T09:48:04.767 INFO:tasks.rados.rados.0.smithi076.stdout:2321: snap_create 2024-09-21T09:48:04.768 INFO:tasks.rados.rados.0.smithi076.stdout:2319: expect (ObjNum 768 snap 263 seq_num 768) 2024-09-21T09:48:04.895 INFO:tasks.rados.rados.0.smithi076.stdout:2320: finishing write tid 1 to smithi07685715-21 2024-09-21T09:48:04.895 INFO:tasks.rados.rados.0.smithi076.stdout:2320: finishing write tid 2 to smithi07685715-21 2024-09-21T09:48:04.895 INFO:tasks.rados.rados.0.smithi076.stdout:2320: finishing write tid 3 to smithi07685715-21 2024-09-21T09:48:04.896 INFO:tasks.rados.rados.0.smithi076.stdout:2320: finishing write tid 4 to smithi07685715-21 2024-09-21T09:48:04.896 INFO:tasks.rados.rados.0.smithi076.stdout:2320: oid 21 updating version 0 to 1751 2024-09-21T09:48:04.896 INFO:tasks.rados.rados.0.smithi076.stdout:2320: oid 21 updating version 1751 to 1752 2024-09-21T09:48:04.896 INFO:tasks.rados.rados.0.smithi076.stdout:2320: oid 21 updating version 1752 to 1753 2024-09-21T09:48:04.896 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1753 (ObjNum 870 snap 304 seq_num 870) dirty exists 2024-09-21T09:48:04.896 INFO:tasks.rados.rados.0.smithi076.stdout:2320: left oid 21 (ObjNum 870 snap 304 seq_num 870) 2024-09-21T09:48:05.042 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:04 smithi191 ceph-mon[102206]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:48:05.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:04 smithi076 ceph-mon[110963]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:48:05.070 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:04 smithi076 ceph-mon[106482]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T09:48:05.762 INFO:tasks.rados.rados.0.smithi076.stdout:2319: done (2 left) 2024-09-21T09:48:05.762 INFO:tasks.rados.rados.0.smithi076.stdout:2320: done (1 left) 2024-09-21T09:48:05.762 INFO:tasks.rados.rados.0.smithi076.stdout:2321: done (0 left) 2024-09-21T09:48:05.762 INFO:tasks.rados.rados.0.smithi076.stdout:2322: snap_create 2024-09-21T09:48:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:05 smithi191 ceph-mon[102206]: pgmap v476: 89 pgs: 89 active+clean; 189 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 604 KiB/s wr, 15 op/s 2024-09-21T09:48:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:05 smithi191 ceph-mon[102206]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:48:06.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:05 smithi076 ceph-mon[106482]: pgmap v476: 89 pgs: 89 active+clean; 189 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 604 KiB/s wr, 15 op/s 2024-09-21T09:48:06.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:05 smithi076 ceph-mon[106482]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:48:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:05 smithi076 ceph-mon[110963]: pgmap v476: 89 pgs: 89 active+clean; 189 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 604 KiB/s wr, 15 op/s 2024-09-21T09:48:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:05 smithi076 ceph-mon[110963]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T09:48:06.761 INFO:tasks.rados.rados.0.smithi076.stdout:2322: done (0 left) 2024-09-21T09:48:06.761 INFO:tasks.rados.rados.0.smithi076.stdout:2323: snap_remove snap 292 2024-09-21T09:48:07.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:06 smithi191 ceph-mon[102206]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:48:07.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:06 smithi076 ceph-mon[106482]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:48:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:06 smithi076 ceph-mon[110963]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T09:48:07.763 INFO:tasks.rados.rados.0.smithi076.stdout:2323: done (0 left) 2024-09-21T09:48:07.764 INFO:tasks.rados.rados.0.smithi076.stdout:2324: snap_remove snap 295 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: pgmap v479: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 670 KiB/s wr, 13 op/s 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:07.951 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:07 smithi191 ceph-mon[102206]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: pgmap v479: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 670 KiB/s wr, 13 op/s 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[106482]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: pgmap v479: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 670 KiB/s wr, 13 op/s 2024-09-21T09:48:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T09:48:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:07 smithi076 ceph-mon[110963]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T09:48:08.766 INFO:tasks.rados.rados.0.smithi076.stdout:2324: done (0 left) 2024-09-21T09:48:08.766 INFO:tasks.rados.rados.0.smithi076.stdout:2325: snap_create 2024-09-21T09:48:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:08 smithi191 ceph-mon[102206]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[106482]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:08 smithi076 ceph-mon[110963]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T09:48:09.767 INFO:tasks.rados.rados.0.smithi076.stdout:2325: done (0 left) 2024-09-21T09:48:09.767 INFO:tasks.rados.rados.0.smithi076.stdout:2326: snap_remove snap 290 2024-09-21T09:48:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:10 smithi191 ceph-mon[102206]: pgmap v482: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 670 KiB/s wr, 13 op/s 2024-09-21T09:48:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:10 smithi191 ceph-mon[102206]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:48:10.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:10 smithi076 ceph-mon[106482]: pgmap v482: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 670 KiB/s wr, 13 op/s 2024-09-21T09:48:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:10 smithi076 ceph-mon[106482]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:48:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:10 smithi076 ceph-mon[110963]: pgmap v482: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 670 KiB/s wr, 13 op/s 2024-09-21T09:48:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:10 smithi076 ceph-mon[110963]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T09:48:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:2326: done (0 left) 2024-09-21T09:48:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:2327: write oid 50 current snap is 307 2024-09-21T09:48:10.768 INFO:tasks.rados.rados.0.smithi076.stdout:2327: seq_num 871 ranges {638897=406121,1818875=750965} 2024-09-21T09:48:10.774 INFO:tasks.rados.rados.0.smithi076.stdout:2327: writing smithi07685715-50 from 638897 to 1045018 tid 1 2024-09-21T09:48:10.784 INFO:tasks.rados.rados.0.smithi076.stdout:2327: writing smithi07685715-50 from 1818875 to 2569840 tid 2 2024-09-21T09:48:10.784 INFO:tasks.rados.rados.0.smithi076.stdout:2328: read oid 4 snap -1 2024-09-21T09:48:10.784 INFO:tasks.rados.rados.0.smithi076.stdout:2328: expect deleted 2024-09-21T09:48:10.784 INFO:tasks.rados.rados.0.smithi076.stdout:2329: delete oid 12 current snap is 307 2024-09-21T09:48:10.784 INFO:tasks.rados.rados.0.smithi076.stdout:2327: finishing write tid 1 to smithi07685715-50 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2329: done (2 left) 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2330: read oid 1 snap -1 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2330: expect (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2331: read oid 39 snap -1 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2331: expect (ObjNum 848 snap 292 seq_num 848) 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2332: read oid 19 snap -1 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2332: expect (ObjNum 860 snap 294 seq_num 860) 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2333: write oid 18 current snap is 307 2024-09-21T09:48:10.786 INFO:tasks.rados.rados.0.smithi076.stdout:2333: seq_num 873 ranges {672511=556619,1819486=754634} 2024-09-21T09:48:10.792 INFO:tasks.rados.rados.0.smithi076.stdout:2333: writing smithi07685715-18 from 672511 to 1229130 tid 1 2024-09-21T09:48:10.800 INFO:tasks.rados.rados.0.smithi076.stdout:2333: writing smithi07685715-18 from 1819486 to 2574120 tid 2 2024-09-21T09:48:10.800 INFO:tasks.rados.rados.0.smithi076.stdout:2334: write oid 12 current snap is 307 2024-09-21T09:48:10.800 INFO:tasks.rados.rados.0.smithi076.stdout:2334: seq_num 874 ranges {689318=631868,1877968=498292} 2024-09-21T09:48:10.807 INFO:tasks.rados.rados.0.smithi076.stdout:2334: writing smithi07685715-12 from 689318 to 1321186 tid 1 2024-09-21T09:48:10.813 INFO:tasks.rados.rados.0.smithi076.stdout:2334: writing smithi07685715-12 from 1877968 to 2376260 tid 2 2024-09-21T09:48:10.813 INFO:tasks.rados.rados.0.smithi076.stdout:2335: snap_remove snap 305 2024-09-21T09:48:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:11 smithi191 ceph-mon[102206]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:48:11.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:11 smithi076 ceph-mon[106482]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:48:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:11 smithi076 ceph-mon[110963]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T09:48:11.772 INFO:tasks.rados.rados.0.smithi076.stdout:2335: done (7 left) 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:2336: rollback oid 3 current snap is 307 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 293 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:2327: finishing write tid 2 to smithi07685715-50 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:2327: finishing write tid 3 to smithi07685715-50 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:2327: finishing write tid 4 to smithi07685715-50 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:2327: oid 50 updating version 0 to 711 2024-09-21T09:48:11.773 INFO:tasks.rados.rados.0.smithi076.stdout:2327: oid 50 updating version 711 to 712 2024-09-21T09:48:11.774 INFO:tasks.rados.rados.0.smithi076.stdout:2327: oid 50 version 712 is already newer than 710 2024-09-21T09:48:11.774 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 712 (ObjNum 871 snap 307 seq_num 871) dirty exists 2024-09-21T09:48:11.774 INFO:tasks.rados.rados.0.smithi076.stdout:2327: left oid 50 (ObjNum 871 snap 307 seq_num 871) 2024-09-21T09:48:11.774 INFO:tasks.rados.rados.0.smithi076.stdout:2332: expect (ObjNum 860 snap 294 seq_num 860) 2024-09-21T09:48:11.849 INFO:tasks.rados.rados.0.smithi076.stdout:2333: finishing write tid 1 to smithi07685715-18 2024-09-21T09:48:11.849 INFO:tasks.rados.rados.0.smithi076.stdout:2330: expect (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:48:11.962 INFO:tasks.rados.rados.0.smithi076.stdout:2331: expect (ObjNum 848 snap 292 seq_num 848) 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: finishing write tid 2 to smithi07685715-18 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: finishing write tid 3 to smithi07685715-18 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: finishing write tid 4 to smithi07685715-18 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: oid 18 updating version 0 to 666 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: oid 18 updating version 666 to 667 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: oid 18 updating version 667 to 668 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 668 (ObjNum 873 snap 307 seq_num 873) dirty exists 2024-09-21T09:48:12.036 INFO:tasks.rados.rados.0.smithi076.stdout:2333: left oid 18 (ObjNum 873 snap 307 seq_num 873) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2334: finishing write tid 1 to smithi07685715-12 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2334: finishing write tid 2 to smithi07685715-12 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2334: finishing write tid 3 to smithi07685715-12 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2327: done (7 left) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2328: done (6 left) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2330: done (5 left) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2331: done (4 left) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2332: done (3 left) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2333: done (2 left) 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2337: rollback oid 23 current snap is 307 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 304 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2334: finishing write tid 4 to smithi07685715-12 2024-09-21T09:48:12.037 INFO:tasks.rados.rados.0.smithi076.stdout:2334: oid 12 updating version 0 to 842 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2334: oid 12 version 842 is already newer than 841 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2334: oid 12 updating version 842 to 843 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 843 (ObjNum 874 snap 307 seq_num 874) dirty exists 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2334: left oid 12 (ObjNum 874 snap 307 seq_num 874) 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2336: finishing rollback tid 0 to smithi07685715-3 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2336: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2336: finishing rollback tid 2 to smithi07685715-3 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 132 (ObjNum 773 snap 267 seq_num 773) dirty exists 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2334: done (2 left) 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2336: done (1 left) 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2338: rollback oid 4 current snap is 307 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 301 2024-09-21T09:48:12.038 INFO:tasks.rados.rados.0.smithi076.stdout:2339: read oid 48 snap 296 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2339: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2340: read oid 10 snap 294 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2340: expect deleted 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2341: rollback oid 16 current snap is 307 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 293 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2342: delete oid 18 current snap is 307 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2342: done (5 left) 2024-09-21T09:48:12.039 INFO:tasks.rados.rados.0.smithi076.stdout:2343: write oid 6 current snap is 307 2024-09-21T09:48:12.040 INFO:tasks.rados.rados.0.smithi076.stdout:2343: seq_num 876 ranges {722932=498719,1994932=501961} 2024-09-21T09:48:12.044 INFO:tasks.rados.rados.0.smithi076.stdout:2343: writing smithi07685715-6 from 722932 to 1221651 tid 1 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2343: writing smithi07685715-6 from 1994932 to 2496893 tid 2 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2344: rollback oid 35 current snap is 307 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 293 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2341: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 955 (ObjNum 49 snap 0 seq_num 830378992) dirty dne 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2337: finishing rollback tid 0 to smithi07685715-23 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2340: done (6 left) 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2341: done (5 left) 2024-09-21T09:48:12.052 INFO:tasks.rados.rados.0.smithi076.stdout:2345: snap_remove snap 300 2024-09-21T09:48:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:12 smithi191 ceph-mon[102206]: pgmap v485: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:12 smithi191 ceph-mon[102206]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:48:12.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:12 smithi076 ceph-mon[106482]: pgmap v485: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:12 smithi076 ceph-mon[106482]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:48:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:12 smithi076 ceph-mon[110963]: pgmap v485: 89 pgs: 89 active+clean; 192 MiB data, 729 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:48:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:12 smithi076 ceph-mon[110963]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:2345: done (5 left) 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:2346: write oid 14 current snap is 307 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:2337: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:2337: finishing rollback tid 2 to smithi07685715-23 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1834 (ObjNum 778 snap 271 seq_num 778) dirty exists 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:2338: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 700 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:48:13.034 INFO:tasks.rados.rados.0.smithi076.stdout:2346: seq_num 877 ranges {739739=573968,2053414=686325} 2024-09-21T09:48:13.041 INFO:tasks.rados.rados.0.smithi076.stdout:2346: writing smithi07685715-14 from 739739 to 1313707 tid 1 2024-09-21T09:48:13.049 INFO:tasks.rados.rados.0.smithi076.stdout:2346: writing smithi07685715-14 from 2053414 to 2739739 tid 2 2024-09-21T09:48:13.050 INFO:tasks.rados.rados.0.smithi076.stdout:2337: done (5 left) 2024-09-21T09:48:13.050 INFO:tasks.rados.rados.0.smithi076.stdout:2338: done (4 left) 2024-09-21T09:48:13.050 INFO:tasks.rados.rados.0.smithi076.stdout:2347: snap_remove snap 299 2024-09-21T09:48:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:48:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:48:13] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.51.0" 2024-09-21T09:48:13.717 INFO:tasks.rados.rados.0.smithi076.stdout:2339: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:13.868 INFO:tasks.rados.rados.0.smithi076.stdout:2343: finishing write tid 1 to smithi07685715-6 2024-09-21T09:48:13.868 INFO:tasks.rados.rados.0.smithi076.stdout:2343: finishing write tid 2 to smithi07685715-6 2024-09-21T09:48:13.868 INFO:tasks.rados.rados.0.smithi076.stdout:2344: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:48:13.868 INFO:tasks.rados.rados.0.smithi076.stdout:2343: finishing write tid 3 to smithi07685715-6 2024-09-21T09:48:13.868 INFO:tasks.rados.rados.0.smithi076.stdout:2344: finishing rollback tid 2 to smithi07685715-35 2024-09-21T09:48:13.868 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1767 (ObjNum 850 snap 292 seq_num 850) dirty exists 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2343: finishing write tid 4 to smithi07685715-6 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2343: oid 6 updating version 0 to 540 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2343: oid 6 version 540 is already newer than 538 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2343: oid 6 version 540 is already newer than 539 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 540 (ObjNum 876 snap 307 seq_num 876) dirty exists 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2343: left oid 6 (ObjNum 876 snap 307 seq_num 876) 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2346: finishing write tid 1 to smithi07685715-14 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2339: done (4 left) 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2343: done (3 left) 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2344: done (2 left) 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2347: done (1 left) 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2348: snap_create 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2346: finishing write tid 2 to smithi07685715-14 2024-09-21T09:48:13.869 INFO:tasks.rados.rados.0.smithi076.stdout:2346: finishing write tid 3 to smithi07685715-14 2024-09-21T09:48:13.870 INFO:tasks.rados.rados.0.smithi076.stdout:2346: finishing write tid 4 to smithi07685715-14 2024-09-21T09:48:13.870 INFO:tasks.rados.rados.0.smithi076.stdout:2346: oid 14 updating version 0 to 1344 2024-09-21T09:48:13.870 INFO:tasks.rados.rados.0.smithi076.stdout:2346: oid 14 updating version 1344 to 1346 2024-09-21T09:48:13.870 INFO:tasks.rados.rados.0.smithi076.stdout:2346: oid 14 version 1346 is already newer than 1345 2024-09-21T09:48:13.870 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1346 (ObjNum 877 snap 307 seq_num 877) dirty exists 2024-09-21T09:48:13.870 INFO:tasks.rados.rados.0.smithi076.stdout:2346: left oid 14 (ObjNum 877 snap 307 seq_num 877) 2024-09-21T09:48:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:14 smithi191 ceph-mon[102206]: pgmap v488: 89 pgs: 89 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 879 KiB/s wr, 9 op/s 2024-09-21T09:48:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:14 smithi191 ceph-mon[102206]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:48:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:14 smithi191 ceph-mon[102206]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:48:14.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:14 smithi076 ceph-mon[106482]: pgmap v488: 89 pgs: 89 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 879 KiB/s wr, 9 op/s 2024-09-21T09:48:14.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:14 smithi076 ceph-mon[106482]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:48:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:14 smithi076 ceph-mon[106482]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:48:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:14 smithi076 ceph-mon[110963]: pgmap v488: 89 pgs: 89 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 879 KiB/s wr, 9 op/s 2024-09-21T09:48:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:14 smithi076 ceph-mon[110963]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T09:48:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:14 smithi076 ceph-mon[110963]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T09:48:14.716 INFO:tasks.rados.rados.0.smithi076.stdout:2346: done (1 left) 2024-09-21T09:48:14.716 INFO:tasks.rados.rados.0.smithi076.stdout:2348: done (0 left) 2024-09-21T09:48:14.716 INFO:tasks.rados.rados.0.smithi076.stdout:2349: delete oid 31 current snap is 308 2024-09-21T09:48:14.718 INFO:tasks.rados.rados.0.smithi076.stdout:2349: done (0 left) 2024-09-21T09:48:14.719 INFO:tasks.rados.rados.0.smithi076.stdout:2350: write oid 40 current snap is 308 2024-09-21T09:48:14.719 INFO:tasks.rados.rados.0.smithi076.stdout:2350: seq_num 879 ranges {773353=724466,2170378=602975} 2024-09-21T09:48:14.726 INFO:tasks.rados.rados.0.smithi076.stdout:2350: writing smithi07685715-40 from 773353 to 1497819 tid 1 2024-09-21T09:48:14.734 INFO:tasks.rados.rados.0.smithi076.stdout:2350: writing smithi07685715-40 from 2170378 to 2773353 tid 2 2024-09-21T09:48:14.734 INFO:tasks.rados.rados.0.smithi076.stdout:2351: read oid 6 snap -1 2024-09-21T09:48:14.734 INFO:tasks.rados.rados.0.smithi076.stdout:2351: expect (ObjNum 876 snap 307 seq_num 876) 2024-09-21T09:48:14.734 INFO:tasks.rados.rados.0.smithi076.stdout:2352: write oid 34 current snap is 308 2024-09-21T09:48:14.734 INFO:tasks.rados.rados.0.smithi076.stdout:2350: finishing write tid 1 to smithi07685715-40 2024-09-21T09:48:14.734 INFO:tasks.rados.rados.0.smithi076.stdout:2352: seq_num 880 ranges {790160=799715,2112507=509299} 2024-09-21T09:48:14.742 INFO:tasks.rados.rados.0.smithi076.stdout:2352: writing smithi07685715-34 from 790160 to 1589875 tid 1 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2352: writing smithi07685715-34 from 2112507 to 2621806 tid 2 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2353: read oid 29 snap 294 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2353: expect (ObjNum 818 snap 284 seq_num 818) 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2354: rollback oid 30 current snap is 308 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 304 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2350: finishing write tid 2 to smithi07685715-40 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2350: finishing write tid 3 to smithi07685715-40 2024-09-21T09:48:14.749 INFO:tasks.rados.rados.0.smithi076.stdout:2350: finishing write tid 4 to smithi07685715-40 2024-09-21T09:48:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2350: oid 40 updating version 0 to 1008 2024-09-21T09:48:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2350: oid 40 updating version 1008 to 1009 2024-09-21T09:48:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2350: oid 40 updating version 1009 to 1010 2024-09-21T09:48:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1010 (ObjNum 879 snap 308 seq_num 879) dirty exists 2024-09-21T09:48:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2350: left oid 40 (ObjNum 879 snap 308 seq_num 879) 2024-09-21T09:48:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2351: expect (ObjNum 876 snap 307 seq_num 876) 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: finishing write tid 1 to smithi07685715-34 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: finishing write tid 2 to smithi07685715-34 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: finishing write tid 3 to smithi07685715-34 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: finishing write tid 4 to smithi07685715-34 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: oid 34 updating version 0 to 491 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: oid 34 updating version 491 to 492 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:2352: oid 34 version 492 is already newer than 490 2024-09-21T09:48:14.983 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 492 (ObjNum 880 snap 308 seq_num 880) dirty exists 2024-09-21T09:48:14.984 INFO:tasks.rados.rados.0.smithi076.stdout:2352: left oid 34 (ObjNum 880 snap 308 seq_num 880) 2024-09-21T09:48:14.984 INFO:tasks.rados.rados.0.smithi076.stdout:2354: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:48:14.984 INFO:tasks.rados.rados.0.smithi076.stdout:2354: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:48:14.984 INFO:tasks.rados.rados.0.smithi076.stdout:2354: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:48:14.984 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1842 (ObjNum 842 snap 291 seq_num 842) dirty exists 2024-09-21T09:48:14.984 INFO:tasks.rados.rados.0.smithi076.stdout:2353: expect (ObjNum 818 snap 284 seq_num 818) 2024-09-21T09:48:15.103 INFO:tasks.rados.rados.0.smithi076.stdout:2350: done (4 left) 2024-09-21T09:48:15.103 INFO:tasks.rados.rados.0.smithi076.stdout:2351: done (3 left) 2024-09-21T09:48:15.103 INFO:tasks.rados.rados.0.smithi076.stdout:2352: done (2 left) 2024-09-21T09:48:15.103 INFO:tasks.rados.rados.0.smithi076.stdout:2353: done (1 left) 2024-09-21T09:48:15.104 INFO:tasks.rados.rados.0.smithi076.stdout:2354: done (0 left) 2024-09-21T09:48:15.104 INFO:tasks.rados.rados.0.smithi076.stdout:2355: delete oid 29 current snap is 308 2024-09-21T09:48:15.105 INFO:tasks.rados.rados.0.smithi076.stdout:2355: done (0 left) 2024-09-21T09:48:15.105 INFO:tasks.rados.rados.0.smithi076.stdout:2356: snap_create 2024-09-21T09:48:16.038 INFO:tasks.rados.rados.0.smithi076.stdout:2356: done (0 left) 2024-09-21T09:48:16.039 INFO:tasks.rados.rados.0.smithi076.stdout:2357: read oid 3 snap -1 2024-09-21T09:48:16.039 INFO:tasks.rados.rados.0.smithi076.stdout:2357: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:48:16.039 INFO:tasks.rados.rados.0.smithi076.stdout:2358: snap_remove snap 288 2024-09-21T09:48:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:15 smithi191 ceph-mon[102206]: pgmap v491: 89 pgs: 89 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 879 KiB/s wr, 9 op/s 2024-09-21T09:48:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:15 smithi191 ceph-mon[102206]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:48:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:15 smithi076 ceph-mon[106482]: pgmap v491: 89 pgs: 89 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 879 KiB/s wr, 9 op/s 2024-09-21T09:48:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:15 smithi076 ceph-mon[106482]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:48:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:15 smithi076 ceph-mon[110963]: pgmap v491: 89 pgs: 89 active+clean; 197 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 879 KiB/s wr, 9 op/s 2024-09-21T09:48:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:15 smithi076 ceph-mon[110963]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T09:48:17.047 INFO:tasks.rados.rados.0.smithi076.stdout:2358: done (1 left) 2024-09-21T09:48:17.047 INFO:tasks.rados.rados.0.smithi076.stdout:2359: read oid 10 snap 308 2024-09-21T09:48:17.047 INFO:tasks.rados.rados.0.smithi076.stdout:2359: expect (ObjNum 866 snap 296 seq_num 866) 2024-09-21T09:48:17.047 INFO:tasks.rados.rados.0.smithi076.stdout:2360: snap_remove snap 294 2024-09-21T09:48:17.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:17 smithi191 ceph-mon[102206]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:48:17.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:17 smithi076 ceph-mon[106482]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:48:17.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:17 smithi076 ceph-mon[110963]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T09:48:18.049 INFO:tasks.rados.rados.0.smithi076.stdout:2360: done (2 left) 2024-09-21T09:48:18.049 INFO:tasks.rados.rados.0.smithi076.stdout:2361: snap_remove snap 301 2024-09-21T09:48:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:18 smithi191 ceph-mon[102206]: pgmap v494: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:48:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:18 smithi191 ceph-mon[102206]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:48:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:18 smithi076 ceph-mon[106482]: pgmap v494: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:48:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:18 smithi076 ceph-mon[106482]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:48:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:18 smithi076 ceph-mon[110963]: pgmap v494: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:48:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:18 smithi076 ceph-mon[110963]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T09:48:18.721 INFO:tasks.rados.rados.0.smithi076.stdout:2361: done (2 left) 2024-09-21T09:48:18.721 INFO:tasks.rados.rados.0.smithi076.stdout:2362: snap_create 2024-09-21T09:48:18.721 INFO:tasks.rados.rados.0.smithi076.stdout:2357: expect (ObjNum 773 snap 267 seq_num 773) 2024-09-21T09:48:18.899 INFO:tasks.rados.rados.0.smithi076.stdout:2359: expect (ObjNum 866 snap 296 seq_num 866) 2024-09-21T09:48:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:19 smithi191 ceph-mon[102206]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:48:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:19 smithi191 ceph-mon[102206]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:48:19.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:19 smithi076 ceph-mon[110963]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:48:19.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:19 smithi076 ceph-mon[110963]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:48:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:19 smithi076 ceph-mon[106482]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T09:48:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:19 smithi076 ceph-mon[106482]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T09:48:19.721 INFO:tasks.rados.rados.0.smithi076.stdout:2357: done (2 left) 2024-09-21T09:48:19.721 INFO:tasks.rados.rados.0.smithi076.stdout:2359: done (1 left) 2024-09-21T09:48:19.721 INFO:tasks.rados.rados.0.smithi076.stdout:2362: done (0 left) 2024-09-21T09:48:19.721 INFO:tasks.rados.rados.0.smithi076.stdout:2363: write oid 41 current snap is 310 2024-09-21T09:48:19.722 INFO:tasks.rados.rados.0.smithi076.stdout:2363: seq_num 882 ranges {423774=666566,1545824=512968,2704609=119165} 2024-09-21T09:48:19.727 INFO:tasks.rados.rados.0.smithi076.stdout:2363: writing smithi07685715-41 from 423774 to 1090340 tid 1 2024-09-21T09:48:19.732 INFO:tasks.rados.rados.0.smithi076.stdout:2363: writing smithi07685715-41 from 1545824 to 2058792 tid 2 2024-09-21T09:48:19.737 INFO:tasks.rados.rados.0.smithi076.stdout:2363: writing smithi07685715-41 from 2704609 to 2823774 tid 3 2024-09-21T09:48:19.737 INFO:tasks.rados.rados.0.smithi076.stdout:2364: write oid 32 current snap is 310 2024-09-21T09:48:19.737 INFO:tasks.rados.rados.0.smithi076.stdout:2363: finishing write tid 1 to smithi07685715-41 2024-09-21T09:48:19.737 INFO:tasks.rados.rados.0.smithi076.stdout:2363: finishing write tid 2 to smithi07685715-41 2024-09-21T09:48:19.737 INFO:tasks.rados.rados.0.smithi076.stdout:2364: seq_num 883 ranges {440581=741815,1604306=656626,2732032=108549} 2024-09-21T09:48:19.743 INFO:tasks.rados.rados.0.smithi076.stdout:2364: writing smithi07685715-32 from 440581 to 1182396 tid 1 2024-09-21T09:48:19.749 INFO:tasks.rados.rados.0.smithi076.stdout:2364: writing smithi07685715-32 from 1604306 to 2260932 tid 2 2024-09-21T09:48:19.753 INFO:tasks.rados.rados.0.smithi076.stdout:2364: writing smithi07685715-32 from 2732032 to 2840581 tid 3 2024-09-21T09:48:19.753 INFO:tasks.rados.rados.0.smithi076.stdout:2365: write oid 15 current snap is 310 2024-09-21T09:48:19.753 INFO:tasks.rados.rados.0.smithi076.stdout:2363: finishing write tid 3 to smithi07685715-41 2024-09-21T09:48:19.753 INFO:tasks.rados.rados.0.smithi076.stdout:2365: seq_num 884 ranges {457388=417064,1546435=516637,2643102=214286} 2024-09-21T09:48:19.757 INFO:tasks.rados.rados.0.smithi076.stdout:2365: writing smithi07685715-15 from 457388 to 874452 tid 1 2024-09-21T09:48:19.764 INFO:tasks.rados.rados.0.smithi076.stdout:2365: writing smithi07685715-15 from 1546435 to 2063072 tid 2 2024-09-21T09:48:19.769 INFO:tasks.rados.rados.0.smithi076.stdout:2365: writing smithi07685715-15 from 2643102 to 2857388 tid 3 2024-09-21T09:48:19.769 INFO:tasks.rados.rados.0.smithi076.stdout:2366: read oid 38 snap -1 2024-09-21T09:48:19.769 INFO:tasks.rados.rados.0.smithi076.stdout:2366: expect (ObjNum 869 snap 302 seq_num 869) 2024-09-21T09:48:19.769 INFO:tasks.rados.rados.0.smithi076.stdout:2367: snap_create 2024-09-21T09:48:19.769 INFO:tasks.rados.rados.0.smithi076.stdout:2363: finishing write tid 4 to smithi07685715-41 2024-09-21T09:48:19.769 INFO:tasks.rados.rados.0.smithi076.stdout:2363: finishing write tid 5 to smithi07685715-41 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2363: oid 41 updating version 0 to 1029 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2363: oid 41 version 1029 is already newer than 1026 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2363: oid 41 version 1029 is already newer than 1027 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2363: oid 41 version 1029 is already newer than 1028 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1029 (ObjNum 882 snap 310 seq_num 882) dirty exists 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2363: left oid 41 (ObjNum 882 snap 310 seq_num 882) 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2364: finishing write tid 1 to smithi07685715-32 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2364: finishing write tid 2 to smithi07685715-32 2024-09-21T09:48:19.770 INFO:tasks.rados.rados.0.smithi076.stdout:2364: finishing write tid 3 to smithi07685715-32 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: finishing write tid 4 to smithi07685715-32 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: finishing write tid 5 to smithi07685715-32 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: oid 32 updating version 0 to 762 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: oid 32 version 762 is already newer than 760 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: oid 32 version 762 is already newer than 761 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: oid 32 version 762 is already newer than 759 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 762 (ObjNum 883 snap 310 seq_num 883) dirty exists 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2364: left oid 32 (ObjNum 883 snap 310 seq_num 883) 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2365: finishing write tid 1 to smithi07685715-15 2024-09-21T09:48:19.771 INFO:tasks.rados.rados.0.smithi076.stdout:2365: finishing write tid 2 to smithi07685715-15 2024-09-21T09:48:19.772 INFO:tasks.rados.rados.0.smithi076.stdout:2365: finishing write tid 3 to smithi07685715-15 2024-09-21T09:48:19.772 INFO:tasks.rados.rados.0.smithi076.stdout:2365: finishing write tid 4 to smithi07685715-15 2024-09-21T09:48:19.772 INFO:tasks.rados.rados.0.smithi076.stdout:2365: finishing write tid 5 to smithi07685715-15 2024-09-21T09:48:19.772 INFO:tasks.rados.rados.0.smithi076.stdout:2365: oid 15 updating version 0 to 522 2024-09-21T09:48:19.772 INFO:tasks.rados.rados.0.smithi076.stdout:2365: oid 15 version 522 is already newer than 520 2024-09-21T09:48:19.772 INFO:tasks.rados.rados.0.smithi076.stdout:2365: oid 15 updating version 522 to 523 2024-09-21T09:48:19.773 INFO:tasks.rados.rados.0.smithi076.stdout:2365: oid 15 version 523 is already newer than 521 2024-09-21T09:48:19.773 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 523 (ObjNum 884 snap 310 seq_num 884) dirty exists 2024-09-21T09:48:19.773 INFO:tasks.rados.rados.0.smithi076.stdout:2365: left oid 15 (ObjNum 884 snap 310 seq_num 884) 2024-09-21T09:48:19.773 INFO:tasks.rados.rados.0.smithi076.stdout:2366: expect (ObjNum 869 snap 302 seq_num 869) 2024-09-21T09:48:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:20 smithi191 ceph-mon[102206]: pgmap v497: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:48:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:20 smithi191 ceph-mon[102206]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:48:20.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:20 smithi076 ceph-mon[106482]: pgmap v497: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:48:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:20 smithi076 ceph-mon[106482]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:48:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:20 smithi076 ceph-mon[110963]: pgmap v497: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:48:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:20 smithi076 ceph-mon[110963]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2363: done (4 left) 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2364: done (3 left) 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2365: done (2 left) 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2366: done (1 left) 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2367: done (0 left) 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2368: write oid 34 current snap is 311 2024-09-21T09:48:20.723 INFO:tasks.rados.rados.0.smithi076.stdout:2368: seq_num 885 ranges {474195=492313,1604917=660295,2670525=203670} 2024-09-21T09:48:20.728 INFO:tasks.rados.rados.0.smithi076.stdout:2368: writing smithi07685715-34 from 474195 to 966508 tid 1 2024-09-21T09:48:20.735 INFO:tasks.rados.rados.0.smithi076.stdout:2368: writing smithi07685715-34 from 1604917 to 2265212 tid 2 2024-09-21T09:48:20.739 INFO:tasks.rados.rados.0.smithi076.stdout:2368: writing smithi07685715-34 from 2670525 to 2874195 tid 3 2024-09-21T09:48:20.739 INFO:tasks.rados.rados.0.smithi076.stdout:2369: write oid 13 current snap is 311 2024-09-21T09:48:20.739 INFO:tasks.rados.rados.0.smithi076.stdout:2368: finishing write tid 1 to smithi07685715-34 2024-09-21T09:48:20.739 INFO:tasks.rados.rados.0.smithi076.stdout:2368: finishing write tid 2 to smithi07685715-34 2024-09-21T09:48:20.739 INFO:tasks.rados.rados.0.smithi076.stdout:2369: seq_num 886 ranges {491002=567562,1663399=520306,2814301=76701} 2024-09-21T09:48:20.744 INFO:tasks.rados.rados.0.smithi076.stdout:2369: writing smithi07685715-13 from 491002 to 1058564 tid 1 2024-09-21T09:48:20.751 INFO:tasks.rados.rados.0.smithi076.stdout:2369: writing smithi07685715-13 from 1663399 to 2183705 tid 2 2024-09-21T09:48:20.755 INFO:tasks.rados.rados.0.smithi076.stdout:2369: writing smithi07685715-13 from 2814301 to 2891002 tid 3 2024-09-21T09:48:20.755 INFO:tasks.rados.rados.0.smithi076.stdout:2370: write oid 36 current snap is 311 2024-09-21T09:48:20.755 INFO:tasks.rados.rados.0.smithi076.stdout:2368: finishing write tid 3 to smithi07685715-34 2024-09-21T09:48:20.755 INFO:tasks.rados.rados.0.smithi076.stdout:2368: finishing write tid 4 to smithi07685715-34 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2368: finishing write tid 5 to smithi07685715-34 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2368: oid 34 updating version 0 to 503 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2368: oid 34 version 503 is already newer than 502 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2368: oid 34 version 503 is already newer than 501 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2368: oid 34 version 503 is already newer than 500 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 503 (ObjNum 885 snap 311 seq_num 885) dirty exists 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2368: left oid 34 (ObjNum 885 snap 311 seq_num 885) 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2369: finishing write tid 1 to smithi07685715-13 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2369: finishing write tid 2 to smithi07685715-13 2024-09-21T09:48:20.756 INFO:tasks.rados.rados.0.smithi076.stdout:2370: seq_num 887 ranges {507809=642811,1721881=663964} 2024-09-21T09:48:20.761 INFO:tasks.rados.rados.0.smithi076.stdout:2370: writing smithi07685715-36 from 507809 to 1150620 tid 1 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2370: writing smithi07685715-36 from 1721881 to 2385845 tid 2 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2368: done (2 left) 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2371: read oid 39 snap -1 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2371: expect (ObjNum 848 snap 292 seq_num 848) 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2372: write oid 46 current snap is 311 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2369: finishing write tid 3 to smithi07685715-13 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2369: finishing write tid 4 to smithi07685715-13 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2369: finishing write tid 5 to smithi07685715-13 2024-09-21T09:48:20.768 INFO:tasks.rados.rados.0.smithi076.stdout:2369: oid 13 updating version 0 to 386 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:2369: oid 13 version 386 is already newer than 385 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:2369: oid 13 updating version 386 to 388 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:2369: oid 13 version 388 is already newer than 387 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 388 (ObjNum 886 snap 311 seq_num 886) dirty exists 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:2369: left oid 13 (ObjNum 886 snap 311 seq_num 886) 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:2370: finishing write tid 1 to smithi07685715-36 2024-09-21T09:48:20.769 INFO:tasks.rados.rados.0.smithi076.stdout:2372: seq_num 888 ranges {524616=718060,1664010=523975,2752794=171822} 2024-09-21T09:48:20.774 INFO:tasks.rados.rados.0.smithi076.stdout:2372: writing smithi07685715-46 from 524616 to 1242676 tid 1 2024-09-21T09:48:20.780 INFO:tasks.rados.rados.0.smithi076.stdout:2372: writing smithi07685715-46 from 1664010 to 2187985 tid 2 2024-09-21T09:48:20.784 INFO:tasks.rados.rados.0.smithi076.stdout:2372: writing smithi07685715-46 from 2752794 to 2924616 tid 3 2024-09-21T09:48:20.785 INFO:tasks.rados.rados.0.smithi076.stdout:2369: done (3 left) 2024-09-21T09:48:20.785 INFO:tasks.rados.rados.0.smithi076.stdout:2373: write oid 22 current snap is 311 2024-09-21T09:48:20.785 INFO:tasks.rados.rados.0.smithi076.stdout:2373: seq_num 889 ranges {541423=793309,2122492=667633} 2024-09-21T09:48:20.791 INFO:tasks.rados.rados.0.smithi076.stdout:2373: writing smithi07685715-22 from 541423 to 1334732 tid 1 2024-09-21T09:48:20.800 INFO:tasks.rados.rados.0.smithi076.stdout:2373: writing smithi07685715-22 from 2122492 to 2790125 tid 2 2024-09-21T09:48:20.800 INFO:tasks.rados.rados.0.smithi076.stdout:2374: snap_create 2024-09-21T09:48:20.800 INFO:tasks.rados.rados.0.smithi076.stdout:2370: finishing write tid 2 to smithi07685715-36 2024-09-21T09:48:20.800 INFO:tasks.rados.rados.0.smithi076.stdout:2370: finishing write tid 3 to smithi07685715-36 2024-09-21T09:48:20.800 INFO:tasks.rados.rados.0.smithi076.stdout:2370: finishing write tid 4 to smithi07685715-36 2024-09-21T09:48:20.800 INFO:tasks.rados.rados.0.smithi076.stdout:2370: oid 36 updating version 0 to 733 2024-09-21T09:48:20.801 INFO:tasks.rados.rados.0.smithi076.stdout:2370: oid 36 updating version 733 to 734 2024-09-21T09:48:20.801 INFO:tasks.rados.rados.0.smithi076.stdout:2370: oid 36 updating version 734 to 735 2024-09-21T09:48:20.801 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 735 (ObjNum 887 snap 311 seq_num 887) dirty exists 2024-09-21T09:48:20.801 INFO:tasks.rados.rados.0.smithi076.stdout:2370: left oid 36 (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:48:20.801 INFO:tasks.rados.rados.0.smithi076.stdout:2371: expect (ObjNum 848 snap 292 seq_num 848) 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: finishing write tid 1 to smithi07685715-46 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: finishing write tid 2 to smithi07685715-46 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: finishing write tid 3 to smithi07685715-46 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: finishing write tid 4 to smithi07685715-46 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: finishing write tid 5 to smithi07685715-46 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: oid 46 updating version 0 to 1784 2024-09-21T09:48:20.871 INFO:tasks.rados.rados.0.smithi076.stdout:2372: oid 46 version 1784 is already newer than 1783 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2372: oid 46 updating version 1784 to 1785 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2372: oid 46 updating version 1785 to 1786 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1786 (ObjNum 888 snap 311 seq_num 888) dirty exists 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2372: left oid 46 (ObjNum 888 snap 311 seq_num 888) 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: finishing write tid 1 to smithi07685715-22 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: finishing write tid 2 to smithi07685715-22 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: finishing write tid 3 to smithi07685715-22 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: finishing write tid 4 to smithi07685715-22 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: oid 22 updating version 0 to 1033 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: oid 22 version 1033 is already newer than 1031 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: oid 22 version 1033 is already newer than 1032 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1033 (ObjNum 889 snap 311 seq_num 889) dirty exists 2024-09-21T09:48:20.872 INFO:tasks.rados.rados.0.smithi076.stdout:2373: left oid 22 (ObjNum 889 snap 311 seq_num 889) 2024-09-21T09:48:21.729 INFO:tasks.rados.rados.0.smithi076.stdout:2370: done (4 left) 2024-09-21T09:48:21.729 INFO:tasks.rados.rados.0.smithi076.stdout:2371: done (3 left) 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2372: done (2 left) 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2373: done (1 left) 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2374: done (0 left) 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2375: read oid 25 snap 304 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2375: expect (ObjNum 826 snap 285 seq_num 826) 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2376: write oid 39 current snap is 312 2024-09-21T09:48:21.730 INFO:tasks.rados.rados.0.smithi076.stdout:2376: seq_num 890 ranges {558230=584911,1897327=527644,2923993=34237} 2024-09-21T09:48:21.736 INFO:tasks.rados.rados.0.smithi076.stdout:2376: writing smithi07685715-39 from 558230 to 1143141 tid 1 2024-09-21T09:48:21.743 INFO:tasks.rados.rados.0.smithi076.stdout:2376: writing smithi07685715-39 from 1897327 to 2424971 tid 2 2024-09-21T09:48:21.746 INFO:tasks.rados.rados.0.smithi076.stdout:2376: writing smithi07685715-39 from 2923993 to 2958230 tid 3 2024-09-21T09:48:21.746 INFO:tasks.rados.rados.0.smithi076.stdout:2377: read oid 42 snap -1 2024-09-21T09:48:21.746 INFO:tasks.rados.rados.0.smithi076.stdout:2377: expect deleted 2024-09-21T09:48:21.746 INFO:tasks.rados.rados.0.smithi076.stdout:2378: write oid 33 current snap is 312 2024-09-21T09:48:21.747 INFO:tasks.rados.rados.0.smithi076.stdout:2378: seq_num 891 ranges {575037=660160,1955809=671302} 2024-09-21T09:48:21.753 INFO:tasks.rados.rados.0.smithi076.stdout:2378: writing smithi07685715-33 from 575037 to 1235197 tid 1 2024-09-21T09:48:21.761 INFO:tasks.rados.rados.0.smithi076.stdout:2378: writing smithi07685715-33 from 1955809 to 2627111 tid 2 2024-09-21T09:48:21.761 INFO:tasks.rados.rados.0.smithi076.stdout:2379: write oid 23 current snap is 312 2024-09-21T09:48:21.761 INFO:tasks.rados.rados.0.smithi076.stdout:2379: seq_num 892 ranges {591844=735409,1897938=531313,2862486=129358} 2024-09-21T09:48:21.768 INFO:tasks.rados.rados.0.smithi076.stdout:2379: writing smithi07685715-23 from 591844 to 1327253 tid 1 2024-09-21T09:48:21.774 INFO:tasks.rados.rados.0.smithi076.stdout:2379: writing smithi07685715-23 from 1897938 to 2429251 tid 2 2024-09-21T09:48:21.778 INFO:tasks.rados.rados.0.smithi076.stdout:2379: writing smithi07685715-23 from 2862486 to 2991844 tid 3 2024-09-21T09:48:21.778 INFO:tasks.rados.rados.0.smithi076.stdout:2380: read oid 8 snap -1 2024-09-21T09:48:21.778 INFO:tasks.rados.rados.0.smithi076.stdout:2380: expect (ObjNum 781 snap 272 seq_num 781) 2024-09-21T09:48:21.778 INFO:tasks.rados.rados.0.smithi076.stdout:2381: delete oid 21 current snap is 312 2024-09-21T09:48:21.778 INFO:tasks.rados.rados.0.smithi076.stdout:2375: expect (ObjNum 826 snap 285 seq_num 826) 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: finishing write tid 1 to smithi07685715-39 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: finishing write tid 2 to smithi07685715-39 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: finishing write tid 3 to smithi07685715-39 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: finishing write tid 4 to smithi07685715-39 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: finishing write tid 5 to smithi07685715-39 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: oid 39 updating version 0 to 1789 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: oid 39 updating version 1789 to 1790 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: oid 39 updating version 1790 to 1791 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: oid 39 version 1791 is already newer than 1788 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1791 (ObjNum 890 snap 312 seq_num 890) dirty exists 2024-09-21T09:48:22.008 INFO:tasks.rados.rados.0.smithi076.stdout:2376: left oid 39 (ObjNum 890 snap 312 seq_num 890) 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: finishing write tid 1 to smithi07685715-33 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2375: done (6 left) 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2376: done (5 left) 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2377: done (4 left) 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2381: done (3 left) 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2382: rollback oid 20 current snap is 312 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 293 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: finishing write tid 2 to smithi07685715-33 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: finishing write tid 3 to smithi07685715-33 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: finishing write tid 4 to smithi07685715-33 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: oid 33 updating version 0 to 573 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: oid 33 updating version 573 to 574 2024-09-21T09:48:22.009 INFO:tasks.rados.rados.0.smithi076.stdout:2378: oid 33 version 574 is already newer than 572 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 574 (ObjNum 891 snap 312 seq_num 891) dirty exists 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2378: left oid 33 (ObjNum 891 snap 312 seq_num 891) 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: finishing write tid 1 to smithi07685715-23 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: finishing write tid 2 to smithi07685715-23 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: finishing write tid 3 to smithi07685715-23 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: finishing write tid 4 to smithi07685715-23 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: finishing write tid 5 to smithi07685715-23 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: oid 23 updating version 0 to 1858 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: oid 23 updating version 1858 to 1860 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: oid 23 version 1860 is already newer than 1859 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: oid 23 updating version 1860 to 1861 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 1861 (ObjNum 892 snap 312 seq_num 892) dirty exists 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2379: left oid 23 (ObjNum 892 snap 312 seq_num 892) 2024-09-21T09:48:22.010 INFO:tasks.rados.rados.0.smithi076.stdout:2380: expect (ObjNum 781 snap 272 seq_num 781) 2024-09-21T09:48:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:21 smithi191 ceph-mon[102206]: pgmap v500: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:21 smithi191 ceph-mon[102206]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:48:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:21 smithi076 ceph-mon[106482]: pgmap v500: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:21 smithi076 ceph-mon[106482]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:48:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:21 smithi076 ceph-mon[110963]: pgmap v500: 89 pgs: 89 active+clean; 203 MiB data, 755 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:22.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:21 smithi076 ceph-mon[110963]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2382: finishing rollback tid 0 to smithi07685715-20 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2382: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2382: finishing rollback tid 2 to smithi07685715-20 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 694 (ObjNum 806 snap 282 seq_num 806) dirty exists 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2378: done (3 left) 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2379: done (2 left) 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2380: done (1 left) 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2382: done (0 left) 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2383: rollback oid 33 current snap is 312 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 308 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2384: write oid 10 current snap is 312 2024-09-21T09:48:22.193 INFO:tasks.rados.rados.0.smithi076.stdout:2384: seq_num 894 ranges {625458=485907,1614902=534982,2917332=108126} 2024-09-21T09:48:22.198 INFO:tasks.rados.rados.0.smithi076.stdout:2384: writing smithi07685715-10 from 625458 to 1111365 tid 1 2024-09-21T09:48:22.204 INFO:tasks.rados.rados.0.smithi076.stdout:2384: writing smithi07685715-10 from 1614902 to 2149884 tid 2 2024-09-21T09:48:22.209 INFO:tasks.rados.rados.0.smithi076.stdout:2384: writing smithi07685715-10 from 2917332 to 3025458 tid 3 2024-09-21T09:48:22.209 INFO:tasks.rados.rados.0.smithi076.stdout:2385: delete oid 16 current snap is 312 2024-09-21T09:48:22.209 INFO:tasks.rados.rados.0.smithi076.stdout:2383: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:48:22.210 INFO:tasks.rados.rados.0.smithi076.stdout:2383: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:48:22.210 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 576 (ObjNum 305 snap 0 seq_num 0) dirty dne 2024-09-21T09:48:22.210 INFO:tasks.rados.rados.0.smithi076.stdout:2384: finishing write tid 1 to smithi07685715-10 2024-09-21T09:48:22.210 INFO:tasks.rados.rados.0.smithi076.stdout:2384: finishing write tid 2 to smithi07685715-10 2024-09-21T09:48:22.211 INFO:tasks.rados.rados.0.smithi076.stdout:2383: done (2 left) 2024-09-21T09:48:22.211 INFO:tasks.rados.rados.0.smithi076.stdout:2385: done (1 left) 2024-09-21T09:48:22.211 INFO:tasks.rados.rados.0.smithi076.stdout:2386: write oid 19 current snap is 312 2024-09-21T09:48:22.211 INFO:tasks.rados.rados.0.smithi076.stdout:2386: seq_num 896 ranges {659072=636405,1731866=422298,2855825=203247} 2024-09-21T09:48:22.218 INFO:tasks.rados.rados.0.smithi076.stdout:2386: writing smithi07685715-19 from 659072 to 1295477 tid 1 2024-09-21T09:48:22.223 INFO:tasks.rados.rados.0.smithi076.stdout:2386: writing smithi07685715-19 from 1731866 to 2154164 tid 2 2024-09-21T09:48:22.228 INFO:tasks.rados.rados.0.smithi076.stdout:2386: writing smithi07685715-19 from 2855825 to 3059072 tid 3 2024-09-21T09:48:22.228 INFO:tasks.rados.rados.0.smithi076.stdout:2387: read oid 41 snap -1 2024-09-21T09:48:22.228 INFO:tasks.rados.rados.0.smithi076.stdout:2387: expect (ObjNum 882 snap 310 seq_num 882) 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2388: write oid 22 current snap is 312 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: finishing write tid 3 to smithi07685715-10 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: finishing write tid 4 to smithi07685715-10 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: finishing write tid 5 to smithi07685715-10 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: oid 10 updating version 0 to 1370 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: oid 10 version 1370 is already newer than 1368 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: oid 10 updating version 1370 to 1371 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: oid 10 version 1371 is already newer than 1369 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1371 (ObjNum 894 snap 312 seq_num 894) dirty exists 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2384: left oid 10 (ObjNum 894 snap 312 seq_num 894) 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2386: finishing write tid 1 to smithi07685715-19 2024-09-21T09:48:22.229 INFO:tasks.rados.rados.0.smithi076.stdout:2388: seq_num 897 ranges {675879=711654,2073995=682309} 2024-09-21T09:48:22.236 INFO:tasks.rados.rados.0.smithi076.stdout:2388: writing smithi07685715-22 from 675879 to 1387533 tid 1 2024-09-21T09:48:22.243 INFO:tasks.rados.rados.0.smithi076.stdout:2388: writing smithi07685715-22 from 2073995 to 2756304 tid 2 2024-09-21T09:48:22.243 INFO:tasks.rados.rados.0.smithi076.stdout:2384: done (3 left) 2024-09-21T09:48:22.243 INFO:tasks.rados.rados.0.smithi076.stdout:2389: snap_create 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: finishing write tid 2 to smithi07685715-19 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: finishing write tid 3 to smithi07685715-19 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: finishing write tid 4 to smithi07685715-19 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: finishing write tid 5 to smithi07685715-19 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: oid 19 updating version 0 to 764 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: oid 19 updating version 764 to 766 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: oid 19 updating version 766 to 767 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: oid 19 version 767 is already newer than 765 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 767 (ObjNum 896 snap 312 seq_num 896) dirty exists 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2386: left oid 19 (ObjNum 896 snap 312 seq_num 896) 2024-09-21T09:48:22.244 INFO:tasks.rados.rados.0.smithi076.stdout:2387: expect (ObjNum 882 snap 310 seq_num 882) 2024-09-21T09:48:22.365 INFO:tasks.rados.rados.0.smithi076.stdout:2388: finishing write tid 1 to smithi07685715-22 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: finishing write tid 2 to smithi07685715-22 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: finishing write tid 3 to smithi07685715-22 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: finishing write tid 4 to smithi07685715-22 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: oid 22 updating version 0 to 1037 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: oid 22 version 1037 is already newer than 1036 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: oid 22 updating version 1037 to 1038 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1038 (ObjNum 897 snap 312 seq_num 897) dirty exists 2024-09-21T09:48:22.366 INFO:tasks.rados.rados.0.smithi076.stdout:2388: left oid 22 (ObjNum 897 snap 312 seq_num 897) 2024-09-21T09:48:22.740 INFO:tasks.rados.rados.0.smithi076.stdout:2386: done (3 left) 2024-09-21T09:48:22.740 INFO:tasks.rados.rados.0.smithi076.stdout:2387: done (2 left) 2024-09-21T09:48:22.740 INFO:tasks.rados.rados.0.smithi076.stdout:2388: done (1 left) 2024-09-21T09:48:22.740 INFO:tasks.rados.rados.0.smithi076.stdout:2389: done (0 left) 2024-09-21T09:48:22.740 INFO:tasks.rados.rados.0.smithi076.stdout:2390: write oid 9 current snap is 313 2024-09-21T09:48:22.740 INFO:tasks.rados.rados.0.smithi076.stdout:2390: seq_num 898 ranges {692686=503256,1848830=425967,2910671=182015} 2024-09-21T09:48:22.746 INFO:tasks.rados.rados.0.smithi076.stdout:2390: writing smithi07685715-9 from 692686 to 1195942 tid 1 2024-09-21T09:48:22.752 INFO:tasks.rados.rados.0.smithi076.stdout:2390: writing smithi07685715-9 from 1848830 to 2274797 tid 2 2024-09-21T09:48:22.757 INFO:tasks.rados.rados.0.smithi076.stdout:2390: writing smithi07685715-9 from 2910671 to 3092686 tid 3 2024-09-21T09:48:22.757 INFO:tasks.rados.rados.0.smithi076.stdout:2391: delete oid 25 current snap is 313 2024-09-21T09:48:22.757 INFO:tasks.rados.rados.0.smithi076.stdout:2390: finishing write tid 1 to smithi07685715-9 2024-09-21T09:48:22.757 INFO:tasks.rados.rados.0.smithi076.stdout:2390: finishing write tid 2 to smithi07685715-9 2024-09-21T09:48:22.759 INFO:tasks.rados.rados.0.smithi076.stdout:2390: finishing write tid 3 to smithi07685715-9 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: finishing write tid 4 to smithi07685715-9 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: finishing write tid 5 to smithi07685715-9 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: oid 9 updating version 0 to 851 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: oid 9 version 851 is already newer than 850 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: oid 9 version 851 is already newer than 848 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: oid 9 version 851 is already newer than 849 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 851 (ObjNum 898 snap 313 seq_num 898) dirty exists 2024-09-21T09:48:22.760 INFO:tasks.rados.rados.0.smithi076.stdout:2390: left oid 9 (ObjNum 898 snap 313 seq_num 898) 2024-09-21T09:48:22.761 INFO:tasks.rados.rados.0.smithi076.stdout:2390: done (1 left) 2024-09-21T09:48:22.761 INFO:tasks.rados.rados.0.smithi076.stdout:2391: done (0 left) 2024-09-21T09:48:22.761 INFO:tasks.rados.rados.0.smithi076.stdout:2392: read oid 14 snap -1 2024-09-21T09:48:22.762 INFO:tasks.rados.rados.0.smithi076.stdout:2392: expect (ObjNum 877 snap 307 seq_num 877) 2024-09-21T09:48:22.762 INFO:tasks.rados.rados.0.smithi076.stdout:2393: write oid 2 current snap is 313 2024-09-21T09:48:22.762 INFO:tasks.rados.rados.0.smithi076.stdout:2393: seq_num 900 ranges {726300=653754,1965794=429636,2965517=160783} 2024-09-21T09:48:22.769 INFO:tasks.rados.rados.0.smithi076.stdout:2393: writing smithi07685715-2 from 726300 to 1380054 tid 1 2024-09-21T09:48:22.774 INFO:tasks.rados.rados.0.smithi076.stdout:2393: writing smithi07685715-2 from 1965794 to 2395430 tid 2 2024-09-21T09:48:22.778 INFO:tasks.rados.rados.0.smithi076.stdout:2393: writing smithi07685715-2 from 2965517 to 3126300 tid 3 2024-09-21T09:48:22.779 INFO:tasks.rados.rados.0.smithi076.stdout:2394: write oid 19 current snap is 313 2024-09-21T09:48:22.779 INFO:tasks.rados.rados.0.smithi076.stdout:2394: seq_num 901 ranges {743107=729003,1907923=689647} 2024-09-21T09:48:22.786 INFO:tasks.rados.rados.0.smithi076.stdout:2394: writing smithi07685715-19 from 743107 to 1472110 tid 1 2024-09-21T09:48:22.793 INFO:tasks.rados.rados.0.smithi076.stdout:2394: writing smithi07685715-19 from 1907923 to 2597570 tid 2 2024-09-21T09:48:22.793 INFO:tasks.rados.rados.0.smithi076.stdout:2395: read oid 46 snap -1 2024-09-21T09:48:22.793 INFO:tasks.rados.rados.0.smithi076.stdout:2395: expect (ObjNum 888 snap 311 seq_num 888) 2024-09-21T09:48:22.793 INFO:tasks.rados.rados.0.smithi076.stdout:2396: read oid 32 snap -1 2024-09-21T09:48:22.794 INFO:tasks.rados.rados.0.smithi076.stdout:2396: expect (ObjNum 883 snap 310 seq_num 883) 2024-09-21T09:48:22.794 INFO:tasks.rados.rados.0.smithi076.stdout:2397: read oid 45 snap -1 2024-09-21T09:48:22.794 INFO:tasks.rados.rados.0.smithi076.stdout:2397: expect (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:48:22.794 INFO:tasks.rados.rados.0.smithi076.stdout:2398: write oid 8 current snap is 313 2024-09-21T09:48:22.794 INFO:tasks.rados.rados.0.smithi076.stdout:2398: seq_num 902 ranges {759914=404252,1566405=433305,2504010=655904} 2024-09-21T09:48:22.800 INFO:tasks.rados.rados.0.smithi076.stdout:2398: writing smithi07685715-8 from 759914 to 1164166 tid 1 2024-09-21T09:48:22.805 INFO:tasks.rados.rados.0.smithi076.stdout:2398: writing smithi07685715-8 from 1566405 to 1999710 tid 2 2024-09-21T09:48:22.811 INFO:tasks.rados.rados.0.smithi076.stdout:2398: writing smithi07685715-8 from 2504010 to 3159914 tid 3 2024-09-21T09:48:22.811 INFO:tasks.rados.rados.0.smithi076.stdout:2399: delete oid 31 current snap is 313 2024-09-21T09:48:22.812 INFO:tasks.rados.rados.0.smithi076.stdout:2392: expect (ObjNum 877 snap 307 seq_num 877) 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: finishing write tid 1 to smithi07685715-2 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: finishing write tid 2 to smithi07685715-2 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: finishing write tid 3 to smithi07685715-2 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: finishing write tid 4 to smithi07685715-2 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: finishing write tid 5 to smithi07685715-2 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: oid 2 updating version 0 to 392 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: oid 2 version 392 is already newer than 390 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: oid 2 updating version 392 to 393 2024-09-21T09:48:22.908 INFO:tasks.rados.rados.0.smithi076.stdout:2393: oid 2 version 393 is already newer than 391 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 393 (ObjNum 900 snap 313 seq_num 900) dirty exists 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:2393: left oid 2 (ObjNum 900 snap 313 seq_num 900) 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:2394: finishing write tid 1 to smithi07685715-19 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:2392: done (7 left) 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:2393: done (6 left) 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:2399: done (5 left) 2024-09-21T09:48:22.909 INFO:tasks.rados.rados.0.smithi076.stdout:2400: snap_remove snap 283 2024-09-21T09:48:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:22 smithi191 ceph-mon[102206]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:48:23.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:22 smithi076 ceph-mon[106482]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:48:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:22 smithi076 ceph-mon[110963]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T09:48:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:48:23] "GET /metrics HTTP/1.1" 200 36135 "" "Prometheus/2.51.0" 2024-09-21T09:48:23.724 INFO:tasks.rados.rados.0.smithi076.stdout:2400: done (5 left) 2024-09-21T09:48:23.724 INFO:tasks.rados.rados.0.smithi076.stdout:2401: read oid 26 snap -1 2024-09-21T09:48:23.724 INFO:tasks.rados.rados.0.smithi076.stdout:2401: expect (ObjNum 792 snap 274 seq_num 792) 2024-09-21T09:48:23.724 INFO:tasks.rados.rados.0.smithi076.stdout:2402: read oid 22 snap -1 2024-09-21T09:48:23.724 INFO:tasks.rados.rados.0.smithi076.stdout:2402: expect (ObjNum 897 snap 312 seq_num 897) 2024-09-21T09:48:23.724 INFO:tasks.rados.rados.0.smithi076.stdout:2403: read oid 20 snap 302 2024-09-21T09:48:23.725 INFO:tasks.rados.rados.0.smithi076.stdout:2403: expect (ObjNum 806 snap 282 seq_num 806) 2024-09-21T09:48:23.725 INFO:tasks.rados.rados.0.smithi076.stdout:2404: delete oid 41 current snap is 313 2024-09-21T09:48:23.725 INFO:tasks.rados.rados.0.smithi076.stdout:2397: expect (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2394: finishing write tid 2 to smithi07685715-19 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2394: finishing write tid 3 to smithi07685715-19 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2397: done (8 left) 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2404: done (7 left) 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2405: write oid 7 current snap is 313 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2394: finishing write tid 4 to smithi07685715-19 2024-09-21T09:48:23.842 INFO:tasks.rados.rados.0.smithi076.stdout:2394: oid 19 updating version 0 to 771 2024-09-21T09:48:23.843 INFO:tasks.rados.rados.0.smithi076.stdout:2394: oid 19 version 771 is already newer than 770 2024-09-21T09:48:23.843 INFO:tasks.rados.rados.0.smithi076.stdout:2394: oid 19 version 771 is already newer than 769 2024-09-21T09:48:23.843 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 771 (ObjNum 901 snap 313 seq_num 901) dirty exists 2024-09-21T09:48:23.843 INFO:tasks.rados.rados.0.smithi076.stdout:2394: left oid 19 (ObjNum 901 snap 313 seq_num 901) 2024-09-21T09:48:23.843 INFO:tasks.rados.rados.0.smithi076.stdout:2396: expect (ObjNum 883 snap 310 seq_num 883) 2024-09-21T09:48:23.930 INFO:tasks.rados.rados.0.smithi076.stdout:2395: expect (ObjNum 888 snap 311 seq_num 888) 2024-09-21T09:48:24.058 INFO:tasks.rados.rados.0.smithi076.stdout:2398: finishing write tid 1 to smithi07685715-8 2024-09-21T09:48:24.058 INFO:tasks.rados.rados.0.smithi076.stdout:2398: finishing write tid 2 to smithi07685715-8 2024-09-21T09:48:24.058 INFO:tasks.rados.rados.0.smithi076.stdout:2405: seq_num 905 ranges {410335=746352,1741851=696985,3102632=107703} 2024-09-21T09:48:24.063 INFO:tasks.rados.rados.0.smithi076.stdout:2405: writing smithi07685715-7 from 410335 to 1156687 tid 1 2024-09-21T09:48:24.070 INFO:tasks.rados.rados.0.smithi076.stdout:2405: writing smithi07685715-7 from 1741851 to 2438836 tid 2 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2405: writing smithi07685715-7 from 3102632 to 3210335 tid 3 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2394: done (7 left) 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2395: done (6 left) 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2396: done (5 left) 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2406: read oid 40 snap 306 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2406: expect (ObjNum 855 snap 292 seq_num 855) 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2407: write oid 3 current snap is 313 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2398: finishing write tid 3 to smithi07685715-8 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2398: finishing write tid 4 to smithi07685715-8 2024-09-21T09:48:24.076 INFO:tasks.rados.rados.0.smithi076.stdout:2398: finishing write tid 5 to smithi07685715-8 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:2398: oid 8 updating version 0 to 1864 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:2398: oid 8 updating version 1864 to 1865 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:2398: oid 8 version 1865 is already newer than 1863 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:2398: oid 8 updating version 1865 to 1866 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1866 (ObjNum 902 snap 313 seq_num 902) dirty exists 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:2398: left oid 8 (ObjNum 902 snap 313 seq_num 902) 2024-09-21T09:48:24.077 INFO:tasks.rados.rados.0.smithi076.stdout:2401: expect (ObjNum 792 snap 274 seq_num 792) 2024-09-21T09:48:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:23 smithi191 ceph-mon[102206]: pgmap v503: 89 pgs: 89 active+clean; 216 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:48:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:23 smithi191 ceph-mon[102206]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:48:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:23 smithi191 ceph-mon[102206]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:48:24.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-mon[110963]: pgmap v503: 89 pgs: 89 active+clean; 216 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:48:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-mon[110963]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:48:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-mon[110963]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:48:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-mon[106482]: pgmap v503: 89 pgs: 89 active+clean; 216 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:48:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-mon[106482]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T09:48:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:23 smithi076 ceph-mon[106482]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T09:48:24.287 INFO:tasks.rados.rados.0.smithi076.stdout:2402: expect (ObjNum 897 snap 312 seq_num 897) 2024-09-21T09:48:24.467 INFO:tasks.rados.rados.0.smithi076.stdout:2403: expect (ObjNum 806 snap 282 seq_num 806) 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: finishing write tid 1 to smithi07685715-7 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: finishing write tid 2 to smithi07685715-7 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: finishing write tid 3 to smithi07685715-7 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: finishing write tid 4 to smithi07685715-7 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: finishing write tid 5 to smithi07685715-7 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: oid 7 updating version 0 to 1380 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: oid 7 updating version 1380 to 1382 2024-09-21T09:48:24.615 INFO:tasks.rados.rados.0.smithi076.stdout:2405: oid 7 version 1382 is already newer than 1379 2024-09-21T09:48:24.616 INFO:tasks.rados.rados.0.smithi076.stdout:2405: oid 7 version 1382 is already newer than 1381 2024-09-21T09:48:24.616 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1382 (ObjNum 905 snap 313 seq_num 905) dirty exists 2024-09-21T09:48:24.616 INFO:tasks.rados.rados.0.smithi076.stdout:2405: left oid 7 (ObjNum 905 snap 313 seq_num 905) 2024-09-21T09:48:24.616 INFO:tasks.rados.rados.0.smithi076.stdout:2407: seq_num 906 ranges {427142=421601,1400333=440643,2613702=450326} 2024-09-21T09:48:24.619 INFO:tasks.rados.rados.0.smithi076.stdout:2407: writing smithi07685715-3 from 427142 to 848743 tid 1 2024-09-21T09:48:24.624 INFO:tasks.rados.rados.0.smithi076.stdout:2407: writing smithi07685715-3 from 1400333 to 1840976 tid 2 2024-09-21T09:48:24.631 INFO:tasks.rados.rados.0.smithi076.stdout:2407: writing smithi07685715-3 from 2613702 to 3064028 tid 3 2024-09-21T09:48:24.632 INFO:tasks.rados.rados.0.smithi076.stdout:2398: done (6 left) 2024-09-21T09:48:24.632 INFO:tasks.rados.rados.0.smithi076.stdout:2401: done (5 left) 2024-09-21T09:48:24.632 INFO:tasks.rados.rados.0.smithi076.stdout:2402: done (4 left) 2024-09-21T09:48:24.632 INFO:tasks.rados.rados.0.smithi076.stdout:2403: done (3 left) 2024-09-21T09:48:24.632 INFO:tasks.rados.rados.0.smithi076.stdout:2405: done (2 left) 2024-09-21T09:48:24.632 INFO:tasks.rados.rados.0.smithi076.stdout:2408: rollback oid 48 current snap is 313 2024-09-21T09:48:24.633 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 298 2024-09-21T09:48:24.633 INFO:tasks.rados.rados.0.smithi076.stdout:2406: expect (ObjNum 855 snap 292 seq_num 855) 2024-09-21T09:48:24.768 INFO:tasks.rados.rados.0.smithi076.stdout:2407: finishing write tid 1 to smithi07685715-3 2024-09-21T09:48:24.768 INFO:tasks.rados.rados.0.smithi076.stdout:2407: finishing write tid 2 to smithi07685715-3 2024-09-21T09:48:24.768 INFO:tasks.rados.rados.0.smithi076.stdout:2407: finishing write tid 3 to smithi07685715-3 2024-09-21T09:48:24.768 INFO:tasks.rados.rados.0.smithi076.stdout:2407: finishing write tid 4 to smithi07685715-3 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: finishing write tid 5 to smithi07685715-3 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: oid 3 updating version 0 to 148 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: oid 3 updating version 148 to 149 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: oid 3 version 149 is already newer than 146 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: oid 3 version 149 is already newer than 147 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 149 (ObjNum 906 snap 313 seq_num 906) dirty exists 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: left oid 3 (ObjNum 906 snap 313 seq_num 906) 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2408: finishing rollback tid 0 to smithi07685715-48 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2408: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2408: finishing rollback tid 2 to smithi07685715-48 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 1878 (ObjNum 858 snap 294 seq_num 858) dirty exists 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2406: done (2 left) 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2407: done (1 left) 2024-09-21T09:48:24.769 INFO:tasks.rados.rados.0.smithi076.stdout:2408: done (0 left) 2024-09-21T09:48:24.770 INFO:tasks.rados.rados.0.smithi076.stdout:2409: read oid 22 snap -1 2024-09-21T09:48:24.770 INFO:tasks.rados.rados.0.smithi076.stdout:2409: expect (ObjNum 897 snap 312 seq_num 897) 2024-09-21T09:48:24.770 INFO:tasks.rados.rados.0.smithi076.stdout:2410: delete oid 16 current snap is 313 2024-09-21T09:48:24.775 INFO:tasks.rados.rados.0.smithi076.stdout:2410: done (1 left) 2024-09-21T09:48:24.776 INFO:tasks.rados.rados.0.smithi076.stdout:2411: snap_remove snap 293 2024-09-21T09:48:25.748 INFO:tasks.rados.rados.0.smithi076.stdout:2411: done (1 left) 2024-09-21T09:48:25.748 INFO:tasks.rados.rados.0.smithi076.stdout:2409: expect (ObjNum 897 snap 312 seq_num 897) 2024-09-21T09:48:25.933 INFO:tasks.rados.rados.0.smithi076.stdout:2412: read oid 5 snap 311 2024-09-21T09:48:25.933 INFO:tasks.rados.rados.0.smithi076.stdout:2412: expect deleted 2024-09-21T09:48:25.933 INFO:tasks.rados.rados.0.smithi076.stdout:2409: done (1 left) 2024-09-21T09:48:25.933 INFO:tasks.rados.rados.0.smithi076.stdout:2413: delete oid 9 current snap is 313 2024-09-21T09:48:25.934 INFO:tasks.rados.rados.0.smithi076.stdout:2412: done (1 left) 2024-09-21T09:48:25.934 INFO:tasks.rados.rados.0.smithi076.stdout:2413: done (0 left) 2024-09-21T09:48:25.935 INFO:tasks.rados.rados.0.smithi076.stdout:2414: rollback oid 37 current snap is 313 2024-09-21T09:48:25.935 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 289 2024-09-21T09:48:25.935 INFO:tasks.rados.rados.0.smithi076.stdout:2415: rollback oid 34 current snap is 313 2024-09-21T09:48:25.935 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 312 2024-09-21T09:48:25.935 INFO:tasks.rados.rados.0.smithi076.stdout:2416: write oid 50 current snap is 313 2024-09-21T09:48:25.935 INFO:tasks.rados.rados.0.smithi076.stdout:2416: seq_num 909 ranges {477563=647348,1859426=587970,2979618=297945} 2024-09-21T09:48:25.940 INFO:tasks.rados.rados.0.smithi076.stdout:2416: writing smithi07685715-50 from 477563 to 1124911 tid 1 2024-09-21T09:48:25.947 INFO:tasks.rados.rados.0.smithi076.stdout:2416: writing smithi07685715-50 from 1859426 to 2447396 tid 2 2024-09-21T09:48:25.952 INFO:tasks.rados.rados.0.smithi076.stdout:2416: writing smithi07685715-50 from 2979618 to 3277563 tid 3 2024-09-21T09:48:25.952 INFO:tasks.rados.rados.0.smithi076.stdout:2417: snap_create 2024-09-21T09:48:25.952 INFO:tasks.rados.rados.0.smithi076.stdout:2415: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:48:25.952 INFO:tasks.rados.rados.0.smithi076.stdout:2414: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:48:25.952 INFO:tasks.rados.rados.0.smithi076.stdout:2415: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:2415: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 509 (ObjNum 885 snap 311 seq_num 885) dirty exists 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:2414: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:2414: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 706 (ObjNum 697 snap 224 seq_num 697) dirty exists 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:2416: finishing write tid 1 to smithi07685715-50 2024-09-21T09:48:25.953 INFO:tasks.rados.rados.0.smithi076.stdout:2416: finishing write tid 2 to smithi07685715-50 2024-09-21T09:48:25.955 INFO:tasks.rados.rados.0.smithi076.stdout:2416: finishing write tid 3 to smithi07685715-50 2024-09-21T09:48:25.955 INFO:tasks.rados.rados.0.smithi076.stdout:2416: finishing write tid 4 to smithi07685715-50 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:2416: finishing write tid 5 to smithi07685715-50 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:2416: oid 50 updating version 0 to 749 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:2416: oid 50 updating version 749 to 752 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:2416: oid 50 version 752 is already newer than 750 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:2416: oid 50 version 752 is already newer than 751 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 752 (ObjNum 909 snap 313 seq_num 909) dirty exists 2024-09-21T09:48:25.956 INFO:tasks.rados.rados.0.smithi076.stdout:2416: left oid 50 (ObjNum 909 snap 313 seq_num 909) 2024-09-21T09:48:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:25 smithi191 ceph-mon[102206]: pgmap v506: 89 pgs: 89 active+clean; 216 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:48:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:26.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:25 smithi076 ceph-mon[106482]: pgmap v506: 89 pgs: 89 active+clean; 216 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:48:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:25 smithi076 ceph-mon[110963]: pgmap v506: 89 pgs: 89 active+clean; 216 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 2.5 MiB/s wr, 17 op/s 2024-09-21T09:48:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:26.760 INFO:tasks.rados.rados.0.smithi076.stdout:2414: done (3 left) 2024-09-21T09:48:26.760 INFO:tasks.rados.rados.0.smithi076.stdout:2415: done (2 left) 2024-09-21T09:48:26.760 INFO:tasks.rados.rados.0.smithi076.stdout:2416: done (1 left) 2024-09-21T09:48:26.760 INFO:tasks.rados.rados.0.smithi076.stdout:2417: done (0 left) 2024-09-21T09:48:26.760 INFO:tasks.rados.rados.0.smithi076.stdout:2418: delete oid 24 current snap is 314 2024-09-21T09:48:26.763 INFO:tasks.rados.rados.0.smithi076.stdout:2418: done (0 left) 2024-09-21T09:48:26.763 INFO:tasks.rados.rados.0.smithi076.stdout:2419: rollback oid 18 current snap is 314 2024-09-21T09:48:26.763 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 18 to 310 2024-09-21T09:48:26.763 INFO:tasks.rados.rados.0.smithi076.stdout:2420: snap_remove snap 302 2024-09-21T09:48:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:26 smithi191 ceph-mon[102206]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:48:27.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:26 smithi076 ceph-mon[106482]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:48:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:26 smithi076 ceph-mon[110963]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T09:48:27.758 INFO:tasks.rados.rados.0.smithi076.stdout:2419: finishing rollback tid 1 to smithi07685715-18 2024-09-21T09:48:27.759 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 671 (ObjNum 129 snap 0 seq_num 864220192) dirty dne 2024-09-21T09:48:27.759 INFO:tasks.rados.rados.0.smithi076.stdout:2419: done (1 left) 2024-09-21T09:48:27.759 INFO:tasks.rados.rados.0.smithi076.stdout:2420: done (0 left) 2024-09-21T09:48:27.759 INFO:tasks.rados.rados.0.smithi076.stdout:2421: write oid 44 current snap is 314 2024-09-21T09:48:27.759 INFO:tasks.rados.rados.0.smithi076.stdout:2421: seq_num 911 ranges {511177=797846,1976390=591639,3034464=276713} 2024-09-21T09:48:27.766 INFO:tasks.rados.rados.0.smithi076.stdout:2421: writing smithi07685715-44 from 511177 to 1309023 tid 1 2024-09-21T09:48:27.774 INFO:tasks.rados.rados.0.smithi076.stdout:2421: writing smithi07685715-44 from 1976390 to 2568029 tid 2 2024-09-21T09:48:27.779 INFO:tasks.rados.rados.0.smithi076.stdout:2421: writing smithi07685715-44 from 3034464 to 3311177 tid 3 2024-09-21T09:48:27.779 INFO:tasks.rados.rados.0.smithi076.stdout:2422: write oid 14 current snap is 314 2024-09-21T09:48:27.779 INFO:tasks.rados.rados.0.smithi076.stdout:2421: finishing write tid 1 to smithi07685715-44 2024-09-21T09:48:27.779 INFO:tasks.rados.rados.0.smithi076.stdout:2421: finishing write tid 2 to smithi07685715-44 2024-09-21T09:48:27.779 INFO:tasks.rados.rados.0.smithi076.stdout:2422: seq_num 912 ranges {527984=473095,1634872=451650,2661887=634311} 2024-09-21T09:48:27.784 INFO:tasks.rados.rados.0.smithi076.stdout:2422: writing smithi07685715-14 from 527984 to 1001079 tid 1 2024-09-21T09:48:27.790 INFO:tasks.rados.rados.0.smithi076.stdout:2422: writing smithi07685715-14 from 1634872 to 2086522 tid 2 2024-09-21T09:48:27.797 INFO:tasks.rados.rados.0.smithi076.stdout:2422: writing smithi07685715-14 from 2661887 to 3296198 tid 3 2024-09-21T09:48:27.797 INFO:tasks.rados.rados.0.smithi076.stdout:2423: read oid 10 snap 298 2024-09-21T09:48:27.797 INFO:tasks.rados.rados.0.smithi076.stdout:2423: expect (ObjNum 866 snap 296 seq_num 866) 2024-09-21T09:48:27.797 INFO:tasks.rados.rados.0.smithi076.stdout:2424: rollback oid 43 current snap is 314 2024-09-21T09:48:27.797 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 297 2024-09-21T09:48:27.797 INFO:tasks.rados.rados.0.smithi076.stdout:2421: finishing write tid 3 to smithi07685715-44 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: finishing write tid 4 to smithi07685715-44 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: finishing write tid 5 to smithi07685715-44 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: oid 44 updating version 0 to 569 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: oid 44 version 569 is already newer than 567 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: oid 44 version 569 is already newer than 568 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: oid 44 version 569 is already newer than 566 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 569 (ObjNum 911 snap 314 seq_num 911) dirty exists 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: left oid 44 (ObjNum 911 snap 314 seq_num 911) 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2422: finishing write tid 1 to smithi07685715-14 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2422: finishing write tid 2 to smithi07685715-14 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2421: done (3 left) 2024-09-21T09:48:27.798 INFO:tasks.rados.rados.0.smithi076.stdout:2425: snap_remove snap 304 2024-09-21T09:48:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:27 smithi191 ceph-mon[102206]: pgmap v508: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 42 op/s 2024-09-21T09:48:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:27 smithi191 ceph-mon[102206]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:48:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:27 smithi076 ceph-mon[106482]: pgmap v508: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 42 op/s 2024-09-21T09:48:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:27 smithi076 ceph-mon[106482]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:48:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:27 smithi076 ceph-mon[110963]: pgmap v508: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 42 op/s 2024-09-21T09:48:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:27 smithi076 ceph-mon[110963]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T09:48:28.727 INFO:tasks.rados.rados.0.smithi076.stdout:2424: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:48:28.727 INFO:tasks.rados.rados.0.smithi076.stdout:2424: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:48:28.727 INFO:tasks.rados.rados.0.smithi076.stdout:2424: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 515 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: finishing write tid 3 to smithi07685715-14 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: finishing write tid 4 to smithi07685715-14 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: finishing write tid 5 to smithi07685715-14 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: oid 14 updating version 0 to 1399 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: oid 14 updating version 1399 to 1400 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: oid 14 updating version 1400 to 1401 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: oid 14 version 1401 is already newer than 1398 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1401 (ObjNum 912 snap 314 seq_num 912) dirty exists 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2422: left oid 14 (ObjNum 912 snap 314 seq_num 912) 2024-09-21T09:48:28.728 INFO:tasks.rados.rados.0.smithi076.stdout:2423: expect (ObjNum 866 snap 296 seq_num 866) 2024-09-21T09:48:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:2422: done (3 left) 2024-09-21T09:48:28.814 INFO:tasks.rados.rados.0.smithi076.stdout:2423: done (2 left) 2024-09-21T09:48:28.814 INFO:tasks.rados.rados.0.smithi076.stdout:2424: done (1 left) 2024-09-21T09:48:28.814 INFO:tasks.rados.rados.0.smithi076.stdout:2425: done (0 left) 2024-09-21T09:48:28.814 INFO:tasks.rados.rados.0.smithi076.stdout:2426: snap_remove snap 297 2024-09-21T09:48:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:28 smithi191 ceph-mon[102206]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:48:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:28 smithi191 ceph-mon[102206]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:48:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:28 smithi076 ceph-mon[106482]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:48:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:28 smithi076 ceph-mon[106482]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:48:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:28 smithi076 ceph-mon[110963]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T09:48:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:28 smithi076 ceph-mon[110963]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T09:48:29.778 INFO:tasks.rados.rados.0.smithi076.stdout:2426: done (0 left) 2024-09-21T09:48:29.778 INFO:tasks.rados.rados.0.smithi076.stdout:2427: read oid 45 snap 312 2024-09-21T09:48:29.778 INFO:tasks.rados.rados.0.smithi076.stdout:2427: expect (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:48:29.778 INFO:tasks.rados.rados.0.smithi076.stdout:2428: snap_remove snap 309 2024-09-21T09:48:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:29 smithi191 ceph-mon[102206]: pgmap v511: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 41 op/s 2024-09-21T09:48:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:29 smithi076 ceph-mon[106482]: pgmap v511: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 41 op/s 2024-09-21T09:48:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:29 smithi076 ceph-mon[110963]: pgmap v511: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.6 MiB/s wr, 41 op/s 2024-09-21T09:48:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2428: done (1 left) 2024-09-21T09:48:30.780 INFO:tasks.rados.rados.0.smithi076.stdout:2429: write oid 46 current snap is 314 2024-09-21T09:48:30.780 INFO:tasks.rados.rados.0.smithi076.stdout:2429: seq_num 913 ranges {544791=664697,1693354=595308,2689310=445583} 2024-09-21T09:48:30.787 INFO:tasks.rados.rados.0.smithi076.stdout:2429: writing smithi07685715-46 from 544791 to 1209488 tid 1 2024-09-21T09:48:30.794 INFO:tasks.rados.rados.0.smithi076.stdout:2429: writing smithi07685715-46 from 1693354 to 2288662 tid 2 2024-09-21T09:48:30.799 INFO:tasks.rados.rados.0.smithi076.stdout:2429: writing smithi07685715-46 from 2689310 to 3134893 tid 3 2024-09-21T09:48:30.799 INFO:tasks.rados.rados.0.smithi076.stdout:2430: write oid 9 current snap is 314 2024-09-21T09:48:30.799 INFO:tasks.rados.rados.0.smithi076.stdout:2430: seq_num 914 ranges {561598=739946,1751836=455319,2716733=644865} 2024-09-21T09:48:30.806 INFO:tasks.rados.rados.0.smithi076.stdout:2430: writing smithi07685715-9 from 561598 to 1301544 tid 1 2024-09-21T09:48:30.811 INFO:tasks.rados.rados.0.smithi076.stdout:2430: writing smithi07685715-9 from 1751836 to 2207155 tid 2 2024-09-21T09:48:30.818 INFO:tasks.rados.rados.0.smithi076.stdout:2430: writing smithi07685715-9 from 2716733 to 3361598 tid 3 2024-09-21T09:48:30.818 INFO:tasks.rados.rados.0.smithi076.stdout:2431: snap_create 2024-09-21T09:48:30.818 INFO:tasks.rados.rados.0.smithi076.stdout:2427: expect (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: finishing write tid 1 to smithi07685715-46 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: finishing write tid 2 to smithi07685715-46 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: finishing write tid 3 to smithi07685715-46 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: finishing write tid 4 to smithi07685715-46 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: finishing write tid 5 to smithi07685715-46 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: oid 46 updating version 0 to 1833 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: oid 46 updating version 1833 to 1834 2024-09-21T09:48:30.930 INFO:tasks.rados.rados.0.smithi076.stdout:2429: oid 46 version 1834 is already newer than 1832 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2429: oid 46 version 1834 is already newer than 1829 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1834 (ObjNum 913 snap 314 seq_num 913) dirty exists 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2429: left oid 46 (ObjNum 913 snap 314 seq_num 913) 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: finishing write tid 1 to smithi07685715-9 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: finishing write tid 2 to smithi07685715-9 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: finishing write tid 3 to smithi07685715-9 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: finishing write tid 4 to smithi07685715-9 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: finishing write tid 5 to smithi07685715-9 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: oid 9 updating version 0 to 860 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: oid 9 updating version 860 to 861 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: oid 9 version 861 is already newer than 859 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: oid 9 updating version 861 to 862 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 862 (ObjNum 914 snap 314 seq_num 914) dirty exists 2024-09-21T09:48:30.931 INFO:tasks.rados.rados.0.smithi076.stdout:2430: left oid 9 (ObjNum 914 snap 314 seq_num 914) 2024-09-21T09:48:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:30 smithi191 ceph-mon[102206]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:48:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:30 smithi191 ceph-mon[102206]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:48:31.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:30 smithi076 ceph-mon[106482]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:48:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:30 smithi076 ceph-mon[106482]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:48:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:30 smithi076 ceph-mon[110963]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T09:48:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:30 smithi076 ceph-mon[110963]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T09:48:31.805 INFO:tasks.rados.rados.0.smithi076.stdout:2427: done (3 left) 2024-09-21T09:48:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2429: done (2 left) 2024-09-21T09:48:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2430: done (1 left) 2024-09-21T09:48:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2431: done (0 left) 2024-09-21T09:48:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2432: write oid 38 current snap is 315 2024-09-21T09:48:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2432: seq_num 915 ranges {578405=415195,1410318=598977,2744156=584480} 2024-09-21T09:48:31.810 INFO:tasks.rados.rados.0.smithi076.stdout:2432: writing smithi07685715-38 from 578405 to 993600 tid 1 2024-09-21T09:48:31.817 INFO:tasks.rados.rados.0.smithi076.stdout:2432: writing smithi07685715-38 from 1410318 to 2009295 tid 2 2024-09-21T09:48:31.824 INFO:tasks.rados.rados.0.smithi076.stdout:2432: writing smithi07685715-38 from 2744156 to 3328636 tid 3 2024-09-21T09:48:31.824 INFO:tasks.rados.rados.0.smithi076.stdout:2433: delete oid 17 current snap is 315 2024-09-21T09:48:31.824 INFO:tasks.rados.rados.0.smithi076.stdout:2432: finishing write tid 1 to smithi07685715-38 2024-09-21T09:48:31.824 INFO:tasks.rados.rados.0.smithi076.stdout:2432: finishing write tid 2 to smithi07685715-38 2024-09-21T09:48:31.827 INFO:tasks.rados.rados.0.smithi076.stdout:2432: finishing write tid 3 to smithi07685715-38 2024-09-21T09:48:31.828 INFO:tasks.rados.rados.0.smithi076.stdout:2432: finishing write tid 4 to smithi07685715-38 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: finishing write tid 5 to smithi07685715-38 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: oid 38 updating version 0 to 1426 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: oid 38 version 1426 is already newer than 1425 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: oid 38 version 1426 is already newer than 1424 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: oid 38 version 1426 is already newer than 1423 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1426 (ObjNum 915 snap 315 seq_num 915) dirty exists 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: left oid 38 (ObjNum 915 snap 315 seq_num 915) 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2432: done (1 left) 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2433: done (0 left) 2024-09-21T09:48:31.829 INFO:tasks.rados.rados.0.smithi076.stdout:2434: rollback oid 47 current snap is 315 2024-09-21T09:48:31.830 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 311 2024-09-21T09:48:31.830 INFO:tasks.rados.rados.0.smithi076.stdout:2435: snap_create 2024-09-21T09:48:31.830 INFO:tasks.rados.rados.0.smithi076.stdout:2434: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:48:31.830 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 0 (ObjNum 864216048 snap 21884 seq_num 304) dirty dne 2024-09-21T09:48:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:31 smithi191 ceph-mon[102206]: pgmap v514: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:31 smithi076 ceph-mon[110963]: pgmap v514: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:31 smithi076 ceph-mon[106482]: pgmap v514: 89 pgs: 89 active+clean; 225 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:32.811 INFO:tasks.rados.rados.0.smithi076.stdout:2434: done (1 left) 2024-09-21T09:48:32.812 INFO:tasks.rados.rados.0.smithi076.stdout:2435: done (0 left) 2024-09-21T09:48:32.812 INFO:tasks.rados.rados.0.smithi076.stdout:2436: read oid 24 snap -1 2024-09-21T09:48:32.812 INFO:tasks.rados.rados.0.smithi076.stdout:2436: expect deleted 2024-09-21T09:48:32.812 INFO:tasks.rados.rados.0.smithi076.stdout:2437: delete oid 2 current snap is 316 2024-09-21T09:48:32.816 INFO:tasks.rados.rados.0.smithi076.stdout:2436: done (1 left) 2024-09-21T09:48:32.816 INFO:tasks.rados.rados.0.smithi076.stdout:2437: done (0 left) 2024-09-21T09:48:32.816 INFO:tasks.rados.rados.0.smithi076.stdout:2438: delete oid 40 current snap is 316 2024-09-21T09:48:32.820 INFO:tasks.rados.rados.0.smithi076.stdout:2438: done (0 left) 2024-09-21T09:48:32.821 INFO:tasks.rados.rados.0.smithi076.stdout:2439: read oid 48 snap -1 2024-09-21T09:48:32.821 INFO:tasks.rados.rados.0.smithi076.stdout:2439: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:32.821 INFO:tasks.rados.rados.0.smithi076.stdout:2440: write oid 41 current snap is 316 2024-09-21T09:48:32.821 INFO:tasks.rados.rados.0.smithi076.stdout:2440: seq_num 919 ranges {645633=716191,1927893=606315,3137495=308138} 2024-09-21T09:48:32.828 INFO:tasks.rados.rados.0.smithi076.stdout:2440: writing smithi07685715-41 from 645633 to 1361824 tid 1 2024-09-21T09:48:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:2440: writing smithi07685715-41 from 1927893 to 2534208 tid 2 2024-09-21T09:48:32.840 INFO:tasks.rados.rados.0.smithi076.stdout:2440: writing smithi07685715-41 from 3137495 to 3445633 tid 3 2024-09-21T09:48:32.840 INFO:tasks.rados.rados.0.smithi076.stdout:2441: write oid 3 current snap is 316 2024-09-21T09:48:32.840 INFO:tasks.rados.rados.0.smithi076.stdout:2441: seq_num 920 ranges {662440=507793,1702728=749973,2881271=557193} 2024-09-21T09:48:32.846 INFO:tasks.rados.rados.0.smithi076.stdout:2441: writing smithi07685715-3 from 662440 to 1170233 tid 1 2024-09-21T09:48:32.853 INFO:tasks.rados.rados.0.smithi076.stdout:2441: writing smithi07685715-3 from 1702728 to 2452701 tid 2 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2441: writing smithi07685715-3 from 2881271 to 3438464 tid 3 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2442: read oid 33 snap -1 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2442: expect deleted 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2443: read oid 32 snap -1 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2443: expect (ObjNum 883 snap 310 seq_num 883) 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2444: rollback oid 43 current snap is 316 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 303 2024-09-21T09:48:32.859 INFO:tasks.rados.rados.0.smithi076.stdout:2439: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:33.013 INFO:tasks.rados.rados.0.smithi076.stdout:2440: finishing write tid 1 to smithi07685715-41 2024-09-21T09:48:33.013 INFO:tasks.rados.rados.0.smithi076.stdout:2440: finishing write tid 2 to smithi07685715-41 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: finishing write tid 3 to smithi07685715-41 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: finishing write tid 4 to smithi07685715-41 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: finishing write tid 5 to smithi07685715-41 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: oid 41 updating version 0 to 1076 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: oid 41 updating version 1076 to 1078 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: oid 41 updating version 1078 to 1079 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: oid 41 version 1079 is already newer than 1077 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1079 (ObjNum 919 snap 316 seq_num 919) dirty exists 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2440: left oid 41 (ObjNum 919 snap 316 seq_num 919) 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2441: finishing write tid 1 to smithi07685715-3 2024-09-21T09:48:33.014 INFO:tasks.rados.rados.0.smithi076.stdout:2439: done (5 left) 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2440: done (4 left) 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2445: delete oid 6 current snap is 316 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: finishing write tid 2 to smithi07685715-3 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: finishing write tid 3 to smithi07685715-3 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: finishing write tid 4 to smithi07685715-3 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: finishing write tid 5 to smithi07685715-3 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: oid 3 updating version 0 to 161 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: oid 3 updating version 161 to 164 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: oid 3 version 164 is already newer than 163 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: oid 3 version 164 is already newer than 162 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 164 (ObjNum 920 snap 316 seq_num 920) dirty exists 2024-09-21T09:48:33.015 INFO:tasks.rados.rados.0.smithi076.stdout:2441: left oid 3 (ObjNum 920 snap 316 seq_num 920) 2024-09-21T09:48:33.016 INFO:tasks.rados.rados.0.smithi076.stdout:2443: expect (ObjNum 883 snap 310 seq_num 883) 2024-09-21T09:48:33.105 INFO:tasks.rados.rados.0.smithi076.stdout:2444: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:48:33.105 INFO:tasks.rados.rados.0.smithi076.stdout:2444: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:48:33.105 INFO:tasks.rados.rados.0.smithi076.stdout:2441: done (4 left) 2024-09-21T09:48:33.105 INFO:tasks.rados.rados.0.smithi076.stdout:2442: done (3 left) 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2443: done (2 left) 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2445: done (1 left) 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2446: read oid 13 snap -1 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2446: expect (ObjNum 886 snap 311 seq_num 886) 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2447: read oid 50 snap 314 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2447: expect (ObjNum 909 snap 313 seq_num 909) 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2448: snap_create 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:2444: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:48:33.106 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 531 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-09-21T09:48:33.109 INFO:tasks.rados.rados.0.smithi076.stdout:2446: expect (ObjNum 886 snap 311 seq_num 886) 2024-09-21T09:48:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:32 smithi191 ceph-mon[102206]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:48:33.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:32 smithi076 ceph-mon[106482]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:48:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:32 smithi076 ceph-mon[110963]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T09:48:33.209 INFO:tasks.rados.rados.0.smithi076.stdout:2447: expect (ObjNum 909 snap 313 seq_num 909) 2024-09-21T09:48:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:48:33] "GET /metrics HTTP/1.1" 200 36134 "" "Prometheus/2.51.0" 2024-09-21T09:48:33.723 INFO:tasks.rados.rados.0.smithi076.stdout:2444: done (3 left) 2024-09-21T09:48:33.723 INFO:tasks.rados.rados.0.smithi076.stdout:2446: done (2 left) 2024-09-21T09:48:33.723 INFO:tasks.rados.rados.0.smithi076.stdout:2447: done (1 left) 2024-09-21T09:48:33.723 INFO:tasks.rados.rados.0.smithi076.stdout:2448: done (0 left) 2024-09-21T09:48:33.724 INFO:tasks.rados.rados.0.smithi076.stdout:2449: write oid 17 current snap is 317 2024-09-21T09:48:33.724 INFO:tasks.rados.rados.0.smithi076.stdout:2449: seq_num 922 ranges {696054=658291,2103339=753642} 2024-09-21T09:48:33.732 INFO:tasks.rados.rados.0.smithi076.stdout:2449: writing smithi07685715-17 from 696054 to 1354345 tid 1 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2449: writing smithi07685715-17 from 2103339 to 2856981 tid 2 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2450: read oid 39 snap -1 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2450: expect (ObjNum 890 snap 312 seq_num 890) 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2451: read oid 42 snap -1 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2451: expect deleted 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2452: delete oid 1 current snap is 317 2024-09-21T09:48:33.742 INFO:tasks.rados.rados.0.smithi076.stdout:2449: finishing write tid 1 to smithi07685715-17 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: finishing write tid 2 to smithi07685715-17 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: finishing write tid 3 to smithi07685715-17 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: finishing write tid 4 to smithi07685715-17 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: oid 17 updating version 0 to 1430 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: oid 17 version 1430 is already newer than 1429 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: oid 17 version 1430 is already newer than 1428 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1430 (ObjNum 922 snap 317 seq_num 922) dirty exists 2024-09-21T09:48:33.747 INFO:tasks.rados.rados.0.smithi076.stdout:2449: left oid 17 (ObjNum 922 snap 317 seq_num 922) 2024-09-21T09:48:33.756 INFO:tasks.rados.rados.0.smithi076.stdout:2450: expect (ObjNum 890 snap 312 seq_num 890) 2024-09-21T09:48:33.893 INFO:tasks.rados.rados.0.smithi076.stdout:2449: done (3 left) 2024-09-21T09:48:33.893 INFO:tasks.rados.rados.0.smithi076.stdout:2450: done (2 left) 2024-09-21T09:48:33.893 INFO:tasks.rados.rados.0.smithi076.stdout:2451: done (1 left) 2024-09-21T09:48:33.893 INFO:tasks.rados.rados.0.smithi076.stdout:2452: done (0 left) 2024-09-21T09:48:33.893 INFO:tasks.rados.rados.0.smithi076.stdout:2453: delete oid 32 current snap is 317 2024-09-21T09:48:33.895 INFO:tasks.rados.rados.0.smithi076.stdout:2453: done (0 left) 2024-09-21T09:48:33.895 INFO:tasks.rados.rados.0.smithi076.stdout:2454: snap_remove snap 312 2024-09-21T09:48:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:33 smithi191 ceph-mon[102206]: pgmap v517: 89 pgs: 89 active+clean; 233 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:48:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:33 smithi191 ceph-mon[102206]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:48:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:33 smithi191 ceph-mon[102206]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:48:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-mon[106482]: pgmap v517: 89 pgs: 89 active+clean; 233 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:48:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-mon[106482]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:48:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-mon[106482]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:48:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-mon[110963]: pgmap v517: 89 pgs: 89 active+clean; 233 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:48:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-mon[110963]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T09:48:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:33 smithi076 ceph-mon[110963]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T09:48:34.819 INFO:tasks.rados.rados.0.smithi076.stdout:2454: done (0 left) 2024-09-21T09:48:34.820 INFO:tasks.rados.rados.0.smithi076.stdout:2455: snap_remove snap 313 2024-09-21T09:48:35.825 INFO:tasks.rados.rados.0.smithi076.stdout:2455: done (0 left) 2024-09-21T09:48:35.825 INFO:tasks.rados.rados.0.smithi076.stdout:2456: read oid 43 snap -1 2024-09-21T09:48:35.825 INFO:tasks.rados.rados.0.smithi076.stdout:2456: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:48:35.825 INFO:tasks.rados.rados.0.smithi076.stdout:2457: read oid 19 snap -1 2024-09-21T09:48:35.826 INFO:tasks.rados.rados.0.smithi076.stdout:2457: expect (ObjNum 901 snap 313 seq_num 901) 2024-09-21T09:48:35.826 INFO:tasks.rados.rados.0.smithi076.stdout:2458: read oid 29 snap 316 2024-09-21T09:48:35.826 INFO:tasks.rados.rados.0.smithi076.stdout:2458: expect deleted 2024-09-21T09:48:35.826 INFO:tasks.rados.rados.0.smithi076.stdout:2459: write oid 14 current snap is 317 2024-09-21T09:48:35.826 INFO:tasks.rados.rados.0.smithi076.stdout:2459: seq_num 925 ranges {746475=484038,1762432=617322,2785680=529906} 2024-09-21T09:48:35.834 INFO:tasks.rados.rados.0.smithi076.stdout:2459: writing smithi07685715-14 from 746475 to 1230513 tid 1 2024-09-21T09:48:35.839 INFO:tasks.rados.rados.0.smithi076.stdout:2459: writing smithi07685715-14 from 1762432 to 2379754 tid 2 2024-09-21T09:48:35.845 INFO:tasks.rados.rados.0.smithi076.stdout:2459: writing smithi07685715-14 from 2785680 to 3315586 tid 3 2024-09-21T09:48:35.845 INFO:tasks.rados.rados.0.smithi076.stdout:2460: snap_remove snap 311 2024-09-21T09:48:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:35 smithi191 ceph-mon[102206]: pgmap v520: 89 pgs: 89 active+clean; 233 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:48:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:35 smithi191 ceph-mon[102206]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:48:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:35 smithi076 ceph-mon[106482]: pgmap v520: 89 pgs: 89 active+clean; 233 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:48:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:35 smithi076 ceph-mon[106482]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:48:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:35 smithi076 ceph-mon[110963]: pgmap v520: 89 pgs: 89 active+clean; 233 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:48:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:35 smithi076 ceph-mon[110963]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T09:48:36.831 INFO:tasks.rados.rados.0.smithi076.stdout:2460: done (4 left) 2024-09-21T09:48:36.831 INFO:tasks.rados.rados.0.smithi076.stdout:2461: delete oid 25 current snap is 317 2024-09-21T09:48:36.831 INFO:tasks.rados.rados.0.smithi076.stdout:2456: expect (ObjNum 429 snap 135 seq_num 429) 2024-09-21T09:48:36.935 INFO:tasks.rados.rados.0.smithi076.stdout:2457: expect (ObjNum 901 snap 313 seq_num 901) 2024-09-21T09:48:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:36 smithi191 ceph-mon[102206]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:48:37.116 INFO:tasks.rados.rados.0.smithi076.stdout:2459: finishing write tid 1 to smithi07685715-14 2024-09-21T09:48:37.116 INFO:tasks.rados.rados.0.smithi076.stdout:2459: finishing write tid 2 to smithi07685715-14 2024-09-21T09:48:37.116 INFO:tasks.rados.rados.0.smithi076.stdout:2459: finishing write tid 3 to smithi07685715-14 2024-09-21T09:48:37.116 INFO:tasks.rados.rados.0.smithi076.stdout:2459: finishing write tid 4 to smithi07685715-14 2024-09-21T09:48:37.116 INFO:tasks.rados.rados.0.smithi076.stdout:2459: finishing write tid 5 to smithi07685715-14 2024-09-21T09:48:37.116 INFO:tasks.rados.rados.0.smithi076.stdout:2459: oid 14 updating version 0 to 1443 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2459: oid 14 version 1443 is already newer than 1438 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2459: oid 14 updating version 1443 to 1444 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2459: oid 14 updating version 1444 to 1445 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1445 (ObjNum 925 snap 317 seq_num 925) dirty exists 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2459: left oid 14 (ObjNum 925 snap 317 seq_num 925) 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2456: done (4 left) 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2457: done (3 left) 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2458: done (2 left) 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2459: done (1 left) 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2461: done (0 left) 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2462: write oid 15 current snap is 317 2024-09-21T09:48:37.117 INFO:tasks.rados.rados.0.smithi076.stdout:2462: seq_num 927 ranges {780089=634536,1879396=620991,3240526=339563} 2024-09-21T09:48:37.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:36 smithi076 ceph-mon[106482]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:48:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:36 smithi076 ceph-mon[110963]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T09:48:37.123 INFO:tasks.rados.rados.0.smithi076.stdout:2462: writing smithi07685715-15 from 780089 to 1414625 tid 1 2024-09-21T09:48:37.130 INFO:tasks.rados.rados.0.smithi076.stdout:2462: writing smithi07685715-15 from 1879396 to 2500387 tid 2 2024-09-21T09:48:37.135 INFO:tasks.rados.rados.0.smithi076.stdout:2462: writing smithi07685715-15 from 3240526 to 3580089 tid 3 2024-09-21T09:48:37.136 INFO:tasks.rados.rados.0.smithi076.stdout:2463: snap_remove snap 310 2024-09-21T09:48:37.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-21T09:48:37.399 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2463: done (1 left) 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2464: read oid 11 snap -1 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2464: expect deleted 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2465: delete oid 27 current snap is 317 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: finishing write tid 1 to smithi07685715-15 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: finishing write tid 2 to smithi07685715-15 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: finishing write tid 3 to smithi07685715-15 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: finishing write tid 4 to smithi07685715-15 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: finishing write tid 5 to smithi07685715-15 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: oid 15 updating version 0 to 541 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: oid 15 version 541 is already newer than 540 2024-09-21T09:48:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:2462: oid 15 version 541 is already newer than 539 2024-09-21T09:48:37.836 INFO:tasks.rados.rados.0.smithi076.stdout:2462: oid 15 updating version 541 to 542 2024-09-21T09:48:37.836 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 542 (ObjNum 927 snap 317 seq_num 927) dirty exists 2024-09-21T09:48:37.836 INFO:tasks.rados.rados.0.smithi076.stdout:2462: left oid 15 (ObjNum 927 snap 317 seq_num 927) 2024-09-21T09:48:37.844 INFO:tasks.rados.rados.0.smithi076.stdout:2462: done (2 left) 2024-09-21T09:48:37.845 INFO:tasks.rados.rados.0.smithi076.stdout:2465: done (1 left) 2024-09-21T09:48:37.845 INFO:tasks.rados.rados.0.smithi076.stdout:2466: write oid 33 current snap is 317 2024-09-21T09:48:37.845 INFO:tasks.rados.rados.0.smithi076.stdout:2466: seq_num 929 ranges {413703=501387,1596360=624660,2895372=691347} 2024-09-21T09:48:37.850 INFO:tasks.rados.rados.0.smithi076.stdout:2466: writing smithi07685715-33 from 413703 to 915090 tid 1 2024-09-21T09:48:37.857 INFO:tasks.rados.rados.0.smithi076.stdout:2466: writing smithi07685715-33 from 1596360 to 2221020 tid 2 2024-09-21T09:48:37.864 INFO:tasks.rados.rados.0.smithi076.stdout:2466: writing smithi07685715-33 from 2895372 to 3586719 tid 3 2024-09-21T09:48:37.865 INFO:tasks.rados.rados.0.smithi076.stdout:2467: delete oid 47 current snap is 317 2024-09-21T09:48:37.865 INFO:tasks.rados.rados.0.smithi076.stdout:2466: finishing write tid 1 to smithi07685715-33 2024-09-21T09:48:37.865 INFO:tasks.rados.rados.0.smithi076.stdout:2466: finishing write tid 2 to smithi07685715-33 2024-09-21T09:48:37.867 INFO:tasks.rados.rados.0.smithi076.stdout:2466: finishing write tid 3 to smithi07685715-33 2024-09-21T09:48:37.867 INFO:tasks.rados.rados.0.smithi076.stdout:2466: finishing write tid 4 to smithi07685715-33 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: finishing write tid 5 to smithi07685715-33 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: oid 33 updating version 0 to 603 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: oid 33 version 603 is already newer than 600 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: oid 33 version 603 is already newer than 601 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: oid 33 version 603 is already newer than 602 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 603 (ObjNum 929 snap 317 seq_num 929) dirty exists 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: left oid 33 (ObjNum 929 snap 317 seq_num 929) 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2464: done (2 left) 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2466: done (1 left) 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2467: done (0 left) 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2468: read oid 2 snap -1 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2468: expect deleted 2024-09-21T09:48:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:2469: delete oid 35 current snap is 317 2024-09-21T09:48:37.869 INFO:tasks.rados.rados.0.smithi076.stdout:2468: done (1 left) 2024-09-21T09:48:37.869 INFO:tasks.rados.rados.0.smithi076.stdout:2469: done (0 left) 2024-09-21T09:48:37.869 INFO:tasks.rados.rados.0.smithi076.stdout:2470: delete oid 40 current snap is 317 2024-09-21T09:48:37.871 INFO:tasks.rados.rados.0.smithi076.stdout:2470: done (0 left) 2024-09-21T09:48:37.871 INFO:tasks.rados.rados.0.smithi076.stdout:2471: rollback oid 15 current snap is 317 2024-09-21T09:48:37.871 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 298 2024-09-21T09:48:37.871 INFO:tasks.rados.rados.0.smithi076.stdout:2472: delete oid 33 current snap is 317 2024-09-21T09:48:37.872 INFO:tasks.rados.rados.0.smithi076.stdout:2472: done (1 left) 2024-09-21T09:48:37.873 INFO:tasks.rados.rados.0.smithi076.stdout:2473: write oid 8 current snap is 317 2024-09-21T09:48:37.873 INFO:tasks.rados.rados.0.smithi076.stdout:2473: seq_num 934 ranges {497738=477632,1772417=775656,3316134=381604} 2024-09-21T09:48:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:2473: writing smithi07685715-8 from 497738 to 975370 tid 1 2024-09-21T09:48:37.886 INFO:tasks.rados.rados.0.smithi076.stdout:2473: writing smithi07685715-8 from 1772417 to 2548073 tid 2 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2473: writing smithi07685715-8 from 3316134 to 3697738 tid 3 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2474: delete oid 5 current snap is 317 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2471: finishing rollback tid 0 to smithi07685715-15 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2471: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2471: finishing rollback tid 2 to smithi07685715-15 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 547 (ObjNum 853 snap 292 seq_num 853) dirty exists 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2473: finishing write tid 1 to smithi07685715-8 2024-09-21T09:48:37.893 INFO:tasks.rados.rados.0.smithi076.stdout:2473: finishing write tid 2 to smithi07685715-8 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2471: done (2 left) 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2474: done (1 left) 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2475: rollback oid 21 current snap is 317 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 289 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2476: read oid 20 snap 315 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2476: expect (ObjNum 806 snap 282 seq_num 806) 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2477: read oid 17 snap 315 2024-09-21T09:48:37.896 INFO:tasks.rados.rados.0.smithi076.stdout:2477: expect deleted 2024-09-21T09:48:37.897 INFO:tasks.rados.rados.0.smithi076.stdout:2478: write oid 10 current snap is 317 2024-09-21T09:48:37.897 INFO:tasks.rados.rados.0.smithi076.stdout:2478: seq_num 936 ranges {531352=744483,2005734=779325,3487333=244019} 2024-09-21T09:48:37.903 INFO:tasks.rados.rados.0.smithi076.stdout:2478: writing smithi07685715-10 from 531352 to 1275835 tid 1 2024-09-21T09:48:37.911 INFO:tasks.rados.rados.0.smithi076.stdout:2478: writing smithi07685715-10 from 2005734 to 2785059 tid 2 2024-09-21T09:48:37.916 INFO:tasks.rados.rados.0.smithi076.stdout:2478: writing smithi07685715-10 from 3487333 to 3731352 tid 3 2024-09-21T09:48:37.916 INFO:tasks.rados.rados.0.smithi076.stdout:2479: snap_remove snap 296 2024-09-21T09:48:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:37 smithi191 ceph-mon[102206]: pgmap v523: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:48:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:37 smithi191 ceph-mon[102206]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:48:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:37 smithi076 ceph-mon[106482]: pgmap v523: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:48:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:37 smithi076 ceph-mon[106482]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:48:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:37 smithi076 ceph-mon[110963]: pgmap v523: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:48:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:37 smithi076 ceph-mon[110963]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2479: done (5 left) 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2480: snap_create 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: finishing write tid 3 to smithi07685715-8 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: finishing write tid 4 to smithi07685715-8 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: finishing write tid 5 to smithi07685715-8 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: oid 8 updating version 0 to 1936 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: oid 8 version 1936 is already newer than 1935 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: oid 8 updating version 1936 to 1938 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: oid 8 version 1938 is already newer than 1937 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 1938 (ObjNum 934 snap 317 seq_num 934) dirty exists 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: left oid 8 (ObjNum 934 snap 317 seq_num 934) 2024-09-21T09:48:38.727 INFO:tasks.rados.rados.0.smithi076.stdout:2478: finishing write tid 1 to smithi07685715-10 2024-09-21T09:48:38.728 INFO:tasks.rados.rados.0.smithi076.stdout:2476: expect (ObjNum 806 snap 282 seq_num 806) 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2475: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2475: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1865 (ObjNum 579 snap 184 seq_num 579) dirty exists 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: finishing write tid 2 to smithi07685715-10 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: finishing write tid 3 to smithi07685715-10 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: finishing write tid 4 to smithi07685715-10 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: finishing write tid 5 to smithi07685715-10 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: oid 10 updating version 0 to 1464 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: oid 10 updating version 1464 to 1465 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: oid 10 version 1465 is already newer than 1463 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: oid 10 updating version 1465 to 1466 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1466 (ObjNum 936 snap 317 seq_num 936) dirty exists 2024-09-21T09:48:38.888 INFO:tasks.rados.rados.0.smithi076.stdout:2478: left oid 10 (ObjNum 936 snap 317 seq_num 936) 2024-09-21T09:48:38.988 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:38 smithi076 ceph-mon[106482]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:48:38.988 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:38 smithi076 ceph-mon[106482]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:48:38.988 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:38 smithi076 ceph-mon[110963]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:48:38.989 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:38 smithi076 ceph-mon[110963]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:48:39.097 INFO:teuthology.orchestra.run.smithi076.stdout:wait for half of osds to upgrade 2024-09-21T09:48:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:38 smithi191 ceph-mon[102206]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T09:48:39.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:38 smithi191 ceph-mon[102206]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T09:48:39.727 INFO:tasks.rados.rados.0.smithi076.stdout:2473: done (5 left) 2024-09-21T09:48:39.728 INFO:tasks.rados.rados.0.smithi076.stdout:2475: done (4 left) 2024-09-21T09:48:39.728 INFO:tasks.rados.rados.0.smithi076.stdout:2476: done (3 left) 2024-09-21T09:48:39.728 INFO:tasks.rados.rados.0.smithi076.stdout:2477: done (2 left) 2024-09-21T09:48:39.728 INFO:tasks.rados.rados.0.smithi076.stdout:2478: done (1 left) 2024-09-21T09:48:39.728 INFO:tasks.rados.rados.0.smithi076.stdout:2480: done (0 left) 2024-09-21T09:48:39.729 INFO:tasks.rados.rados.0.smithi076.stdout:2481: read oid 7 snap -1 2024-09-21T09:48:39.729 INFO:tasks.rados.rados.0.smithi076.stdout:2481: expect (ObjNum 905 snap 313 seq_num 905) 2024-09-21T09:48:39.729 INFO:tasks.rados.rados.0.smithi076.stdout:2482: rollback oid 40 current snap is 318 2024-09-21T09:48:39.729 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 317 2024-09-21T09:48:39.729 INFO:tasks.rados.rados.0.smithi076.stdout:2483: snap_remove snap 315 2024-09-21T09:48:39.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-09-21T09:48:39.945 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:40.065 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:39 smithi076 ceph-mon[106482]: pgmap v526: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:48:40.065 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:39 smithi076 ceph-mon[106482]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:48:40.066 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:39 smithi076 ceph-mon[110963]: pgmap v526: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:48:40.066 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:39 smithi076 ceph-mon[110963]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:48:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:39 smithi191 ceph-mon[102206]: pgmap v526: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 24 op/s 2024-09-21T09:48:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:39 smithi191 ceph-mon[102206]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T09:48:40.729 INFO:tasks.rados.rados.0.smithi076.stdout:2483: done (2 left) 2024-09-21T09:48:40.729 INFO:tasks.rados.rados.0.smithi076.stdout:2484: snap_remove snap 306 2024-09-21T09:48:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:40 smithi191 ceph-mon[102206]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:48:41.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:40 smithi076 ceph-mon[106482]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:48:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:40 smithi076 ceph-mon[110963]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T09:48:41.732 INFO:tasks.rados.rados.0.smithi076.stdout:2484: done (2 left) 2024-09-21T09:48:41.732 INFO:tasks.rados.rados.0.smithi076.stdout:2485: rollback oid 41 current snap is 318 2024-09-21T09:48:41.732 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 316 2024-09-21T09:48:41.732 INFO:tasks.rados.rados.0.smithi076.stdout:2486: read oid 2 snap -1 2024-09-21T09:48:41.732 INFO:tasks.rados.rados.0.smithi076.stdout:2486: expect deleted 2024-09-21T09:48:41.732 INFO:tasks.rados.rados.0.smithi076.stdout:2487: snap_remove snap 289 2024-09-21T09:48:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:41 smithi191 ceph-mon[102206]: pgmap v529: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:41 smithi191 ceph-mon[102206]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:48:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:41 smithi076 ceph-mon[106482]: pgmap v529: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:41 smithi076 ceph-mon[106482]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:48:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:41 smithi076 ceph-mon[110963]: pgmap v529: 89 pgs: 89 active+clean; 244 MiB data, 822 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:41 smithi076 ceph-mon[110963]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T09:48:42.735 INFO:tasks.rados.rados.0.smithi076.stdout:2487: done (4 left) 2024-09-21T09:48:42.735 INFO:tasks.rados.rados.0.smithi076.stdout:2488: snap_create 2024-09-21T09:48:42.735 INFO:tasks.rados.rados.0.smithi076.stdout:2482: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:48:42.735 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1075 (ObjNum 129 snap 0 seq_num 865257720) dirty dne 2024-09-21T09:48:42.735 INFO:tasks.rados.rados.0.smithi076.stdout:2481: expect (ObjNum 905 snap 313 seq_num 905) 2024-09-21T09:48:42.922 INFO:tasks.rados.rados.0.smithi076.stdout:2485: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:48:42.922 INFO:tasks.rados.rados.0.smithi076.stdout:2485: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:48:42.922 INFO:tasks.rados.rados.0.smithi076.stdout:2485: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:48:42.922 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1117 (ObjNum 919 snap 316 seq_num 919) dirty exists 2024-09-21T09:48:42.980 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:42 smithi076 ceph-mon[106482]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:42.980 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:42 smithi076 ceph-mon[106482]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:48:42.980 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:42 smithi076 ceph-mon[110963]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:42.980 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:42 smithi076 ceph-mon[110963]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:48:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:42 smithi191 ceph-mon[102206]: from='client.34293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:42 smithi191 ceph-mon[102206]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T09:48:43.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:48:43] "GET /metrics HTTP/1.1" 200 36133 "" "Prometheus/2.51.0" 2024-09-21T09:48:43.726 INFO:tasks.rados.rados.0.smithi076.stdout:2481: done (4 left) 2024-09-21T09:48:43.727 INFO:tasks.rados.rados.0.smithi076.stdout:2482: done (3 left) 2024-09-21T09:48:43.727 INFO:tasks.rados.rados.0.smithi076.stdout:2485: done (2 left) 2024-09-21T09:48:43.727 INFO:tasks.rados.rados.0.smithi076.stdout:2486: done (1 left) 2024-09-21T09:48:43.727 INFO:tasks.rados.rados.0.smithi076.stdout:2488: done (0 left) 2024-09-21T09:48:43.727 INFO:tasks.rados.rados.0.smithi076.stdout:2489: write oid 39 current snap is 319 2024-09-21T09:48:43.727 INFO:tasks.rados.rados.0.smithi076.stdout:2489: seq_num 937 ranges {548159=419732,1664216=522983,2714756=614229} 2024-09-21T09:48:43.732 INFO:tasks.rados.rados.0.smithi076.stdout:2489: writing smithi07685715-39 from 548159 to 967891 tid 1 2024-09-21T09:48:43.738 INFO:tasks.rados.rados.0.smithi076.stdout:2489: writing smithi07685715-39 from 1664216 to 2187199 tid 2 2024-09-21T09:48:43.745 INFO:tasks.rados.rados.0.smithi076.stdout:2489: writing smithi07685715-39 from 2714756 to 3328985 tid 3 2024-09-21T09:48:43.745 INFO:tasks.rados.rados.0.smithi076.stdout:2490: snap_remove snap 314 2024-09-21T09:48:43.800 INFO:teuthology.orchestra.run.smithi076.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: pgmap v532: 89 pgs: 89 active+clean; 247 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: pgmap v532: 89 pgs: 89 active+clean; 247 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:48:44.107 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:44.108 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:44.108 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:44.108 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:44.108 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: pgmap v532: 89 pgs: 89 active+clean; 247 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:48:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T09:48:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:48:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:44.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:48:44.644 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:44.728 INFO:tasks.rados.rados.0.smithi076.stdout:2490: done (1 left) 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2491: snap_create 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: finishing write tid 1 to smithi07685715-39 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: finishing write tid 2 to smithi07685715-39 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: finishing write tid 3 to smithi07685715-39 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: finishing write tid 4 to smithi07685715-39 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: finishing write tid 5 to smithi07685715-39 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: oid 39 updating version 0 to 1899 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: oid 39 updating version 1899 to 1900 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: oid 39 version 1900 is already newer than 1898 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: oid 39 version 1900 is already newer than 1897 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1900 (ObjNum 937 snap 319 seq_num 937) dirty exists 2024-09-21T09:48:44.729 INFO:tasks.rados.rados.0.smithi076.stdout:2489: left oid 39 (ObjNum 937 snap 319 seq_num 937) 2024-09-21T09:48:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:44 smithi191 ceph-mon[102206]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:44 smithi191 ceph-mon[102206]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:44 smithi191 ceph-mon[102206]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:48:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:44 smithi076 ceph-mon[106482]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:44 smithi076 ceph-mon[106482]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:44 smithi076 ceph-mon[106482]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:48:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:44 smithi076 ceph-mon[110963]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:44 smithi076 ceph-mon[110963]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:48:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:44 smithi076 ceph-mon[110963]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T09:48:45.732 INFO:tasks.rados.rados.0.smithi076.stdout:2489: done (1 left) 2024-09-21T09:48:45.732 INFO:tasks.rados.rados.0.smithi076.stdout:2491: done (0 left) 2024-09-21T09:48:45.733 INFO:tasks.rados.rados.0.smithi076.stdout:2492: write oid 13 current snap is 320 2024-09-21T09:48:45.733 INFO:tasks.rados.rados.0.smithi076.stdout:2492: seq_num 938 ranges {564966=494981,1606345=782994,3025826=425501} 2024-09-21T09:48:45.738 INFO:tasks.rados.rados.0.smithi076.stdout:2492: writing smithi07685715-13 from 564966 to 1059947 tid 1 2024-09-21T09:48:45.746 INFO:tasks.rados.rados.0.smithi076.stdout:2492: writing smithi07685715-13 from 1606345 to 2389339 tid 2 2024-09-21T09:48:45.752 INFO:tasks.rados.rados.0.smithi076.stdout:2492: writing smithi07685715-13 from 3025826 to 3451327 tid 3 2024-09-21T09:48:45.752 INFO:tasks.rados.rados.0.smithi076.stdout:2493: delete oid 35 current snap is 320 2024-09-21T09:48:45.752 INFO:tasks.rados.rados.0.smithi076.stdout:2492: finishing write tid 1 to smithi07685715-13 2024-09-21T09:48:45.752 INFO:tasks.rados.rados.0.smithi076.stdout:2492: finishing write tid 2 to smithi07685715-13 2024-09-21T09:48:45.755 INFO:tasks.rados.rados.0.smithi076.stdout:2492: finishing write tid 3 to smithi07685715-13 2024-09-21T09:48:45.755 INFO:tasks.rados.rados.0.smithi076.stdout:2492: finishing write tid 4 to smithi07685715-13 2024-09-21T09:48:45.755 INFO:tasks.rados.rados.0.smithi076.stdout:2492: finishing write tid 5 to smithi07685715-13 2024-09-21T09:48:45.755 INFO:tasks.rados.rados.0.smithi076.stdout:2492: oid 13 updating version 0 to 431 2024-09-21T09:48:45.755 INFO:tasks.rados.rados.0.smithi076.stdout:2492: oid 13 updating version 431 to 433 2024-09-21T09:48:45.756 INFO:tasks.rados.rados.0.smithi076.stdout:2492: oid 13 version 433 is already newer than 432 2024-09-21T09:48:45.756 INFO:tasks.rados.rados.0.smithi076.stdout:2492: oid 13 updating version 433 to 434 2024-09-21T09:48:45.756 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 434 (ObjNum 938 snap 320 seq_num 938) dirty exists 2024-09-21T09:48:45.756 INFO:tasks.rados.rados.0.smithi076.stdout:2492: left oid 13 (ObjNum 938 snap 320 seq_num 938) 2024-09-21T09:48:45.757 INFO:tasks.rados.rados.0.smithi076.stdout:2492: done (1 left) 2024-09-21T09:48:45.758 INFO:tasks.rados.rados.0.smithi076.stdout:2493: done (0 left) 2024-09-21T09:48:45.758 INFO:tasks.rados.rados.0.smithi076.stdout:2494: snap_create 2024-09-21T09:48:45.871 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:45 smithi076 ceph-mon[106482]: pgmap v535: 89 pgs: 89 active+clean; 247 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:48:45.871 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:45 smithi076 ceph-mon[106482]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:48:45.871 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:45 smithi076 ceph-mon[110963]: pgmap v535: 89 pgs: 89 active+clean; 247 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:48:45.871 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:45 smithi076 ceph-mon[110963]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:48:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:45 smithi191 ceph-mon[102206]: pgmap v535: 89 pgs: 89 active+clean; 247 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:48:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:45 smithi191 ceph-mon[102206]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T09:48:46.730 INFO:tasks.rados.rados.0.smithi076.stdout:2494: done (0 left) 2024-09-21T09:48:46.730 INFO:tasks.rados.rados.0.smithi076.stdout:2495: snap_remove snap 318 2024-09-21T09:48:47.694 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:2495: done (0 left) 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:2496: read oid 30 snap -1 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:2496: expect (ObjNum 842 snap 291 seq_num 842) 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:2497: read oid 44 snap -1 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:2497: expect (ObjNum 911 snap 314 seq_num 911) 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:2498: rollback oid 20 current snap is 321 2024-09-21T09:48:47.733 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 307 2024-09-21T09:48:47.734 INFO:tasks.rados.rados.0.smithi076.stdout:2499: delete oid 23 current snap is 321 2024-09-21T09:48:47.742 INFO:tasks.rados.rados.0.smithi076.stdout:2498: finishing rollback tid 0 to smithi07685715-20 2024-09-21T09:48:47.743 INFO:tasks.rados.rados.0.smithi076.stdout:2498: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:48:47.743 INFO:tasks.rados.rados.0.smithi076.stdout:2498: finishing rollback tid 2 to smithi07685715-20 2024-09-21T09:48:47.743 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 760 (ObjNum 806 snap 282 seq_num 806) dirty exists 2024-09-21T09:48:47.744 INFO:tasks.rados.rados.0.smithi076.stdout:2496: expect (ObjNum 842 snap 291 seq_num 842) 2024-09-21T09:48:47.840 INFO:tasks.rados.rados.0.smithi076.stdout:2497: expect (ObjNum 911 snap 314 seq_num 911) 2024-09-21T09:48:47.986 INFO:tasks.rados.rados.0.smithi076.stdout:2496: done (3 left) 2024-09-21T09:48:47.987 INFO:tasks.rados.rados.0.smithi076.stdout:2497: done (2 left) 2024-09-21T09:48:47.987 INFO:tasks.rados.rados.0.smithi076.stdout:2498: done (1 left) 2024-09-21T09:48:47.987 INFO:tasks.rados.rados.0.smithi076.stdout:2499: done (0 left) 2024-09-21T09:48:47.987 INFO:tasks.rados.rados.0.smithi076.stdout:2500: read oid 50 snap 275 2024-09-21T09:48:47.987 INFO:tasks.rados.rados.0.smithi076.stdout:2500: expect (ObjNum 670 snap 214 seq_num 670) 2024-09-21T09:48:47.987 INFO:tasks.rados.rados.0.smithi076.stdout:2501: snap_create 2024-09-21T09:48:47.997 INFO:tasks.rados.rados.0.smithi076.stdout:2500: expect (ObjNum 670 snap 214 seq_num 670) 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: pgmap v538: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 796 KiB/s wr, 3 op/s 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.037 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: pgmap v538: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 796 KiB/s wr, 3 op/s 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:48:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: pgmap v538: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 796 KiB/s wr, 3 op/s 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (7m) 41s ago 13m 19.7M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (6m) 40s ago 13m 79.3M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (12m) 41s ago 12m 22.2M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (11m) 40s ago 11m 18.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (8m) 40s ago 17m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (8m) 41s ago 18m 516M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (6m) 41s ago 18m 66.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:48:48.289 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (3m) 40s ago 17m 40.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 bb6de71151ea 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (5m) 41s ago 17m 51.4M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (8m) 41s ago 12m 11.2M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (8m) 40s ago 12m 11.5M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (16m) 41s ago 16m 121M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 1cb83dae177f 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (16m) 41s ago 16m 93.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (16m) 41s ago 16m 109M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (15m) 41s ago 15m 89.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (15m) 40s ago 15m 111M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (14m) 40s ago 14m 123M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (14m) 40s ago 14m 87.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (14m) 40s ago 14m 91.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:48:48.290 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (7m) 40s ago 13m 48.1M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:48:48.631 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 10, 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:48:48.632 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:48:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2500: done (1 left) 2024-09-21T09:48:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2501: done (0 left) 2024-09-21T09:48:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2502: write oid 21 current snap is 322 2024-09-21T09:48:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:2502: seq_num 941 ranges {615387=720728,1781791=530321,3108095=707292} 2024-09-21T09:48:48.744 INFO:tasks.rados.rados.0.smithi076.stdout:2502: writing smithi07685715-21 from 615387 to 1336115 tid 1 2024-09-21T09:48:48.749 INFO:tasks.rados.rados.0.smithi076.stdout:2502: writing smithi07685715-21 from 1781791 to 2312112 tid 2 2024-09-21T09:48:48.758 INFO:tasks.rados.rados.0.smithi076.stdout:2502: writing smithi07685715-21 from 3108095 to 3815387 tid 3 2024-09-21T09:48:48.758 INFO:tasks.rados.rados.0.smithi076.stdout:2503: write oid 30 current snap is 322 2024-09-21T09:48:48.758 INFO:tasks.rados.rados.0.smithi076.stdout:2502: finishing write tid 1 to smithi07685715-21 2024-09-21T09:48:48.758 INFO:tasks.rados.rados.0.smithi076.stdout:2502: finishing write tid 2 to smithi07685715-21 2024-09-21T09:48:48.758 INFO:tasks.rados.rados.0.smithi076.stdout:2503: seq_num 942 ranges {632194=795977,2123920=790332,3419165=413029} 2024-09-21T09:48:48.765 INFO:tasks.rados.rados.0.smithi076.stdout:2503: writing smithi07685715-30 from 632194 to 1428171 tid 1 2024-09-21T09:48:48.775 INFO:tasks.rados.rados.0.smithi076.stdout:2503: writing smithi07685715-30 from 2123920 to 2914252 tid 2 2024-09-21T09:48:48.780 INFO:tasks.rados.rados.0.smithi076.stdout:2503: writing smithi07685715-30 from 3419165 to 3832194 tid 3 2024-09-21T09:48:48.780 INFO:tasks.rados.rados.0.smithi076.stdout:2504: write oid 17 current snap is 322 2024-09-21T09:48:48.780 INFO:tasks.rados.rados.0.smithi076.stdout:2502: finishing write tid 3 to smithi07685715-21 2024-09-21T09:48:48.780 INFO:tasks.rados.rados.0.smithi076.stdout:2502: finishing write tid 4 to smithi07685715-21 2024-09-21T09:48:48.780 INFO:tasks.rados.rados.0.smithi076.stdout:2502: finishing write tid 5 to smithi07685715-21 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2502: oid 21 updating version 0 to 1914 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2502: oid 21 version 1914 is already newer than 1913 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2502: oid 21 version 1914 is already newer than 1912 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2502: oid 21 version 1914 is already newer than 1911 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 1914 (ObjNum 941 snap 322 seq_num 941) dirty exists 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2502: left oid 21 (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2503: finishing write tid 1 to smithi07685715-30 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2503: finishing write tid 2 to smithi07685715-30 2024-09-21T09:48:48.781 INFO:tasks.rados.rados.0.smithi076.stdout:2504: seq_num 943 ranges {649001=587579,1898755=533990,3162941=686060} 2024-09-21T09:48:48.787 INFO:tasks.rados.rados.0.smithi076.stdout:2504: writing smithi07685715-17 from 649001 to 1236580 tid 1 2024-09-21T09:48:48.793 INFO:tasks.rados.rados.0.smithi076.stdout:2504: writing smithi07685715-17 from 1898755 to 2432745 tid 2 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2504: writing smithi07685715-17 from 3162941 to 3849001 tid 3 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2502: done (2 left) 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2505: rollback oid 9 current snap is 322 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 319 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2503: finishing write tid 3 to smithi07685715-30 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2503: finishing write tid 4 to smithi07685715-30 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2503: finishing write tid 5 to smithi07685715-30 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2503: oid 30 updating version 0 to 1972 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2503: oid 30 updating version 1972 to 1973 2024-09-21T09:48:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:2503: oid 30 version 1973 is already newer than 1970 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:2503: oid 30 version 1973 is already newer than 1971 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1973 (ObjNum 942 snap 322 seq_num 942) dirty exists 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:2503: left oid 30 (ObjNum 942 snap 322 seq_num 942) 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:2504: finishing write tid 1 to smithi07685715-17 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:2504: finishing write tid 2 to smithi07685715-17 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:2503: done (2 left) 2024-09-21T09:48:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:2506: snap_create 2024-09-21T09:48:48.806 INFO:tasks.rados.rados.0.smithi076.stdout:2504: finishing write tid 3 to smithi07685715-17 2024-09-21T09:48:48.806 INFO:tasks.rados.rados.0.smithi076.stdout:2504: finishing write tid 4 to smithi07685715-17 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2504: finishing write tid 5 to smithi07685715-17 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2504: oid 17 updating version 0 to 1503 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2504: oid 17 version 1503 is already newer than 1501 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2504: oid 17 version 1503 is already newer than 1500 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2504: oid 17 version 1503 is already newer than 1502 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1503 (ObjNum 943 snap 322 seq_num 943) dirty exists 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2504: left oid 17 (ObjNum 943 snap 322 seq_num 943) 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2505: finishing rollback tid 0 to smithi07685715-9 2024-09-21T09:48:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:2505: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:48:48.809 INFO:tasks.rados.rados.0.smithi076.stdout:2505: finishing rollback tid 2 to smithi07685715-9 2024-09-21T09:48:48.809 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 870 (ObjNum 914 snap 314 seq_num 914) dirty exists 2024-09-21T09:48:48.947 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:48:48.947 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "0/8 daemons upgraded", 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:48:48.948 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:48:48.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:48:48.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all mgr 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all mon 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all crash 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: Upgrade: osd.0 is safe to restart 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2735641650' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[106482]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:48:48.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all mgr 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all mon 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all crash 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: Upgrade: osd.0 is safe to restart 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2735641650' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:48.950 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:48 smithi076 ceph-mon[110963]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:48:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (squid) 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all mgr 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all mon 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all crash 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: Upgrade: osd.0 is safe to restart 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: from='client.34299 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: from='client.44234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2735641650' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:48:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:48 smithi191 ceph-mon[102206]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T09:48:49.753 INFO:tasks.rados.rados.0.smithi076.stdout:2504: done (2 left) 2024-09-21T09:48:49.753 INFO:tasks.rados.rados.0.smithi076.stdout:2505: done (1 left) 2024-09-21T09:48:49.753 INFO:tasks.rados.rados.0.smithi076.stdout:2506: done (0 left) 2024-09-21T09:48:49.753 INFO:tasks.rados.rados.0.smithi076.stdout:2507: snap_create 2024-09-21T09:48:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: pgmap v541: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 796 KiB/s wr, 3 op/s 2024-09-21T09:48:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: Upgrade: Updating osd.0 2024-09-21T09:48:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:48:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:50.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: Deploying daemon osd.0 on smithi076 2024-09-21T09:48:50.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:49 smithi191 ceph-mon[102206]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: pgmap v541: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 796 KiB/s wr, 3 op/s 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: Upgrade: Updating osd.0 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: Deploying daemon osd.0 on smithi076 2024-09-21T09:48:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[110963]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:48:50.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: pgmap v541: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 796 KiB/s wr, 3 op/s 2024-09-21T09:48:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: Upgrade: Updating osd.0 2024-09-21T09:48:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:48:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:48:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: Deploying daemon osd.0 on smithi076 2024-09-21T09:48:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: from='client.34323 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:50.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:49 smithi076 ceph-mon[106482]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T09:48:50.755 INFO:tasks.rados.rados.0.smithi076.stdout:2507: done (0 left) 2024-09-21T09:48:50.755 INFO:tasks.rados.rados.0.smithi076.stdout:2508: write oid 35 current snap is 324 2024-09-21T09:48:50.755 INFO:tasks.rados.rados.0.smithi076.stdout:2508: seq_num 944 ranges {665808=662828,1957237=677648,3074011=609486} 2024-09-21T09:48:50.762 INFO:tasks.rados.rados.0.smithi076.stdout:2508: writing smithi07685715-35 from 665808 to 1328636 tid 1 2024-09-21T09:48:50.769 INFO:tasks.rados.rados.0.smithi076.stdout:2508: writing smithi07685715-35 from 1957237 to 2634885 tid 2 2024-09-21T09:48:50.775 INFO:tasks.rados.rados.0.smithi076.stdout:2508: writing smithi07685715-35 from 3074011 to 3683497 tid 3 2024-09-21T09:48:50.776 INFO:tasks.rados.rados.0.smithi076.stdout:2509: write oid 26 current snap is 324 2024-09-21T09:48:50.776 INFO:tasks.rados.rados.0.smithi076.stdout:2508: finishing write tid 1 to smithi07685715-35 2024-09-21T09:48:50.776 INFO:tasks.rados.rados.0.smithi076.stdout:2508: finishing write tid 2 to smithi07685715-35 2024-09-21T09:48:50.776 INFO:tasks.rados.rados.0.smithi076.stdout:2509: seq_num 945 ranges {682615=738077,2015719=537659,3217787=420758} 2024-09-21T09:48:50.783 INFO:tasks.rados.rados.0.smithi076.stdout:2509: writing smithi07685715-26 from 682615 to 1420692 tid 1 2024-09-21T09:48:50.789 INFO:tasks.rados.rados.0.smithi076.stdout:2509: writing smithi07685715-26 from 2015719 to 2553378 tid 2 2024-09-21T09:48:50.796 INFO:tasks.rados.rados.0.smithi076.stdout:2509: writing smithi07685715-26 from 3217787 to 3638545 tid 3 2024-09-21T09:48:50.796 INFO:tasks.rados.rados.0.smithi076.stdout:2510: read oid 34 snap 303 2024-09-21T09:48:50.796 INFO:tasks.rados.rados.0.smithi076.stdout:2510: expect (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:48:50.796 INFO:tasks.rados.rados.0.smithi076.stdout:2511: snap_remove snap 322 2024-09-21T09:48:51.757 INFO:tasks.rados.rados.0.smithi076.stdout:2511: done (3 left) 2024-09-21T09:48:51.757 INFO:tasks.rados.rados.0.smithi076.stdout:2512: rollback oid 14 current snap is 324 2024-09-21T09:48:51.757 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 319 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: finishing write tid 3 to smithi07685715-35 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: finishing write tid 4 to smithi07685715-35 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: finishing write tid 5 to smithi07685715-35 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: oid 35 updating version 0 to 1915 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: oid 35 updating version 1915 to 1918 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: oid 35 version 1918 is already newer than 1916 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: oid 35 version 1918 is already newer than 1917 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1918 (ObjNum 944 snap 324 seq_num 944) dirty exists 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2508: left oid 35 (ObjNum 944 snap 324 seq_num 944) 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2509: finishing write tid 1 to smithi07685715-26 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2509: finishing write tid 2 to smithi07685715-26 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2509: finishing write tid 3 to smithi07685715-26 2024-09-21T09:48:51.758 INFO:tasks.rados.rados.0.smithi076.stdout:2509: finishing write tid 4 to smithi07685715-26 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2509: finishing write tid 5 to smithi07685715-26 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2509: oid 26 updating version 0 to 874 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2509: oid 26 version 874 is already newer than 873 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2509: oid 26 version 874 is already newer than 872 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2509: oid 26 updating version 874 to 875 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 875 (ObjNum 945 snap 324 seq_num 945) dirty exists 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2509: left oid 26 (ObjNum 945 snap 324 seq_num 945) 2024-09-21T09:48:51.759 INFO:tasks.rados.rados.0.smithi076.stdout:2510: expect (ObjNum 836 snap 290 seq_num 836) 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2512: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2512: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2512: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1507 (ObjNum 925 snap 317 seq_num 925) dirty exists 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2508: done (3 left) 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2509: done (2 left) 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2510: done (1 left) 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2512: done (0 left) 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2513: read oid 36 snap -1 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2513: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:48:51.840 INFO:tasks.rados.rados.0.smithi076.stdout:2514: rollback oid 7 current snap is 324 2024-09-21T09:48:51.841 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 316 2024-09-21T09:48:51.841 INFO:tasks.rados.rados.0.smithi076.stdout:2515: write oid 38 current snap is 324 2024-09-21T09:48:51.841 INFO:tasks.rados.rados.0.smithi076.stdout:2515: seq_num 946 ranges {699422=413326,1557848=681317,3012504=748383} 2024-09-21T09:48:51.845 INFO:tasks.rados.rados.0.smithi076.stdout:2515: writing smithi07685715-38 from 699422 to 1112748 tid 1 2024-09-21T09:48:51.852 INFO:tasks.rados.rados.0.smithi076.stdout:2515: writing smithi07685715-38 from 1557848 to 2239165 tid 2 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2515: writing smithi07685715-38 from 3012504 to 3760887 tid 3 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2516: read oid 25 snap -1 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2516: expect deleted 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2517: read oid 48 snap -1 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2517: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2518: delete oid 1 current snap is 324 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2514: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2514: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2514: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1511 (ObjNum 905 snap 313 seq_num 905) dirty exists 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2515: finishing write tid 1 to smithi07685715-38 2024-09-21T09:48:51.861 INFO:tasks.rados.rados.0.smithi076.stdout:2513: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:48:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:51 smithi076 ceph-mon[106482]: pgmap v544: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:51 smithi076 ceph-mon[106482]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:48:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:51 smithi076 ceph-mon[110963]: pgmap v544: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:51.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:51 smithi076 ceph-mon[110963]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: finishing write tid 2 to smithi07685715-38 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: finishing write tid 3 to smithi07685715-38 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: finishing write tid 4 to smithi07685715-38 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: finishing write tid 5 to smithi07685715-38 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: oid 38 updating version 0 to 1515 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: oid 38 updating version 1515 to 1516 2024-09-21T09:48:52.039 INFO:tasks.rados.rados.0.smithi076.stdout:2515: oid 38 version 1516 is already newer than 1513 2024-09-21T09:48:52.040 INFO:tasks.rados.rados.0.smithi076.stdout:2515: oid 38 version 1516 is already newer than 1514 2024-09-21T09:48:52.040 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1516 (ObjNum 946 snap 324 seq_num 946) dirty exists 2024-09-21T09:48:52.040 INFO:tasks.rados.rados.0.smithi076.stdout:2515: left oid 38 (ObjNum 946 snap 324 seq_num 946) 2024-09-21T09:48:52.040 INFO:tasks.rados.rados.0.smithi076.stdout:2517: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:51 smithi191 ceph-mon[102206]: pgmap v544: 89 pgs: 89 active+clean; 248 MiB data, 825 MiB used, 714 GiB / 715 GiB avail 2024-09-21T09:48:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:51 smithi191 ceph-mon[102206]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T09:48:52.198 INFO:tasks.rados.rados.0.smithi076.stdout:2513: done (5 left) 2024-09-21T09:48:52.198 INFO:tasks.rados.rados.0.smithi076.stdout:2514: done (4 left) 2024-09-21T09:48:52.199 INFO:tasks.rados.rados.0.smithi076.stdout:2515: done (3 left) 2024-09-21T09:48:52.199 INFO:tasks.rados.rados.0.smithi076.stdout:2516: done (2 left) 2024-09-21T09:48:52.199 INFO:tasks.rados.rados.0.smithi076.stdout:2517: done (1 left) 2024-09-21T09:48:52.199 INFO:tasks.rados.rados.0.smithi076.stdout:2518: done (0 left) 2024-09-21T09:48:52.199 INFO:tasks.rados.rados.0.smithi076.stdout:2519: write oid 5 current snap is 324 2024-09-21T09:48:52.199 INFO:tasks.rados.rados.0.smithi076.stdout:2519: seq_num 948 ranges {733036=563824,2074812=684986,3183703=749333} 2024-09-21T09:48:52.200 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:51 smithi076 systemd[1]: Stopping Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:48:52.205 INFO:tasks.rados.rados.0.smithi076.stdout:2519: writing smithi07685715-5 from 733036 to 1296860 tid 1 2024-09-21T09:48:52.213 INFO:tasks.rados.rados.0.smithi076.stdout:2519: writing smithi07685715-5 from 2074812 to 2759798 tid 2 2024-09-21T09:48:52.220 INFO:tasks.rados.rados.0.smithi076.stdout:2519: writing smithi07685715-5 from 3183703 to 3933036 tid 3 2024-09-21T09:48:52.220 INFO:tasks.rados.rados.0.smithi076.stdout:2520: delete oid 18 current snap is 324 2024-09-21T09:48:52.220 INFO:tasks.rados.rados.0.smithi076.stdout:2519: finishing write tid 1 to smithi07685715-5 2024-09-21T09:48:52.220 INFO:tasks.rados.rados.0.smithi076.stdout:2519: finishing write tid 2 to smithi07685715-5 2024-09-21T09:48:52.222 INFO:tasks.rados.rados.0.smithi076.stdout:2520: done (1 left) 2024-09-21T09:48:52.222 INFO:tasks.rados.rados.0.smithi076.stdout:2521: delete oid 50 current snap is 324 2024-09-21T09:48:52.224 INFO:tasks.rados.rados.0.smithi076.stdout:2521: done (1 left) 2024-09-21T09:48:52.224 INFO:tasks.rados.rados.0.smithi076.stdout:2522: write oid 27 current snap is 324 2024-09-21T09:48:52.224 INFO:tasks.rados.rados.0.smithi076.stdout:2522: seq_num 951 ranges {783457=505924,1850258=548666,2865972=721096} 2024-09-21T09:48:52.231 INFO:tasks.rados.rados.0.smithi076.stdout:2522: writing smithi07685715-27 from 783457 to 1289381 tid 1 2024-09-21T09:48:52.237 INFO:tasks.rados.rados.0.smithi076.stdout:2522: writing smithi07685715-27 from 1850258 to 2398924 tid 2 2024-09-21T09:48:52.244 INFO:tasks.rados.rados.0.smithi076.stdout:2522: writing smithi07685715-27 from 2865972 to 3587068 tid 3 2024-09-21T09:48:52.244 INFO:tasks.rados.rados.0.smithi076.stdout:2523: delete oid 19 current snap is 324 2024-09-21T09:48:52.244 INFO:tasks.rados.rados.0.smithi076.stdout:2519: finishing write tid 3 to smithi07685715-5 2024-09-21T09:48:52.244 INFO:tasks.rados.rados.0.smithi076.stdout:2519: finishing write tid 4 to smithi07685715-5 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2519: finishing write tid 5 to smithi07685715-5 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2519: oid 5 updating version 0 to 1923 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2519: oid 5 version 1923 is already newer than 1921 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2519: oid 5 version 1923 is already newer than 1920 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2519: oid 5 version 1923 is already newer than 1922 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1923 (ObjNum 948 snap 324 seq_num 948) dirty exists 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2519: left oid 5 (ObjNum 948 snap 324 seq_num 948) 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2522: finishing write tid 1 to smithi07685715-27 2024-09-21T09:48:52.245 INFO:tasks.rados.rados.0.smithi076.stdout:2522: finishing write tid 2 to smithi07685715-27 2024-09-21T09:48:52.246 INFO:tasks.rados.rados.0.smithi076.stdout:2519: done (2 left) 2024-09-21T09:48:52.246 INFO:tasks.rados.rados.0.smithi076.stdout:2523: done (1 left) 2024-09-21T09:48:52.246 INFO:tasks.rados.rados.0.smithi076.stdout:2524: read oid 45 snap -1 2024-09-21T09:48:52.247 INFO:tasks.rados.rados.0.smithi076.stdout:2524: expect (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:48:52.247 INFO:tasks.rados.rados.0.smithi076.stdout:2525: snap_remove snap 298 2024-09-21T09:48:52.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[45597]: 2024-09-21T09:48:52.197+0000 7f2264c45640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:48:52.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[45597]: 2024-09-21T09:48:52.197+0000 7f2264c45640 -1 osd.0 717 *** Got signal Terminated *** 2024-09-21T09:48:52.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[45597]: 2024-09-21T09:48:52.197+0000 7f2264c45640 -1 osd.0 717 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:48:52.766 INFO:tasks.rados.rados.0.smithi076.stdout:2522: finishing write tid 3 to smithi07685715-27 2024-09-21T09:48:52.766 INFO:tasks.rados.rados.0.smithi076.stdout:2524: expect (ObjNum 862 snap 295 seq_num 862) 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: finishing write tid 4 to smithi07685715-27 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: finishing write tid 5 to smithi07685715-27 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: oid 27 updating version 0 to 1139 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: oid 27 version 1139 is already newer than 1137 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: oid 27 version 1139 is already newer than 1136 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: oid 27 version 1139 is already newer than 1138 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1139 (ObjNum 951 snap 324 seq_num 951) dirty exists 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: left oid 27 (ObjNum 951 snap 324 seq_num 951) 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2522: done (2 left) 2024-09-21T09:48:52.882 INFO:tasks.rados.rados.0.smithi076.stdout:2524: done (1 left) 2024-09-21T09:48:52.883 INFO:tasks.rados.rados.0.smithi076.stdout:2525: done (0 left) 2024-09-21T09:48:52.883 INFO:tasks.rados.rados.0.smithi076.stdout:2526: delete oid 7 current snap is 324 2024-09-21T09:48:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:52 smithi191 ceph-mon[102206]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:48:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:52 smithi191 ceph-mon[102206]: osd.0 marked itself down and dead 2024-09-21T09:48:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-mon[110963]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:48:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-mon[110963]: osd.0 marked itself down and dead 2024-09-21T09:48:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-mon[106482]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T09:48:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:52 smithi076 ceph-mon[106482]: osd.0 marked itself down and dead 2024-09-21T09:48:53.582 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:48:53] "GET /metrics HTTP/1.1" 200 36133 "" "Prometheus/2.51.0" 2024-09-21T09:48:53.583 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:53 smithi076 podman[122132]: 2024-09-21 09:48:53.30088493 +0000 UTC m=+1.222965570 container died 1cb83dae177f47a1511a197fae305ae9d9ff52a4e419dd7c2efebc5a66f56021 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:48:53.749 INFO:tasks.rados.rados.0.smithi076.stdout:2526: done (0 left) 2024-09-21T09:48:53.749 INFO:tasks.rados.rados.0.smithi076.stdout:2527: snap_create 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[106482]: pgmap v547: 89 pgs: 89 active+clean; 259 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[106482]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[106482]: osdmap e718: 8 total, 7 up, 8 in 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[106482]: osdmap e719: 8 total, 7 up, 8 in 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[110963]: pgmap v547: 89 pgs: 89 active+clean; 259 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[110963]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[110963]: osdmap e718: 8 total, 7 up, 8 in 2024-09-21T09:48:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:53 smithi076 ceph-mon[110963]: osdmap e719: 8 total, 7 up, 8 in 2024-09-21T09:48:53.874 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:53 smithi076 podman[122132]: 2024-09-21 09:48:53.680215016 +0000 UTC m=+1.602295636 container cleanup 1cb83dae177f47a1511a197fae305ae9d9ff52a4e419dd7c2efebc5a66f56021 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:48:53.874 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:53 smithi076 bash[122132]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0 2024-09-21T09:48:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:53 smithi191 ceph-mon[102206]: pgmap v547: 89 pgs: 89 active+clean; 259 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:48:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:53 smithi191 ceph-mon[102206]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:48:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:53 smithi191 ceph-mon[102206]: osdmap e718: 8 total, 7 up, 8 in 2024-09-21T09:48:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:53 smithi191 ceph-mon[102206]: osdmap e719: 8 total, 7 up, 8 in 2024-09-21T09:48:54.343 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:53 smithi076 podman[122152]: 2024-09-21 09:48:53.982529593 +0000 UTC m=+0.675745053 container remove 1cb83dae177f47a1511a197fae305ae9d9ff52a4e419dd7c2efebc5a66f56021 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:48:54.596 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:54 smithi076 podman[122245]: 2024-09-21 09:48:54.244473362 +0000 UTC m=+0.015879846 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:48:54.596 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:54 smithi076 podman[122245]: 2024-09-21 09:48:54.353535651 +0000 UTC m=+0.124942174 container create e41cd09951e5a822a91cdb908d9d3d0f376c903a4f87fddb96758773701186c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:48:54.597 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:54 smithi076 podman[122245]: 2024-09-21 09:48:54.590181114 +0000 UTC m=+0.361587596 container init e41cd09951e5a822a91cdb908d9d3d0f376c903a4f87fddb96758773701186c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:48:54.597 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:54 smithi076 podman[122245]: 2024-09-21 09:48:54.595261995 +0000 UTC m=+0.366668478 container start e41cd09951e5a822a91cdb908d9d3d0f376c903a4f87fddb96758773701186c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T09:48:54.736 INFO:tasks.rados.rados.0.smithi076.stdout:2527: done (0 left) 2024-09-21T09:48:54.736 INFO:tasks.rados.rados.0.smithi076.stdout:2528: delete oid 11 current snap is 325 2024-09-21T09:48:54.739 INFO:tasks.rados.rados.0.smithi076.stdout:2528: done (0 left) 2024-09-21T09:48:54.739 INFO:tasks.rados.rados.0.smithi076.stdout:2529: read oid 13 snap -1 2024-09-21T09:48:54.739 INFO:tasks.rados.rados.0.smithi076.stdout:2529: expect (ObjNum 938 snap 320 seq_num 938) 2024-09-21T09:48:54.739 INFO:tasks.rados.rados.0.smithi076.stdout:2530: snap_create 2024-09-21T09:48:54.746 INFO:tasks.rados.rados.0.smithi076.stdout:2529: expect (ObjNum 938 snap 320 seq_num 938) 2024-09-21T09:48:54.873 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:54 smithi076 podman[122245]: 2024-09-21 09:48:54.681549537 +0000 UTC m=+0.452956019 container attach e41cd09951e5a822a91cdb908d9d3d0f376c903a4f87fddb96758773701186c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T09:48:54.873 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:54 smithi076 podman[122245]: 2024-09-21 09:48:54.789376397 +0000 UTC m=+0.560782872 container died e41cd09951e5a822a91cdb908d9d3d0f376c903a4f87fddb96758773701186c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T09:48:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 podman[122289]: 2024-09-21 09:48:55.369319981 +0000 UTC m=+0.574249541 container remove e41cd09951e5a822a91cdb908d9d3d0f376c903a4f87fddb96758773701186c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:48:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T09:48:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 systemd[1]: Stopped Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:48:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0.service: Consumed 18.364s CPU time. 2024-09-21T09:48:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 systemd[1]: Starting Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:48:55.738 INFO:tasks.rados.rados.0.smithi076.stdout:2529: done (1 left) 2024-09-21T09:48:55.738 INFO:tasks.rados.rados.0.smithi076.stdout:2530: done (0 left) 2024-09-21T09:48:55.738 INFO:tasks.rados.rados.0.smithi076.stdout:2531: write oid 7 current snap is 326 2024-09-21T09:48:55.739 INFO:tasks.rados.rados.0.smithi076.stdout:2531: seq_num 955 ranges {450685=406920,1567833=482852} 2024-09-21T09:48:55.743 INFO:tasks.rados.rados.0.smithi076.stdout:2531: writing smithi07685715-7 from 450685 to 857605 tid 1 2024-09-21T09:48:55.749 INFO:tasks.rados.rados.0.smithi076.stdout:2531: writing smithi07685715-7 from 1567833 to 2050685 tid 2 2024-09-21T09:48:55.749 INFO:tasks.rados.rados.0.smithi076.stdout:2532: read oid 10 snap -1 2024-09-21T09:48:55.749 INFO:tasks.rados.rados.0.smithi076.stdout:2532: expect (ObjNum 936 snap 317 seq_num 936) 2024-09-21T09:48:55.750 INFO:tasks.rados.rados.0.smithi076.stdout:2533: snap_remove snap 321 2024-09-21T09:48:56.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[106482]: pgmap v550: 89 pgs: 13 stale+active+clean, 76 active+clean; 259 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:48:56.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[106482]: osdmap e720: 8 total, 7 up, 8 in 2024-09-21T09:48:56.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:56.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:56.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[110963]: pgmap v550: 89 pgs: 13 stale+active+clean, 76 active+clean; 259 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:48:56.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[110963]: osdmap e720: 8 total, 7 up, 8 in 2024-09-21T09:48:56.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:56.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:56.069 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 podman[122393]: 2024-09-21 09:48:55.697548821 +0000 UTC m=+0.015741064 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:48:56.069 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:55 smithi076 podman[122393]: 2024-09-21 09:48:55.846396297 +0000 UTC m=+0.164588527 container create 4ed42a795c8c4e0c7f908357b753b1054b782e897ec603163ec7031b31a7d7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2) 2024-09-21T09:48:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:55 smithi191 ceph-mon[102206]: pgmap v550: 89 pgs: 13 stale+active+clean, 76 active+clean; 259 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 17 op/s 2024-09-21T09:48:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:55 smithi191 ceph-mon[102206]: osdmap e720: 8 total, 7 up, 8 in 2024-09-21T09:48:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 podman[122393]: 2024-09-21 09:48:56.067144476 +0000 UTC m=+0.385336715 container init 4ed42a795c8c4e0c7f908357b753b1054b782e897ec603163ec7031b31a7d7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 podman[122393]: 2024-09-21 09:48:56.072110473 +0000 UTC m=+0.390302720 container start 4ed42a795c8c4e0c7f908357b753b1054b782e897ec603163ec7031b31a7d7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True) 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 podman[122393]: 2024-09-21 09:48:56.174378646 +0000 UTC m=+0.492570877 container attach 4ed42a795c8c4e0c7f908357b753b1054b782e897ec603163ec7031b31a7d7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:48:56.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:48:56.744 INFO:tasks.rados.rados.0.smithi076.stdout:2533: done (2 left) 2024-09-21T09:48:56.744 INFO:tasks.rados.rados.0.smithi076.stdout:2534: snap_remove snap 317 2024-09-21T09:48:56.972 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-mon[106482]: osdmap e721: 8 total, 7 up, 8 in 2024-09-21T09:48:56.972 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-mon[110963]: osdmap e721: 8 total, 7 up, 8 in 2024-09-21T09:48:56.972 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:48:56.972 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:48:56.972 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:48:56.972 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:48:56.973 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:48:56.973 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:48:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:56 smithi191 ceph-mon[102206]: osdmap e721: 8 total, 7 up, 8 in 2024-09-21T09:48:57.296 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:48:57.296 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[122408]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:56 smithi076 bash[122393]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:48:57.297 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:57 smithi076 podman[122393]: 2024-09-21 09:48:57.001547341 +0000 UTC m=+1.319739584 container died 4ed42a795c8c4e0c7f908357b753b1054b782e897ec603163ec7031b31a7d7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T09:48:57.751 INFO:tasks.rados.rados.0.smithi076.stdout:2534: done (2 left) 2024-09-21T09:48:57.751 INFO:tasks.rados.rados.0.smithi076.stdout:2535: write oid 34 current snap is 326 2024-09-21T09:48:57.751 INFO:tasks.rados.rados.0.smithi076.stdout:2531: finishing write tid 1 to smithi07685715-7 2024-09-21T09:48:57.751 INFO:tasks.rados.rados.0.smithi076.stdout:2535: seq_num 956 ranges {467492=482169,1626315=441177} 2024-09-21T09:48:57.755 INFO:tasks.rados.rados.0.smithi076.stdout:2535: writing smithi07685715-34 from 467492 to 949661 tid 1 2024-09-21T09:48:57.762 INFO:tasks.rados.rados.0.smithi076.stdout:2535: writing smithi07685715-34 from 1626315 to 2067492 tid 2 2024-09-21T09:48:57.762 INFO:tasks.rados.rados.0.smithi076.stdout:2536: write oid 20 current snap is 326 2024-09-21T09:48:57.762 INFO:tasks.rados.rados.0.smithi076.stdout:2536: seq_num 957 ranges {484299=557418,1684797=399502} 2024-09-21T09:48:57.767 INFO:tasks.rados.rados.0.smithi076.stdout:2536: writing smithi07685715-20 from 484299 to 1041717 tid 1 2024-09-21T09:48:57.773 INFO:tasks.rados.rados.0.smithi076.stdout:2536: writing smithi07685715-20 from 1684797 to 2084299 tid 2 2024-09-21T09:48:57.773 INFO:tasks.rados.rados.0.smithi076.stdout:2537: read oid 48 snap -1 2024-09-21T09:48:57.773 INFO:tasks.rados.rados.0.smithi076.stdout:2537: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:48:57.773 INFO:tasks.rados.rados.0.smithi076.stdout:2538: snap_remove snap 323 2024-09-21T09:48:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:57 smithi076 ceph-mon[110963]: pgmap v553: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s; 102/519 objects degraded (19.653%) 2024-09-21T09:48:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:57 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 102/519 objects degraded (19.653%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:48:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:57 smithi076 ceph-mon[110963]: osdmap e722: 8 total, 7 up, 8 in 2024-09-21T09:48:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:57 smithi076 ceph-mon[106482]: pgmap v553: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s; 102/519 objects degraded (19.653%) 2024-09-21T09:48:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:57 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 102/519 objects degraded (19.653%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:48:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:57 smithi076 ceph-mon[106482]: osdmap e722: 8 total, 7 up, 8 in 2024-09-21T09:48:57.874 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:57 smithi076 podman[122571]: 2024-09-21 09:48:57.58785274 +0000 UTC m=+0.576270505 container remove 4ed42a795c8c4e0c7f908357b753b1054b782e897ec603163ec7031b31a7d7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:48:57.874 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:57 smithi076 podman[122636]: 2024-09-21 09:48:57.694036126 +0000 UTC m=+0.015719487 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:48:57.874 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:57 smithi076 podman[122636]: 2024-09-21 09:48:57.805799578 +0000 UTC m=+0.127482936 container create f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:48:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:57 smithi191 ceph-mon[102206]: pgmap v553: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s; 102/519 objects degraded (19.653%) 2024-09-21T09:48:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:57 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 102/519 objects degraded (19.653%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:48:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:57 smithi191 ceph-mon[102206]: osdmap e722: 8 total, 7 up, 8 in 2024-09-21T09:48:58.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:58 smithi076 podman[122636]: 2024-09-21 09:48:58.042004159 +0000 UTC m=+0.363687527 container init f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:48:58.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:58 smithi076 podman[122636]: 2024-09-21 09:48:58.046628802 +0000 UTC m=+0.368312144 container start f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T09:48:58.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-osd[122675]: -- 172.21.15.76:0/3122189961 <== mon.2 v2:172.21.15.76:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5635317cba40 con 0x56353259a000 2024-09-21T09:48:58.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:58 smithi076 bash[122636]: f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 2024-09-21T09:48:58.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:58 smithi076 systemd[1]: Started Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:48:58.738 INFO:tasks.rados.rados.0.smithi076.stdout:2538: done (5 left) 2024-09-21T09:48:58.738 INFO:tasks.rados.rados.0.smithi076.stdout:2539: write oid 16 current snap is 326 2024-09-21T09:48:58.739 INFO:tasks.rados.rados.0.smithi076.stdout:2539: seq_num 958 ranges {501106=749020,1743279=357827} 2024-09-21T09:48:58.745 INFO:tasks.rados.rados.0.smithi076.stdout:2539: writing smithi07685715-16 from 501106 to 1250126 tid 1 2024-09-21T09:48:58.750 INFO:tasks.rados.rados.0.smithi076.stdout:2539: writing smithi07685715-16 from 1743279 to 2101106 tid 2 2024-09-21T09:48:58.750 INFO:tasks.rados.rados.0.smithi076.stdout:2540: write oid 30 current snap is 326 2024-09-21T09:48:58.750 INFO:tasks.rados.rados.0.smithi076.stdout:2540: seq_num 959 ranges {517913=424269,1401761=446989} 2024-09-21T09:48:58.755 INFO:tasks.rados.rados.0.smithi076.stdout:2540: writing smithi07685715-30 from 517913 to 942182 tid 1 2024-09-21T09:48:58.760 INFO:tasks.rados.rados.0.smithi076.stdout:2540: writing smithi07685715-30 from 1401761 to 1848750 tid 2 2024-09-21T09:48:58.760 INFO:tasks.rados.rados.0.smithi076.stdout:2541: snap_remove snap 319 2024-09-21T09:48:59.080 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[106482]: osdmap e723: 8 total, 7 up, 8 in 2024-09-21T09:48:59.080 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.080 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.080 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.080 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:59.081 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[106482]: osdmap e724: 8 total, 7 up, 8 in 2024-09-21T09:48:59.081 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[110963]: osdmap e723: 8 total, 7 up, 8 in 2024-09-21T09:48:59.081 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.081 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.081 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.081 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:59.082 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:58 smithi076 ceph-mon[110963]: osdmap e724: 8 total, 7 up, 8 in 2024-09-21T09:48:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:58 smithi191 ceph-mon[102206]: osdmap e723: 8 total, 7 up, 8 in 2024-09-21T09:48:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:48:59.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:48:59.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:58 smithi191 ceph-mon[102206]: osdmap e724: 8 total, 7 up, 8 in 2024-09-21T09:48:59.760 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:48:59 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[122652]: 2024-09-21T09:48:59.451+0000 7f199b75b640 -1 Falling back to public interface 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2541: done (7 left) 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2542: read oid 37 snap -1 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2542: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2543: read oid 1 snap 307 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2543: expect (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2544: write oid 43 current snap is 326 2024-09-21T09:48:59.763 INFO:tasks.rados.rados.0.smithi076.stdout:2531: finishing write tid 2 to smithi07685715-7 2024-09-21T09:48:59.764 INFO:tasks.rados.rados.0.smithi076.stdout:2544: seq_num 960 ranges {534720=499518,1460243=674477} 2024-09-21T09:48:59.766 INFO:tasks.rados.rados.0.smithi076.stdout:2544: writing smithi07685715-43 from 534720 to 1034238 tid 1 2024-09-21T09:48:59.775 INFO:tasks.rados.rados.0.smithi076.stdout:2544: writing smithi07685715-43 from 1460243 to 2134720 tid 2 2024-09-21T09:48:59.775 INFO:tasks.rados.rados.0.smithi076.stdout:2545: write oid 50 current snap is 326 2024-09-21T09:48:59.775 INFO:tasks.rados.rados.0.smithi076.stdout:2545: seq_num 961 ranges {551527=574767,1918725=232802} 2024-09-21T09:48:59.781 INFO:tasks.rados.rados.0.smithi076.stdout:2545: writing smithi07685715-50 from 551527 to 1126294 tid 1 2024-09-21T09:48:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2545: writing smithi07685715-50 from 1918725 to 2151527 tid 2 2024-09-21T09:48:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2546: delete oid 12 current snap is 326 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:2531: finishing write tid 3 to smithi07685715-7 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:2531: finishing write tid 4 to smithi07685715-7 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:2531: oid 7 updating version 0 to 1519 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:2531: oid 7 updating version 1519 to 1520 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:2531: oid 7 version 1520 is already newer than 1518 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1520 (ObjNum 955 snap 326 seq_num 955) dirty exists 2024-09-21T09:48:59.790 INFO:tasks.rados.rados.0.smithi076.stdout:2531: left oid 7 (ObjNum 955 snap 326 seq_num 955) 2024-09-21T09:48:59.791 INFO:tasks.rados.rados.0.smithi076.stdout:2532: expect (ObjNum 936 snap 317 seq_num 936) 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: finishing write tid 1 to smithi07685715-34 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: finishing write tid 2 to smithi07685715-34 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: finishing write tid 3 to smithi07685715-34 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: finishing write tid 4 to smithi07685715-34 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: oid 34 updating version 0 to 564 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: oid 34 version 564 is already newer than 563 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: oid 34 updating version 564 to 565 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 565 (ObjNum 956 snap 326 seq_num 956) dirty exists 2024-09-21T09:48:59.999 INFO:tasks.rados.rados.0.smithi076.stdout:2535: left oid 34 (ObjNum 956 snap 326 seq_num 956) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: finishing write tid 1 to smithi07685715-20 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: finishing write tid 2 to smithi07685715-20 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: finishing write tid 3 to smithi07685715-20 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: finishing write tid 4 to smithi07685715-20 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: oid 20 updating version 0 to 764 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: oid 20 version 764 is already newer than 762 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: oid 20 version 764 is already newer than 763 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 764 (ObjNum 957 snap 326 seq_num 957) dirty exists 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: left oid 20 (ObjNum 957 snap 326 seq_num 957) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2531: done (11 left) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2532: done (10 left) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2535: done (9 left) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2536: done (8 left) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2546: done (7 left) 2024-09-21T09:49:00.000 INFO:tasks.rados.rados.0.smithi076.stdout:2547: read oid 11 snap 320 2024-09-21T09:49:00.001 INFO:tasks.rados.rados.0.smithi076.stdout:2547: expect deleted 2024-09-21T09:49:00.001 INFO:tasks.rados.rados.0.smithi076.stdout:2548: delete oid 10 current snap is 326 2024-09-21T09:49:00.001 INFO:tasks.rados.rados.0.smithi076.stdout:2537: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:49:00.027 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:48:59 smithi076 ceph-mon[106482]: pgmap v556: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s; 102/519 objects degraded (19.653%) 2024-09-21T09:49:00.027 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:48:59 smithi076 ceph-mon[110963]: pgmap v556: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s; 102/519 objects degraded (19.653%) 2024-09-21T09:49:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:48:59 smithi191 ceph-mon[102206]: pgmap v556: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 15 op/s; 102/519 objects degraded (19.653%) 2024-09-21T09:49:00.153 INFO:tasks.rados.rados.0.smithi076.stdout:2539: finishing write tid 1 to smithi07685715-16 2024-09-21T09:49:00.153 INFO:tasks.rados.rados.0.smithi076.stdout:2539: finishing write tid 2 to smithi07685715-16 2024-09-21T09:49:00.154 INFO:tasks.rados.rados.0.smithi076.stdout:2537: done (8 left) 2024-09-21T09:49:00.154 INFO:tasks.rados.rados.0.smithi076.stdout:2548: done (7 left) 2024-09-21T09:49:00.154 INFO:tasks.rados.rados.0.smithi076.stdout:2549: read oid 10 snap 285 2024-09-21T09:49:00.154 INFO:tasks.rados.rados.0.smithi076.stdout:2549: expect deleted 2024-09-21T09:49:00.154 INFO:tasks.rados.rados.0.smithi076.stdout:2550: snap_remove snap 324 2024-09-21T09:49:00.771 INFO:tasks.rados.rados.0.smithi076.stdout:2550: done (8 left) 2024-09-21T09:49:00.771 INFO:tasks.rados.rados.0.smithi076.stdout:2551: rollback oid 47 current snap is 326 2024-09-21T09:49:00.771 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 307 2024-09-21T09:49:00.771 INFO:tasks.rados.rados.0.smithi076.stdout:2539: finishing write tid 3 to smithi07685715-16 2024-09-21T09:49:00.771 INFO:tasks.rados.rados.0.smithi076.stdout:2539: finishing write tid 4 to smithi07685715-16 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2539: oid 16 updating version 0 to 1140 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2539: oid 16 updating version 1140 to 1141 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2539: oid 16 updating version 1141 to 1142 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1142 (ObjNum 958 snap 326 seq_num 958) dirty exists 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2539: left oid 16 (ObjNum 958 snap 326 seq_num 958) 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: finishing write tid 1 to smithi07685715-30 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: finishing write tid 2 to smithi07685715-30 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: finishing write tid 3 to smithi07685715-30 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: finishing write tid 4 to smithi07685715-30 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: oid 30 updating version 0 to 1977 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: oid 30 version 1977 is already newer than 1976 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: oid 30 version 1977 is already newer than 1975 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 1977 (ObjNum 959 snap 326 seq_num 959) dirty exists 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2540: left oid 30 (ObjNum 959 snap 326 seq_num 959) 2024-09-21T09:49:00.772 INFO:tasks.rados.rados.0.smithi076.stdout:2543: expect (ObjNum 810 snap 282 seq_num 810) 2024-09-21T09:49:00.888 INFO:tasks.rados.rados.0.smithi076.stdout:2544: finishing write tid 1 to smithi07685715-43 2024-09-21T09:49:00.888 INFO:tasks.rados.rados.0.smithi076.stdout:2544: finishing write tid 2 to smithi07685715-43 2024-09-21T09:49:00.888 INFO:tasks.rados.rados.0.smithi076.stdout:2544: finishing write tid 3 to smithi07685715-43 2024-09-21T09:49:00.888 INFO:tasks.rados.rados.0.smithi076.stdout:2539: done (8 left) 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2540: done (7 left) 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2543: done (6 left) 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2552: write oid 4 current snap is 326 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2544: finishing write tid 4 to smithi07685715-43 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2544: oid 43 updating version 0 to 569 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2544: oid 43 version 569 is already newer than 567 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2544: oid 43 version 569 is already newer than 568 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 569 (ObjNum 960 snap 326 seq_num 960) dirty exists 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2544: left oid 43 (ObjNum 960 snap 326 seq_num 960) 2024-09-21T09:49:00.889 INFO:tasks.rados.rados.0.smithi076.stdout:2542: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:49:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:00 smithi191 ceph-mon[102206]: osdmap e725: 8 total, 7 up, 8 in 2024-09-21T09:49:01.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:00 smithi076 ceph-mon[106482]: osdmap e725: 8 total, 7 up, 8 in 2024-09-21T09:49:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:00 smithi076 ceph-mon[110963]: osdmap e725: 8 total, 7 up, 8 in 2024-09-21T09:49:01.146 INFO:tasks.rados.rados.0.smithi076.stdout:2545: finishing write tid 1 to smithi07685715-50 2024-09-21T09:49:01.146 INFO:tasks.rados.rados.0.smithi076.stdout:2545: finishing write tid 2 to smithi07685715-50 2024-09-21T09:49:01.146 INFO:tasks.rados.rados.0.smithi076.stdout:2552: seq_num 964 ranges {601948=400514,1577818=624130} 2024-09-21T09:49:01.151 INFO:tasks.rados.rados.0.smithi076.stdout:2552: writing smithi07685715-4 from 601948 to 1002462 tid 1 2024-09-21T09:49:01.158 INFO:tasks.rados.rados.0.smithi076.stdout:2552: writing smithi07685715-4 from 1577818 to 2201948 tid 2 2024-09-21T09:49:01.158 INFO:tasks.rados.rados.0.smithi076.stdout:2542: done (6 left) 2024-09-21T09:49:01.158 INFO:tasks.rados.rados.0.smithi076.stdout:2544: done (5 left) 2024-09-21T09:49:01.158 INFO:tasks.rados.rados.0.smithi076.stdout:2553: delete oid 33 current snap is 326 2024-09-21T09:49:01.158 INFO:tasks.rados.rados.0.smithi076.stdout:2545: finishing write tid 3 to smithi07685715-50 2024-09-21T09:49:01.158 INFO:tasks.rados.rados.0.smithi076.stdout:2545: finishing write tid 4 to smithi07685715-50 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:2545: oid 50 updating version 0 to 806 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:2545: oid 50 updating version 806 to 807 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:2545: oid 50 version 807 is already newer than 803 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 807 (ObjNum 961 snap 326 seq_num 961) dirty exists 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:2545: left oid 50 (ObjNum 961 snap 326 seq_num 961) 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:2551: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:01.159 INFO:tasks.rados.rados.0.smithi076.stdout:2552: finishing write tid 1 to smithi07685715-4 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2545: done (5 left) 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2547: done (4 left) 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2549: done (3 left) 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2551: done (2 left) 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2553: done (1 left) 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2554: read oid 37 snap 285 2024-09-21T09:49:01.160 INFO:tasks.rados.rados.0.smithi076.stdout:2554: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:49:01.161 INFO:tasks.rados.rados.0.smithi076.stdout:2555: write oid 44 current snap is 326 2024-09-21T09:49:01.161 INFO:tasks.rados.rados.0.smithi076.stdout:2555: seq_num 966 ranges {635562=667365,2094782=140780} 2024-09-21T09:49:01.167 INFO:tasks.rados.rados.0.smithi076.stdout:2555: writing smithi07685715-44 from 635562 to 1302927 tid 1 2024-09-21T09:49:01.172 INFO:tasks.rados.rados.0.smithi076.stdout:2555: writing smithi07685715-44 from 2094782 to 2235562 tid 2 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2556: write oid 7 current snap is 326 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: finishing write tid 2 to smithi07685715-4 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: finishing write tid 3 to smithi07685715-4 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: finishing write tid 4 to smithi07685715-4 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: oid 4 updating version 0 to 810 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: oid 4 version 810 is already newer than 809 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: oid 4 version 810 is already newer than 808 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 810 (ObjNum 964 snap 326 seq_num 964) dirty exists 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2552: left oid 4 (ObjNum 964 snap 326 seq_num 964) 2024-09-21T09:49:01.173 INFO:tasks.rados.rados.0.smithi076.stdout:2556: seq_num 967 ranges {652369=742614,2153264=99105} 2024-09-21T09:49:01.179 INFO:tasks.rados.rados.0.smithi076.stdout:2556: writing smithi07685715-7 from 652369 to 1394983 tid 1 2024-09-21T09:49:01.184 INFO:tasks.rados.rados.0.smithi076.stdout:2556: writing smithi07685715-7 from 2153264 to 2252369 tid 2 2024-09-21T09:49:01.184 INFO:tasks.rados.rados.0.smithi076.stdout:2552: done (3 left) 2024-09-21T09:49:01.184 INFO:tasks.rados.rados.0.smithi076.stdout:2557: delete oid 3 current snap is 326 2024-09-21T09:49:01.184 INFO:tasks.rados.rados.0.smithi076.stdout:2554: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:49:01.443 INFO:tasks.rados.rados.0.smithi076.stdout:2555: finishing write tid 1 to smithi07685715-44 2024-09-21T09:49:01.443 INFO:tasks.rados.rados.0.smithi076.stdout:2555: finishing write tid 2 to smithi07685715-44 2024-09-21T09:49:01.443 INFO:tasks.rados.rados.0.smithi076.stdout:2555: finishing write tid 3 to smithi07685715-44 2024-09-21T09:49:01.443 INFO:tasks.rados.rados.0.smithi076.stdout:2555: finishing write tid 4 to smithi07685715-44 2024-09-21T09:49:01.443 INFO:tasks.rados.rados.0.smithi076.stdout:2555: oid 44 updating version 0 to 621 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2555: oid 44 version 621 is already newer than 619 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2555: oid 44 version 621 is already newer than 620 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 621 (ObjNum 966 snap 326 seq_num 966) dirty exists 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2555: left oid 44 (ObjNum 966 snap 326 seq_num 966) 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2556: finishing write tid 1 to smithi07685715-7 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2556: finishing write tid 2 to smithi07685715-7 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2556: finishing write tid 3 to smithi07685715-7 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2554: done (3 left) 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2555: done (2 left) 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2557: done (1 left) 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2558: snap_create 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2556: finishing write tid 4 to smithi07685715-7 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2556: oid 7 updating version 0 to 1523 2024-09-21T09:49:01.444 INFO:tasks.rados.rados.0.smithi076.stdout:2556: oid 7 updating version 1523 to 1524 2024-09-21T09:49:01.445 INFO:tasks.rados.rados.0.smithi076.stdout:2556: oid 7 updating version 1524 to 1525 2024-09-21T09:49:01.445 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1525 (ObjNum 967 snap 326 seq_num 967) dirty exists 2024-09-21T09:49:01.445 INFO:tasks.rados.rados.0.smithi076.stdout:2556: left oid 7 (ObjNum 967 snap 326 seq_num 967) 2024-09-21T09:49:01.773 INFO:tasks.rados.rados.0.smithi076.stdout:2556: done (1 left) 2024-09-21T09:49:01.773 INFO:tasks.rados.rados.0.smithi076.stdout:2558: done (0 left) 2024-09-21T09:49:01.773 INFO:tasks.rados.rados.0.smithi076.stdout:2559: write oid 4 current snap is 327 2024-09-21T09:49:01.773 INFO:tasks.rados.rados.0.smithi076.stdout:2559: seq_num 969 ranges {685983=493112,1870228=415755} 2024-09-21T09:49:01.779 INFO:tasks.rados.rados.0.smithi076.stdout:2559: writing smithi07685715-4 from 685983 to 1179095 tid 1 2024-09-21T09:49:01.786 INFO:tasks.rados.rados.0.smithi076.stdout:2559: writing smithi07685715-4 from 1870228 to 2285983 tid 2 2024-09-21T09:49:01.786 INFO:tasks.rados.rados.0.smithi076.stdout:2560: snap_remove snap 325 2024-09-21T09:49:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:01 smithi191 ceph-mon[102206]: pgmap v559: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 102/519 objects degraded (19.653%) 2024-09-21T09:49:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:01 smithi191 ceph-mon[102206]: osdmap e726: 8 total, 7 up, 8 in 2024-09-21T09:49:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:01 smithi076 ceph-mon[106482]: pgmap v559: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 102/519 objects degraded (19.653%) 2024-09-21T09:49:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:01 smithi076 ceph-mon[106482]: osdmap e726: 8 total, 7 up, 8 in 2024-09-21T09:49:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:01 smithi076 ceph-mon[110963]: pgmap v559: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 271 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 102/519 objects degraded (19.653%) 2024-09-21T09:49:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:01 smithi076 ceph-mon[110963]: osdmap e726: 8 total, 7 up, 8 in 2024-09-21T09:49:02.777 INFO:tasks.rados.rados.0.smithi076.stdout:2560: done (1 left) 2024-09-21T09:49:02.778 INFO:tasks.rados.rados.0.smithi076.stdout:2561: snap_remove snap 316 2024-09-21T09:49:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[106482]: osdmap e727: 8 total, 7 up, 8 in 2024-09-21T09:49:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.024 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[110963]: osdmap e727: 8 total, 7 up, 8 in 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.025 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:02 smithi191 ceph-mon[102206]: osdmap e727: 8 total, 7 up, 8 in 2024-09-21T09:49:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:03.374 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:49:03] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.51.0" 2024-09-21T09:49:03.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[122652]: 2024-09-21T09:49:03.174+0000 7f199b75b640 -1 osd.0 717 log_to_monitors true 2024-09-21T09:49:03.783 INFO:tasks.rados.rados.0.smithi076.stdout:2561: done (1 left) 2024-09-21T09:49:03.783 INFO:tasks.rados.rados.0.smithi076.stdout:2562: snap_remove snap 307 2024-09-21T09:49:04.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[106482]: pgmap v562: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 282 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 20 op/s; 108/543 objects degraded (19.890%) 2024-09-21T09:49:04.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[106482]: osdmap e728: 8 total, 7 up, 8 in 2024-09-21T09:49:04.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 108/543 objects degraded (19.890%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:04.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[106482]: from='osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:49:04.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[106482]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:49:04.092 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[110963]: pgmap v562: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 282 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 20 op/s; 108/543 objects degraded (19.890%) 2024-09-21T09:49:04.092 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[110963]: osdmap e728: 8 total, 7 up, 8 in 2024-09-21T09:49:04.092 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 108/543 objects degraded (19.890%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:04.092 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[110963]: from='osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:49:04.092 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-mon[110963]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:49:04.092 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 09:49:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[122652]: 2024-09-21T09:49:03.808+0000 7f198f4fd640 -1 osd.0 717 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:49:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:03 smithi191 ceph-mon[102206]: pgmap v562: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 282 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 20 op/s; 108/543 objects degraded (19.890%) 2024-09-21T09:49:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:03 smithi191 ceph-mon[102206]: osdmap e728: 8 total, 7 up, 8 in 2024-09-21T09:49:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:03 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 108/543 objects degraded (19.890%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:03 smithi191 ceph-mon[102206]: from='osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:49:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:03 smithi191 ceph-mon[102206]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:49:04.803 INFO:tasks.rados.rados.0.smithi076.stdout:2562: done (1 left) 2024-09-21T09:49:04.803 INFO:tasks.rados.rados.0.smithi076.stdout:2563: write oid 45 current snap is 327 2024-09-21T09:49:04.804 INFO:tasks.rados.rados.0.smithi076.stdout:2563: seq_num 970 ranges {702790=568361,1812357=490433} 2024-09-21T09:49:04.806 INFO:tasks.rados.rados.0.smithi076.stdout:2563: writing smithi07685715-45 from 702790 to 1271151 tid 1 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2563: writing smithi07685715-45 from 1812357 to 2302790 tid 2 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2564: snap_create 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: finishing write tid 1 to smithi07685715-4 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: finishing write tid 2 to smithi07685715-4 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: finishing write tid 3 to smithi07685715-4 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: finishing write tid 4 to smithi07685715-4 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: oid 4 updating version 0 to 814 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: oid 4 version 814 is already newer than 813 2024-09-21T09:49:04.813 INFO:tasks.rados.rados.0.smithi076.stdout:2559: oid 4 version 814 is already newer than 812 2024-09-21T09:49:04.814 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 814 (ObjNum 969 snap 327 seq_num 969) dirty exists 2024-09-21T09:49:04.814 INFO:tasks.rados.rados.0.smithi076.stdout:2559: left oid 4 (ObjNum 969 snap 327 seq_num 969) 2024-09-21T09:49:04.814 INFO:tasks.rados.rados.0.smithi076.stdout:2563: finishing write tid 1 to smithi07685715-45 2024-09-21T09:49:04.816 INFO:tasks.rados.rados.0.smithi076.stdout:2563: finishing write tid 2 to smithi07685715-45 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:2563: finishing write tid 3 to smithi07685715-45 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:2563: finishing write tid 4 to smithi07685715-45 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:2563: oid 45 updating version 0 to 568 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:2563: oid 45 version 568 is already newer than 567 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:2563: oid 45 updating version 568 to 569 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 569 (ObjNum 970 snap 327 seq_num 970) dirty exists 2024-09-21T09:49:04.817 INFO:tasks.rados.rados.0.smithi076.stdout:2563: left oid 45 (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:49:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:04 smithi191 ceph-mon[102206]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:49:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:04 smithi191 ceph-mon[102206]: osdmap e729: 8 total, 7 up, 8 in 2024-09-21T09:49:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:04 smithi191 ceph-mon[102206]: from='osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:04 smithi191 ceph-mon[102206]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[106482]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[106482]: osdmap e729: 8 total, 7 up, 8 in 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[106482]: from='osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[106482]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[110963]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[110963]: osdmap e729: 8 total, 7 up, 8 in 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[110963]: from='osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:04 smithi076 ceph-mon[110963]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:05.793 INFO:tasks.rados.rados.0.smithi076.stdout:2559: done (2 left) 2024-09-21T09:49:05.793 INFO:tasks.rados.rados.0.smithi076.stdout:2563: done (1 left) 2024-09-21T09:49:05.793 INFO:tasks.rados.rados.0.smithi076.stdout:2564: done (0 left) 2024-09-21T09:49:05.793 INFO:tasks.rados.rados.0.smithi076.stdout:2565: delete oid 44 current snap is 328 2024-09-21T09:49:05.797 INFO:tasks.rados.rados.0.smithi076.stdout:2565: done (0 left) 2024-09-21T09:49:05.797 INFO:tasks.rados.rados.0.smithi076.stdout:2566: write oid 25 current snap is 328 2024-09-21T09:49:05.797 INFO:tasks.rados.rados.0.smithi076.stdout:2566: seq_num 972 ranges {736404=718859,1929321=407083} 2024-09-21T09:49:05.807 INFO:tasks.rados.rados.0.smithi076.stdout:2566: writing smithi07685715-25 from 736404 to 1455263 tid 1 2024-09-21T09:49:05.812 INFO:tasks.rados.rados.0.smithi076.stdout:2566: writing smithi07685715-25 from 1929321 to 2336404 tid 2 2024-09-21T09:49:05.813 INFO:tasks.rados.rados.0.smithi076.stdout:2567: read oid 49 snap -1 2024-09-21T09:49:05.813 INFO:tasks.rados.rados.0.smithi076.stdout:2567: expect (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:49:05.813 INFO:tasks.rados.rados.0.smithi076.stdout:2568: snap_create 2024-09-21T09:49:05.819 INFO:tasks.rados.rados.0.smithi076.stdout:2566: finishing write tid 1 to smithi07685715-25 2024-09-21T09:49:05.820 INFO:tasks.rados.rados.0.smithi076.stdout:2566: finishing write tid 2 to smithi07685715-25 2024-09-21T09:49:05.820 INFO:tasks.rados.rados.0.smithi076.stdout:2566: finishing write tid 3 to smithi07685715-25 2024-09-21T09:49:05.820 INFO:tasks.rados.rados.0.smithi076.stdout:2566: finishing write tid 4 to smithi07685715-25 2024-09-21T09:49:05.821 INFO:tasks.rados.rados.0.smithi076.stdout:2566: oid 25 updating version 0 to 616 2024-09-21T09:49:05.821 INFO:tasks.rados.rados.0.smithi076.stdout:2566: oid 25 updating version 616 to 618 2024-09-21T09:49:05.821 INFO:tasks.rados.rados.0.smithi076.stdout:2566: oid 25 version 618 is already newer than 617 2024-09-21T09:49:05.821 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 618 (ObjNum 972 snap 328 seq_num 972) dirty exists 2024-09-21T09:49:05.821 INFO:tasks.rados.rados.0.smithi076.stdout:2566: left oid 25 (ObjNum 972 snap 328 seq_num 972) 2024-09-21T09:49:05.832 INFO:tasks.rados.rados.0.smithi076.stdout:2567: expect (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:49:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: pgmap v565: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 282 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 20 op/s; 108/543 objects degraded (19.890%) 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944] boot 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:49:05.872+0000 7f86bc22c640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: pgmap v565: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 282 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 20 op/s; 108/543 objects degraded (19.890%) 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944] boot 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: pgmap v565: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 282 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 20 op/s; 108/543 objects degraded (19.890%) 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: osd.0 [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944] boot 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:06.802 INFO:tasks.rados.rados.0.smithi076.stdout:2566: done (2 left) 2024-09-21T09:49:06.803 INFO:tasks.rados.rados.0.smithi076.stdout:2567: done (1 left) 2024-09-21T09:49:06.803 INFO:tasks.rados.rados.0.smithi076.stdout:2568: done (0 left) 2024-09-21T09:49:06.803 INFO:tasks.rados.rados.0.smithi076.stdout:2569: snap_create 2024-09-21T09:49:07.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:49:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:06 smithi191 ceph-mon[102206]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[110963]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:07.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:49:07.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:49:07.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:06 smithi076 ceph-mon[106482]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:49:07.805 INFO:tasks.rados.rados.0.smithi076.stdout:2569: done (0 left) 2024-09-21T09:49:07.805 INFO:tasks.rados.rados.0.smithi076.stdout:2570: snap_create 2024-09-21T09:49:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:07 smithi191 ceph-mon[102206]: pgmap v568: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 277 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 756 KiB/s wr, 6 op/s; 27/528 objects degraded (5.114%); 2.3 MiB/s, 2 objects/s recovering 2024-09-21T09:49:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:07 smithi191 ceph-mon[102206]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:49:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:07 smithi076 ceph-mon[106482]: pgmap v568: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 277 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 756 KiB/s wr, 6 op/s; 27/528 objects degraded (5.114%); 2.3 MiB/s, 2 objects/s recovering 2024-09-21T09:49:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:07 smithi076 ceph-mon[106482]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:49:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:07 smithi076 ceph-mon[110963]: pgmap v568: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 277 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 756 KiB/s wr, 6 op/s; 27/528 objects degraded (5.114%); 2.3 MiB/s, 2 objects/s recovering 2024-09-21T09:49:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:07 smithi076 ceph-mon[110963]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T09:49:08.807 INFO:tasks.rados.rados.0.smithi076.stdout:2570: done (0 left) 2024-09-21T09:49:08.808 INFO:tasks.rados.rados.0.smithi076.stdout:2571: read oid 32 snap -1 2024-09-21T09:49:08.808 INFO:tasks.rados.rados.0.smithi076.stdout:2571: expect deleted 2024-09-21T09:49:08.808 INFO:tasks.rados.rados.0.smithi076.stdout:2572: write oid 41 current snap is 331 2024-09-21T09:49:08.808 INFO:tasks.rados.rados.0.smithi076.stdout:2572: seq_num 973 ranges {753211=794108,1987803=365408} 2024-09-21T09:49:08.816 INFO:tasks.rados.rados.0.smithi076.stdout:2572: writing smithi07685715-41 from 753211 to 1547319 tid 1 2024-09-21T09:49:08.821 INFO:tasks.rados.rados.0.smithi076.stdout:2572: writing smithi07685715-41 from 1987803 to 2353211 tid 2 2024-09-21T09:49:08.821 INFO:tasks.rados.rados.0.smithi076.stdout:2573: snap_create 2024-09-21T09:49:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:2572: finishing write tid 1 to smithi07685715-41 2024-09-21T09:49:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:2572: finishing write tid 2 to smithi07685715-41 2024-09-21T09:49:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:2572: finishing write tid 3 to smithi07685715-41 2024-09-21T09:49:08.826 INFO:tasks.rados.rados.0.smithi076.stdout:2572: finishing write tid 4 to smithi07685715-41 2024-09-21T09:49:08.826 INFO:tasks.rados.rados.0.smithi076.stdout:2572: oid 41 updating version 0 to 1156 2024-09-21T09:49:08.826 INFO:tasks.rados.rados.0.smithi076.stdout:2572: oid 41 version 1156 is already newer than 1155 2024-09-21T09:49:08.826 INFO:tasks.rados.rados.0.smithi076.stdout:2572: oid 41 version 1156 is already newer than 1154 2024-09-21T09:49:08.826 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1156 (ObjNum 973 snap 331 seq_num 973) dirty exists 2024-09-21T09:49:08.826 INFO:tasks.rados.rados.0.smithi076.stdout:2572: left oid 41 (ObjNum 973 snap 331 seq_num 973) 2024-09-21T09:49:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:08 smithi191 ceph-mon[102206]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:49:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:08 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 27/528 objects degraded (5.114%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:08 smithi076 ceph-mon[110963]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:49:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:08 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 27/528 objects degraded (5.114%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:08 smithi076 ceph-mon[106482]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T09:49:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:08 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 27/528 objects degraded (5.114%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:09.825 INFO:tasks.rados.rados.0.smithi076.stdout:2571: done (2 left) 2024-09-21T09:49:09.826 INFO:tasks.rados.rados.0.smithi076.stdout:2572: done (1 left) 2024-09-21T09:49:09.826 INFO:tasks.rados.rados.0.smithi076.stdout:2573: done (0 left) 2024-09-21T09:49:09.826 INFO:tasks.rados.rados.0.smithi076.stdout:2574: write oid 16 current snap is 332 2024-09-21T09:49:09.826 INFO:tasks.rados.rados.0.smithi076.stdout:2574: seq_num 974 ranges {770018=585710,2046285=323733} 2024-09-21T09:49:09.832 INFO:tasks.rados.rados.0.smithi076.stdout:2574: writing smithi07685715-16 from 770018 to 1355728 tid 1 2024-09-21T09:49:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:2574: writing smithi07685715-16 from 2046285 to 2370018 tid 2 2024-09-21T09:49:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:2575: read oid 5 snap -1 2024-09-21T09:49:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:2575: expect (ObjNum 948 snap 324 seq_num 948) 2024-09-21T09:49:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:2576: snap_remove snap 275 2024-09-21T09:49:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:09 smithi191 ceph-mon[102206]: pgmap v571: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 277 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 756 KiB/s wr, 6 op/s; 27/528 objects degraded (5.114%); 2.3 MiB/s, 2 objects/s recovering 2024-09-21T09:49:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:09 smithi191 ceph-mon[102206]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:49:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:09 smithi076 ceph-mon[106482]: pgmap v571: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 277 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 756 KiB/s wr, 6 op/s; 27/528 objects degraded (5.114%); 2.3 MiB/s, 2 objects/s recovering 2024-09-21T09:49:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:09 smithi076 ceph-mon[106482]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:49:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:09 smithi076 ceph-mon[110963]: pgmap v571: 89 pgs: 9 active+undersized, 4 active+undersized+degraded, 76 active+clean; 277 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 756 KiB/s wr, 6 op/s; 27/528 objects degraded (5.114%); 2.3 MiB/s, 2 objects/s recovering 2024-09-21T09:49:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:09 smithi076 ceph-mon[110963]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T09:49:10.842 INFO:tasks.rados.rados.0.smithi076.stdout:2576: done (2 left) 2024-09-21T09:49:10.842 INFO:tasks.rados.rados.0.smithi076.stdout:2577: write oid 34 current snap is 332 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: finishing write tid 1 to smithi07685715-16 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: finishing write tid 2 to smithi07685715-16 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: finishing write tid 3 to smithi07685715-16 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: finishing write tid 4 to smithi07685715-16 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: oid 16 updating version 0 to 1160 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: oid 16 version 1160 is already newer than 1159 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: oid 16 version 1160 is already newer than 1158 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1160 (ObjNum 974 snap 332 seq_num 974) dirty exists 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2574: left oid 16 (ObjNum 974 snap 332 seq_num 974) 2024-09-21T09:49:10.843 INFO:tasks.rados.rados.0.smithi076.stdout:2575: expect (ObjNum 948 snap 324 seq_num 948) 2024-09-21T09:49:10.968 INFO:tasks.rados.rados.0.smithi076.stdout:2577: seq_num 975 ranges {786825=660959,2104767=282058} 2024-09-21T09:49:10.975 INFO:tasks.rados.rados.0.smithi076.stdout:2577: writing smithi07685715-34 from 786825 to 1447784 tid 1 2024-09-21T09:49:10.980 INFO:tasks.rados.rados.0.smithi076.stdout:2577: writing smithi07685715-34 from 2104767 to 2386825 tid 2 2024-09-21T09:49:10.980 INFO:tasks.rados.rados.0.smithi076.stdout:2574: done (2 left) 2024-09-21T09:49:10.980 INFO:tasks.rados.rados.0.smithi076.stdout:2575: done (1 left) 2024-09-21T09:49:10.980 INFO:tasks.rados.rados.0.smithi076.stdout:2578: snap_create 2024-09-21T09:49:10.980 INFO:tasks.rados.rados.0.smithi076.stdout:2577: finishing write tid 1 to smithi07685715-34 2024-09-21T09:49:10.982 INFO:tasks.rados.rados.0.smithi076.stdout:2577: finishing write tid 2 to smithi07685715-34 2024-09-21T09:49:10.982 INFO:tasks.rados.rados.0.smithi076.stdout:2577: finishing write tid 3 to smithi07685715-34 2024-09-21T09:49:10.983 INFO:tasks.rados.rados.0.smithi076.stdout:2577: finishing write tid 4 to smithi07685715-34 2024-09-21T09:49:10.983 INFO:tasks.rados.rados.0.smithi076.stdout:2577: oid 34 updating version 0 to 582 2024-09-21T09:49:10.983 INFO:tasks.rados.rados.0.smithi076.stdout:2577: oid 34 updating version 582 to 583 2024-09-21T09:49:10.983 INFO:tasks.rados.rados.0.smithi076.stdout:2577: oid 34 version 583 is already newer than 581 2024-09-21T09:49:10.983 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 583 (ObjNum 975 snap 332 seq_num 975) dirty exists 2024-09-21T09:49:10.983 INFO:tasks.rados.rados.0.smithi076.stdout:2577: left oid 34 (ObjNum 975 snap 332 seq_num 975) 2024-09-21T09:49:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:10 smithi191 ceph-mon[102206]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:49:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:10 smithi076 ceph-mon[106482]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:49:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:10 smithi076 ceph-mon[110963]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T09:49:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:11.842 INFO:tasks.rados.rados.0.smithi076.stdout:2577: done (1 left) 2024-09-21T09:49:11.843 INFO:tasks.rados.rados.0.smithi076.stdout:2578: done (0 left) 2024-09-21T09:49:11.843 INFO:tasks.rados.rados.0.smithi076.stdout:2579: write oid 33 current snap is 333 2024-09-21T09:49:11.843 INFO:tasks.rados.rados.0.smithi076.stdout:2579: seq_num 976 ranges {403632=736208,1763249=619999} 2024-09-21T09:49:11.848 INFO:tasks.rados.rados.0.smithi076.stdout:2579: writing smithi07685715-33 from 403632 to 1139840 tid 1 2024-09-21T09:49:11.855 INFO:tasks.rados.rados.0.smithi076.stdout:2579: writing smithi07685715-33 from 1763249 to 2383248 tid 2 2024-09-21T09:49:11.856 INFO:tasks.rados.rados.0.smithi076.stdout:2580: delete oid 20 current snap is 333 2024-09-21T09:49:11.856 INFO:tasks.rados.rados.0.smithi076.stdout:2579: finishing write tid 1 to smithi07685715-33 2024-09-21T09:49:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:2579: finishing write tid 2 to smithi07685715-33 2024-09-21T09:49:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:2580: done (1 left) 2024-09-21T09:49:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:2581: snap_create 2024-09-21T09:49:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:2579: finishing write tid 3 to smithi07685715-33 2024-09-21T09:49:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:2579: finishing write tid 4 to smithi07685715-33 2024-09-21T09:49:11.860 INFO:tasks.rados.rados.0.smithi076.stdout:2579: oid 33 updating version 0 to 623 2024-09-21T09:49:11.860 INFO:tasks.rados.rados.0.smithi076.stdout:2579: oid 33 updating version 623 to 625 2024-09-21T09:49:11.860 INFO:tasks.rados.rados.0.smithi076.stdout:2579: oid 33 version 625 is already newer than 624 2024-09-21T09:49:11.860 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 625 (ObjNum 976 snap 333 seq_num 976) dirty exists 2024-09-21T09:49:11.860 INFO:tasks.rados.rados.0.smithi076.stdout:2579: left oid 33 (ObjNum 976 snap 333 seq_num 976) 2024-09-21T09:49:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:11 smithi191 ceph-mon[102206]: pgmap v574: 89 pgs: 89 active+clean; 274 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 804 KiB/s, 0 objects/s recovering 2024-09-21T09:49:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:11 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/528 objects degraded (5.114%), 4 pgs degraded) 2024-09-21T09:49:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:11 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T09:49:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:11 smithi191 ceph-mon[102206]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[106482]: pgmap v574: 89 pgs: 89 active+clean; 274 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 804 KiB/s, 0 objects/s recovering 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/528 objects degraded (5.114%), 4 pgs degraded) 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[106482]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[110963]: pgmap v574: 89 pgs: 89 active+clean; 274 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 804 KiB/s, 0 objects/s recovering 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/528 objects degraded (5.114%), 4 pgs degraded) 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T09:49:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:11 smithi076 ceph-mon[110963]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2579: done (1 left) 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2581: done (0 left) 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2582: rollback oid 15 current snap is 334 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 333 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2583: read oid 28 snap -1 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2583: expect deleted 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2584: read oid 11 snap -1 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2584: expect deleted 2024-09-21T09:49:12.851 INFO:tasks.rados.rados.0.smithi076.stdout:2585: snap_create 2024-09-21T09:49:12.855 INFO:tasks.rados.rados.0.smithi076.stdout:2582: finishing rollback tid 0 to smithi07685715-15 2024-09-21T09:49:12.855 INFO:tasks.rados.rados.0.smithi076.stdout:2582: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:49:12.855 INFO:tasks.rados.rados.0.smithi076.stdout:2582: finishing rollback tid 2 to smithi07685715-15 2024-09-21T09:49:12.855 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 575 (ObjNum 853 snap 292 seq_num 853) dirty exists 2024-09-21T09:49:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:12 smithi191 ceph-mon[102206]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:49:13.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:12 smithi076 ceph-mon[106482]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:49:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:12 smithi076 ceph-mon[110963]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T09:49:13.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:49:13] "GET /metrics HTTP/1.1" 200 36274 "" "Prometheus/2.51.0" 2024-09-21T09:49:13.738 INFO:tasks.rados.rados.0.smithi076.stdout:2582: done (3 left) 2024-09-21T09:49:13.739 INFO:tasks.rados.rados.0.smithi076.stdout:2583: done (2 left) 2024-09-21T09:49:13.739 INFO:tasks.rados.rados.0.smithi076.stdout:2584: done (1 left) 2024-09-21T09:49:13.739 INFO:tasks.rados.rados.0.smithi076.stdout:2585: done (0 left) 2024-09-21T09:49:13.739 INFO:tasks.rados.rados.0.smithi076.stdout:2586: snap_create 2024-09-21T09:49:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:13 smithi191 ceph-mon[102206]: pgmap v577: 89 pgs: 89 active+clean; 256 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 736 KiB/s wr, 5 op/s; 804 KiB/s, 0 objects/s recovering 2024-09-21T09:49:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:13 smithi191 ceph-mon[102206]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:49:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:13 smithi191 ceph-mon[102206]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:49:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-mon[106482]: pgmap v577: 89 pgs: 89 active+clean; 256 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 736 KiB/s wr, 5 op/s; 804 KiB/s, 0 objects/s recovering 2024-09-21T09:49:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-mon[106482]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:49:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-mon[106482]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:49:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-mon[110963]: pgmap v577: 89 pgs: 89 active+clean; 256 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 736 KiB/s wr, 5 op/s; 804 KiB/s, 0 objects/s recovering 2024-09-21T09:49:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-mon[110963]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T09:49:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:13 smithi076 ceph-mon[110963]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T09:49:14.738 INFO:tasks.rados.rados.0.smithi076.stdout:2586: done (0 left) 2024-09-21T09:49:14.738 INFO:tasks.rados.rados.0.smithi076.stdout:2587: read oid 7 snap -1 2024-09-21T09:49:14.739 INFO:tasks.rados.rados.0.smithi076.stdout:2587: expect (ObjNum 967 snap 326 seq_num 967) 2024-09-21T09:49:14.739 INFO:tasks.rados.rados.0.smithi076.stdout:2588: read oid 5 snap -1 2024-09-21T09:49:14.739 INFO:tasks.rados.rados.0.smithi076.stdout:2588: expect (ObjNum 948 snap 324 seq_num 948) 2024-09-21T09:49:14.739 INFO:tasks.rados.rados.0.smithi076.stdout:2589: write oid 35 current snap is 336 2024-09-21T09:49:14.739 INFO:tasks.rados.rados.0.smithi076.stdout:2589: seq_num 978 ranges {437246=486706,1363860=623668} 2024-09-21T09:49:14.744 INFO:tasks.rados.rados.0.smithi076.stdout:2589: writing smithi07685715-35 from 437246 to 923952 tid 1 2024-09-21T09:49:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2589: writing smithi07685715-35 from 1363860 to 1987528 tid 2 2024-09-21T09:49:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2590: write oid 17 current snap is 336 2024-09-21T09:49:14.750 INFO:tasks.rados.rados.0.smithi076.stdout:2590: seq_num 979 ranges {454053=561955,1422342=483679,2368404=85649} 2024-09-21T09:49:14.756 INFO:tasks.rados.rados.0.smithi076.stdout:2590: writing smithi07685715-17 from 454053 to 1016008 tid 1 2024-09-21T09:49:14.761 INFO:tasks.rados.rados.0.smithi076.stdout:2590: writing smithi07685715-17 from 1422342 to 1906021 tid 2 2024-09-21T09:49:14.764 INFO:tasks.rados.rados.0.smithi076.stdout:2590: writing smithi07685715-17 from 2368404 to 2454053 tid 3 2024-09-21T09:49:14.764 INFO:tasks.rados.rados.0.smithi076.stdout:2591: delete oid 33 current snap is 336 2024-09-21T09:49:14.764 INFO:tasks.rados.rados.0.smithi076.stdout:2587: expect (ObjNum 967 snap 326 seq_num 967) 2024-09-21T09:49:14.864 INFO:tasks.rados.rados.0.smithi076.stdout:2588: expect (ObjNum 948 snap 324 seq_num 948) 2024-09-21T09:49:14.987 INFO:tasks.rados.rados.0.smithi076.stdout:2589: finishing write tid 1 to smithi07685715-35 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: finishing write tid 2 to smithi07685715-35 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: finishing write tid 3 to smithi07685715-35 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: finishing write tid 4 to smithi07685715-35 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: oid 35 updating version 0 to 1964 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: oid 35 updating version 1964 to 1965 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: oid 35 updating version 1965 to 1966 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 1966 (ObjNum 978 snap 336 seq_num 978) dirty exists 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2589: left oid 35 (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2590: finishing write tid 1 to smithi07685715-17 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2590: finishing write tid 2 to smithi07685715-17 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2590: finishing write tid 3 to smithi07685715-17 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2590: finishing write tid 4 to smithi07685715-17 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2587: done (4 left) 2024-09-21T09:49:14.988 INFO:tasks.rados.rados.0.smithi076.stdout:2588: done (3 left) 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2589: done (2 left) 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2591: done (1 left) 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2592: read oid 44 snap -1 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2592: expect deleted 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2593: read oid 24 snap -1 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2593: expect deleted 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2594: read oid 19 snap -1 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2594: expect deleted 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2595: read oid 47 snap 285 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2595: expect deleted 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2596: read oid 34 snap 326 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2596: expect (ObjNum 956 snap 326 seq_num 956) 2024-09-21T09:49:14.989 INFO:tasks.rados.rados.0.smithi076.stdout:2597: read oid 21 snap -1 2024-09-21T09:49:14.990 INFO:tasks.rados.rados.0.smithi076.stdout:2597: expect (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:49:14.990 INFO:tasks.rados.rados.0.smithi076.stdout:2598: snap_remove snap 330 2024-09-21T09:49:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2598: done (7 left) 2024-09-21T09:49:15.857 INFO:tasks.rados.rados.0.smithi076.stdout:2599: write oid 12 current snap is 336 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2590: finishing write tid 5 to smithi07685715-17 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2590: oid 17 updating version 0 to 1560 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2590: oid 17 updating version 1560 to 1562 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2590: oid 17 version 1562 is already newer than 1561 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2590: oid 17 version 1562 is already newer than 1559 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1562 (ObjNum 979 snap 336 seq_num 979) dirty exists 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2590: left oid 17 (ObjNum 979 snap 336 seq_num 979) 2024-09-21T09:49:15.858 INFO:tasks.rados.rados.0.smithi076.stdout:2599: seq_num 981 ranges {487667=428806,1655659=770995} 2024-09-21T09:49:15.862 INFO:tasks.rados.rados.0.smithi076.stdout:2599: writing smithi07685715-12 from 487667 to 916473 tid 1 2024-09-21T09:49:15.871 INFO:tasks.rados.rados.0.smithi076.stdout:2599: writing smithi07685715-12 from 1655659 to 2426654 tid 2 2024-09-21T09:49:15.871 INFO:tasks.rados.rados.0.smithi076.stdout:2590: done (7 left) 2024-09-21T09:49:15.871 INFO:tasks.rados.rados.0.smithi076.stdout:2600: rollback oid 5 current snap is 336 2024-09-21T09:49:15.871 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 328 2024-09-21T09:49:15.871 INFO:tasks.rados.rados.0.smithi076.stdout:2596: expect (ObjNum 956 snap 326 seq_num 956) 2024-09-21T09:49:15.987 INFO:tasks.rados.rados.0.smithi076.stdout:2597: expect (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:49:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:15 smithi191 ceph-mon[102206]: pgmap v580: 89 pgs: 89 active+clean; 252 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 736 KiB/s wr, 5 op/s 2024-09-21T09:49:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:15 smithi191 ceph-mon[102206]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:49:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:16.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:15 smithi076 ceph-mon[106482]: pgmap v580: 89 pgs: 89 active+clean; 252 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 736 KiB/s wr, 5 op/s 2024-09-21T09:49:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:15 smithi076 ceph-mon[106482]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:49:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:15 smithi076 ceph-mon[110963]: pgmap v580: 89 pgs: 89 active+clean; 252 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 736 KiB/s wr, 5 op/s 2024-09-21T09:49:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:15 smithi076 ceph-mon[110963]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T09:49:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2599: finishing write tid 1 to smithi07685715-12 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2599: finishing write tid 2 to smithi07685715-12 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2592: done (7 left) 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2593: done (6 left) 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2594: done (5 left) 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2595: done (4 left) 2024-09-21T09:49:16.258 INFO:tasks.rados.rados.0.smithi076.stdout:2596: done (3 left) 2024-09-21T09:49:16.259 INFO:tasks.rados.rados.0.smithi076.stdout:2597: done (2 left) 2024-09-21T09:49:16.259 INFO:tasks.rados.rados.0.smithi076.stdout:2601: write oid 6 current snap is 336 2024-09-21T09:49:16.259 INFO:tasks.rados.rados.0.smithi076.stdout:2599: finishing write tid 3 to smithi07685715-12 2024-09-21T09:49:16.259 INFO:tasks.rados.rados.0.smithi076.stdout:2601: seq_num 982 ranges {504474=504055,1714141=631006} 2024-09-21T09:49:16.263 INFO:tasks.rados.rados.0.smithi076.stdout:2601: writing smithi07685715-6 from 504474 to 1008529 tid 1 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2601: writing smithi07685715-6 from 1714141 to 2345147 tid 2 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2602: write oid 2 current snap is 336 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2600: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2599: finishing write tid 4 to smithi07685715-12 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2599: oid 12 updating version 0 to 892 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2599: oid 12 updating version 892 to 894 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2599: oid 12 version 894 is already newer than 893 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 894 (ObjNum 981 snap 336 seq_num 981) dirty exists 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2599: left oid 12 (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2600: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:2600: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:49:16.271 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 1972 (ObjNum 948 snap 324 seq_num 948) dirty exists 2024-09-21T09:49:16.272 INFO:tasks.rados.rados.0.smithi076.stdout:2601: finishing write tid 1 to smithi07685715-6 2024-09-21T09:49:16.272 INFO:tasks.rados.rados.0.smithi076.stdout:2602: seq_num 983 ranges {521281=579304,1656270=774664} 2024-09-21T09:49:16.276 INFO:tasks.rados.rados.0.smithi076.stdout:2602: writing smithi07685715-2 from 521281 to 1100585 tid 1 2024-09-21T09:49:16.284 INFO:tasks.rados.rados.0.smithi076.stdout:2602: writing smithi07685715-2 from 1656270 to 2430934 tid 2 2024-09-21T09:49:16.284 INFO:tasks.rados.rados.0.smithi076.stdout:2599: done (3 left) 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2600: done (2 left) 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2603: read oid 49 snap -1 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2603: expect (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2604: rollback oid 46 current snap is 336 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 303 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: finishing write tid 2 to smithi07685715-6 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: finishing write tid 3 to smithi07685715-6 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: finishing write tid 4 to smithi07685715-6 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: oid 6 updating version 0 to 638 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: oid 6 version 638 is already newer than 636 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: oid 6 version 638 is already newer than 637 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 638 (ObjNum 982 snap 336 seq_num 982) dirty exists 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: left oid 6 (ObjNum 982 snap 336 seq_num 982) 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2602: finishing write tid 1 to smithi07685715-2 2024-09-21T09:49:16.285 INFO:tasks.rados.rados.0.smithi076.stdout:2601: done (3 left) 2024-09-21T09:49:16.286 INFO:tasks.rados.rados.0.smithi076.stdout:2605: snap_remove snap 327 2024-09-21T09:49:16.860 INFO:tasks.rados.rados.0.smithi076.stdout:2605: done (3 left) 2024-09-21T09:49:16.860 INFO:tasks.rados.rados.0.smithi076.stdout:2606: read oid 19 snap 328 2024-09-21T09:49:16.860 INFO:tasks.rados.rados.0.smithi076.stdout:2606: expect deleted 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2607: rollback oid 29 current snap is 336 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 335 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2608: read oid 23 snap -1 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2608: expect deleted 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2609: read oid 11 snap -1 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2609: expect deleted 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2610: rollback oid 4 current snap is 336 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 285 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2611: write oid 39 current snap is 336 2024-09-21T09:49:16.861 INFO:tasks.rados.rados.0.smithi076.stdout:2611: seq_num 984 ranges {538088=654553,1714752=634675} 2024-09-21T09:49:16.867 INFO:tasks.rados.rados.0.smithi076.stdout:2611: writing smithi07685715-39 from 538088 to 1192641 tid 1 2024-09-21T09:49:16.873 INFO:tasks.rados.rados.0.smithi076.stdout:2611: writing smithi07685715-39 from 1714752 to 2349427 tid 2 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2612: snap_create 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: finishing write tid 2 to smithi07685715-2 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: finishing write tid 3 to smithi07685715-2 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: finishing write tid 4 to smithi07685715-2 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: oid 2 updating version 0 to 450 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: oid 2 version 450 is already newer than 449 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: oid 2 updating version 450 to 451 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 451 (ObjNum 983 snap 336 seq_num 983) dirty exists 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2602: left oid 2 (ObjNum 983 snap 336 seq_num 983) 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2604: finishing rollback tid 0 to smithi07685715-46 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2604: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2604: finishing rollback tid 2 to smithi07685715-46 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 1976 (ObjNum 832 snap 289 seq_num 832) dirty exists 2024-09-21T09:49:16.874 INFO:tasks.rados.rados.0.smithi076.stdout:2603: expect (ObjNum 796 snap 275 seq_num 796) 2024-09-21T09:49:16.980 INFO:tasks.rados.rados.0.smithi076.stdout:2607: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:49:16.980 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 682 (ObjNum 593 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:16.980 INFO:tasks.rados.rados.0.smithi076.stdout:2610: finishing rollback tid 0 to smithi07685715-4 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2610: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2610: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 828 (ObjNum 816 snap 283 seq_num 816) dirty exists 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: finishing write tid 1 to smithi07685715-39 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: finishing write tid 2 to smithi07685715-39 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: finishing write tid 3 to smithi07685715-39 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: finishing write tid 4 to smithi07685715-39 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: oid 39 updating version 0 to 1982 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: oid 39 updating version 1982 to 1986 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: oid 39 version 1986 is already newer than 1985 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 1986 (ObjNum 984 snap 336 seq_num 984) dirty exists 2024-09-21T09:49:16.981 INFO:tasks.rados.rados.0.smithi076.stdout:2611: left oid 39 (ObjNum 984 snap 336 seq_num 984) 2024-09-21T09:49:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:16 smithi191 ceph-mon[102206]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:49:17.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:16 smithi076 ceph-mon[106482]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:49:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:16 smithi076 ceph-mon[110963]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2602: done (9 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2603: done (8 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2604: done (7 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2606: done (6 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2607: done (5 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2608: done (4 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2609: done (3 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2610: done (2 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2611: done (1 left) 2024-09-21T09:49:17.866 INFO:tasks.rados.rados.0.smithi076.stdout:2612: done (0 left) 2024-09-21T09:49:17.867 INFO:tasks.rados.rados.0.smithi076.stdout:2613: write oid 10 current snap is 337 2024-09-21T09:49:17.867 INFO:tasks.rados.rados.0.smithi076.stdout:2613: seq_num 985 ranges {554895=729802,1773234=778333} 2024-09-21T09:49:17.872 INFO:tasks.rados.rados.0.smithi076.stdout:2613: writing smithi07685715-10 from 554895 to 1284697 tid 1 2024-09-21T09:49:17.880 INFO:tasks.rados.rados.0.smithi076.stdout:2613: writing smithi07685715-10 from 1773234 to 2551567 tid 2 2024-09-21T09:49:17.880 INFO:tasks.rados.rados.0.smithi076.stdout:2614: read oid 7 snap 308 2024-09-21T09:49:17.880 INFO:tasks.rados.rados.0.smithi076.stdout:2614: expect (ObjNum 859 snap 294 seq_num 859) 2024-09-21T09:49:17.880 INFO:tasks.rados.rados.0.smithi076.stdout:2615: rollback oid 15 current snap is 337 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 331 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:2613: finishing write tid 1 to smithi07685715-10 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:2616: read oid 29 snap 331 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:2616: expect deleted 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:2617: read oid 35 snap 335 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:2617: expect (ObjNum 944 snap 324 seq_num 944) 2024-09-21T09:49:17.881 INFO:tasks.rados.rados.0.smithi076.stdout:2618: delete oid 13 current snap is 337 2024-09-21T09:49:17.883 INFO:tasks.rados.rados.0.smithi076.stdout:2618: done (5 left) 2024-09-21T09:49:17.883 INFO:tasks.rados.rados.0.smithi076.stdout:2619: snap_remove snap 303 2024-09-21T09:49:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:17 smithi076 ceph-mon[106482]: pgmap v583: 89 pgs: 89 active+clean; 258 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 896 KiB/s wr, 12 op/s 2024-09-21T09:49:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:17 smithi076 ceph-mon[106482]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:49:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:17 smithi076 ceph-mon[110963]: pgmap v583: 89 pgs: 89 active+clean; 258 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 896 KiB/s wr, 12 op/s 2024-09-21T09:49:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:17 smithi076 ceph-mon[110963]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:49:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:17 smithi191 ceph-mon[102206]: pgmap v583: 89 pgs: 89 active+clean; 258 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 896 KiB/s wr, 12 op/s 2024-09-21T09:49:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:17 smithi191 ceph-mon[102206]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2619: done (5 left) 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2620: rollback oid 44 current snap is 337 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 333 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2615: finishing rollback tid 0 to smithi07685715-15 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2615: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2615: finishing rollback tid 2 to smithi07685715-15 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 583 (ObjNum 853 snap 292 seq_num 853) dirty exists 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2613: finishing write tid 2 to smithi07685715-10 2024-09-21T09:49:18.745 INFO:tasks.rados.rados.0.smithi076.stdout:2613: finishing write tid 3 to smithi07685715-10 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:2613: finishing write tid 4 to smithi07685715-10 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:2613: oid 10 updating version 0 to 1567 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:2613: oid 10 updating version 1567 to 1569 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:2613: oid 10 version 1569 is already newer than 1568 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1569 (ObjNum 985 snap 337 seq_num 985) dirty exists 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:2613: left oid 10 (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:49:18.746 INFO:tasks.rados.rados.0.smithi076.stdout:2614: expect (ObjNum 859 snap 294 seq_num 859) 2024-09-21T09:49:18.898 INFO:tasks.rados.rados.0.smithi076.stdout:2617: expect (ObjNum 944 snap 324 seq_num 944) 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:2620: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 631 (ObjNum 177 snap 0 seq_num 864023504) dirty dne 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:2613: done (5 left) 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:2614: done (4 left) 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:2615: done (3 left) 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:2616: done (2 left) 2024-09-21T09:49:19.021 INFO:tasks.rados.rados.0.smithi076.stdout:2617: done (1 left) 2024-09-21T09:49:19.022 INFO:tasks.rados.rados.0.smithi076.stdout:2620: done (0 left) 2024-09-21T09:49:19.022 INFO:tasks.rados.rados.0.smithi076.stdout:2621: delete oid 19 current snap is 337 2024-09-21T09:49:19.023 INFO:tasks.rados.rados.0.smithi076.stdout:2621: done (0 left) 2024-09-21T09:49:19.023 INFO:tasks.rados.rados.0.smithi076.stdout:2622: snap_remove snap 308 2024-09-21T09:49:19.248 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:18 smithi076 ceph-mon[106482]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:49:19.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:18 smithi076 ceph-mon[106482]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:49:19.249 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:18 smithi076 ceph-mon[110963]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:49:19.249 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:18 smithi076 ceph-mon[110963]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:49:19.249 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:49:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:18 smithi191 ceph-mon[102206]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T09:49:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:18 smithi191 ceph-mon[102206]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T09:49:19.819 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:49:19.819 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (7m) 17s ago 13m 19.6M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:49:19.819 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (7m) 72s ago 13m 79.3M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:49:19.819 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (12m) 17s ago 12m 22.8M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:49:19.819 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (12m) 72s ago 12m 18.0M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:49:19.819 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (8m) 72s ago 17m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (9m) 17s ago 19m 516M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (6m) 17s ago 19m 71.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (4m) 72s ago 18m 40.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 bb6de71151ea 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (6m) 17s ago 17m 54.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (8m) 17s ago 13m 10.5M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (8m) 72s ago 13m 11.5M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (22s) 17s ago 17m 14.3M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 f0b24f5d8966 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (17m) 17s ago 17m 98.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 46510f5f4502 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (16m) 17s ago 16m 120M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (16m) 17s ago 16m 93.8M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (15m) 72s ago 15m 111M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (15m) 72s ago 15m 123M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (15m) 72s ago 15m 87.6M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:49:19.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (14m) 72s ago 14m 91.1M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:49:19.821 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (8m) 72s ago 14m 48.1M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:49:19.883 INFO:tasks.rados.rados.0.smithi076.stdout:2622: done (0 left) 2024-09-21T09:49:19.883 INFO:tasks.rados.rados.0.smithi076.stdout:2623: delete oid 38 current snap is 337 2024-09-21T09:49:19.899 INFO:tasks.rados.rados.0.smithi076.stdout:2623: done (0 left) 2024-09-21T09:49:19.899 INFO:tasks.rados.rados.0.smithi076.stdout:2624: delete oid 32 current snap is 337 2024-09-21T09:49:19.901 INFO:tasks.rados.rados.0.smithi076.stdout:2624: done (0 left) 2024-09-21T09:49:19.901 INFO:tasks.rados.rados.0.smithi076.stdout:2625: snap_remove snap 331 2024-09-21T09:49:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:19 smithi076 ceph-mon[106482]: pgmap v586: 89 pgs: 89 active+clean; 258 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 896 KiB/s wr, 12 op/s 2024-09-21T09:49:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:19 smithi076 ceph-mon[110963]: pgmap v586: 89 pgs: 89 active+clean; 258 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 896 KiB/s wr, 12 op/s 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:49:20.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:20.173 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:49:20.173 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 7, 2024-09-21T09:49:20.173 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:49:20.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:20.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:49:20.173 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:49:20.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:20.174 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:49:20.174 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 9, 2024-09-21T09:49:20.174 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 6 2024-09-21T09:49:20.174 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:49:20.174 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:49:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:19 smithi191 ceph-mon[102206]: pgmap v586: 89 pgs: 89 active+clean; 258 MiB data, 850 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 896 KiB/s wr, 12 op/s 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "1/8 daemons upgraded", 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:49:20.458 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:49:20.891 INFO:tasks.rados.rados.0.smithi076.stdout:2625: done (0 left) 2024-09-21T09:49:20.892 INFO:tasks.rados.rados.0.smithi076.stdout:2626: snap_create 2024-09-21T09:49:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[106482]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[106482]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[106482]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[106482]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:49:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/595541124' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:21.239 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[110963]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.240 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[110963]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.240 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[110963]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.240 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[110963]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:49:21.240 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:20 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/595541124' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:20 smithi191 ceph-mon[102206]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:20 smithi191 ceph-mon[102206]: from='client.44270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:20 smithi191 ceph-mon[102206]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:20 smithi191 ceph-mon[102206]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T09:49:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:20 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/595541124' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:21.891 INFO:tasks.rados.rados.0.smithi076.stdout:2626: done (0 left) 2024-09-21T09:49:21.891 INFO:tasks.rados.rados.0.smithi076.stdout:2627: snap_remove snap 332 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[106482]: pgmap v589: 89 pgs: 89 active+clean; 262 MiB data, 854 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 645 KiB/s wr, 6 op/s 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[106482]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[106482]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[106482]: Upgrade: osd.1 is safe to restart 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[110963]: pgmap v589: 89 pgs: 89 active+clean; 262 MiB data, 854 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 645 KiB/s wr, 6 op/s 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[110963]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[110963]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:49:22.165 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:49:22.166 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:49:22.166 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:21 smithi076 ceph-mon[110963]: Upgrade: osd.1 is safe to restart 2024-09-21T09:49:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:21 smithi191 ceph-mon[102206]: pgmap v589: 89 pgs: 89 active+clean; 262 MiB data, 854 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 645 KiB/s wr, 6 op/s 2024-09-21T09:49:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:21 smithi191 ceph-mon[102206]: from='client.34359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:21 smithi191 ceph-mon[102206]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T09:49:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:21 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:49:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:21 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:49:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:21 smithi191 ceph-mon[102206]: Upgrade: osd.1 is safe to restart 2024-09-21T09:49:22.893 INFO:tasks.rados.rados.0.smithi076.stdout:2627: done (0 left) 2024-09-21T09:49:22.894 INFO:tasks.rados.rados.0.smithi076.stdout:2628: rollback oid 42 current snap is 338 2024-09-21T09:49:22.894 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 334 2024-09-21T09:49:22.894 INFO:tasks.rados.rados.0.smithi076.stdout:2629: rollback oid 5 current snap is 338 2024-09-21T09:49:22.894 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 329 2024-09-21T09:49:22.894 INFO:tasks.rados.rados.0.smithi076.stdout:2630: delete oid 47 current snap is 338 2024-09-21T09:49:22.897 INFO:tasks.rados.rados.0.smithi076.stdout:2628: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:49:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:49:22.904 INFO:tasks.rados.rados.0.smithi076.stdout:2628: done (2 left) 2024-09-21T09:49:22.904 INFO:tasks.rados.rados.0.smithi076.stdout:2630: done (1 left) 2024-09-21T09:49:22.904 INFO:tasks.rados.rados.0.smithi076.stdout:2631: write oid 43 current snap is 338 2024-09-21T09:49:22.904 INFO:tasks.rados.rados.0.smithi076.stdout:2631: seq_num 991 ranges {655737=497649,1607773=789340} 2024-09-21T09:49:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2631: writing smithi07685715-43 from 655737 to 1153386 tid 1 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2631: writing smithi07685715-43 from 1607773 to 2397113 tid 2 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2632: rollback oid 6 current snap is 338 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 337 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2629: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2629: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2629: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2016 (ObjNum 948 snap 324 seq_num 948) dirty exists 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2631: finishing write tid 1 to smithi07685715-43 2024-09-21T09:49:22.918 INFO:tasks.rados.rados.0.smithi076.stdout:2629: done (2 left) 2024-09-21T09:49:22.919 INFO:tasks.rados.rados.0.smithi076.stdout:2633: rollback oid 13 current snap is 338 2024-09-21T09:49:22.919 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 320 2024-09-21T09:49:22.919 INFO:tasks.rados.rados.0.smithi076.stdout:2634: delete oid 15 current snap is 338 2024-09-21T09:49:22.920 INFO:tasks.rados.rados.0.smithi076.stdout:2634: done (3 left) 2024-09-21T09:49:22.920 INFO:tasks.rados.rados.0.smithi076.stdout:2635: read oid 30 snap -1 2024-09-21T09:49:22.920 INFO:tasks.rados.rados.0.smithi076.stdout:2635: expect (ObjNum 959 snap 326 seq_num 959) 2024-09-21T09:49:22.920 INFO:tasks.rados.rados.0.smithi076.stdout:2636: write oid 42 current snap is 338 2024-09-21T09:49:22.920 INFO:tasks.rados.rados.0.smithi076.stdout:2633: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:49:22.920 INFO:tasks.rados.rados.0.smithi076.stdout:2636: seq_num 993 ranges {689351=648147,2124737=564614} 2024-09-21T09:49:22.927 INFO:tasks.rados.rados.0.smithi076.stdout:2636: writing smithi07685715-42 from 689351 to 1337498 tid 1 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2636: writing smithi07685715-42 from 2124737 to 2689351 tid 2 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2637: read oid 21 snap -1 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2637: expect (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2638: read oid 45 snap -1 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2638: expect (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2639: snap_create 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2633: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 463 (ObjNum 938 snap 320 seq_num 938) dirty exists 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2632: finishing rollback tid 0 to smithi07685715-6 2024-09-21T09:49:22.936 INFO:tasks.rados.rados.0.smithi076.stdout:2632: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: finishing write tid 2 to smithi07685715-43 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2632: finishing rollback tid 2 to smithi07685715-6 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 653 (ObjNum 982 snap 336 seq_num 982) dirty exists 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: finishing write tid 3 to smithi07685715-43 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: finishing write tid 4 to smithi07685715-43 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: oid 43 updating version 0 to 600 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: oid 43 updating version 600 to 601 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: oid 43 version 601 is already newer than 599 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 601 (ObjNum 991 snap 338 seq_num 991) dirty exists 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2631: left oid 43 (ObjNum 991 snap 338 seq_num 991) 2024-09-21T09:49:22.937 INFO:tasks.rados.rados.0.smithi076.stdout:2635: expect (ObjNum 959 snap 326 seq_num 959) 2024-09-21T09:49:23.054 INFO:tasks.rados.rados.0.smithi076.stdout:2636: finishing write tid 1 to smithi07685715-42 2024-09-21T09:49:23.055 INFO:tasks.rados.rados.0.smithi076.stdout:2638: expect (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[106482]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[110963]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:49:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: finishing write tid 2 to smithi07685715-42 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: finishing write tid 3 to smithi07685715-42 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: finishing write tid 4 to smithi07685715-42 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: oid 42 updating version 0 to 704 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: oid 42 version 704 is already newer than 703 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: oid 42 version 704 is already newer than 702 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 704 (ObjNum 993 snap 338 seq_num 993) dirty exists 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2636: left oid 42 (ObjNum 993 snap 338 seq_num 993) 2024-09-21T09:49:23.150 INFO:tasks.rados.rados.0.smithi076.stdout:2637: expect (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:49:23.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:22 smithi191 ceph-mon[102206]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T09:49:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:49:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:23.589 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:49:23] "GET /metrics HTTP/1.1" 200 36274 "" "Prometheus/2.51.0" 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2631: done (7 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2632: done (6 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2633: done (5 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2635: done (4 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2636: done (3 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2637: done (2 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2638: done (1 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2639: done (0 left) 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2640: write oid 20 current snap is 339 2024-09-21T09:49:23.740 INFO:tasks.rados.rados.0.smithi076.stdout:2640: seq_num 994 ranges {706158=723396,2183219=522939} 2024-09-21T09:49:23.747 INFO:tasks.rados.rados.0.smithi076.stdout:2640: writing smithi07685715-20 from 706158 to 1429554 tid 1 2024-09-21T09:49:23.754 INFO:tasks.rados.rados.0.smithi076.stdout:2640: writing smithi07685715-20 from 2183219 to 2706158 tid 2 2024-09-21T09:49:23.755 INFO:tasks.rados.rados.0.smithi076.stdout:2641: rollback oid 39 current snap is 339 2024-09-21T09:49:23.755 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 337 2024-09-21T09:49:23.755 INFO:tasks.rados.rados.0.smithi076.stdout:2640: finishing write tid 1 to smithi07685715-20 2024-09-21T09:49:23.755 INFO:tasks.rados.rados.0.smithi076.stdout:2642: delete oid 9 current snap is 339 2024-09-21T09:49:23.759 INFO:tasks.rados.rados.0.smithi076.stdout:2640: finishing write tid 2 to smithi07685715-20 2024-09-21T09:49:23.759 INFO:tasks.rados.rados.0.smithi076.stdout:2640: finishing write tid 3 to smithi07685715-20 2024-09-21T09:49:23.759 INFO:tasks.rados.rados.0.smithi076.stdout:2640: finishing write tid 4 to smithi07685715-20 2024-09-21T09:49:23.759 INFO:tasks.rados.rados.0.smithi076.stdout:2640: oid 20 updating version 0 to 799 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2640: oid 20 updating version 799 to 800 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2640: oid 20 version 800 is already newer than 798 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 800 (ObjNum 994 snap 339 seq_num 994) dirty exists 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2640: left oid 20 (ObjNum 994 snap 339 seq_num 994) 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2640: done (2 left) 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2642: done (1 left) 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2643: read oid 12 snap -1 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2643: expect (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2644: write oid 30 current snap is 339 2024-09-21T09:49:23.760 INFO:tasks.rados.rados.0.smithi076.stdout:2644: seq_num 996 ranges {739772=590247,1900183=540336} 2024-09-21T09:49:23.766 INFO:tasks.rados.rados.0.smithi076.stdout:2644: writing smithi07685715-30 from 739772 to 1330019 tid 1 2024-09-21T09:49:23.773 INFO:tasks.rados.rados.0.smithi076.stdout:2644: writing smithi07685715-30 from 1900183 to 2440519 tid 2 2024-09-21T09:49:23.773 INFO:tasks.rados.rados.0.smithi076.stdout:2645: read oid 3 snap -1 2024-09-21T09:49:23.773 INFO:tasks.rados.rados.0.smithi076.stdout:2645: expect deleted 2024-09-21T09:49:23.773 INFO:tasks.rados.rados.0.smithi076.stdout:2646: snap_remove snap 285 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[110963]: Upgrade: Updating osd.1 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[110963]: Deploying daemon osd.1 on smithi076 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[110963]: pgmap v592: 89 pgs: 89 active+clean; 248 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[110963]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[110963]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[106482]: Upgrade: Updating osd.1 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[106482]: Deploying daemon osd.1 on smithi076 2024-09-21T09:49:24.161 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[106482]: pgmap v592: 89 pgs: 89 active+clean; 248 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:49:24.162 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[106482]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:49:24.162 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:23 smithi076 ceph-mon[106482]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:49:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:23 smithi191 ceph-mon[102206]: Upgrade: Updating osd.1 2024-09-21T09:49:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:23 smithi191 ceph-mon[102206]: Deploying daemon osd.1 on smithi076 2024-09-21T09:49:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:23 smithi191 ceph-mon[102206]: pgmap v592: 89 pgs: 89 active+clean; 248 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.5 MiB/s wr, 16 op/s 2024-09-21T09:49:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:23 smithi191 ceph-mon[102206]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T09:49:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:23 smithi191 ceph-mon[102206]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T09:49:24.744 INFO:tasks.rados.rados.0.smithi076.stdout:2646: done (4 left) 2024-09-21T09:49:24.744 INFO:tasks.rados.rados.0.smithi076.stdout:2647: write oid 4 current snap is 339 2024-09-21T09:49:24.744 INFO:tasks.rados.rados.0.smithi076.stdout:2647: seq_num 997 ranges {756579=665496,1958665=400347} 2024-09-21T09:49:24.753 INFO:tasks.rados.rados.0.smithi076.stdout:2647: writing smithi07685715-4 from 756579 to 1422075 tid 1 2024-09-21T09:49:24.759 INFO:tasks.rados.rados.0.smithi076.stdout:2647: writing smithi07685715-4 from 1958665 to 2359012 tid 2 2024-09-21T09:49:24.759 INFO:tasks.rados.rados.0.smithi076.stdout:2648: write oid 25 current snap is 339 2024-09-21T09:49:24.759 INFO:tasks.rados.rados.0.smithi076.stdout:2648: seq_num 998 ranges {773386=740745,2017147=544005} 2024-09-21T09:49:24.767 INFO:tasks.rados.rados.0.smithi076.stdout:2648: writing smithi07685715-25 from 773386 to 1514131 tid 1 2024-09-21T09:49:24.773 INFO:tasks.rados.rados.0.smithi076.stdout:2648: writing smithi07685715-25 from 2017147 to 2561152 tid 2 2024-09-21T09:49:24.774 INFO:tasks.rados.rados.0.smithi076.stdout:2649: snap_create 2024-09-21T09:49:24.774 INFO:tasks.rados.rados.0.smithi076.stdout:2641: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:49:24.774 INFO:tasks.rados.rados.0.smithi076.stdout:2641: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:49:24.774 INFO:tasks.rados.rados.0.smithi076.stdout:2641: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:49:24.774 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2028 (ObjNum 984 snap 336 seq_num 984) dirty exists 2024-09-21T09:49:24.774 INFO:tasks.rados.rados.0.smithi076.stdout:2643: expect (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:49:24.855 INFO:tasks.rados.rados.0.smithi076.stdout:2644: finishing write tid 1 to smithi07685715-30 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: finishing write tid 2 to smithi07685715-30 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: finishing write tid 3 to smithi07685715-30 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: finishing write tid 4 to smithi07685715-30 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: oid 30 updating version 0 to 2025 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: oid 30 version 2025 is already newer than 2024 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: oid 30 version 2025 is already newer than 2023 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2025 (ObjNum 996 snap 339 seq_num 996) dirty exists 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2644: left oid 30 (ObjNum 996 snap 339 seq_num 996) 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2647: finishing write tid 1 to smithi07685715-4 2024-09-21T09:49:24.980 INFO:tasks.rados.rados.0.smithi076.stdout:2647: finishing write tid 2 to smithi07685715-4 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2647: finishing write tid 3 to smithi07685715-4 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2647: finishing write tid 4 to smithi07685715-4 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2647: oid 4 updating version 0 to 847 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2647: oid 4 version 847 is already newer than 846 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2647: oid 4 updating version 847 to 848 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 848 (ObjNum 997 snap 339 seq_num 997) dirty exists 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2647: left oid 4 (ObjNum 997 snap 339 seq_num 997) 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2648: finishing write tid 1 to smithi07685715-25 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2648: finishing write tid 2 to smithi07685715-25 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2648: finishing write tid 3 to smithi07685715-25 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2648: finishing write tid 4 to smithi07685715-25 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2648: oid 25 updating version 0 to 637 2024-09-21T09:49:24.981 INFO:tasks.rados.rados.0.smithi076.stdout:2648: oid 25 version 637 is already newer than 636 2024-09-21T09:49:24.982 INFO:tasks.rados.rados.0.smithi076.stdout:2648: oid 25 version 637 is already newer than 635 2024-09-21T09:49:24.982 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 637 (ObjNum 998 snap 339 seq_num 998) dirty exists 2024-09-21T09:49:24.982 INFO:tasks.rados.rados.0.smithi076.stdout:2648: left oid 25 (ObjNum 998 snap 339 seq_num 998) 2024-09-21T09:49:25.448 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:25 smithi076 systemd[1]: Stopping Ceph osd.1 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:49:25.742 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[51605]: 2024-09-21T09:49:25.445+0000 7ff89ac3a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:49:25.742 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[51605]: 2024-09-21T09:49:25.445+0000 7ff89ac3a640 -1 osd.1 750 *** Got signal Terminated *** 2024-09-21T09:49:25.742 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[51605]: 2024-09-21T09:49:25.445+0000 7ff89ac3a640 -1 osd.1 750 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2641: done (6 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2643: done (5 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2644: done (4 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2645: done (3 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2647: done (2 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2648: done (1 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2649: done (0 left) 2024-09-21T09:49:25.758 INFO:tasks.rados.rados.0.smithi076.stdout:2650: delete oid 37 current snap is 340 2024-09-21T09:49:25.765 INFO:tasks.rados.rados.0.smithi076.stdout:2650: done (0 left) 2024-09-21T09:49:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2651: snap_create 2024-09-21T09:49:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:25 smithi191 ceph-mon[102206]: pgmap v595: 89 pgs: 89 active+clean; 243 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 923 KiB/s wr, 9 op/s 2024-09-21T09:49:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:25 smithi191 ceph-mon[102206]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:49:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:25 smithi191 ceph-mon[102206]: osd.1 marked itself down and dead 2024-09-21T09:49:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:26.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[110963]: pgmap v595: 89 pgs: 89 active+clean; 243 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 923 KiB/s wr, 9 op/s 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[110963]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[110963]: osd.1 marked itself down and dead 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[106482]: pgmap v595: 89 pgs: 89 active+clean; 243 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 923 KiB/s wr, 9 op/s 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[106482]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[106482]: osd.1 marked itself down and dead 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:26.623 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:26 smithi076 podman[127173]: 2024-09-21 09:49:26.302596536 +0000 UTC m=+0.971975686 container died 46510f5f4502dc3f295bd2b66b0358c50fe596f619771f4198455e9ae9bc50b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:49:26.760 INFO:tasks.rados.rados.0.smithi076.stdout:2651: done (0 left) 2024-09-21T09:49:26.761 INFO:tasks.rados.rados.0.smithi076.stdout:2652: write oid 43 current snap is 341 2024-09-21T09:49:26.761 INFO:tasks.rados.rados.0.smithi076.stdout:2652: seq_num 1000 ranges {407000=491243,1617758=547674,2595228=211772} 2024-09-21T09:49:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2652: writing smithi07685715-43 from 407000 to 898243 tid 1 2024-09-21T09:49:26.773 INFO:tasks.rados.rados.0.smithi076.stdout:2652: writing smithi07685715-43 from 1617758 to 2165432 tid 2 2024-09-21T09:49:26.777 INFO:tasks.rados.rados.0.smithi076.stdout:2652: writing smithi07685715-43 from 2595228 to 2807000 tid 3 2024-09-21T09:49:26.777 INFO:tasks.rados.rados.0.smithi076.stdout:2653: delete oid 14 current snap is 341 2024-09-21T09:49:26.777 INFO:tasks.rados.rados.0.smithi076.stdout:2652: finishing write tid 1 to smithi07685715-43 2024-09-21T09:49:26.778 INFO:tasks.rados.rados.0.smithi076.stdout:2652: finishing write tid 2 to smithi07685715-43 2024-09-21T09:49:26.779 INFO:tasks.rados.rados.0.smithi076.stdout:2652: finishing write tid 3 to smithi07685715-43 2024-09-21T09:49:26.779 INFO:tasks.rados.rados.0.smithi076.stdout:2652: finishing write tid 4 to smithi07685715-43 2024-09-21T09:49:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2652: finishing write tid 5 to smithi07685715-43 2024-09-21T09:49:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2652: oid 43 updating version 0 to 608 2024-09-21T09:49:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2652: oid 43 updating version 608 to 610 2024-09-21T09:49:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2652: oid 43 version 610 is already newer than 609 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2652: oid 43 version 610 is already newer than 607 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 610 (ObjNum 1000 snap 341 seq_num 1000) dirty exists 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2652: left oid 43 (ObjNum 1000 snap 341 seq_num 1000) 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2652: done (1 left) 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2653: done (0 left) 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2654: read oid 2 snap -1 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2654: expect (ObjNum 983 snap 336 seq_num 983) 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2655: read oid 44 snap -1 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2655: expect deleted 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2656: write oid 20 current snap is 341 2024-09-21T09:49:26.781 INFO:tasks.rados.rados.0.smithi076.stdout:2656: seq_num 1002 ranges {440614=641741,1734722=551343} 2024-09-21T09:49:26.786 INFO:tasks.rados.rados.0.smithi076.stdout:2656: writing smithi07685715-20 from 440614 to 1082355 tid 1 2024-09-21T09:49:26.793 INFO:tasks.rados.rados.0.smithi076.stdout:2656: writing smithi07685715-20 from 1734722 to 2286065 tid 2 2024-09-21T09:49:26.793 INFO:tasks.rados.rados.0.smithi076.stdout:2657: delete oid 5 current snap is 341 2024-09-21T09:49:26.793 INFO:tasks.rados.rados.0.smithi076.stdout:2654: expect (ObjNum 983 snap 336 seq_num 983) 2024-09-21T09:49:26.881 INFO:tasks.rados.rados.0.smithi076.stdout:2656: finishing write tid 1 to smithi07685715-20 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2654: done (3 left) 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2655: done (2 left) 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2657: done (1 left) 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2658: rollback oid 13 current snap is 341 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 320 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: finishing write tid 2 to smithi07685715-20 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: finishing write tid 3 to smithi07685715-20 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: finishing write tid 4 to smithi07685715-20 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: oid 20 updating version 0 to 810 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: oid 20 updating version 810 to 812 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: oid 20 version 812 is already newer than 811 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 812 (ObjNum 1002 snap 341 seq_num 1002) dirty exists 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: left oid 20 (ObjNum 1002 snap 341 seq_num 1002) 2024-09-21T09:49:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:2656: done (1 left) 2024-09-21T09:49:26.883 INFO:tasks.rados.rados.0.smithi076.stdout:2659: snap_create 2024-09-21T09:49:26.883 INFO:tasks.rados.rados.0.smithi076.stdout:2658: finishing rollback tid 0 to smithi07685715-13 2024-09-21T09:49:26.883 INFO:tasks.rados.rados.0.smithi076.stdout:2658: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:49:26.884 INFO:tasks.rados.rados.0.smithi076.stdout:2658: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:49:26.884 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 471 (ObjNum 938 snap 320 seq_num 938) dirty exists 2024-09-21T09:49:27.056 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:26 smithi076 ceph-mon[110963]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:49:27.056 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:26 smithi076 ceph-mon[110963]: osdmap e751: 8 total, 7 up, 8 in 2024-09-21T09:49:27.056 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:26 smithi076 podman[127173]: 2024-09-21 09:49:26.769297817 +0000 UTC m=+1.438676966 container cleanup 46510f5f4502dc3f295bd2b66b0358c50fe596f619771f4198455e9ae9bc50b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:49:27.056 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:26 smithi076 bash[127173]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1 2024-09-21T09:49:27.056 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:26 smithi076 ceph-mon[106482]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:49:27.057 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:26 smithi076 ceph-mon[106482]: osdmap e751: 8 total, 7 up, 8 in 2024-09-21T09:49:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:26 smithi191 ceph-mon[102206]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:49:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:26 smithi191 ceph-mon[102206]: osdmap e751: 8 total, 7 up, 8 in 2024-09-21T09:49:27.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127193]: 2024-09-21 09:49:27.054380966 +0000 UTC m=+0.746971749 container remove 46510f5f4502dc3f295bd2b66b0358c50fe596f619771f4198455e9ae9bc50b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:49:27.710 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127287]: 2024-09-21 09:49:27.285888307 +0000 UTC m=+0.015603717 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:49:27.710 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127287]: 2024-09-21 09:49:27.389001995 +0000 UTC m=+0.118717470 container create cfbbd046c25faab39cecb0c335e965d5c3c0e4efc44f418556d56c67ced89d72 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:49:27.710 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127287]: 2024-09-21 09:49:27.600372292 +0000 UTC m=+0.330087701 container init cfbbd046c25faab39cecb0c335e965d5c3c0e4efc44f418556d56c67ced89d72 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:49:27.710 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127287]: 2024-09-21 09:49:27.605271251 +0000 UTC m=+0.334986661 container start cfbbd046c25faab39cecb0c335e965d5c3c0e4efc44f418556d56c67ced89d72 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T09:49:27.710 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127287]: 2024-09-21 09:49:27.70856524 +0000 UTC m=+0.438280666 container attach cfbbd046c25faab39cecb0c335e965d5c3c0e4efc44f418556d56c67ced89d72 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-21T09:49:27.768 INFO:tasks.rados.rados.0.smithi076.stdout:2658: done (1 left) 2024-09-21T09:49:27.768 INFO:tasks.rados.rados.0.smithi076.stdout:2659: done (0 left) 2024-09-21T09:49:27.768 INFO:tasks.rados.rados.0.smithi076.stdout:2660: read oid 46 snap -1 2024-09-21T09:49:27.768 INFO:tasks.rados.rados.0.smithi076.stdout:2660: expect (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:49:27.768 INFO:tasks.rados.rados.0.smithi076.stdout:2661: delete oid 17 current snap is 342 2024-09-21T09:49:27.772 INFO:tasks.rados.rados.0.smithi076.stdout:2661: done (1 left) 2024-09-21T09:49:27.772 INFO:tasks.rados.rados.0.smithi076.stdout:2662: write oid 27 current snap is 342 2024-09-21T09:49:27.773 INFO:tasks.rados.rados.0.smithi076.stdout:2662: seq_num 1005 ranges {491035=583841,1510168=698670,2732343=158692} 2024-09-21T09:49:27.778 INFO:tasks.rados.rados.0.smithi076.stdout:2662: writing smithi07685715-27 from 491035 to 1074876 tid 1 2024-09-21T09:49:27.784 INFO:tasks.rados.rados.0.smithi076.stdout:2662: writing smithi07685715-27 from 1510168 to 2208838 tid 2 2024-09-21T09:49:27.788 INFO:tasks.rados.rados.0.smithi076.stdout:2662: writing smithi07685715-27 from 2732343 to 2891035 tid 3 2024-09-21T09:49:27.788 INFO:tasks.rados.rados.0.smithi076.stdout:2663: write oid 16 current snap is 342 2024-09-21T09:49:27.788 INFO:tasks.rados.rados.0.smithi076.stdout:2663: seq_num 1006 ranges {507842=659090,1568650=558681,2759766=148076} 2024-09-21T09:49:27.794 INFO:tasks.rados.rados.0.smithi076.stdout:2663: writing smithi07685715-16 from 507842 to 1166932 tid 1 2024-09-21T09:49:27.799 INFO:tasks.rados.rados.0.smithi076.stdout:2663: writing smithi07685715-16 from 1568650 to 2127331 tid 2 2024-09-21T09:49:27.804 INFO:tasks.rados.rados.0.smithi076.stdout:2663: writing smithi07685715-16 from 2759766 to 2907842 tid 3 2024-09-21T09:49:27.804 INFO:tasks.rados.rados.0.smithi076.stdout:2664: rollback oid 26 current snap is 342 2024-09-21T09:49:27.804 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 326 2024-09-21T09:49:27.804 INFO:tasks.rados.rados.0.smithi076.stdout:2660: expect (ObjNum 832 snap 289 seq_num 832) 2024-09-21T09:49:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2662: finishing write tid 1 to smithi07685715-27 2024-09-21T09:49:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2662: finishing write tid 2 to smithi07685715-27 2024-09-21T09:49:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2660: done (3 left) 2024-09-21T09:49:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2665: read oid 36 snap 338 2024-09-21T09:49:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2665: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2666: delete oid 7 current snap is 342 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: finishing write tid 3 to smithi07685715-27 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: finishing write tid 4 to smithi07685715-27 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: finishing write tid 5 to smithi07685715-27 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: oid 27 updating version 0 to 1201 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: oid 27 version 1201 is already newer than 1200 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: oid 27 updating version 1201 to 1203 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: oid 27 version 1203 is already newer than 1202 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1203 (ObjNum 1005 snap 342 seq_num 1005) dirty exists 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2662: left oid 27 (ObjNum 1005 snap 342 seq_num 1005) 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2663: finishing write tid 1 to smithi07685715-16 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2663: finishing write tid 2 to smithi07685715-16 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2663: finishing write tid 3 to smithi07685715-16 2024-09-21T09:49:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2663: finishing write tid 4 to smithi07685715-16 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2663: finishing write tid 5 to smithi07685715-16 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2663: oid 16 updating version 0 to 1207 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2663: oid 16 version 1207 is already newer than 1206 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2663: oid 16 updating version 1207 to 1208 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2663: oid 16 version 1208 is already newer than 1205 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1208 (ObjNum 1006 snap 342 seq_num 1006) dirty exists 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2663: left oid 16 (ObjNum 1006 snap 342 seq_num 1006) 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2664: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2664: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:2664: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:49:27.947 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 912 (ObjNum 945 snap 324 seq_num 945) dirty exists 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2662: done (4 left) 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2663: done (3 left) 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2664: done (2 left) 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2666: done (1 left) 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2667: read oid 31 snap -1 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2667: expect deleted 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2668: read oid 35 snap -1 2024-09-21T09:49:27.948 INFO:tasks.rados.rados.0.smithi076.stdout:2668: expect (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:27.949 INFO:tasks.rados.rados.0.smithi076.stdout:2669: write oid 25 current snap is 342 2024-09-21T09:49:27.949 INFO:tasks.rados.rados.0.smithi076.stdout:2669: seq_num 1008 ranges {541456=409588,1569261=562350,2814612=126844} 2024-09-21T09:49:27.953 INFO:tasks.rados.rados.0.smithi076.stdout:2669: writing smithi07685715-25 from 541456 to 951044 tid 1 2024-09-21T09:49:27.959 INFO:tasks.rados.rados.0.smithi076.stdout:2669: writing smithi07685715-25 from 1569261 to 2131611 tid 2 2024-09-21T09:49:27.964 INFO:tasks.rados.rados.0.smithi076.stdout:2669: writing smithi07685715-25 from 2814612 to 2941456 tid 3 2024-09-21T09:49:27.964 INFO:tasks.rados.rados.0.smithi076.stdout:2670: write oid 16 current snap is 342 2024-09-21T09:49:27.964 INFO:tasks.rados.rados.0.smithi076.stdout:2670: seq_num 1009 ranges {558263=484837,1627743=706008} 2024-09-21T09:49:27.969 INFO:tasks.rados.rados.0.smithi076.stdout:2670: writing smithi07685715-16 from 558263 to 1043100 tid 1 2024-09-21T09:49:27.976 INFO:tasks.rados.rados.0.smithi076.stdout:2670: writing smithi07685715-16 from 1627743 to 2333751 tid 2 2024-09-21T09:49:27.976 INFO:tasks.rados.rados.0.smithi076.stdout:2671: write oid 9 current snap is 342 2024-09-21T09:49:27.976 INFO:tasks.rados.rados.0.smithi076.stdout:2671: seq_num 1010 ranges {575070=560086,1686225=566019,2869458=105612} 2024-09-21T09:49:27.982 INFO:tasks.rados.rados.0.smithi076.stdout:2671: writing smithi07685715-9 from 575070 to 1135156 tid 1 2024-09-21T09:49:27.988 INFO:tasks.rados.rados.0.smithi076.stdout:2671: writing smithi07685715-9 from 1686225 to 2252244 tid 2 2024-09-21T09:49:27.992 INFO:tasks.rados.rados.0.smithi076.stdout:2671: writing smithi07685715-9 from 2869458 to 2975070 tid 3 2024-09-21T09:49:27.993 INFO:tasks.rados.rados.0.smithi076.stdout:2672: snap_remove snap 329 2024-09-21T09:49:28.096 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:27 smithi076 ceph-mon[106482]: pgmap v598: 89 pgs: 33 peering, 1 stale+active+clean, 55 active+clean; 174 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:49:28.096 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:27 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-21T09:49:28.096 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:27 smithi076 ceph-mon[106482]: osdmap e752: 8 total, 7 up, 8 in 2024-09-21T09:49:28.097 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:27 smithi076 ceph-mon[110963]: pgmap v598: 89 pgs: 33 peering, 1 stale+active+clean, 55 active+clean; 174 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:49:28.097 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:27 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-21T09:49:28.097 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:27 smithi076 ceph-mon[110963]: osdmap e752: 8 total, 7 up, 8 in 2024-09-21T09:49:28.097 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:27 smithi076 podman[127330]: 2024-09-21 09:49:27.831738462 +0000 UTC m=+0.025701893 container died cfbbd046c25faab39cecb0c335e965d5c3c0e4efc44f418556d56c67ced89d72 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T09:49:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:27 smithi191 ceph-mon[102206]: pgmap v598: 89 pgs: 33 peering, 1 stale+active+clean, 55 active+clean; 174 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 19 op/s 2024-09-21T09:49:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:27 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 8 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-21T09:49:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:27 smithi191 ceph-mon[102206]: osdmap e752: 8 total, 7 up, 8 in 2024-09-21T09:49:28.746 INFO:tasks.rados.rados.0.smithi076.stdout:2672: done (6 left) 2024-09-21T09:49:28.746 INFO:tasks.rados.rados.0.smithi076.stdout:2673: rollback oid 3 current snap is 342 2024-09-21T09:49:28.746 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 328 2024-09-21T09:49:28.747 INFO:tasks.rados.rados.0.smithi076.stdout:2665: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:49:28.765 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 podman[127330]: 2024-09-21 09:49:28.413106799 +0000 UTC m=+0.607070264 container remove cfbbd046c25faab39cecb0c335e965d5c3c0e4efc44f418556d56c67ced89d72 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b) 2024-09-21T09:49:28.765 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T09:49:28.765 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 systemd[1]: Stopped Ceph osd.1 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:49:28.765 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.1.service: Consumed 15.308s CPU time. 2024-09-21T09:49:28.765 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 systemd[1]: Starting Ceph osd.1 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:49:28.908 INFO:tasks.rados.rados.0.smithi076.stdout:2668: expect (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2669: finishing write tid 1 to smithi07685715-25 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2669: finishing write tid 2 to smithi07685715-25 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2665: done (6 left) 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2667: done (5 left) 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2668: done (4 left) 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2674: write oid 13 current snap is 342 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2669: finishing write tid 3 to smithi07685715-25 2024-09-21T09:49:29.019 INFO:tasks.rados.rados.0.smithi076.stdout:2674: seq_num 1011 ranges {591877=635335,1628354=709677} 2024-09-21T09:49:29.025 INFO:tasks.rados.rados.0.smithi076.stdout:2674: writing smithi07685715-13 from 591877 to 1227212 tid 1 2024-09-21T09:49:29.031 INFO:tasks.rados.rados.0.smithi076.stdout:2674: writing smithi07685715-13 from 1628354 to 2338031 tid 2 2024-09-21T09:49:29.031 INFO:tasks.rados.rados.0.smithi076.stdout:2675: snap_remove snap 341 2024-09-21T09:49:29.086 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:28 smithi076 ceph-mon[106482]: osdmap e753: 8 total, 7 up, 8 in 2024-09-21T09:49:29.086 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:28 smithi076 ceph-mon[106482]: osdmap e754: 8 total, 7 up, 8 in 2024-09-21T09:49:29.086 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:28 smithi076 ceph-mon[110963]: osdmap e753: 8 total, 7 up, 8 in 2024-09-21T09:49:29.086 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:28 smithi076 ceph-mon[110963]: osdmap e754: 8 total, 7 up, 8 in 2024-09-21T09:49:29.086 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 podman[127436]: 2024-09-21 09:49:28.741498624 +0000 UTC m=+0.016351458 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:49:29.087 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:28 smithi076 podman[127436]: 2024-09-21 09:49:28.881863134 +0000 UTC m=+0.156715962 container create 03db3c266ab8c1d90c3a9036749ac67d7957da1fd035c90610877732cca1b524 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T09:49:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:28 smithi191 ceph-mon[102206]: osdmap e753: 8 total, 7 up, 8 in 2024-09-21T09:49:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:28 smithi191 ceph-mon[102206]: osdmap e754: 8 total, 7 up, 8 in 2024-09-21T09:49:29.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 podman[127436]: 2024-09-21 09:49:29.084837096 +0000 UTC m=+0.359689933 container init 03db3c266ab8c1d90c3a9036749ac67d7957da1fd035c90610877732cca1b524 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b) 2024-09-21T09:49:29.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 podman[127436]: 2024-09-21 09:49:29.08988536 +0000 UTC m=+0.364738197 container start 03db3c266ab8c1d90c3a9036749ac67d7957da1fd035c90610877732cca1b524 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:49:29.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 podman[127436]: 2024-09-21 09:49:29.209947887 +0000 UTC m=+0.484800724 container attach 03db3c266ab8c1d90c3a9036749ac67d7957da1fd035c90610877732cca1b524 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T09:49:29.374 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:49:29.374 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:49:29.374 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:49:29.374 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:49:29.772 INFO:tasks.rados.rados.0.smithi076.stdout:2675: done (5 left) 2024-09-21T09:49:29.772 INFO:tasks.rados.rados.0.smithi076.stdout:2676: read oid 10 snap -1 2024-09-21T09:49:29.773 INFO:tasks.rados.rados.0.smithi076.stdout:2676: expect (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:49:29.773 INFO:tasks.rados.rados.0.smithi076.stdout:2677: write oid 24 current snap is 342 2024-09-21T09:49:29.773 INFO:tasks.rados.rados.0.smithi076.stdout:2677: seq_num 1012 ranges {608684=426937,1803189=569688,2924304=84380} 2024-09-21T09:49:29.778 INFO:tasks.rados.rados.0.smithi076.stdout:2677: writing smithi07685715-24 from 608684 to 1035621 tid 1 2024-09-21T09:49:29.786 INFO:tasks.rados.rados.0.smithi076.stdout:2677: writing smithi07685715-24 from 1803189 to 2372877 tid 2 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2677: writing smithi07685715-24 from 2924304 to 3008684 tid 3 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2678: read oid 26 snap 328 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2678: expect (ObjNum 945 snap 324 seq_num 945) 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2679: write oid 1 current snap is 342 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2669: finishing write tid 4 to smithi07685715-25 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2669: finishing write tid 5 to smithi07685715-25 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2669: oid 25 updating version 0 to 640 2024-09-21T09:49:29.790 INFO:tasks.rados.rados.0.smithi076.stdout:2669: oid 25 updating version 640 to 641 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2669: oid 25 version 641 is already newer than 639 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2669: oid 25 updating version 641 to 642 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 642 (ObjNum 1008 snap 342 seq_num 1008) dirty exists 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2669: left oid 25 (ObjNum 1008 snap 342 seq_num 1008) 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: finishing write tid 1 to smithi07685715-16 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: finishing write tid 2 to smithi07685715-16 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: finishing write tid 3 to smithi07685715-16 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: finishing write tid 4 to smithi07685715-16 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: oid 16 updating version 0 to 1211 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: oid 16 version 1211 is already newer than 1210 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: oid 16 version 1211 is already newer than 1209 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1211 (ObjNum 1009 snap 342 seq_num 1009) dirty exists 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2670: left oid 16 (ObjNum 1009 snap 342 seq_num 1009) 2024-09-21T09:49:29.791 INFO:tasks.rados.rados.0.smithi076.stdout:2671: finishing write tid 1 to smithi07685715-9 2024-09-21T09:49:29.792 INFO:tasks.rados.rados.0.smithi076.stdout:2671: finishing write tid 2 to smithi07685715-9 2024-09-21T09:49:29.792 INFO:tasks.rados.rados.0.smithi076.stdout:2671: finishing write tid 3 to smithi07685715-9 2024-09-21T09:49:29.792 INFO:tasks.rados.rados.0.smithi076.stdout:2679: seq_num 1013 ranges {625491=502186,1861671=713346} 2024-09-21T09:49:29.795 INFO:tasks.rados.rados.0.smithi076.stdout:2679: writing smithi07685715-1 from 625491 to 1127677 tid 1 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2679: writing smithi07685715-1 from 1861671 to 2575017 tid 2 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2669: done (8 left) 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2670: done (7 left) 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2680: snap_create 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2671: finishing write tid 4 to smithi07685715-9 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2671: finishing write tid 5 to smithi07685715-9 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2671: oid 9 updating version 0 to 916 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2671: oid 9 version 916 is already newer than 913 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2671: oid 9 version 916 is already newer than 915 2024-09-21T09:49:29.805 INFO:tasks.rados.rados.0.smithi076.stdout:2671: oid 9 version 916 is already newer than 914 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 916 (ObjNum 1010 snap 342 seq_num 1010) dirty exists 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2671: left oid 9 (ObjNum 1010 snap 342 seq_num 1010) 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2673: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 186 (ObjNum 145 snap 0 seq_num 865251888) dirty dne 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: finishing write tid 1 to smithi07685715-13 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: finishing write tid 2 to smithi07685715-13 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: finishing write tid 3 to smithi07685715-13 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: finishing write tid 4 to smithi07685715-13 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: oid 13 updating version 0 to 477 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: oid 13 version 477 is already newer than 475 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: oid 13 version 477 is already newer than 476 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 477 (ObjNum 1011 snap 342 seq_num 1011) dirty exists 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2674: left oid 13 (ObjNum 1011 snap 342 seq_num 1011) 2024-09-21T09:49:29.806 INFO:tasks.rados.rados.0.smithi076.stdout:2676: expect (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:49:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:2677: finishing write tid 1 to smithi07685715-24 2024-09-21T09:49:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:2677: finishing write tid 2 to smithi07685715-24 2024-09-21T09:49:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:2677: finishing write tid 3 to smithi07685715-24 2024-09-21T09:49:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:2677: finishing write tid 4 to smithi07685715-24 2024-09-21T09:49:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:2677: finishing write tid 5 to smithi07685715-24 2024-09-21T09:49:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:2677: oid 24 updating version 0 to 258 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:2677: oid 24 version 258 is already newer than 257 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:2677: oid 24 updating version 258 to 259 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:2677: oid 24 version 259 is already newer than 256 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 259 (ObjNum 1012 snap 342 seq_num 1012) dirty exists 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:2677: left oid 24 (ObjNum 1012 snap 342 seq_num 1012) 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:2679: finishing write tid 1 to smithi07685715-1 2024-09-21T09:49:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:2678: expect (ObjNum 945 snap 324 seq_num 945) 2024-09-21T09:49:29.983 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-mon[106482]: pgmap v601: 89 pgs: 33 peering, 1 stale+active+clean, 55 active+clean; 174 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:49:29.983 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-mon[110963]: pgmap v601: 89 pgs: 33 peering, 1 stale+active+clean, 55 active+clean; 174 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:49:29.983 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:49:29.983 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:49:29.983 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:49:29.984 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:49:30.119 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:29 smithi191 ceph-mon[102206]: pgmap v601: 89 pgs: 33 peering, 1 stale+active+clean, 55 active+clean; 174 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T09:49:30.151 INFO:tasks.rados.rados.0.smithi076.stdout:2679: finishing write tid 2 to smithi07685715-1 2024-09-21T09:49:30.151 INFO:tasks.rados.rados.0.smithi076.stdout:2679: finishing write tid 3 to smithi07685715-1 2024-09-21T09:49:30.151 INFO:tasks.rados.rados.0.smithi076.stdout:2679: finishing write tid 4 to smithi07685715-1 2024-09-21T09:49:30.152 INFO:tasks.rados.rados.0.smithi076.stdout:2679: oid 1 updating version 0 to 2045 2024-09-21T09:49:30.152 INFO:tasks.rados.rados.0.smithi076.stdout:2679: oid 1 updating version 2045 to 2047 2024-09-21T09:49:30.152 INFO:tasks.rados.rados.0.smithi076.stdout:2679: oid 1 version 2047 is already newer than 2046 2024-09-21T09:49:30.152 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2047 (ObjNum 1013 snap 342 seq_num 1013) dirty exists 2024-09-21T09:49:30.152 INFO:tasks.rados.rados.0.smithi076.stdout:2679: left oid 1 (ObjNum 1013 snap 342 seq_num 1013) 2024-09-21T09:49:30.315 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate[127451]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:49:30.316 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:29 smithi076 bash[127436]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:49:30.317 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:30 smithi076 podman[127436]: 2024-09-21 09:49:30.014627867 +0000 UTC m=+1.289480701 container died 03db3c266ab8c1d90c3a9036749ac67d7957da1fd035c90610877732cca1b524 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T09:49:30.778 INFO:tasks.rados.rados.0.smithi076.stdout:2671: done (7 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2673: done (6 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2674: done (5 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2676: done (4 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2677: done (3 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2678: done (2 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2679: done (1 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2680: done (0 left) 2024-09-21T09:49:30.779 INFO:tasks.rados.rados.0.smithi076.stdout:2681: delete oid 4 current snap is 343 2024-09-21T09:49:30.781 INFO:tasks.rados.rados.0.smithi076.stdout:2681: done (0 left) 2024-09-21T09:49:30.781 INFO:tasks.rados.rados.0.smithi076.stdout:2682: rollback oid 2 current snap is 343 2024-09-21T09:49:30.781 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 342 2024-09-21T09:49:30.781 INFO:tasks.rados.rados.0.smithi076.stdout:2683: snap_create 2024-09-21T09:49:30.785 INFO:tasks.rados.rados.0.smithi076.stdout:2682: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:49:30.785 INFO:tasks.rados.rados.0.smithi076.stdout:2682: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:49:30.785 INFO:tasks.rados.rados.0.smithi076.stdout:2682: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:49:30.785 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 483 (ObjNum 983 snap 336 seq_num 983) dirty exists 2024-09-21T09:49:30.976 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:30 smithi076 ceph-mon[106482]: osdmap e755: 8 total, 7 up, 8 in 2024-09-21T09:49:30.976 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:30 smithi076 ceph-mon[110963]: osdmap e755: 8 total, 7 up, 8 in 2024-09-21T09:49:30.977 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:30 smithi076 podman[127601]: 2024-09-21 09:49:30.62295692 +0000 UTC m=+0.599239621 container remove 03db3c266ab8c1d90c3a9036749ac67d7957da1fd035c90610877732cca1b524 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:49:30.977 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:30 smithi076 podman[127666]: 2024-09-21 09:49:30.730712729 +0000 UTC m=+0.015883221 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:49:30.977 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:30 smithi076 podman[127666]: 2024-09-21 09:49:30.857896581 +0000 UTC m=+0.143067104 container create c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T09:49:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:30 smithi191 ceph-mon[102206]: osdmap e755: 8 total, 7 up, 8 in 2024-09-21T09:49:31.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:31 smithi076 podman[127666]: 2024-09-21 09:49:31.060230391 +0000 UTC m=+0.345400877 container init c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T09:49:31.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:31 smithi076 podman[127666]: 2024-09-21 09:49:31.064973847 +0000 UTC m=+0.350144328 container start c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS) 2024-09-21T09:49:31.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-osd[127705]: -- 172.21.15.76:0/1147069411 <== mon.2 v2:172.21.15.76:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ac8983ba40 con 0x55ac8a60a000 2024-09-21T09:49:31.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:31 smithi076 bash[127666]: c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 2024-09-21T09:49:31.373 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:31 smithi076 systemd[1]: Started Ceph osd.1 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:49:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:2682: done (1 left) 2024-09-21T09:49:31.783 INFO:tasks.rados.rados.0.smithi076.stdout:2683: done (0 left) 2024-09-21T09:49:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:2684: read oid 35 snap -1 2024-09-21T09:49:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:2684: expect (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:2685: write oid 26 current snap is 344 2024-09-21T09:49:31.784 INFO:tasks.rados.rados.0.smithi076.stdout:2685: seq_num 1015 ranges {659105=652684,1862282=717015} 2024-09-21T09:49:31.790 INFO:tasks.rados.rados.0.smithi076.stdout:2685: writing smithi07685715-26 from 659105 to 1311789 tid 1 2024-09-21T09:49:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:2685: writing smithi07685715-26 from 1862282 to 2579297 tid 2 2024-09-21T09:49:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:2686: write oid 41 current snap is 344 2024-09-21T09:49:31.798 INFO:tasks.rados.rados.0.smithi076.stdout:2686: seq_num 1016 ranges {675912=727933,1920764=460673,2801290=274622} 2024-09-21T09:49:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2686: writing smithi07685715-41 from 675912 to 1403845 tid 1 2024-09-21T09:49:31.811 INFO:tasks.rados.rados.0.smithi076.stdout:2686: writing smithi07685715-41 from 1920764 to 2381437 tid 2 2024-09-21T09:49:31.816 INFO:tasks.rados.rados.0.smithi076.stdout:2686: writing smithi07685715-41 from 2801290 to 3075912 tid 3 2024-09-21T09:49:31.816 INFO:tasks.rados.rados.0.smithi076.stdout:2687: read oid 32 snap 339 2024-09-21T09:49:31.816 INFO:tasks.rados.rados.0.smithi076.stdout:2687: expect deleted 2024-09-21T09:49:31.816 INFO:tasks.rados.rados.0.smithi076.stdout:2688: write oid 19 current snap is 344 2024-09-21T09:49:31.816 INFO:tasks.rados.rados.0.smithi076.stdout:2684: expect (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:31.926 INFO:tasks.rados.rados.0.smithi076.stdout:2685: finishing write tid 1 to smithi07685715-26 2024-09-21T09:49:31.926 INFO:tasks.rados.rados.0.smithi076.stdout:2685: finishing write tid 2 to smithi07685715-26 2024-09-21T09:49:31.926 INFO:tasks.rados.rados.0.smithi076.stdout:2685: finishing write tid 3 to smithi07685715-26 2024-09-21T09:49:31.927 INFO:tasks.rados.rados.0.smithi076.stdout:2688: seq_num 1017 ranges {692719=403182,1579246=720684,2828713=264006} 2024-09-21T09:49:31.932 INFO:tasks.rados.rados.0.smithi076.stdout:2688: writing smithi07685715-19 from 692719 to 1095901 tid 1 2024-09-21T09:49:31.938 INFO:tasks.rados.rados.0.smithi076.stdout:2688: writing smithi07685715-19 from 1579246 to 2299930 tid 2 2024-09-21T09:49:31.944 INFO:tasks.rados.rados.0.smithi076.stdout:2688: writing smithi07685715-19 from 2828713 to 3092719 tid 3 2024-09-21T09:49:31.944 INFO:tasks.rados.rados.0.smithi076.stdout:2684: done (4 left) 2024-09-21T09:49:31.944 INFO:tasks.rados.rados.0.smithi076.stdout:2689: read oid 48 snap -1 2024-09-21T09:49:31.944 INFO:tasks.rados.rados.0.smithi076.stdout:2689: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2690: rollback oid 31 current snap is 344 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 320 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2685: finishing write tid 4 to smithi07685715-26 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2685: oid 26 updating version 0 to 918 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2685: oid 26 updating version 918 to 919 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2685: oid 26 updating version 919 to 920 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 920 (ObjNum 1015 snap 344 seq_num 1015) dirty exists 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2685: left oid 26 (ObjNum 1015 snap 344 seq_num 1015) 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2686: finishing write tid 1 to smithi07685715-41 2024-09-21T09:49:31.945 INFO:tasks.rados.rados.0.smithi076.stdout:2686: finishing write tid 2 to smithi07685715-41 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: finishing write tid 3 to smithi07685715-41 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: finishing write tid 4 to smithi07685715-41 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: finishing write tid 5 to smithi07685715-41 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: oid 41 updating version 0 to 1226 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: oid 41 version 1226 is already newer than 1223 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: oid 41 version 1226 is already newer than 1224 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: oid 41 version 1226 is already newer than 1225 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1226 (ObjNum 1016 snap 344 seq_num 1016) dirty exists 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: left oid 41 (ObjNum 1016 snap 344 seq_num 1016) 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2685: done (5 left) 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2686: done (4 left) 2024-09-21T09:49:31.946 INFO:tasks.rados.rados.0.smithi076.stdout:2687: done (3 left) 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2691: read oid 20 snap -1 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2691: expect (ObjNum 1002 snap 341 seq_num 1002) 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2692: read oid 40 snap 320 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2692: expect deleted 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2693: read oid 9 snap -1 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2693: expect (ObjNum 1010 snap 342 seq_num 1010) 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2688: finishing write tid 1 to smithi07685715-19 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2688: finishing write tid 2 to smithi07685715-19 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2694: read oid 37 snap 339 2024-09-21T09:49:31.947 INFO:tasks.rados.rados.0.smithi076.stdout:2694: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:49:31.948 INFO:tasks.rados.rados.0.smithi076.stdout:2695: write oid 24 current snap is 344 2024-09-21T09:49:31.948 INFO:tasks.rados.rados.0.smithi076.stdout:2695: seq_num 1018 ranges {709526=478431,1637728=464342,2856136=253390} 2024-09-21T09:49:31.950 INFO:tasks.rados.rados.0.smithi076.stdout:2695: writing smithi07685715-24 from 709526 to 1187957 tid 1 2024-09-21T09:49:31.955 INFO:tasks.rados.rados.0.smithi076.stdout:2695: writing smithi07685715-24 from 1637728 to 2102070 tid 2 2024-09-21T09:49:31.961 INFO:tasks.rados.rados.0.smithi076.stdout:2695: writing smithi07685715-24 from 2856136 to 3109526 tid 3 2024-09-21T09:49:31.961 INFO:tasks.rados.rados.0.smithi076.stdout:2696: snap_remove snap 335 2024-09-21T09:49:32.039 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: pgmap v604: 89 pgs: 3 active+undersized, 33 peering, 1 active+undersized+degraded, 52 active+clean; 167 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s; 1/345 objects degraded (0.290%) 2024-09-21T09:49:32.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 1/345 objects degraded (0.290%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:49:32.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: osdmap e756: 8 total, 7 up, 8 in 2024-09-21T09:49:32.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.040 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:31 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: pgmap v604: 89 pgs: 3 active+undersized, 33 peering, 1 active+undersized+degraded, 52 active+clean; 167 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s; 1/345 objects degraded (0.290%) 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 1/345 objects degraded (0.290%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: osdmap e756: 8 total, 7 up, 8 in 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: pgmap v604: 89 pgs: 3 active+undersized, 33 peering, 1 active+undersized+degraded, 52 active+clean; 167 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s; 1/345 objects degraded (0.290%) 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 1/345 objects degraded (0.290%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: osdmap e756: 8 total, 7 up, 8 in 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:31 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:32.790 INFO:tasks.rados.rados.0.smithi076.stdout:2696: done (8 left) 2024-09-21T09:49:32.790 INFO:tasks.rados.rados.0.smithi076.stdout:2697: read oid 18 snap -1 2024-09-21T09:49:32.790 INFO:tasks.rados.rados.0.smithi076.stdout:2697: expect deleted 2024-09-21T09:49:32.790 INFO:tasks.rados.rados.0.smithi076.stdout:2698: read oid 44 snap -1 2024-09-21T09:49:32.790 INFO:tasks.rados.rados.0.smithi076.stdout:2698: expect deleted 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2699: snap_create 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2690: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 0 (ObjNum 81 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: finishing write tid 3 to smithi07685715-19 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: finishing write tid 4 to smithi07685715-19 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: finishing write tid 5 to smithi07685715-19 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: oid 19 updating version 0 to 852 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: oid 19 version 852 is already newer than 851 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: oid 19 version 852 is already newer than 849 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: oid 19 version 852 is already newer than 850 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 852 (ObjNum 1017 snap 344 seq_num 1017) dirty exists 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2688: left oid 19 (ObjNum 1017 snap 344 seq_num 1017) 2024-09-21T09:49:32.791 INFO:tasks.rados.rados.0.smithi076.stdout:2693: expect (ObjNum 1010 snap 342 seq_num 1010) 2024-09-21T09:49:32.894 INFO:tasks.rados.rados.0.smithi076.stdout:2695: finishing write tid 1 to smithi07685715-24 2024-09-21T09:49:32.894 INFO:tasks.rados.rados.0.smithi076.stdout:2689: expect (ObjNum 858 snap 294 seq_num 858) 2024-09-21T09:49:32.936 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:32 smithi191 ceph-mon[102206]: osdmap e757: 8 total, 7 up, 8 in 2024-09-21T09:49:33.041 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:32 smithi076 ceph-mon[106482]: osdmap e757: 8 total, 7 up, 8 in 2024-09-21T09:49:33.042 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:32 smithi076 ceph-mon[110963]: osdmap e757: 8 total, 7 up, 8 in 2024-09-21T09:49:33.042 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:32 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[127682]: 2024-09-21T09:49:32.937+0000 7f844bf48640 -1 Falling back to public interface 2024-09-21T09:49:33.050 INFO:tasks.rados.rados.0.smithi076.stdout:2691: expect (ObjNum 1002 snap 341 seq_num 1002) 2024-09-21T09:49:33.178 INFO:tasks.rados.rados.0.smithi076.stdout:2695: finishing write tid 2 to smithi07685715-24 2024-09-21T09:49:33.178 INFO:tasks.rados.rados.0.smithi076.stdout:2695: finishing write tid 3 to smithi07685715-24 2024-09-21T09:49:33.178 INFO:tasks.rados.rados.0.smithi076.stdout:2695: finishing write tid 4 to smithi07685715-24 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2695: finishing write tid 5 to smithi07685715-24 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2695: oid 24 updating version 0 to 263 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2695: oid 24 updating version 263 to 264 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2695: oid 24 version 264 is already newer than 262 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2695: oid 24 version 264 is already newer than 261 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 264 (ObjNum 1018 snap 344 seq_num 1018) dirty exists 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2695: left oid 24 (ObjNum 1018 snap 344 seq_num 1018) 2024-09-21T09:49:33.179 INFO:tasks.rados.rados.0.smithi076.stdout:2694: expect (ObjNum 697 snap 224 seq_num 697) 2024-09-21T09:49:33.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:49:33] "GET /metrics HTTP/1.1" 200 36268 "" "Prometheus/2.51.0" 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2688: done (10 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2689: done (9 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2690: done (8 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2691: done (7 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2692: done (6 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2693: done (5 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2694: done (4 left) 2024-09-21T09:49:33.745 INFO:tasks.rados.rados.0.smithi076.stdout:2695: done (3 left) 2024-09-21T09:49:33.746 INFO:tasks.rados.rados.0.smithi076.stdout:2697: done (2 left) 2024-09-21T09:49:33.746 INFO:tasks.rados.rados.0.smithi076.stdout:2698: done (1 left) 2024-09-21T09:49:33.746 INFO:tasks.rados.rados.0.smithi076.stdout:2699: done (0 left) 2024-09-21T09:49:33.746 INFO:tasks.rados.rados.0.smithi076.stdout:2700: write oid 44 current snap is 345 2024-09-21T09:49:33.746 INFO:tasks.rados.rados.0.smithi076.stdout:2700: seq_num 1019 ranges {726333=670033,2096210=724353} 2024-09-21T09:49:33.752 INFO:tasks.rados.rados.0.smithi076.stdout:2700: writing smithi07685715-44 from 726333 to 1396366 tid 1 2024-09-21T09:49:33.761 INFO:tasks.rados.rados.0.smithi076.stdout:2700: writing smithi07685715-44 from 2096210 to 2820563 tid 2 2024-09-21T09:49:33.761 INFO:tasks.rados.rados.0.smithi076.stdout:2701: write oid 47 current snap is 345 2024-09-21T09:49:33.761 INFO:tasks.rados.rados.0.smithi076.stdout:2700: finishing write tid 1 to smithi07685715-44 2024-09-21T09:49:33.761 INFO:tasks.rados.rados.0.smithi076.stdout:2701: seq_num 1020 ranges {743140=745282,2154692=468011} 2024-09-21T09:49:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2701: writing smithi07685715-47 from 743140 to 1488422 tid 1 2024-09-21T09:49:33.775 INFO:tasks.rados.rados.0.smithi076.stdout:2701: writing smithi07685715-47 from 2154692 to 2622703 tid 2 2024-09-21T09:49:33.775 INFO:tasks.rados.rados.0.smithi076.stdout:2702: read oid 12 snap -1 2024-09-21T09:49:33.775 INFO:tasks.rados.rados.0.smithi076.stdout:2702: expect (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:49:33.776 INFO:tasks.rados.rados.0.smithi076.stdout:2703: write oid 14 current snap is 345 2024-09-21T09:49:33.776 INFO:tasks.rados.rados.0.smithi076.stdout:2700: finishing write tid 2 to smithi07685715-44 2024-09-21T09:49:33.776 INFO:tasks.rados.rados.0.smithi076.stdout:2703: seq_num 1021 ranges {759947=420531,1813174=728022} 2024-09-21T09:49:33.781 INFO:tasks.rados.rados.0.smithi076.stdout:2703: writing smithi07685715-14 from 759947 to 1180478 tid 1 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2703: writing smithi07685715-14 from 1813174 to 2541196 tid 2 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2704: rollback oid 16 current snap is 345 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 334 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2700: finishing write tid 3 to smithi07685715-44 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2700: finishing write tid 4 to smithi07685715-44 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2700: oid 44 updating version 0 to 659 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2700: oid 44 updating version 659 to 660 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2700: oid 44 version 660 is already newer than 658 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 660 (ObjNum 1019 snap 345 seq_num 1019) dirty exists 2024-09-21T09:49:33.790 INFO:tasks.rados.rados.0.smithi076.stdout:2700: left oid 44 (ObjNum 1019 snap 345 seq_num 1019) 2024-09-21T09:49:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:2701: finishing write tid 1 to smithi07685715-47 2024-09-21T09:49:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:2700: done (4 left) 2024-09-21T09:49:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:2705: snap_remove snap 333 2024-09-21T09:49:33.944 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:33 smithi191 ceph-mon[102206]: pgmap v607: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 182 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.7 MiB/s wr, 26 op/s; 32/378 objects degraded (8.466%) 2024-09-21T09:49:33.944 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:33 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-09-21T09:49:33.944 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:33 smithi191 ceph-mon[102206]: osdmap e758: 8 total, 7 up, 8 in 2024-09-21T09:49:33.944 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:33 smithi191 ceph-mon[102206]: osdmap e759: 8 total, 7 up, 8 in 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[110963]: pgmap v607: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 182 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.7 MiB/s wr, 26 op/s; 32/378 objects degraded (8.466%) 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[110963]: osdmap e758: 8 total, 7 up, 8 in 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[110963]: osdmap e759: 8 total, 7 up, 8 in 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[106482]: pgmap v607: 89 pgs: 29 active+undersized, 8 active+undersized+degraded, 52 active+clean; 182 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 2.7 MiB/s wr, 26 op/s; 32/378 objects degraded (8.466%) 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 8 pgs peering) 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[106482]: osdmap e758: 8 total, 7 up, 8 in 2024-09-21T09:49:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:33 smithi076 ceph-mon[106482]: osdmap e759: 8 total, 7 up, 8 in 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2705: done (4 left) 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2706: rollback oid 42 current snap is 345 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 344 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: finishing write tid 2 to smithi07685715-47 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: finishing write tid 3 to smithi07685715-47 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: finishing write tid 4 to smithi07685715-47 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: oid 47 updating version 0 to 2034 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: oid 47 updating version 2034 to 2035 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: oid 47 updating version 2035 to 2036 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2036 (ObjNum 1020 snap 345 seq_num 1020) dirty exists 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2701: left oid 47 (ObjNum 1020 snap 345 seq_num 1020) 2024-09-21T09:49:34.796 INFO:tasks.rados.rados.0.smithi076.stdout:2702: expect (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:49:34.879 INFO:tasks.rados.rados.0.smithi076.stdout:2703: finishing write tid 1 to smithi07685715-14 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2703: finishing write tid 2 to smithi07685715-14 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2704: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2704: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2701: done (4 left) 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2702: done (3 left) 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2707: read oid 2 snap -1 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2707: expect (ObjNum 983 snap 336 seq_num 983) 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2708: read oid 4 snap -1 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2708: expect deleted 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2709: rollback oid 21 current snap is 345 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 343 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2703: finishing write tid 3 to smithi07685715-14 2024-09-21T09:49:34.880 INFO:tasks.rados.rados.0.smithi076.stdout:2704: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1236 (ObjNum 974 snap 332 seq_num 974) dirty exists 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2703: finishing write tid 4 to smithi07685715-14 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2703: oid 14 updating version 0 to 1629 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2703: oid 14 updating version 1629 to 1630 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2703: oid 14 version 1630 is already newer than 1628 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1630 (ObjNum 1021 snap 345 seq_num 1021) dirty exists 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2703: left oid 14 (ObjNum 1021 snap 345 seq_num 1021) 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2706: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2706: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2706: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 708 (ObjNum 993 snap 338 seq_num 993) dirty exists 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2703: done (5 left) 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2704: done (4 left) 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2706: done (3 left) 2024-09-21T09:49:34.881 INFO:tasks.rados.rados.0.smithi076.stdout:2710: snap_remove snap 326 2024-09-21T09:49:35.798 INFO:tasks.rados.rados.0.smithi076.stdout:2710: done (3 left) 2024-09-21T09:49:35.799 INFO:tasks.rados.rados.0.smithi076.stdout:2711: read oid 17 snap 336 2024-09-21T09:49:35.799 INFO:tasks.rados.rados.0.smithi076.stdout:2711: expect (ObjNum 979 snap 336 seq_num 979) 2024-09-21T09:49:35.799 INFO:tasks.rados.rados.0.smithi076.stdout:2712: delete oid 6 current snap is 345 2024-09-21T09:49:35.799 INFO:tasks.rados.rados.0.smithi076.stdout:2707: expect (ObjNum 983 snap 336 seq_num 983) 2024-09-21T09:49:35.880 INFO:tasks.rados.rados.0.smithi076.stdout:2709: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:49:35.880 INFO:tasks.rados.rados.0.smithi076.stdout:2709: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:49:35.880 INFO:tasks.rados.rados.0.smithi076.stdout:2709: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:49:35.880 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 2067 (ObjNum 941 snap 322 seq_num 941) dirty exists 2024-09-21T09:49:35.880 INFO:tasks.rados.rados.0.smithi076.stdout:2711: expect (ObjNum 979 snap 336 seq_num 979) 2024-09-21T09:49:36.014 INFO:tasks.rados.rados.0.smithi076.stdout:2707: done (4 left) 2024-09-21T09:49:36.014 INFO:tasks.rados.rados.0.smithi076.stdout:2708: done (3 left) 2024-09-21T09:49:36.014 INFO:tasks.rados.rados.0.smithi076.stdout:2709: done (2 left) 2024-09-21T09:49:36.014 INFO:tasks.rados.rados.0.smithi076.stdout:2711: done (1 left) 2024-09-21T09:49:36.015 INFO:tasks.rados.rados.0.smithi076.stdout:2712: done (0 left) 2024-09-21T09:49:36.015 INFO:tasks.rados.rados.0.smithi076.stdout:2713: write oid 48 current snap is 345 2024-09-21T09:49:36.015 INFO:tasks.rados.rados.0.smithi076.stdout:2713: seq_num 1023 ranges {793561=571029,1813785=731691,2993251=200310} 2024-09-21T09:49:36.021 INFO:tasks.rados.rados.0.smithi076.stdout:2713: writing smithi07685715-48 from 793561 to 1364590 tid 1 2024-09-21T09:49:36.028 INFO:tasks.rados.rados.0.smithi076.stdout:2713: writing smithi07685715-48 from 1813785 to 2545476 tid 2 2024-09-21T09:49:36.031 INFO:tasks.rados.rados.0.smithi076.stdout:2713: writing smithi07685715-48 from 2993251 to 3193561 tid 3 2024-09-21T09:49:36.032 INFO:tasks.rados.rados.0.smithi076.stdout:2714: snap_remove snap 337 2024-09-21T09:49:36.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: pgmap v610: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 189 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 31 op/s; 34/384 objects degraded (8.854%) 2024-09-21T09:49:36.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: osdmap e760: 8 total, 7 up, 8 in 2024-09-21T09:49:36.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: pgmap v610: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 189 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 31 op/s; 34/384 objects degraded (8.854%) 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: osdmap e760: 8 total, 7 up, 8 in 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:35 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: pgmap v610: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 189 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.5 MiB/s wr, 31 op/s; 34/384 objects degraded (8.854%) 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: osdmap e760: 8 total, 7 up, 8 in 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:35 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2714: done (1 left) 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2715: delete oid 2 current snap is 345 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: finishing write tid 1 to smithi07685715-48 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: finishing write tid 2 to smithi07685715-48 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: finishing write tid 3 to smithi07685715-48 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: finishing write tid 4 to smithi07685715-48 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: finishing write tid 5 to smithi07685715-48 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: oid 48 updating version 0 to 2040 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: oid 48 version 2040 is already newer than 2039 2024-09-21T09:49:36.807 INFO:tasks.rados.rados.0.smithi076.stdout:2713: oid 48 updating version 2040 to 2041 2024-09-21T09:49:36.808 INFO:tasks.rados.rados.0.smithi076.stdout:2713: oid 48 version 2041 is already newer than 2038 2024-09-21T09:49:36.808 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2041 (ObjNum 1023 snap 345 seq_num 1023) dirty exists 2024-09-21T09:49:36.808 INFO:tasks.rados.rados.0.smithi076.stdout:2713: left oid 48 (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:49:36.810 INFO:tasks.rados.rados.0.smithi076.stdout:2713: done (1 left) 2024-09-21T09:49:36.810 INFO:tasks.rados.rados.0.smithi076.stdout:2715: done (0 left) 2024-09-21T09:49:36.810 INFO:tasks.rados.rados.0.smithi076.stdout:2716: read oid 38 snap 343 2024-09-21T09:49:36.810 INFO:tasks.rados.rados.0.smithi076.stdout:2716: expect deleted 2024-09-21T09:49:36.811 INFO:tasks.rados.rados.0.smithi076.stdout:2717: write oid 26 current snap is 345 2024-09-21T09:49:36.811 INFO:tasks.rados.rados.0.smithi076.stdout:2717: seq_num 1025 ranges {427175=721527,1930749=735360} 2024-09-21T09:49:36.817 INFO:tasks.rados.rados.0.smithi076.stdout:2717: writing smithi07685715-26 from 427175 to 1148702 tid 1 2024-09-21T09:49:36.826 INFO:tasks.rados.rados.0.smithi076.stdout:2717: writing smithi07685715-26 from 1930749 to 2666109 tid 2 2024-09-21T09:49:36.826 INFO:tasks.rados.rados.0.smithi076.stdout:2718: read oid 36 snap -1 2024-09-21T09:49:36.826 INFO:tasks.rados.rados.0.smithi076.stdout:2718: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:49:36.826 INFO:tasks.rados.rados.0.smithi076.stdout:2719: snap_create 2024-09-21T09:49:36.826 INFO:tasks.rados.rados.0.smithi076.stdout:2717: finishing write tid 1 to smithi07685715-26 2024-09-21T09:49:36.829 INFO:tasks.rados.rados.0.smithi076.stdout:2717: finishing write tid 2 to smithi07685715-26 2024-09-21T09:49:36.829 INFO:tasks.rados.rados.0.smithi076.stdout:2717: finishing write tid 3 to smithi07685715-26 2024-09-21T09:49:36.829 INFO:tasks.rados.rados.0.smithi076.stdout:2717: finishing write tid 4 to smithi07685715-26 2024-09-21T09:49:36.830 INFO:tasks.rados.rados.0.smithi076.stdout:2717: oid 26 updating version 0 to 924 2024-09-21T09:49:36.830 INFO:tasks.rados.rados.0.smithi076.stdout:2717: oid 26 version 924 is already newer than 923 2024-09-21T09:49:36.830 INFO:tasks.rados.rados.0.smithi076.stdout:2717: oid 26 version 924 is already newer than 922 2024-09-21T09:49:36.830 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 924 (ObjNum 1025 snap 345 seq_num 1025) dirty exists 2024-09-21T09:49:36.830 INFO:tasks.rados.rados.0.smithi076.stdout:2717: left oid 26 (ObjNum 1025 snap 345 seq_num 1025) 2024-09-21T09:49:36.836 INFO:tasks.rados.rados.0.smithi076.stdout:2718: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:49:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:36 smithi076 ceph-mon[106482]: osdmap e761: 8 total, 7 up, 8 in 2024-09-21T09:49:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:36 smithi076 ceph-mon[106482]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:49:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:36 smithi076 ceph-mon[110963]: osdmap e761: 8 total, 7 up, 8 in 2024-09-21T09:49:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:36 smithi076 ceph-mon[110963]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:49:36.873 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[127682]: 2024-09-21T09:49:36.581+0000 7f844bf48640 -1 osd.1 750 log_to_monitors true 2024-09-21T09:49:36.874 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 09:49:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[127682]: 2024-09-21T09:49:36.832+0000 7f843fcea640 -1 osd.1 750 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:49:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:36 smithi191 ceph-mon[102206]: osdmap e761: 8 total, 7 up, 8 in 2024-09-21T09:49:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:36 smithi191 ceph-mon[102206]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2716: done (3 left) 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2717: done (2 left) 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2718: done (1 left) 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2719: done (0 left) 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2720: read oid 4 snap -1 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2720: expect deleted 2024-09-21T09:49:37.822 INFO:tasks.rados.rados.0.smithi076.stdout:2721: snap_remove snap 343 2024-09-21T09:49:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:37 smithi191 ceph-mon[102206]: pgmap v613: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 203 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 30 op/s; 37/402 objects degraded (9.204%) 2024-09-21T09:49:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:37 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 37/402 objects degraded (9.204%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:37 smithi191 ceph-mon[102206]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:49:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:37 smithi191 ceph-mon[102206]: osdmap e762: 8 total, 7 up, 8 in 2024-09-21T09:49:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:37 smithi191 ceph-mon[102206]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:38.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[106482]: pgmap v613: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 203 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 30 op/s; 37/402 objects degraded (9.204%) 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 37/402 objects degraded (9.204%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[106482]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[106482]: osdmap e762: 8 total, 7 up, 8 in 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[106482]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[110963]: pgmap v613: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 203 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 30 op/s; 37/402 objects degraded (9.204%) 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 37/402 objects degraded (9.204%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[110963]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[110963]: osdmap e762: 8 total, 7 up, 8 in 2024-09-21T09:49:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:37 smithi076 ceph-mon[110963]: from='osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:49:38.826 INFO:tasks.rados.rados.0.smithi076.stdout:2721: done (1 left) 2024-09-21T09:49:38.826 INFO:tasks.rados.rados.0.smithi076.stdout:2722: snap_remove snap 334 2024-09-21T09:49:39.067 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[106482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[106482]: osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950] boot 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[106482]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[110963]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[110963]: osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950] boot 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[110963]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:49:39.068 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:49:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:38 smithi191 ceph-mon[102206]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:49:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:38 smithi191 ceph-mon[102206]: osd.1 [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950] boot 2024-09-21T09:49:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:38 smithi191 ceph-mon[102206]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T09:49:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:49:39.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:49:39.065+0000 7f86bc22c640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:49:39.828 INFO:tasks.rados.rados.0.smithi076.stdout:2722: done (1 left) 2024-09-21T09:49:39.828 INFO:tasks.rados.rados.0.smithi076.stdout:2723: write oid 16 current snap is 346 2024-09-21T09:49:39.828 INFO:tasks.rados.rados.0.smithi076.stdout:2723: seq_num 1026 ranges {443982=796776,1989231=479018,2959167=284815} 2024-09-21T09:49:39.835 INFO:tasks.rados.rados.0.smithi076.stdout:2723: writing smithi07685715-16 from 443982 to 1240758 tid 1 2024-09-21T09:49:39.843 INFO:tasks.rados.rados.0.smithi076.stdout:2723: writing smithi07685715-16 from 1989231 to 2468249 tid 2 2024-09-21T09:49:39.847 INFO:tasks.rados.rados.0.smithi076.stdout:2723: writing smithi07685715-16 from 2959167 to 3243982 tid 3 2024-09-21T09:49:39.848 INFO:tasks.rados.rados.0.smithi076.stdout:2724: snap_create 2024-09-21T09:49:39.848 INFO:tasks.rados.rados.0.smithi076.stdout:2723: finishing write tid 1 to smithi07685715-16 2024-09-21T09:49:39.848 INFO:tasks.rados.rados.0.smithi076.stdout:2723: finishing write tid 2 to smithi07685715-16 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: finishing write tid 3 to smithi07685715-16 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: finishing write tid 4 to smithi07685715-16 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: finishing write tid 5 to smithi07685715-16 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: oid 16 updating version 0 to 1267 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: oid 16 updating version 1267 to 1268 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: oid 16 version 1268 is already newer than 1264 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: oid 16 updating version 1268 to 1269 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1269 (ObjNum 1026 snap 346 seq_num 1026) dirty exists 2024-09-21T09:49:39.851 INFO:tasks.rados.rados.0.smithi076.stdout:2723: left oid 16 (ObjNum 1026 snap 346 seq_num 1026) 2024-09-21T09:49:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: pgmap v616: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 203 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 24 op/s; 37/402 objects degraded (9.204%) 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:49:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:39 smithi191 ceph-mon[102206]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: pgmap v616: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 203 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 24 op/s; 37/402 objects degraded (9.204%) 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[106482]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: pgmap v616: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 203 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 2.0 MiB/s wr, 24 op/s; 37/402 objects degraded (9.204%) 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:40.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:49:40.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:40.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:49:40.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:49:40.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:39 smithi076 ceph-mon[110963]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2720: done (2 left) 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2723: done (1 left) 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2724: done (0 left) 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2725: read oid 27 snap 338 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2725: expect (ObjNum 951 snap 324 seq_num 951) 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2726: write oid 30 current snap is 347 2024-09-21T09:49:40.844 INFO:tasks.rados.rados.0.smithi076.stdout:2726: seq_num 1027 ranges {460789=588378,1764066=739029,3219296=41493} 2024-09-21T09:49:40.849 INFO:tasks.rados.rados.0.smithi076.stdout:2726: writing smithi07685715-30 from 460789 to 1049167 tid 1 2024-09-21T09:49:40.858 INFO:tasks.rados.rados.0.smithi076.stdout:2726: writing smithi07685715-30 from 1764066 to 2503095 tid 2 2024-09-21T09:49:40.862 INFO:tasks.rados.rados.0.smithi076.stdout:2726: writing smithi07685715-30 from 3219296 to 3260789 tid 3 2024-09-21T09:49:40.862 INFO:tasks.rados.rados.0.smithi076.stdout:2727: write oid 24 current snap is 347 2024-09-21T09:49:40.863 INFO:tasks.rados.rados.0.smithi076.stdout:2727: seq_num 1028 ranges {477596=663627,1706195=482687,2614013=567041} 2024-09-21T09:49:40.868 INFO:tasks.rados.rados.0.smithi076.stdout:2727: writing smithi07685715-24 from 477596 to 1141223 tid 1 2024-09-21T09:49:40.874 INFO:tasks.rados.rados.0.smithi076.stdout:2727: writing smithi07685715-24 from 1706195 to 2188882 tid 2 2024-09-21T09:49:40.880 INFO:tasks.rados.rados.0.smithi076.stdout:2727: writing smithi07685715-24 from 2614013 to 3181054 tid 3 2024-09-21T09:49:40.880 INFO:tasks.rados.rados.0.smithi076.stdout:2728: snap_remove snap 340 2024-09-21T09:49:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:40 smithi191 ceph-mon[102206]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:49:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:41.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:40 smithi076 ceph-mon[106482]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:49:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:40 smithi076 ceph-mon[110963]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T09:49:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:41.844 INFO:tasks.rados.rados.0.smithi076.stdout:2726: finishing write tid 1 to smithi07685715-30 2024-09-21T09:49:41.844 INFO:tasks.rados.rados.0.smithi076.stdout:2725: expect (ObjNum 951 snap 324 seq_num 951) 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2726: finishing write tid 2 to smithi07685715-30 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2726: finishing write tid 3 to smithi07685715-30 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2725: done (3 left) 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2728: done (2 left) 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2729: read oid 29 snap -1 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2729: expect deleted 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2730: read oid 43 snap 339 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2730: expect (ObjNum 991 snap 338 seq_num 991) 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2731: rollback oid 16 current snap is 347 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 328 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2726: finishing write tid 4 to smithi07685715-30 2024-09-21T09:49:41.976 INFO:tasks.rados.rados.0.smithi076.stdout:2726: finishing write tid 5 to smithi07685715-30 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2726: oid 30 updating version 0 to 2052 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2726: oid 30 updating version 2052 to 2054 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2726: oid 30 version 2054 is already newer than 2051 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2726: oid 30 version 2054 is already newer than 2053 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2054 (ObjNum 1027 snap 347 seq_num 1027) dirty exists 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2726: left oid 30 (ObjNum 1027 snap 347 seq_num 1027) 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2727: finishing write tid 1 to smithi07685715-24 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2727: finishing write tid 2 to smithi07685715-24 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2727: finishing write tid 3 to smithi07685715-24 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2726: done (4 left) 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2732: snap_create 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2727: finishing write tid 4 to smithi07685715-24 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2727: finishing write tid 5 to smithi07685715-24 2024-09-21T09:49:41.977 INFO:tasks.rados.rados.0.smithi076.stdout:2727: oid 24 updating version 0 to 271 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:2727: oid 24 version 271 is already newer than 270 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:2727: oid 24 version 271 is already newer than 269 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:2727: oid 24 version 271 is already newer than 268 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 271 (ObjNum 1028 snap 347 seq_num 1028) dirty exists 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:2727: left oid 24 (ObjNum 1028 snap 347 seq_num 1028) 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:2731: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:49:41.978 INFO:tasks.rados.rados.0.smithi076.stdout:2731: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:49:41.979 INFO:tasks.rados.rados.0.smithi076.stdout:2731: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:49:41.979 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1279 (ObjNum 958 snap 326 seq_num 958) dirty exists 2024-09-21T09:49:41.979 INFO:tasks.rados.rados.0.smithi076.stdout:2730: expect (ObjNum 991 snap 338 seq_num 991) 2024-09-21T09:49:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:41 smithi191 ceph-mon[102206]: pgmap v619: 89 pgs: 3 peering, 27 active+undersized, 7 active+undersized+degraded, 52 active+clean; 203 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 2 op/s; 34/402 objects degraded (8.458%) 2024-09-21T09:49:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:41 smithi191 ceph-mon[102206]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:49:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:41 smithi076 ceph-mon[106482]: pgmap v619: 89 pgs: 3 peering, 27 active+undersized, 7 active+undersized+degraded, 52 active+clean; 203 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 2 op/s; 34/402 objects degraded (8.458%) 2024-09-21T09:49:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:41 smithi076 ceph-mon[106482]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:49:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:41 smithi076 ceph-mon[110963]: pgmap v619: 89 pgs: 3 peering, 27 active+undersized, 7 active+undersized+degraded, 52 active+clean; 203 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s rd, 2 op/s; 34/402 objects degraded (8.458%) 2024-09-21T09:49:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:41 smithi076 ceph-mon[110963]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2727: done (4 left) 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2729: done (3 left) 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2730: done (2 left) 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2731: done (1 left) 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2732: done (0 left) 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2733: rollback oid 32 current snap is 348 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 336 2024-09-21T09:49:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:2734: delete oid 43 current snap is 348 2024-09-21T09:49:42.866 INFO:tasks.rados.rados.0.smithi076.stdout:2733: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:49:42.866 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 0 (ObjNum 129 snap 0 seq_num 864373296) dirty dne 2024-09-21T09:49:42.869 INFO:tasks.rados.rados.0.smithi076.stdout:2733: done (1 left) 2024-09-21T09:49:42.869 INFO:tasks.rados.rados.0.smithi076.stdout:2734: done (0 left) 2024-09-21T09:49:42.869 INFO:tasks.rados.rados.0.smithi076.stdout:2735: read oid 8 snap -1 2024-09-21T09:49:42.869 INFO:tasks.rados.rados.0.smithi076.stdout:2735: expect (ObjNum 934 snap 317 seq_num 934) 2024-09-21T09:49:42.869 INFO:tasks.rados.rados.0.smithi076.stdout:2736: snap_create 2024-09-21T09:49:42.883 INFO:tasks.rados.rados.0.smithi076.stdout:2735: expect (ObjNum 934 snap 317 seq_num 934) 2024-09-21T09:49:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:42 smithi191 ceph-mon[102206]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:49:43.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:42 smithi076 ceph-mon[106482]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:49:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:42 smithi076 ceph-mon[110963]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T09:49:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:49:43] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.51.0" 2024-09-21T09:49:43.869 INFO:tasks.rados.rados.0.smithi076.stdout:2735: done (1 left) 2024-09-21T09:49:43.869 INFO:tasks.rados.rados.0.smithi076.stdout:2736: done (0 left) 2024-09-21T09:49:43.869 INFO:tasks.rados.rados.0.smithi076.stdout:2737: read oid 7 snap 347 2024-09-21T09:49:43.869 INFO:tasks.rados.rados.0.smithi076.stdout:2737: expect deleted 2024-09-21T09:49:43.869 INFO:tasks.rados.rados.0.smithi076.stdout:2738: write oid 17 current snap is 349 2024-09-21T09:49:43.870 INFO:tasks.rados.rados.0.smithi076.stdout:2738: seq_num 1030 ranges {511210=414125,1423159=486356,2668859=589585} 2024-09-21T09:49:43.873 INFO:tasks.rados.rados.0.smithi076.stdout:2738: writing smithi07685715-17 from 511210 to 925335 tid 1 2024-09-21T09:49:43.880 INFO:tasks.rados.rados.0.smithi076.stdout:2738: writing smithi07685715-17 from 1423159 to 1909515 tid 2 2024-09-21T09:49:43.887 INFO:tasks.rados.rados.0.smithi076.stdout:2738: writing smithi07685715-17 from 2668859 to 3258444 tid 3 2024-09-21T09:49:43.887 INFO:tasks.rados.rados.0.smithi076.stdout:2739: read oid 13 snap -1 2024-09-21T09:49:43.887 INFO:tasks.rados.rados.0.smithi076.stdout:2739: expect (ObjNum 1011 snap 342 seq_num 1011) 2024-09-21T09:49:43.888 INFO:tasks.rados.rados.0.smithi076.stdout:2740: read oid 40 snap -1 2024-09-21T09:49:43.888 INFO:tasks.rados.rados.0.smithi076.stdout:2740: expect deleted 2024-09-21T09:49:43.888 INFO:tasks.rados.rados.0.smithi076.stdout:2741: write oid 11 current snap is 349 2024-09-21T09:49:43.888 INFO:tasks.rados.rados.0.smithi076.stdout:2741: seq_num 1031 ranges {528017=489374,1481641=630014,2696282=400857} 2024-09-21T09:49:43.893 INFO:tasks.rados.rados.0.smithi076.stdout:2741: writing smithi07685715-11 from 528017 to 1017391 tid 1 2024-09-21T09:49:43.899 INFO:tasks.rados.rados.0.smithi076.stdout:2741: writing smithi07685715-11 from 1481641 to 2111655 tid 2 2024-09-21T09:49:43.913 INFO:tasks.rados.rados.0.smithi076.stdout:2741: writing smithi07685715-11 from 2696282 to 3097139 tid 3 2024-09-21T09:49:43.913 INFO:tasks.rados.rados.0.smithi076.stdout:2742: read oid 34 snap -1 2024-09-21T09:49:43.913 INFO:tasks.rados.rados.0.smithi076.stdout:2742: expect (ObjNum 975 snap 332 seq_num 975) 2024-09-21T09:49:43.913 INFO:tasks.rados.rados.0.smithi076.stdout:2743: read oid 35 snap 336 2024-09-21T09:49:43.914 INFO:tasks.rados.rados.0.smithi076.stdout:2743: expect (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:43.914 INFO:tasks.rados.rados.0.smithi076.stdout:2744: write oid 19 current snap is 349 2024-09-21T09:49:43.914 INFO:tasks.rados.rados.0.smithi076.stdout:2738: finishing write tid 1 to smithi07685715-17 2024-09-21T09:49:43.914 INFO:tasks.rados.rados.0.smithi076.stdout:2738: finishing write tid 2 to smithi07685715-17 2024-09-21T09:49:43.914 INFO:tasks.rados.rados.0.smithi076.stdout:2744: seq_num 1032 ranges {544824=564623,1823770=490025,3007352=337472} 2024-09-21T09:49:43.914 INFO:tasks.rados.rados.0.smithi076.stdout:2744: writing smithi07685715-19 from 544824 to 1109447 tid 1 2024-09-21T09:49:43.917 INFO:tasks.rados.rados.0.smithi076.stdout:2744: writing smithi07685715-19 from 1823770 to 2313795 tid 2 2024-09-21T09:49:43.923 INFO:tasks.rados.rados.0.smithi076.stdout:2744: writing smithi07685715-19 from 3007352 to 3344824 tid 3 2024-09-21T09:49:43.923 INFO:tasks.rados.rados.0.smithi076.stdout:2737: done (7 left) 2024-09-21T09:49:43.924 INFO:tasks.rados.rados.0.smithi076.stdout:2745: snap_remove snap 345 2024-09-21T09:49:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:43 smithi191 ceph-mon[102206]: pgmap v622: 89 pgs: 3 peering, 86 active+clean; 206 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 3.6 MiB/s, 2 objects/s recovering 2024-09-21T09:49:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:43 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/402 objects degraded (8.458%), 7 pgs degraded) 2024-09-21T09:49:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:43 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T09:49:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:43 smithi191 ceph-mon[102206]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:49:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[106482]: pgmap v622: 89 pgs: 3 peering, 86 active+clean; 206 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 3.6 MiB/s, 2 objects/s recovering 2024-09-21T09:49:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/402 objects degraded (8.458%), 7 pgs degraded) 2024-09-21T09:49:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T09:49:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[106482]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:49:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[110963]: pgmap v622: 89 pgs: 3 peering, 86 active+clean; 206 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.0 MiB/s wr, 10 op/s; 3.6 MiB/s, 2 objects/s recovering 2024-09-21T09:49:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/402 objects degraded (8.458%), 7 pgs degraded) 2024-09-21T09:49:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T09:49:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:43 smithi076 ceph-mon[110963]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T09:49:44.879 INFO:tasks.rados.rados.0.smithi076.stdout:2745: done (7 left) 2024-09-21T09:49:44.879 INFO:tasks.rados.rados.0.smithi076.stdout:2746: write oid 43 current snap is 349 2024-09-21T09:49:44.879 INFO:tasks.rados.rados.0.smithi076.stdout:2746: seq_num 1033 ranges {561631=639872,1882252=633683,3034775=326856} 2024-09-21T09:49:44.885 INFO:tasks.rados.rados.0.smithi076.stdout:2746: writing smithi07685715-43 from 561631 to 1201503 tid 1 2024-09-21T09:49:44.892 INFO:tasks.rados.rados.0.smithi076.stdout:2746: writing smithi07685715-43 from 1882252 to 2515935 tid 2 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2746: writing smithi07685715-43 from 3034775 to 3361631 tid 3 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2747: rollback oid 2 current snap is 349 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 348 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2738: finishing write tid 3 to smithi07685715-17 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2738: finishing write tid 4 to smithi07685715-17 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2738: finishing write tid 5 to smithi07685715-17 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2738: oid 17 updating version 0 to 1667 2024-09-21T09:49:44.898 INFO:tasks.rados.rados.0.smithi076.stdout:2738: oid 17 updating version 1667 to 1668 2024-09-21T09:49:44.899 INFO:tasks.rados.rados.0.smithi076.stdout:2738: oid 17 updating version 1668 to 1669 2024-09-21T09:49:44.899 INFO:tasks.rados.rados.0.smithi076.stdout:2738: oid 17 updating version 1669 to 1670 2024-09-21T09:49:44.899 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1670 (ObjNum 1030 snap 349 seq_num 1030) dirty exists 2024-09-21T09:49:44.899 INFO:tasks.rados.rados.0.smithi076.stdout:2738: left oid 17 (ObjNum 1030 snap 349 seq_num 1030) 2024-09-21T09:49:44.899 INFO:tasks.rados.rados.0.smithi076.stdout:2739: expect (ObjNum 1011 snap 342 seq_num 1011) 2024-09-21T09:49:45.073 INFO:tasks.rados.rados.0.smithi076.stdout:2741: finishing write tid 1 to smithi07685715-11 2024-09-21T09:49:45.073 INFO:tasks.rados.rados.0.smithi076.stdout:2741: finishing write tid 2 to smithi07685715-11 2024-09-21T09:49:45.073 INFO:tasks.rados.rados.0.smithi076.stdout:2738: done (8 left) 2024-09-21T09:49:45.073 INFO:tasks.rados.rados.0.smithi076.stdout:2739: done (7 left) 2024-09-21T09:49:45.073 INFO:tasks.rados.rados.0.smithi076.stdout:2740: done (6 left) 2024-09-21T09:49:45.073 INFO:tasks.rados.rados.0.smithi076.stdout:2748: read oid 36 snap -1 2024-09-21T09:49:45.074 INFO:tasks.rados.rados.0.smithi076.stdout:2748: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:49:45.074 INFO:tasks.rados.rados.0.smithi076.stdout:2749: snap_remove snap 320 2024-09-21T09:49:45.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:44 smithi191 ceph-mon[102206]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:49:45.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:44 smithi076 ceph-mon[106482]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:49:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:44 smithi076 ceph-mon[110963]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T09:49:45.882 INFO:tasks.rados.rados.0.smithi076.stdout:2749: done (7 left) 2024-09-21T09:49:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:2750: delete oid 31 current snap is 349 2024-09-21T09:49:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:2741: finishing write tid 3 to smithi07685715-11 2024-09-21T09:49:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:2742: expect (ObjNum 975 snap 332 seq_num 975) 2024-09-21T09:49:46.043 INFO:tasks.rados.rados.0.smithi076.stdout:2743: expect (ObjNum 978 snap 336 seq_num 978) 2024-09-21T09:49:46.149 INFO:tasks.rados.rados.0.smithi076.stdout:2741: finishing write tid 4 to smithi07685715-11 2024-09-21T09:49:46.149 INFO:tasks.rados.rados.0.smithi076.stdout:2741: finishing write tid 5 to smithi07685715-11 2024-09-21T09:49:46.149 INFO:tasks.rados.rados.0.smithi076.stdout:2741: oid 11 updating version 0 to 2101 2024-09-21T09:49:46.149 INFO:tasks.rados.rados.0.smithi076.stdout:2741: oid 11 version 2101 is already newer than 2100 2024-09-21T09:49:46.149 INFO:tasks.rados.rados.0.smithi076.stdout:2741: oid 11 updating version 2101 to 2102 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2741: oid 11 updating version 2102 to 2103 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2103 (ObjNum 1031 snap 349 seq_num 1031) dirty exists 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2741: left oid 11 (ObjNum 1031 snap 349 seq_num 1031) 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2744: finishing write tid 1 to smithi07685715-19 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2744: finishing write tid 2 to smithi07685715-19 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2741: done (7 left) 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2742: done (6 left) 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2743: done (5 left) 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2750: done (4 left) 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2751: write oid 41 current snap is 349 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2744: finishing write tid 3 to smithi07685715-19 2024-09-21T09:49:46.150 INFO:tasks.rados.rados.0.smithi076.stdout:2744: finishing write tid 4 to smithi07685715-19 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2744: finishing write tid 5 to smithi07685715-19 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2744: oid 19 updating version 0 to 855 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2744: oid 19 updating version 855 to 856 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2744: oid 19 updating version 856 to 857 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2744: oid 19 version 857 is already newer than 854 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 857 (ObjNum 1032 snap 349 seq_num 1032) dirty exists 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2744: left oid 19 (ObjNum 1032 snap 349 seq_num 1032) 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2746: finishing write tid 1 to smithi07685715-43 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2746: finishing write tid 2 to smithi07685715-43 2024-09-21T09:49:46.151 INFO:tasks.rados.rados.0.smithi076.stdout:2751: seq_num 1035 ranges {595245=506723,1715569=637352,2805974=562298} 2024-09-21T09:49:46.154 INFO:tasks.rados.rados.0.smithi076.stdout:2751: writing smithi07685715-41 from 595245 to 1101968 tid 1 2024-09-21T09:49:46.161 INFO:tasks.rados.rados.0.smithi076.stdout:2751: writing smithi07685715-41 from 1715569 to 2352921 tid 2 2024-09-21T09:49:46.167 INFO:tasks.rados.rados.0.smithi076.stdout:2751: writing smithi07685715-41 from 2805974 to 3368272 tid 3 2024-09-21T09:49:46.167 INFO:tasks.rados.rados.0.smithi076.stdout:2744: done (4 left) 2024-09-21T09:49:46.167 INFO:tasks.rados.rados.0.smithi076.stdout:2752: write oid 49 current snap is 349 2024-09-21T09:49:46.167 INFO:tasks.rados.rados.0.smithi076.stdout:2752: seq_num 1036 ranges {612052=581972,1657698=497363,2717044=695008} 2024-09-21T09:49:46.173 INFO:tasks.rados.rados.0.smithi076.stdout:2752: writing smithi07685715-49 from 612052 to 1194024 tid 1 2024-09-21T09:49:46.179 INFO:tasks.rados.rados.0.smithi076.stdout:2752: writing smithi07685715-49 from 1657698 to 2155061 tid 2 2024-09-21T09:49:46.186 INFO:tasks.rados.rados.0.smithi076.stdout:2752: writing smithi07685715-49 from 2717044 to 3412052 tid 3 2024-09-21T09:49:46.186 INFO:tasks.rados.rados.0.smithi076.stdout:2753: delete oid 9 current snap is 349 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: finishing write tid 3 to smithi07685715-43 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: finishing write tid 4 to smithi07685715-43 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: finishing write tid 5 to smithi07685715-43 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: oid 43 updating version 0 to 624 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: oid 43 updating version 624 to 625 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: oid 43 version 625 is already newer than 623 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: oid 43 updating version 625 to 626 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 626 (ObjNum 1033 snap 349 seq_num 1033) dirty exists 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2746: left oid 43 (ObjNum 1033 snap 349 seq_num 1033) 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:2747: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:49:46.187 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 493 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:46.188 INFO:tasks.rados.rados.0.smithi076.stdout:2748: expect (ObjNum 887 snap 311 seq_num 887) 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: finishing write tid 1 to smithi07685715-41 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: finishing write tid 2 to smithi07685715-41 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: finishing write tid 3 to smithi07685715-41 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: finishing write tid 4 to smithi07685715-41 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: finishing write tid 5 to smithi07685715-41 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: oid 41 updating version 0 to 1287 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: oid 41 updating version 1287 to 1288 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: oid 41 version 1288 is already newer than 1286 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:2751: oid 41 version 1288 is already newer than 1285 2024-09-21T09:49:46.343 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1288 (ObjNum 1035 snap 349 seq_num 1035) dirty exists 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2751: left oid 41 (ObjNum 1035 snap 349 seq_num 1035) 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: finishing write tid 1 to smithi07685715-49 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: finishing write tid 2 to smithi07685715-49 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: finishing write tid 3 to smithi07685715-49 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: finishing write tid 4 to smithi07685715-49 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: finishing write tid 5 to smithi07685715-49 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: oid 49 updating version 0 to 2069 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: oid 49 updating version 2069 to 2071 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: oid 49 version 2071 is already newer than 2068 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:2752: oid 49 version 2071 is already newer than 2070 2024-09-21T09:49:46.344 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2071 (ObjNum 1036 snap 349 seq_num 1036) dirty exists 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2752: left oid 49 (ObjNum 1036 snap 349 seq_num 1036) 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2746: done (5 left) 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2747: done (4 left) 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2748: done (3 left) 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2751: done (2 left) 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2752: done (1 left) 2024-09-21T09:49:46.345 INFO:tasks.rados.rados.0.smithi076.stdout:2753: done (0 left) 2024-09-21T09:49:46.346 INFO:tasks.rados.rados.0.smithi076.stdout:2754: delete oid 17 current snap is 349 2024-09-21T09:49:46.346 INFO:tasks.rados.rados.0.smithi076.stdout:2754: done (0 left) 2024-09-21T09:49:46.347 INFO:tasks.rados.rados.0.smithi076.stdout:2755: snap_remove snap 348 2024-09-21T09:49:46.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:45 smithi191 ceph-mon[102206]: pgmap v625: 89 pgs: 89 active+clean; 208 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 4.1 MiB/s, 3 objects/s recovering 2024-09-21T09:49:46.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:45 smithi191 ceph-mon[102206]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:49:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:45 smithi076 ceph-mon[106482]: pgmap v625: 89 pgs: 89 active+clean; 208 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 4.1 MiB/s, 3 objects/s recovering 2024-09-21T09:49:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:45 smithi076 ceph-mon[106482]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:49:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:45 smithi076 ceph-mon[110963]: pgmap v625: 89 pgs: 89 active+clean; 208 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s; 4.1 MiB/s, 3 objects/s recovering 2024-09-21T09:49:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:45 smithi076 ceph-mon[110963]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T09:49:46.891 INFO:tasks.rados.rados.0.smithi076.stdout:2755: done (0 left) 2024-09-21T09:49:46.891 INFO:tasks.rados.rados.0.smithi076.stdout:2756: snap_remove snap 342 2024-09-21T09:49:47.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:46 smithi191 ceph-mon[102206]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:49:47.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:46 smithi076 ceph-mon[106482]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:49:47.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:46 smithi076 ceph-mon[110963]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T09:49:47.894 INFO:tasks.rados.rados.0.smithi076.stdout:2756: done (0 left) 2024-09-21T09:49:47.895 INFO:tasks.rados.rados.0.smithi076.stdout:2757: snap_create 2024-09-21T09:49:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:47 smithi191 ceph-mon[102206]: pgmap v628: 89 pgs: 89 active+clean; 190 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.9 MiB/s wr, 15 op/s; 556 KiB/s, 0 objects/s recovering 2024-09-21T09:49:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:47 smithi191 ceph-mon[102206]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:49:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:47 smithi076 ceph-mon[106482]: pgmap v628: 89 pgs: 89 active+clean; 190 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.9 MiB/s wr, 15 op/s; 556 KiB/s, 0 objects/s recovering 2024-09-21T09:49:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:47 smithi076 ceph-mon[106482]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:49:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:47 smithi076 ceph-mon[110963]: pgmap v628: 89 pgs: 89 active+clean; 190 MiB data, 734 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 1.9 MiB/s wr, 15 op/s; 556 KiB/s, 0 objects/s recovering 2024-09-21T09:49:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:47 smithi076 ceph-mon[110963]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T09:49:48.757 INFO:tasks.rados.rados.0.smithi076.stdout:2757: done (0 left) 2024-09-21T09:49:48.757 INFO:tasks.rados.rados.0.smithi076.stdout:2758: read oid 24 snap -1 2024-09-21T09:49:48.757 INFO:tasks.rados.rados.0.smithi076.stdout:2758: expect (ObjNum 1028 snap 347 seq_num 1028) 2024-09-21T09:49:48.757 INFO:tasks.rados.rados.0.smithi076.stdout:2759: snap_remove snap 328 2024-09-21T09:49:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:48 smithi191 ceph-mon[102206]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:49:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:48 smithi191 ceph-mon[102206]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:49:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:48 smithi076 ceph-mon[106482]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:49:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:48 smithi076 ceph-mon[106482]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:49:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:48 smithi076 ceph-mon[110963]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T09:49:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:48 smithi076 ceph-mon[110963]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T09:49:49.754 INFO:tasks.rados.rados.0.smithi076.stdout:2759: done (1 left) 2024-09-21T09:49:49.754 INFO:tasks.rados.rados.0.smithi076.stdout:2760: write oid 41 current snap is 350 2024-09-21T09:49:49.754 INFO:tasks.rados.rados.0.smithi076.stdout:2760: seq_num 1039 ranges {662473=407719,1833144=644690,3199313=263160} 2024-09-21T09:49:49.760 INFO:tasks.rados.rados.0.smithi076.stdout:2760: writing smithi07685715-41 from 662473 to 1070192 tid 1 2024-09-21T09:49:49.769 INFO:tasks.rados.rados.0.smithi076.stdout:2760: writing smithi07685715-41 from 1833144 to 2477834 tid 2 2024-09-21T09:49:49.774 INFO:tasks.rados.rados.0.smithi076.stdout:2760: writing smithi07685715-41 from 3199313 to 3462473 tid 3 2024-09-21T09:49:49.774 INFO:tasks.rados.rados.0.smithi076.stdout:2761: delete oid 8 current snap is 350 2024-09-21T09:49:49.774 INFO:tasks.rados.rados.0.smithi076.stdout:2758: expect (ObjNum 1028 snap 347 seq_num 1028) 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: finishing write tid 1 to smithi07685715-41 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: finishing write tid 2 to smithi07685715-41 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: finishing write tid 3 to smithi07685715-41 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: finishing write tid 4 to smithi07685715-41 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: finishing write tid 5 to smithi07685715-41 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: oid 41 updating version 0 to 1305 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: oid 41 version 1305 is already newer than 1303 2024-09-21T09:49:49.961 INFO:tasks.rados.rados.0.smithi076.stdout:2760: oid 41 version 1305 is already newer than 1300 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2760: oid 41 version 1305 is already newer than 1304 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1305 (ObjNum 1039 snap 350 seq_num 1039) dirty exists 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2760: left oid 41 (ObjNum 1039 snap 350 seq_num 1039) 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2758: done (2 left) 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2760: done (1 left) 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2761: done (0 left) 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2762: read oid 41 snap -1 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2762: expect (ObjNum 1039 snap 350 seq_num 1039) 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2763: read oid 4 snap -1 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2763: expect deleted 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2764: rollback oid 45 current snap is 350 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 45 to 347 2024-09-21T09:49:49.962 INFO:tasks.rados.rados.0.smithi076.stdout:2765: snap_create 2024-09-21T09:49:49.966 INFO:tasks.rados.rados.0.smithi076.stdout:2764: finishing rollback tid 0 to smithi07685715-45 2024-09-21T09:49:49.966 INFO:tasks.rados.rados.0.smithi076.stdout:2764: finishing rollback tid 1 to smithi07685715-45 2024-09-21T09:49:49.966 INFO:tasks.rados.rados.0.smithi076.stdout:2764: finishing rollback tid 2 to smithi07685715-45 2024-09-21T09:49:49.966 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 623 (ObjNum 970 snap 327 seq_num 970) dirty exists 2024-09-21T09:49:49.972 INFO:tasks.rados.rados.0.smithi076.stdout:2762: expect (ObjNum 1039 snap 350 seq_num 1039) 2024-09-21T09:49:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:49 smithi191 ceph-mon[102206]: pgmap v631: 89 pgs: 89 active+clean; 190 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:49:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:49 smithi191 ceph-mon[102206]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:49:50.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:49 smithi076 ceph-mon[106482]: pgmap v631: 89 pgs: 89 active+clean; 190 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:49:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:49 smithi076 ceph-mon[106482]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:49:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:49 smithi076 ceph-mon[110963]: pgmap v631: 89 pgs: 89 active+clean; 190 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T09:49:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:49 smithi076 ceph-mon[110963]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T09:49:50.762 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:49:50.918 INFO:tasks.rados.rados.0.smithi076.stdout:2762: done (3 left) 2024-09-21T09:49:50.918 INFO:tasks.rados.rados.0.smithi076.stdout:2763: done (2 left) 2024-09-21T09:49:50.918 INFO:tasks.rados.rados.0.smithi076.stdout:2764: done (1 left) 2024-09-21T09:49:50.918 INFO:tasks.rados.rados.0.smithi076.stdout:2765: done (0 left) 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2766: read oid 42 snap -1 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2766: expect (ObjNum 993 snap 338 seq_num 993) 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2767: read oid 29 snap -1 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2767: expect deleted 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2768: read oid 21 snap -1 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2768: expect (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2769: read oid 45 snap -1 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2769: expect (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2770: rollback oid 34 current snap is 351 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 350 2024-09-21T09:49:50.919 INFO:tasks.rados.rados.0.smithi076.stdout:2771: snap_remove snap 346 2024-09-21T09:49:51.339 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:49:51.339 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (8m) 16s ago 14m 19.6M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:49:51.339 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (8m) 15s ago 14m 79.7M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (13m) 16s ago 13m 23.1M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (13m) 15s ago 13m 18.3M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (9m) 15s ago 18m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (9m) 16s ago 19m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (7m) 16s ago 19m 73.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (5m) 15s ago 18m 46.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 bb6de71151ea 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (6m) 16s ago 18m 57.4M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (9m) 16s ago 13m 11.0M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (9m) 15s ago 13m 11.7M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (53s) 16s ago 17m 66.1M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 f0b24f5d8966 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (20s) 16s ago 17m 12.0M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c1218f57993c 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (17m) 16s ago 17m 120M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e ab907ee95e4b 2024-09-21T09:49:51.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (16m) 16s ago 16m 91.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:49:51.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (16m) 15s ago 16m 118M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:49:51.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (15m) 15s ago 15m 134M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:49:51.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (15m) 15s ago 15m 95.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:49:51.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (15m) 15s ago 15m 94.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:49:51.341 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (8m) 15s ago 14m 48.1M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[106482]: pgmap v634: 89 pgs: 89 active+clean; 179 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 755 KiB/s wr, 9 op/s 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[106482]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[106482]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[106482]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[110963]: pgmap v634: 89 pgs: 89 active+clean; 179 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 755 KiB/s wr, 9 op/s 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[110963]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[110963]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:49:51.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:51 smithi076 ceph-mon[110963]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 6, 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:49:51.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 8, 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 7 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:49:51.683 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:49:51.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:51 smithi191 ceph-mon[102206]: pgmap v634: 89 pgs: 89 active+clean; 179 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 6.9 MiB/s rd, 755 KiB/s wr, 9 op/s 2024-09-21T09:49:51.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:51 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:51.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:51 smithi191 ceph-mon[102206]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:51.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:51 smithi191 ceph-mon[102206]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T09:49:51.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:51 smithi191 ceph-mon[102206]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:51.919 INFO:tasks.rados.rados.0.smithi076.stdout:2771: done (5 left) 2024-09-21T09:49:51.919 INFO:tasks.rados.rados.0.smithi076.stdout:2772: read oid 28 snap -1 2024-09-21T09:49:51.919 INFO:tasks.rados.rados.0.smithi076.stdout:2772: expect deleted 2024-09-21T09:49:51.919 INFO:tasks.rados.rados.0.smithi076.stdout:2773: write oid 41 current snap is 351 2024-09-21T09:49:51.919 INFO:tasks.rados.rados.0.smithi076.stdout:2773: seq_num 1041 ranges {696087=558217,1833755=648359,3137806=358281} 2024-09-21T09:49:51.926 INFO:tasks.rados.rados.0.smithi076.stdout:2773: writing smithi07685715-41 from 696087 to 1254304 tid 1 2024-09-21T09:49:51.933 INFO:tasks.rados.rados.0.smithi076.stdout:2773: writing smithi07685715-41 from 1833755 to 2482114 tid 2 2024-09-21T09:49:51.939 INFO:tasks.rados.rados.0.smithi076.stdout:2773: writing smithi07685715-41 from 3137806 to 3496087 tid 3 2024-09-21T09:49:51.939 INFO:tasks.rados.rados.0.smithi076.stdout:2767: done (6 left) 2024-09-21T09:49:51.939 INFO:tasks.rados.rados.0.smithi076.stdout:2774: write oid 17 current snap is 351 2024-09-21T09:49:51.939 INFO:tasks.rados.rados.0.smithi076.stdout:2770: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:49:51.939 INFO:tasks.rados.rados.0.smithi076.stdout:2774: seq_num 1042 ranges {712894=749819,2008590=792017,3281582=231312} 2024-09-21T09:49:51.947 INFO:tasks.rados.rados.0.smithi076.stdout:2774: writing smithi07685715-17 from 712894 to 1462713 tid 1 2024-09-21T09:49:51.955 INFO:tasks.rados.rados.0.smithi076.stdout:2774: writing smithi07685715-17 from 2008590 to 2800607 tid 2 2024-09-21T09:49:51.959 INFO:tasks.rados.rados.0.smithi076.stdout:2774: writing smithi07685715-17 from 3281582 to 3512894 tid 3 2024-09-21T09:49:51.959 INFO:tasks.rados.rados.0.smithi076.stdout:2775: rollback oid 14 current snap is 351 2024-09-21T09:49:51.959 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 339 2024-09-21T09:49:51.959 INFO:tasks.rados.rados.0.smithi076.stdout:2770: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:49:51.959 INFO:tasks.rados.rados.0.smithi076.stdout:2770: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:49:51.959 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 640 (ObjNum 975 snap 332 seq_num 975) dirty exists 2024-09-21T09:49:51.960 INFO:tasks.rados.rados.0.smithi076.stdout:2769: expect (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:49:51.981 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:49:51.981 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:49:51.981 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:49:51.981 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-21T09:49:51.982 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:49:51.982 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "2/8 daemons upgraded", 2024-09-21T09:49:51.982 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:49:51.982 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:49:51.982 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:49:52.058 INFO:tasks.rados.rados.0.smithi076.stdout:2766: expect (ObjNum 993 snap 338 seq_num 993) 2024-09-21T09:49:52.144 INFO:tasks.rados.rados.0.smithi076.stdout:2768: expect (ObjNum 941 snap 322 seq_num 941) 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: finishing write tid 1 to smithi07685715-41 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: finishing write tid 2 to smithi07685715-41 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: finishing write tid 3 to smithi07685715-41 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: finishing write tid 4 to smithi07685715-41 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: finishing write tid 5 to smithi07685715-41 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: oid 41 updating version 0 to 1307 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: oid 41 updating version 1307 to 1313 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: oid 41 version 1313 is already newer than 1312 2024-09-21T09:49:52.415 INFO:tasks.rados.rados.0.smithi076.stdout:2773: oid 41 updating version 1313 to 1314 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1314 (ObjNum 1041 snap 351 seq_num 1041) dirty exists 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2773: left oid 41 (ObjNum 1041 snap 351 seq_num 1041) 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: finishing write tid 1 to smithi07685715-17 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: finishing write tid 2 to smithi07685715-17 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: finishing write tid 3 to smithi07685715-17 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: finishing write tid 4 to smithi07685715-17 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: finishing write tid 5 to smithi07685715-17 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: oid 17 updating version 0 to 1693 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: oid 17 version 1693 is already newer than 1690 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: oid 17 version 1693 is already newer than 1692 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: oid 17 version 1693 is already newer than 1691 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1693 (ObjNum 1042 snap 351 seq_num 1042) dirty exists 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2774: left oid 17 (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2775: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2775: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:49:52.416 INFO:tasks.rados.rados.0.smithi076.stdout:2775: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1697 (ObjNum 925 snap 317 seq_num 925) dirty exists 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2766: done (7 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2768: done (6 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2769: done (5 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2770: done (4 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2772: done (3 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2773: done (2 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2774: done (1 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2775: done (0 left) 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2776: rollback oid 22 current snap is 351 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 350 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2777: read oid 11 snap -1 2024-09-21T09:49:52.417 INFO:tasks.rados.rados.0.smithi076.stdout:2777: expect (ObjNum 1031 snap 349 seq_num 1031) 2024-09-21T09:49:52.418 INFO:tasks.rados.rados.0.smithi076.stdout:2778: write oid 2 current snap is 351 2024-09-21T09:49:52.418 INFO:tasks.rados.rados.0.smithi076.stdout:2778: seq_num 1043 ranges {729701=425068,1667072=652028,2909005=485180} 2024-09-21T09:49:52.420 INFO:tasks.rados.rados.0.smithi076.stdout:2778: writing smithi07685715-2 from 729701 to 1154769 tid 1 2024-09-21T09:49:52.427 INFO:tasks.rados.rados.0.smithi076.stdout:2778: writing smithi07685715-2 from 1667072 to 2319100 tid 2 2024-09-21T09:49:52.433 INFO:tasks.rados.rados.0.smithi076.stdout:2778: writing smithi07685715-2 from 2909005 to 3394185 tid 3 2024-09-21T09:49:52.433 INFO:tasks.rados.rados.0.smithi076.stdout:2779: read oid 20 snap -1 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:2779: expect (ObjNum 1002 snap 341 seq_num 1002) 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:2780: write oid 4 current snap is 351 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:2776: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:2776: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:2776: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1318 (ObjNum 897 snap 312 seq_num 897) dirty exists 2024-09-21T09:49:52.434 INFO:tasks.rados.rados.0.smithi076.stdout:2777: expect (ObjNum 1031 snap 349 seq_num 1031) 2024-09-21T09:49:52.541 INFO:tasks.rados.rados.0.smithi076.stdout:2778: finishing write tid 1 to smithi07685715-2 2024-09-21T09:49:52.541 INFO:tasks.rados.rados.0.smithi076.stdout:2778: finishing write tid 2 to smithi07685715-2 2024-09-21T09:49:52.541 INFO:tasks.rados.rados.0.smithi076.stdout:2780: seq_num 1044 ranges {746508=500317,2009201=795686,3220075=326433} 2024-09-21T09:49:52.547 INFO:tasks.rados.rados.0.smithi076.stdout:2780: writing smithi07685715-4 from 746508 to 1246825 tid 1 2024-09-21T09:49:52.555 INFO:tasks.rados.rados.0.smithi076.stdout:2780: writing smithi07685715-4 from 2009201 to 2804887 tid 2 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2780: writing smithi07685715-4 from 3220075 to 3546508 tid 3 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2776: done (4 left) 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2777: done (3 left) 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2781: write oid 28 current snap is 351 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: finishing write tid 3 to smithi07685715-2 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: finishing write tid 4 to smithi07685715-2 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: finishing write tid 5 to smithi07685715-2 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: oid 2 updating version 0 to 512 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: oid 2 version 512 is already newer than 510 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: oid 2 version 512 is already newer than 511 2024-09-21T09:49:52.560 INFO:tasks.rados.rados.0.smithi076.stdout:2778: oid 2 version 512 is already newer than 509 2024-09-21T09:49:52.561 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 512 (ObjNum 1043 snap 351 seq_num 1043) dirty exists 2024-09-21T09:49:52.561 INFO:tasks.rados.rados.0.smithi076.stdout:2778: left oid 2 (ObjNum 1043 snap 351 seq_num 1043) 2024-09-21T09:49:52.561 INFO:tasks.rados.rados.0.smithi076.stdout:2779: expect (ObjNum 1002 snap 341 seq_num 1002) 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: finishing write tid 1 to smithi07685715-4 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: finishing write tid 2 to smithi07685715-4 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: finishing write tid 3 to smithi07685715-4 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: finishing write tid 4 to smithi07685715-4 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: finishing write tid 5 to smithi07685715-4 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: oid 4 updating version 0 to 875 2024-09-21T09:49:52.686 INFO:tasks.rados.rados.0.smithi076.stdout:2780: oid 4 version 875 is already newer than 873 2024-09-21T09:49:52.687 INFO:tasks.rados.rados.0.smithi076.stdout:2780: oid 4 updating version 875 to 876 2024-09-21T09:49:52.687 INFO:tasks.rados.rados.0.smithi076.stdout:2780: oid 4 version 876 is already newer than 874 2024-09-21T09:49:52.687 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 876 (ObjNum 1044 snap 351 seq_num 1044) dirty exists 2024-09-21T09:49:52.687 INFO:tasks.rados.rados.0.smithi076.stdout:2780: left oid 4 (ObjNum 1044 snap 351 seq_num 1044) 2024-09-21T09:49:52.687 INFO:tasks.rados.rados.0.smithi076.stdout:2781: seq_num 1045 ranges {763315=575566,2067683=655697,3247498=315817} 2024-09-21T09:49:52.693 INFO:tasks.rados.rados.0.smithi076.stdout:2781: writing smithi07685715-28 from 763315 to 1338881 tid 1 2024-09-21T09:49:52.700 INFO:tasks.rados.rados.0.smithi076.stdout:2781: writing smithi07685715-28 from 2067683 to 2723380 tid 2 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2781: writing smithi07685715-28 from 3247498 to 3563315 tid 3 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2778: done (3 left) 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2779: done (2 left) 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2780: done (1 left) 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2782: read oid 25 snap -1 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2782: expect (ObjNum 1008 snap 342 seq_num 1008) 2024-09-21T09:49:52.706 INFO:tasks.rados.rados.0.smithi076.stdout:2783: rollback oid 39 current snap is 351 2024-09-21T09:49:52.707 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 350 2024-09-21T09:49:52.707 INFO:tasks.rados.rados.0.smithi076.stdout:2781: finishing write tid 1 to smithi07685715-28 2024-09-21T09:49:52.707 INFO:tasks.rados.rados.0.smithi076.stdout:2781: finishing write tid 2 to smithi07685715-28 2024-09-21T09:49:52.707 INFO:tasks.rados.rados.0.smithi076.stdout:2784: read oid 5 snap -1 2024-09-21T09:49:52.707 INFO:tasks.rados.rados.0.smithi076.stdout:2784: expect deleted 2024-09-21T09:49:52.707 INFO:tasks.rados.rados.0.smithi076.stdout:2785: snap_create 2024-09-21T09:49:52.709 INFO:tasks.rados.rados.0.smithi076.stdout:2782: expect (ObjNum 1008 snap 342 seq_num 1008) 2024-09-21T09:49:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:52 smithi191 ceph-mon[102206]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:52.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2187484075' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:52.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:52 smithi191 ceph-mon[102206]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:49:52.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:52 smithi191 ceph-mon[102206]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[106482]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2187484075' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[106482]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[106482]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[110963]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2187484075' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[110963]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T09:49:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:52 smithi076 ceph-mon[110963]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: finishing write tid 3 to smithi07685715-28 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: finishing write tid 4 to smithi07685715-28 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: finishing write tid 5 to smithi07685715-28 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: oid 28 updating version 0 to 2097 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: oid 28 version 2097 is already newer than 2094 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: oid 28 version 2097 is already newer than 2096 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: oid 28 version 2097 is already newer than 2095 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 2097 (ObjNum 1045 snap 351 seq_num 1045) dirty exists 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2781: left oid 28 (ObjNum 1045 snap 351 seq_num 1045) 2024-09-21T09:49:52.878 INFO:tasks.rados.rados.0.smithi076.stdout:2783: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:49:52.879 INFO:tasks.rados.rados.0.smithi076.stdout:2783: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:49:52.879 INFO:tasks.rados.rados.0.smithi076.stdout:2783: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:49:52.879 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2125 (ObjNum 984 snap 336 seq_num 984) dirty exists 2024-09-21T09:49:53.589 INFO:tasks.rados.rados.0.smithi076.stdout:2781: done (4 left) 2024-09-21T09:49:53.589 INFO:tasks.rados.rados.0.smithi076.stdout:2782: done (3 left) 2024-09-21T09:49:53.589 INFO:tasks.rados.rados.0.smithi076.stdout:2783: done (2 left) 2024-09-21T09:49:53.589 INFO:tasks.rados.rados.0.smithi076.stdout:2784: done (1 left) 2024-09-21T09:49:53.589 INFO:tasks.rados.rados.0.smithi076.stdout:2785: done (0 left) 2024-09-21T09:49:53.589 INFO:tasks.rados.rados.0.smithi076.stdout:2786: delete oid 30 current snap is 352 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:2786: done (0 left) 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:2787: rollback oid 50 current snap is 352 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 339 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:2788: read oid 9 snap -1 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:2788: expect deleted 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:2789: write oid 23 current snap is 352 2024-09-21T09:49:53.594 INFO:tasks.rados.rados.0.smithi076.stdout:2789: seq_num 1047 ranges {796929=726064,2184647=659366,3302344=294585} 2024-09-21T09:49:53.602 INFO:tasks.rados.rados.0.smithi076.stdout:2789: writing smithi07685715-23 from 796929 to 1522993 tid 1 2024-09-21T09:49:53.609 INFO:tasks.rados.rados.0.smithi076.stdout:2789: writing smithi07685715-23 from 2184647 to 2844013 tid 2 2024-09-21T09:49:53.613 INFO:tasks.rados.rados.0.smithi076.stdout:2789: writing smithi07685715-23 from 3302344 to 3596929 tid 3 2024-09-21T09:49:53.613 INFO:tasks.rados.rados.0.smithi076.stdout:2790: write oid 13 current snap is 352 2024-09-21T09:49:53.614 INFO:tasks.rados.rados.0.smithi076.stdout:2790: seq_num 1048 ranges {413736=401313,1326776=403024,2413414=457893,3438716=175020} 2024-09-21T09:49:53.618 INFO:tasks.rados.rados.0.smithi076.stdout:2790: writing smithi07685715-13 from 413736 to 815049 tid 1 2024-09-21T09:49:53.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:49:53] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.51.0" 2024-09-21T09:49:53.623 INFO:tasks.rados.rados.0.smithi076.stdout:2790: writing smithi07685715-13 from 1326776 to 1729800 tid 2 2024-09-21T09:49:53.629 INFO:tasks.rados.rados.0.smithi076.stdout:2790: writing smithi07685715-13 from 2413414 to 2871307 tid 3 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2790: writing smithi07685715-13 from 3438716 to 3613736 tid 4 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2791: rollback oid 20 current snap is 352 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 351 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2787: finishing rollback tid 0 to smithi07685715-50 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2787: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2787: finishing rollback tid 2 to smithi07685715-50 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 880 (ObjNum 961 snap 326 seq_num 961) dirty exists 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2789: finishing write tid 1 to smithi07685715-23 2024-09-21T09:49:53.634 INFO:tasks.rados.rados.0.smithi076.stdout:2789: finishing write tid 2 to smithi07685715-23 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: finishing write tid 3 to smithi07685715-23 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: finishing write tid 4 to smithi07685715-23 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: finishing write tid 5 to smithi07685715-23 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: oid 23 updating version 0 to 2100 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: oid 23 updating version 2100 to 2102 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: oid 23 version 2102 is already newer than 2101 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: oid 23 updating version 2102 to 2103 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2103 (ObjNum 1047 snap 352 seq_num 1047) dirty exists 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2789: left oid 23 (ObjNum 1047 snap 352 seq_num 1047) 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2790: finishing write tid 1 to smithi07685715-13 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2790: finishing write tid 2 to smithi07685715-13 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2790: finishing write tid 3 to smithi07685715-13 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2787: done (4 left) 2024-09-21T09:49:53.635 INFO:tasks.rados.rados.0.smithi076.stdout:2788: done (3 left) 2024-09-21T09:49:53.636 INFO:tasks.rados.rados.0.smithi076.stdout:2789: done (2 left) 2024-09-21T09:49:53.636 INFO:tasks.rados.rados.0.smithi076.stdout:2792: write oid 42 current snap is 352 2024-09-21T09:49:53.636 INFO:tasks.rados.rados.0.smithi076.stdout:2792: seq_num 1049 ranges {430543=476562,1385258=663035,2840837=669165} 2024-09-21T09:49:53.638 INFO:tasks.rados.rados.0.smithi076.stdout:2792: writing smithi07685715-42 from 430543 to 907105 tid 1 2024-09-21T09:49:53.644 INFO:tasks.rados.rados.0.smithi076.stdout:2792: writing smithi07685715-42 from 1385258 to 2048293 tid 2 2024-09-21T09:49:53.652 INFO:tasks.rados.rados.0.smithi076.stdout:2792: writing smithi07685715-42 from 2840837 to 3510002 tid 3 2024-09-21T09:49:53.653 INFO:tasks.rados.rados.0.smithi076.stdout:2793: snap_remove snap 339 2024-09-21T09:49:53.757 INFO:tasks.rados.rados.0.smithi076.stdout:2793: done (3 left) 2024-09-21T09:49:53.757 INFO:tasks.rados.rados.0.smithi076.stdout:2794: snap_remove snap 338 2024-09-21T09:49:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:53 smithi191 ceph-mon[102206]: pgmap v637: 89 pgs: 89 active+clean; 166 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 685 KiB/s wr, 15 op/s 2024-09-21T09:49:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:53 smithi191 ceph-mon[102206]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:49:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:53 smithi191 ceph-mon[102206]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:49:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-mon[110963]: pgmap v637: 89 pgs: 89 active+clean; 166 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 685 KiB/s wr, 15 op/s 2024-09-21T09:49:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-mon[110963]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:49:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-mon[110963]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:49:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-mon[106482]: pgmap v637: 89 pgs: 89 active+clean; 166 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 685 KiB/s wr, 15 op/s 2024-09-21T09:49:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-mon[106482]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T09:49:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:53 smithi076 ceph-mon[106482]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T09:49:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:2794: done (3 left) 2024-09-21T09:49:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:2795: write oid 14 current snap is 352 2024-09-21T09:49:54.756 INFO:tasks.rados.rados.0.smithi076.stdout:2795: seq_num 1050 ranges {447350=668164,1560093=406693,2584613=480437} 2024-09-21T09:49:54.762 INFO:tasks.rados.rados.0.smithi076.stdout:2795: writing smithi07685715-14 from 447350 to 1115514 tid 1 2024-09-21T09:49:54.767 INFO:tasks.rados.rados.0.smithi076.stdout:2795: writing smithi07685715-14 from 1560093 to 1966786 tid 2 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2795: writing smithi07685715-14 from 2584613 to 3065050 tid 3 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2796: snap_create 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: finishing write tid 4 to smithi07685715-13 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: finishing write tid 5 to smithi07685715-13 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: finishing write tid 6 to smithi07685715-13 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: oid 13 updating version 0 to 516 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: oid 13 updating version 516 to 518 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: oid 13 version 518 is already newer than 514 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: oid 13 version 518 is already newer than 515 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: oid 13 version 518 is already newer than 517 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 518 (ObjNum 1048 snap 352 seq_num 1048) dirty exists 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2790: left oid 13 (ObjNum 1048 snap 352 seq_num 1048) 2024-09-21T09:49:54.775 INFO:tasks.rados.rados.0.smithi076.stdout:2791: finishing rollback tid 0 to smithi07685715-20 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2791: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2791: finishing rollback tid 2 to smithi07685715-20 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 842 (ObjNum 1002 snap 341 seq_num 1002) dirty exists 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: finishing write tid 1 to smithi07685715-42 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: finishing write tid 2 to smithi07685715-42 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: finishing write tid 3 to smithi07685715-42 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: finishing write tid 4 to smithi07685715-42 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: finishing write tid 5 to smithi07685715-42 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: oid 42 updating version 0 to 717 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: oid 42 updating version 717 to 718 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: oid 42 version 718 is already newer than 716 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: oid 42 updating version 718 to 719 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 719 (ObjNum 1049 snap 352 seq_num 1049) dirty exists 2024-09-21T09:49:54.776 INFO:tasks.rados.rados.0.smithi076.stdout:2792: left oid 42 (ObjNum 1049 snap 352 seq_num 1049) 2024-09-21T09:49:54.777 INFO:tasks.rados.rados.0.smithi076.stdout:2795: finishing write tid 1 to smithi07685715-14 2024-09-21T09:49:54.777 INFO:tasks.rados.rados.0.smithi076.stdout:2795: finishing write tid 2 to smithi07685715-14 2024-09-21T09:49:54.777 INFO:tasks.rados.rados.0.smithi076.stdout:2795: finishing write tid 3 to smithi07685715-14 2024-09-21T09:49:54.777 INFO:tasks.rados.rados.0.smithi076.stdout:2795: finishing write tid 4 to smithi07685715-14 2024-09-21T09:49:54.777 INFO:tasks.rados.rados.0.smithi076.stdout:2795: finishing write tid 5 to smithi07685715-14 2024-09-21T09:49:54.777 INFO:tasks.rados.rados.0.smithi076.stdout:2795: oid 14 updating version 0 to 1714 2024-09-21T09:49:54.778 INFO:tasks.rados.rados.0.smithi076.stdout:2795: oid 14 version 1714 is already newer than 1713 2024-09-21T09:49:54.778 INFO:tasks.rados.rados.0.smithi076.stdout:2795: oid 14 version 1714 is already newer than 1712 2024-09-21T09:49:54.778 INFO:tasks.rados.rados.0.smithi076.stdout:2795: oid 14 version 1714 is already newer than 1709 2024-09-21T09:49:54.778 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1714 (ObjNum 1050 snap 352 seq_num 1050) dirty exists 2024-09-21T09:49:54.778 INFO:tasks.rados.rados.0.smithi076.stdout:2795: left oid 14 (ObjNum 1050 snap 352 seq_num 1050) 2024-09-21T09:49:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:49:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:49:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[110963]: Upgrade: osd.2 is safe to restart 2024-09-21T09:49:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[110963]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:49:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:49:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:49:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[106482]: Upgrade: osd.2 is safe to restart 2024-09-21T09:49:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:54 smithi076 ceph-mon[106482]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:49:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:49:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:54 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:49:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:54 smithi191 ceph-mon[102206]: Upgrade: osd.2 is safe to restart 2024-09-21T09:49:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:54 smithi191 ceph-mon[102206]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T09:49:55.757 INFO:tasks.rados.rados.0.smithi076.stdout:2790: done (4 left) 2024-09-21T09:49:55.757 INFO:tasks.rados.rados.0.smithi076.stdout:2791: done (3 left) 2024-09-21T09:49:55.757 INFO:tasks.rados.rados.0.smithi076.stdout:2792: done (2 left) 2024-09-21T09:49:55.757 INFO:tasks.rados.rados.0.smithi076.stdout:2795: done (1 left) 2024-09-21T09:49:55.758 INFO:tasks.rados.rados.0.smithi076.stdout:2796: done (0 left) 2024-09-21T09:49:55.758 INFO:tasks.rados.rados.0.smithi076.stdout:2797: delete oid 8 current snap is 353 2024-09-21T09:49:55.762 INFO:tasks.rados.rados.0.smithi076.stdout:2797: done (0 left) 2024-09-21T09:49:55.762 INFO:tasks.rados.rados.0.smithi076.stdout:2798: read oid 29 snap -1 2024-09-21T09:49:55.762 INFO:tasks.rados.rados.0.smithi076.stdout:2798: expect deleted 2024-09-21T09:49:55.762 INFO:tasks.rados.rados.0.smithi076.stdout:2799: read oid 43 snap -1 2024-09-21T09:49:55.762 INFO:tasks.rados.rados.0.smithi076.stdout:2799: expect (ObjNum 1033 snap 349 seq_num 1033) 2024-09-21T09:49:55.763 INFO:tasks.rados.rados.0.smithi076.stdout:2800: rollback oid 3 current snap is 353 2024-09-21T09:49:55.763 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 344 2024-09-21T09:49:55.763 INFO:tasks.rados.rados.0.smithi076.stdout:2801: write oid 44 current snap is 353 2024-09-21T09:49:55.763 INFO:tasks.rados.rados.0.smithi076.stdout:2801: seq_num 1052 ranges {480964=418662,1560704=410362,2523106=619334} 2024-09-21T09:49:55.767 INFO:tasks.rados.rados.0.smithi076.stdout:2801: writing smithi07685715-44 from 480964 to 899626 tid 1 2024-09-21T09:49:55.773 INFO:tasks.rados.rados.0.smithi076.stdout:2801: writing smithi07685715-44 from 1560704 to 1971066 tid 2 2024-09-21T09:49:55.780 INFO:tasks.rados.rados.0.smithi076.stdout:2801: writing smithi07685715-44 from 2523106 to 3142440 tid 3 2024-09-21T09:49:55.780 INFO:tasks.rados.rados.0.smithi076.stdout:2802: delete oid 37 current snap is 353 2024-09-21T09:49:55.780 INFO:tasks.rados.rados.0.smithi076.stdout:2800: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:49:55.781 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:49:55.781 INFO:tasks.rados.rados.0.smithi076.stdout:2799: expect (ObjNum 1033 snap 349 seq_num 1033) 2024-09-21T09:49:55.890 INFO:tasks.rados.rados.0.smithi076.stdout:2801: finishing write tid 1 to smithi07685715-44 2024-09-21T09:49:55.890 INFO:tasks.rados.rados.0.smithi076.stdout:2801: finishing write tid 2 to smithi07685715-44 2024-09-21T09:49:55.890 INFO:tasks.rados.rados.0.smithi076.stdout:2801: finishing write tid 3 to smithi07685715-44 2024-09-21T09:49:55.890 INFO:tasks.rados.rados.0.smithi076.stdout:2801: finishing write tid 4 to smithi07685715-44 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: finishing write tid 5 to smithi07685715-44 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: oid 44 updating version 0 to 682 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: oid 44 version 682 is already newer than 681 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: oid 44 version 682 is already newer than 680 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: oid 44 updating version 682 to 683 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 683 (ObjNum 1052 snap 353 seq_num 1052) dirty exists 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: left oid 44 (ObjNum 1052 snap 353 seq_num 1052) 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2798: done (4 left) 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2799: done (3 left) 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2800: done (2 left) 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2801: done (1 left) 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2802: done (0 left) 2024-09-21T09:49:55.891 INFO:tasks.rados.rados.0.smithi076.stdout:2803: write oid 44 current snap is 353 2024-09-21T09:49:55.892 INFO:tasks.rados.rados.0.smithi076.stdout:2803: seq_num 1054 ranges {514578=569160,1677668=414031,2577952=641878} 2024-09-21T09:49:55.896 INFO:tasks.rados.rados.0.smithi076.stdout:2803: writing smithi07685715-44 from 514578 to 1083738 tid 1 2024-09-21T09:49:55.901 INFO:tasks.rados.rados.0.smithi076.stdout:2803: writing smithi07685715-44 from 1677668 to 2091699 tid 2 2024-09-21T09:49:55.908 INFO:tasks.rados.rados.0.smithi076.stdout:2803: writing smithi07685715-44 from 2577952 to 3219830 tid 3 2024-09-21T09:49:55.908 INFO:tasks.rados.rados.0.smithi076.stdout:2804: write oid 13 current snap is 353 2024-09-21T09:49:55.908 INFO:tasks.rados.rados.0.smithi076.stdout:2804: seq_num 1055 ranges {531385=644409,1736150=557689,3005375=569503} 2024-09-21T09:49:55.914 INFO:tasks.rados.rados.0.smithi076.stdout:2804: writing smithi07685715-13 from 531385 to 1175794 tid 1 2024-09-21T09:49:55.920 INFO:tasks.rados.rados.0.smithi076.stdout:2804: writing smithi07685715-13 from 1736150 to 2293839 tid 2 2024-09-21T09:49:55.928 INFO:tasks.rados.rados.0.smithi076.stdout:2804: writing smithi07685715-13 from 3005375 to 3574878 tid 3 2024-09-21T09:49:55.928 INFO:tasks.rados.rados.0.smithi076.stdout:2805: delete oid 49 current snap is 353 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: finishing write tid 1 to smithi07685715-44 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: finishing write tid 2 to smithi07685715-44 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: finishing write tid 3 to smithi07685715-44 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: finishing write tid 4 to smithi07685715-44 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: finishing write tid 5 to smithi07685715-44 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: oid 44 updating version 0 to 685 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: oid 44 updating version 685 to 687 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: oid 44 version 687 is already newer than 686 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: oid 44 version 687 is already newer than 684 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 687 (ObjNum 1054 snap 353 seq_num 1054) dirty exists 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2803: left oid 44 (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:49:55.929 INFO:tasks.rados.rados.0.smithi076.stdout:2804: finishing write tid 1 to smithi07685715-13 2024-09-21T09:49:55.930 INFO:tasks.rados.rados.0.smithi076.stdout:2804: finishing write tid 2 to smithi07685715-13 2024-09-21T09:49:55.930 INFO:tasks.rados.rados.0.smithi076.stdout:2803: done (2 left) 2024-09-21T09:49:55.930 INFO:tasks.rados.rados.0.smithi076.stdout:2805: done (1 left) 2024-09-21T09:49:55.930 INFO:tasks.rados.rados.0.smithi076.stdout:2806: read oid 48 snap -1 2024-09-21T09:49:55.931 INFO:tasks.rados.rados.0.smithi076.stdout:2806: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:49:55.931 INFO:tasks.rados.rados.0.smithi076.stdout:2807: delete oid 35 current snap is 353 2024-09-21T09:49:55.931 INFO:tasks.rados.rados.0.smithi076.stdout:2804: finishing write tid 3 to smithi07685715-13 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: finishing write tid 4 to smithi07685715-13 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: finishing write tid 5 to smithi07685715-13 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: oid 13 updating version 0 to 531 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: oid 13 version 531 is already newer than 528 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: oid 13 version 531 is already newer than 530 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: oid 13 version 531 is already newer than 529 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 531 (ObjNum 1055 snap 353 seq_num 1055) dirty exists 2024-09-21T09:49:55.932 INFO:tasks.rados.rados.0.smithi076.stdout:2804: left oid 13 (ObjNum 1055 snap 353 seq_num 1055) 2024-09-21T09:49:55.934 INFO:tasks.rados.rados.0.smithi076.stdout:2804: done (2 left) 2024-09-21T09:49:55.934 INFO:tasks.rados.rados.0.smithi076.stdout:2807: done (1 left) 2024-09-21T09:49:55.934 INFO:tasks.rados.rados.0.smithi076.stdout:2808: snap_create 2024-09-21T09:49:55.944 INFO:tasks.rados.rados.0.smithi076.stdout:2806: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: pgmap v640: 89 pgs: 89 active+clean; 170 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 755 KiB/s wr, 13 op/s 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:56.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[106482]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: pgmap v640: 89 pgs: 89 active+clean; 170 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 755 KiB/s wr, 13 op/s 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:56.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:55 smithi076 ceph-mon[110963]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: pgmap v640: 89 pgs: 89 active+clean; 170 MiB data, 644 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 755 KiB/s wr, 13 op/s 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:49:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:55 smithi191 ceph-mon[102206]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T09:49:56.936 INFO:tasks.rados.rados.0.smithi076.stdout:2806: done (1 left) 2024-09-21T09:49:56.936 INFO:tasks.rados.rados.0.smithi076.stdout:2808: done (0 left) 2024-09-21T09:49:56.936 INFO:tasks.rados.rados.0.smithi076.stdout:2809: snap_remove snap 349 2024-09-21T09:49:57.082 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:56 smithi076 ceph-mon[106482]: Upgrade: Updating osd.2 2024-09-21T09:49:57.082 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:56 smithi076 ceph-mon[106482]: Deploying daemon osd.2 on smithi076 2024-09-21T09:49:57.082 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:56 smithi076 ceph-mon[110963]: Upgrade: Updating osd.2 2024-09-21T09:49:57.082 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:56 smithi076 ceph-mon[110963]: Deploying daemon osd.2 on smithi076 2024-09-21T09:49:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:56 smithi191 ceph-mon[102206]: Upgrade: Updating osd.2 2024-09-21T09:49:57.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:56 smithi191 ceph-mon[102206]: Deploying daemon osd.2 on smithi076 2024-09-21T09:49:57.945 INFO:tasks.rados.rados.0.smithi076.stdout:2809: done (0 left) 2024-09-21T09:49:57.945 INFO:tasks.rados.rados.0.smithi076.stdout:2810: write oid 4 current snap is 354 2024-09-21T09:49:57.945 INFO:tasks.rados.rados.0.smithi076.stdout:2810: seq_num 1058 ranges {581806=586509,1911596=421369,3087644=403319} 2024-09-21T09:49:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:2810: writing smithi07685715-4 from 581806 to 1168315 tid 1 2024-09-21T09:49:57.959 INFO:tasks.rados.rados.0.smithi076.stdout:2810: writing smithi07685715-4 from 1911596 to 2332965 tid 2 2024-09-21T09:49:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:2810: writing smithi07685715-4 from 3087644 to 3490963 tid 3 2024-09-21T09:49:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:2811: write oid 11 current snap is 354 2024-09-21T09:49:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:2810: finishing write tid 1 to smithi07685715-4 2024-09-21T09:49:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:2810: finishing write tid 2 to smithi07685715-4 2024-09-21T09:49:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:2811: seq_num 1059 ranges {598613=661758,1970078=565027,3115067=614591} 2024-09-21T09:49:57.972 INFO:tasks.rados.rados.0.smithi076.stdout:2811: writing smithi07685715-11 from 598613 to 1260371 tid 1 2024-09-21T09:49:57.979 INFO:tasks.rados.rados.0.smithi076.stdout:2811: writing smithi07685715-11 from 1970078 to 2535105 tid 2 2024-09-21T09:49:57.985 INFO:tasks.rados.rados.0.smithi076.stdout:2811: writing smithi07685715-11 from 3115067 to 3729658 tid 3 2024-09-21T09:49:57.986 INFO:tasks.rados.rados.0.smithi076.stdout:2812: write oid 39 current snap is 354 2024-09-21T09:49:57.986 INFO:tasks.rados.rados.0.smithi076.stdout:2812: seq_num 1060 ranges {615420=737007,1912207=425038,3026137=542216} 2024-09-21T09:49:57.993 INFO:tasks.rados.rados.0.smithi076.stdout:2812: writing smithi07685715-39 from 615420 to 1352427 tid 1 2024-09-21T09:49:57.998 INFO:tasks.rados.rados.0.smithi076.stdout:2812: writing smithi07685715-39 from 1912207 to 2337245 tid 2 2024-09-21T09:49:58.005 INFO:tasks.rados.rados.0.smithi076.stdout:2812: writing smithi07685715-39 from 3026137 to 3568353 tid 3 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2813: delete oid 36 current snap is 354 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: finishing write tid 3 to smithi07685715-4 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: finishing write tid 4 to smithi07685715-4 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: finishing write tid 5 to smithi07685715-4 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: oid 4 updating version 0 to 894 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: oid 4 version 894 is already newer than 893 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: oid 4 version 894 is already newer than 892 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: oid 4 updating version 894 to 895 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 895 (ObjNum 1058 snap 354 seq_num 1058) dirty exists 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2810: left oid 4 (ObjNum 1058 snap 354 seq_num 1058) 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2811: finishing write tid 1 to smithi07685715-11 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2811: finishing write tid 2 to smithi07685715-11 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2811: finishing write tid 3 to smithi07685715-11 2024-09-21T09:49:58.006 INFO:tasks.rados.rados.0.smithi076.stdout:2811: finishing write tid 4 to smithi07685715-11 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2811: finishing write tid 5 to smithi07685715-11 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2811: oid 11 updating version 0 to 2146 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2811: oid 11 updating version 2146 to 2148 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2811: oid 11 version 2148 is already newer than 2147 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2811: oid 11 version 2148 is already newer than 2145 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2148 (ObjNum 1059 snap 354 seq_num 1059) dirty exists 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2811: left oid 11 (ObjNum 1059 snap 354 seq_num 1059) 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2812: finishing write tid 1 to smithi07685715-39 2024-09-21T09:49:58.007 INFO:tasks.rados.rados.0.smithi076.stdout:2812: finishing write tid 2 to smithi07685715-39 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2810: done (3 left) 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2811: done (2 left) 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2813: done (1 left) 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2814: read oid 50 snap -1 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2814: expect (ObjNum 961 snap 326 seq_num 961) 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2815: write oid 24 current snap is 354 2024-09-21T09:49:58.008 INFO:tasks.rados.rados.0.smithi076.stdout:2815: seq_num 1062 ranges {649034=487505,1629171=428707,2680983=564760,3798768=50266} 2024-09-21T09:49:58.014 INFO:tasks.rados.rados.0.smithi076.stdout:2815: writing smithi07685715-24 from 649034 to 1136539 tid 1 2024-09-21T09:49:58.019 INFO:tasks.rados.rados.0.smithi076.stdout:2815: writing smithi07685715-24 from 1629171 to 2057878 tid 2 2024-09-21T09:49:58.026 INFO:tasks.rados.rados.0.smithi076.stdout:2815: writing smithi07685715-24 from 2680983 to 3245743 tid 3 2024-09-21T09:49:58.029 INFO:tasks.rados.rados.0.smithi076.stdout:2815: writing smithi07685715-24 from 3798768 to 3849034 tid 4 2024-09-21T09:49:58.029 INFO:tasks.rados.rados.0.smithi076.stdout:2816: write oid 33 current snap is 354 2024-09-21T09:49:58.029 INFO:tasks.rados.rados.0.smithi076.stdout:2812: finishing write tid 3 to smithi07685715-39 2024-09-21T09:49:58.029 INFO:tasks.rados.rados.0.smithi076.stdout:2816: seq_num 1063 ranges {665841=562754,1687653=572365,2708406=776032} 2024-09-21T09:49:58.036 INFO:tasks.rados.rados.0.smithi076.stdout:2816: writing smithi07685715-33 from 665841 to 1228595 tid 1 2024-09-21T09:49:58.041 INFO:tasks.rados.rados.0.smithi076.stdout:2816: writing smithi07685715-33 from 1687653 to 2260018 tid 2 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2816: writing smithi07685715-33 from 2708406 to 3484438 tid 3 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2817: read oid 8 snap -1 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2817: expect deleted 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2818: rollback oid 32 current snap is 354 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 336 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2812: finishing write tid 4 to smithi07685715-39 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2819: snap_create 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2812: finishing write tid 5 to smithi07685715-39 2024-09-21T09:49:58.049 INFO:tasks.rados.rados.0.smithi076.stdout:2812: oid 39 updating version 0 to 2150 2024-09-21T09:49:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:2812: oid 39 updating version 2150 to 2152 2024-09-21T09:49:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:2812: oid 39 updating version 2152 to 2153 2024-09-21T09:49:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:2812: oid 39 version 2153 is already newer than 2151 2024-09-21T09:49:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2153 (ObjNum 1060 snap 354 seq_num 1060) dirty exists 2024-09-21T09:49:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:2812: left oid 39 (ObjNum 1060 snap 354 seq_num 1060) 2024-09-21T09:49:58.050 INFO:tasks.rados.rados.0.smithi076.stdout:2814: expect (ObjNum 961 snap 326 seq_num 961) 2024-09-21T09:49:58.123 INFO:tasks.rados.rados.0.smithi076.stdout:2815: finishing write tid 1 to smithi07685715-24 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: finishing write tid 2 to smithi07685715-24 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: finishing write tid 3 to smithi07685715-24 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: finishing write tid 4 to smithi07685715-24 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: finishing write tid 5 to smithi07685715-24 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: finishing write tid 6 to smithi07685715-24 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: oid 24 updating version 0 to 283 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: oid 24 version 283 is already newer than 279 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: oid 24 version 283 is already newer than 282 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: oid 24 version 283 is already newer than 281 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: oid 24 version 283 is already newer than 280 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 283 (ObjNum 1062 snap 354 seq_num 1062) dirty exists 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2815: left oid 24 (ObjNum 1062 snap 354 seq_num 1062) 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2816: finishing write tid 1 to smithi07685715-33 2024-09-21T09:49:58.124 INFO:tasks.rados.rados.0.smithi076.stdout:2816: finishing write tid 2 to smithi07685715-33 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2818: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 0 (ObjNum 129 snap 0 seq_num 864570928) dirty dne 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: finishing write tid 3 to smithi07685715-33 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: finishing write tid 4 to smithi07685715-33 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: finishing write tid 5 to smithi07685715-33 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: oid 33 updating version 0 to 662 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: oid 33 version 662 is already newer than 661 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: oid 33 updating version 662 to 664 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: oid 33 version 664 is already newer than 663 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 664 (ObjNum 1063 snap 354 seq_num 1063) dirty exists 2024-09-21T09:49:58.125 INFO:tasks.rados.rados.0.smithi076.stdout:2816: left oid 33 (ObjNum 1063 snap 354 seq_num 1063) 2024-09-21T09:49:58.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:57 smithi076 ceph-mon[106482]: pgmap v643: 89 pgs: 89 active+clean; 185 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 28 op/s 2024-09-21T09:49:58.305 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:57 smithi076 ceph-mon[106482]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:49:58.305 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:57 smithi076 ceph-mon[110963]: pgmap v643: 89 pgs: 89 active+clean; 185 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 28 op/s 2024-09-21T09:49:58.305 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:57 smithi076 ceph-mon[110963]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:49:58.305 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:58 smithi076 systemd[1]: Stopping Ceph osd.2 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:49:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:57 smithi191 ceph-mon[102206]: pgmap v643: 89 pgs: 89 active+clean; 185 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 28 op/s 2024-09-21T09:49:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:57 smithi191 ceph-mon[102206]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2812: done (6 left) 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2814: done (5 left) 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2815: done (4 left) 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2816: done (3 left) 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2817: done (2 left) 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2818: done (1 left) 2024-09-21T09:49:58.757 INFO:tasks.rados.rados.0.smithi076.stdout:2819: done (0 left) 2024-09-21T09:49:58.758 INFO:tasks.rados.rados.0.smithi076.stdout:2820: snap_remove snap 352 2024-09-21T09:49:58.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[57979]: 2024-09-21T09:49:58.559+0000 7fb80447c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:49:58.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[57979]: 2024-09-21T09:49:58.559+0000 7fb80447c640 -1 osd.2 783 *** Got signal Terminated *** 2024-09-21T09:49:58.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[57979]: 2024-09-21T09:49:58.559+0000 7fb80447c640 -1 osd.2 783 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:49:59.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[106482]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:49:59.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[106482]: osd.2 marked itself down and dead 2024-09-21T09:49:59.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[106482]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:49:59.297 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[106482]: osdmap e784: 8 total, 7 up, 8 in 2024-09-21T09:49:59.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[110963]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:49:59.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[110963]: osd.2 marked itself down and dead 2024-09-21T09:49:59.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[110963]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:49:59.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:58 smithi076 ceph-mon[110963]: osdmap e784: 8 total, 7 up, 8 in 2024-09-21T09:49:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:58 smithi191 ceph-mon[102206]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T09:49:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:58 smithi191 ceph-mon[102206]: osd.2 marked itself down and dead 2024-09-21T09:49:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:58 smithi191 ceph-mon[102206]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:49:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:58 smithi191 ceph-mon[102206]: osdmap e784: 8 total, 7 up, 8 in 2024-09-21T09:49:59.611 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:59 smithi076 podman[132242]: 2024-09-21 09:49:59.295081565 +0000 UTC m=+0.848855396 container died ab907ee95e4b06aed38f8c29033d1682c6be165bcc4f038c32879be69b6272e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:49:59.758 INFO:tasks.rados.rados.0.smithi076.stdout:2820: done (0 left) 2024-09-21T09:49:59.758 INFO:tasks.rados.rados.0.smithi076.stdout:2821: rollback oid 26 current snap is 355 2024-09-21T09:49:59.758 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 344 2024-09-21T09:49:59.758 INFO:tasks.rados.rados.0.smithi076.stdout:2822: delete oid 4 current snap is 355 2024-09-21T09:49:59.768 INFO:tasks.rados.rados.0.smithi076.stdout:2822: done (1 left) 2024-09-21T09:49:59.768 INFO:tasks.rados.rados.0.smithi076.stdout:2823: read oid 4 snap 347 2024-09-21T09:49:59.768 INFO:tasks.rados.rados.0.smithi076.stdout:2823: expect deleted 2024-09-21T09:49:59.768 INFO:tasks.rados.rados.0.smithi076.stdout:2824: write oid 29 current snap is 355 2024-09-21T09:49:59.768 INFO:tasks.rados.rados.0.smithi076.stdout:2821: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:49:59.768 INFO:tasks.rados.rados.0.smithi076.stdout:2824: seq_num 1065 ranges {699455=429605,1804617=576034,3163252=514929} 2024-09-21T09:49:59.774 INFO:tasks.rados.rados.0.smithi076.stdout:2824: writing smithi07685715-29 from 699455 to 1129060 tid 1 2024-09-21T09:49:59.781 INFO:tasks.rados.rados.0.smithi076.stdout:2824: writing smithi07685715-29 from 1804617 to 2380651 tid 2 2024-09-21T09:49:59.788 INFO:tasks.rados.rados.0.smithi076.stdout:2824: writing smithi07685715-29 from 3163252 to 3678181 tid 3 2024-09-21T09:49:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2825: snap_create 2024-09-21T09:49:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2821: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:49:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2821: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:49:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 966 (ObjNum 1015 snap 344 seq_num 1015) dirty exists 2024-09-21T09:49:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2824: finishing write tid 1 to smithi07685715-29 2024-09-21T09:49:59.789 INFO:tasks.rados.rados.0.smithi076.stdout:2824: finishing write tid 2 to smithi07685715-29 2024-09-21T09:49:59.791 INFO:tasks.rados.rados.0.smithi076.stdout:2824: finishing write tid 3 to smithi07685715-29 2024-09-21T09:49:59.791 INFO:tasks.rados.rados.0.smithi076.stdout:2824: finishing write tid 4 to smithi07685715-29 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:2824: finishing write tid 5 to smithi07685715-29 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:2824: oid 29 updating version 0 to 855 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:2824: oid 29 version 855 is already newer than 854 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:2824: oid 29 version 855 is already newer than 852 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:2824: oid 29 version 855 is already newer than 853 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 855 (ObjNum 1065 snap 355 seq_num 1065) dirty exists 2024-09-21T09:49:59.792 INFO:tasks.rados.rados.0.smithi076.stdout:2824: left oid 29 (ObjNum 1065 snap 355 seq_num 1065) 2024-09-21T09:49:59.872 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:59 smithi076 podman[132242]: 2024-09-21 09:49:59.717449101 +0000 UTC m=+1.271222922 container cleanup ab907ee95e4b06aed38f8c29033d1682c6be165bcc4f038c32879be69b6272e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0) 2024-09-21T09:49:59.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:59 smithi076 bash[132242]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2 2024-09-21T09:50:00.350 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:59 smithi076 ceph-mon[106482]: pgmap v646: 89 pgs: 89 active+clean; 184 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 3.8 MiB/s wr, 28 op/s 2024-09-21T09:50:00.350 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:49:59 smithi076 ceph-mon[106482]: osdmap e785: 8 total, 7 up, 8 in 2024-09-21T09:50:00.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:59 smithi076 ceph-mon[110963]: pgmap v646: 89 pgs: 89 active+clean; 184 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 3.8 MiB/s wr, 28 op/s 2024-09-21T09:50:00.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:49:59 smithi076 ceph-mon[110963]: osdmap e785: 8 total, 7 up, 8 in 2024-09-21T09:50:00.351 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:49:59 smithi076 podman[132263]: 2024-09-21 09:49:59.92782884 +0000 UTC m=+0.629218506 container remove ab907ee95e4b06aed38f8c29033d1682c6be165bcc4f038c32879be69b6272e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T09:50:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:59 smithi191 ceph-mon[102206]: pgmap v646: 89 pgs: 89 active+clean; 184 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 3.8 MiB/s wr, 28 op/s 2024-09-21T09:50:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:49:59 smithi191 ceph-mon[102206]: osdmap e785: 8 total, 7 up, 8 in 2024-09-21T09:50:00.608 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:00 smithi076 podman[132360]: 2024-09-21 09:50:00.251509381 +0000 UTC m=+0.015940268 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:00.608 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:00 smithi076 podman[132360]: 2024-09-21 09:50:00.36210737 +0000 UTC m=+0.126538283 container create 40481ae96263a78740a5c34825b30ed4b5177726933c51c25a9dc8d3c8be1352 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, RELEASE=main-62da54b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T09:50:00.757 INFO:tasks.rados.rados.0.smithi076.stdout:2821: done (3 left) 2024-09-21T09:50:00.757 INFO:tasks.rados.rados.0.smithi076.stdout:2823: done (2 left) 2024-09-21T09:50:00.757 INFO:tasks.rados.rados.0.smithi076.stdout:2824: done (1 left) 2024-09-21T09:50:00.757 INFO:tasks.rados.rados.0.smithi076.stdout:2825: done (0 left) 2024-09-21T09:50:00.757 INFO:tasks.rados.rados.0.smithi076.stdout:2826: delete oid 25 current snap is 356 2024-09-21T09:50:00.761 INFO:tasks.rados.rados.0.smithi076.stdout:2826: done (0 left) 2024-09-21T09:50:00.761 INFO:tasks.rados.rados.0.smithi076.stdout:2827: snap_create 2024-09-21T09:50:00.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:00 smithi076 podman[132360]: 2024-09-21 09:50:00.60671792 +0000 UTC m=+0.371148810 container init 40481ae96263a78740a5c34825b30ed4b5177726933c51c25a9dc8d3c8be1352 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:00.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:00 smithi076 podman[132360]: 2024-09-21 09:50:00.611627622 +0000 UTC m=+0.376058496 container start 40481ae96263a78740a5c34825b30ed4b5177726933c51c25a9dc8d3c8be1352 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:50:00.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:00 smithi076 podman[132360]: 2024-09-21 09:50:00.698236051 +0000 UTC m=+0.462666930 container attach 40481ae96263a78740a5c34825b30ed4b5177726933c51c25a9dc8d3c8be1352 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:50:00.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:00 smithi076 podman[132360]: 2024-09-21 09:50:00.798888349 +0000 UTC m=+0.563319243 container died 40481ae96263a78740a5c34825b30ed4b5177726933c51c25a9dc8d3c8be1352 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:00 smithi191 ceph-mon[102206]: Health detail: HEALTH_WARN 1 osds down 2024-09-21T09:50:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:00 smithi191 ceph-mon[102206]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T09:50:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:00 smithi191 ceph-mon[102206]: osd.2 (root=default,host=smithi076) is down 2024-09-21T09:50:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:00 smithi191 ceph-mon[102206]: osdmap e786: 8 total, 7 up, 8 in 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[106482]: Health detail: HEALTH_WARN 1 osds down 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[106482]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[106482]: osd.2 (root=default,host=smithi076) is down 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[106482]: osdmap e786: 8 total, 7 up, 8 in 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[110963]: Health detail: HEALTH_WARN 1 osds down 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[110963]: [WRN] OSD_DOWN: 1 osds down 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[110963]: osd.2 (root=default,host=smithi076) is down 2024-09-21T09:50:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:00 smithi076 ceph-mon[110963]: osdmap e786: 8 total, 7 up, 8 in 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2827: done (0 left) 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2828: rollback oid 38 current snap is 357 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 355 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2829: read oid 16 snap 350 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2829: expect (ObjNum 958 snap 326 seq_num 958) 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2830: read oid 33 snap -1 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2830: expect (ObjNum 1063 snap 354 seq_num 1063) 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:2831: rollback oid 34 current snap is 357 2024-09-21T09:50:01.759 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 356 2024-09-21T09:50:01.760 INFO:tasks.rados.rados.0.smithi076.stdout:2832: write oid 3 current snap is 357 2024-09-21T09:50:01.760 INFO:tasks.rados.rados.0.smithi076.stdout:2832: seq_num 1067 ranges {733069=580103,1921581=579703,3218098=537473} 2024-09-21T09:50:01.766 INFO:tasks.rados.rados.0.smithi076.stdout:2832: writing smithi07685715-3 from 733069 to 1313172 tid 1 2024-09-21T09:50:01.773 INFO:tasks.rados.rados.0.smithi076.stdout:2832: writing smithi07685715-3 from 1921581 to 2501284 tid 2 2024-09-21T09:50:01.780 INFO:tasks.rados.rados.0.smithi076.stdout:2832: writing smithi07685715-3 from 3218098 to 3755571 tid 3 2024-09-21T09:50:01.780 INFO:tasks.rados.rados.0.smithi076.stdout:2833: write oid 37 current snap is 357 2024-09-21T09:50:01.780 INFO:tasks.rados.rados.0.smithi076.stdout:2833: seq_num 1068 ranges {749876=655352,1863710=723361,3012815=748745} 2024-09-21T09:50:01.787 INFO:tasks.rados.rados.0.smithi076.stdout:2833: writing smithi07685715-37 from 749876 to 1405228 tid 1 2024-09-21T09:50:01.793 INFO:tasks.rados.rados.0.smithi076.stdout:2833: writing smithi07685715-37 from 1863710 to 2587071 tid 2 2024-09-21T09:50:01.800 INFO:tasks.rados.rados.0.smithi076.stdout:2833: writing smithi07685715-37 from 3012815 to 3761560 tid 3 2024-09-21T09:50:01.800 INFO:tasks.rados.rados.0.smithi076.stdout:2834: write oid 35 current snap is 357 2024-09-21T09:50:01.800 INFO:tasks.rados.rados.0.smithi076.stdout:2834: seq_num 1069 ranges {766683=730601,1922192=583372,3156591=676370} 2024-09-21T09:50:01.808 INFO:tasks.rados.rados.0.smithi076.stdout:2834: writing smithi07685715-35 from 766683 to 1497284 tid 1 2024-09-21T09:50:01.814 INFO:tasks.rados.rados.0.smithi076.stdout:2834: writing smithi07685715-35 from 1922192 to 2505564 tid 2 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2834: writing smithi07685715-35 from 3156591 to 3832961 tid 3 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2835: snap_create 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2831: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2828: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1583 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2831: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2831: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 658 (ObjNum 975 snap 332 seq_num 975) dirty exists 2024-09-21T09:50:01.822 INFO:tasks.rados.rados.0.smithi076.stdout:2830: expect (ObjNum 1063 snap 354 seq_num 1063) 2024-09-21T09:50:01.848 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 podman[132404]: 2024-09-21 09:50:01.411392515 +0000 UTC m=+0.606829274 container remove 40481ae96263a78740a5c34825b30ed4b5177726933c51c25a9dc8d3c8be1352 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T09:50:01.848 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T09:50:01.848 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 systemd[1]: Stopped Ceph osd.2 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:50:01.848 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.2.service: Consumed 16.132s CPU time. 2024-09-21T09:50:01.848 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 systemd[1]: Starting Ceph osd.2 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:50:01.848 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 podman[132509]: 2024-09-21 09:50:01.748828013 +0000 UTC m=+0.015721318 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:01.946 INFO:tasks.rados.rados.0.smithi076.stdout:2829: expect (ObjNum 958 snap 326 seq_num 958) 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: finishing write tid 1 to smithi07685715-3 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: finishing write tid 2 to smithi07685715-3 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: finishing write tid 3 to smithi07685715-3 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: finishing write tid 4 to smithi07685715-3 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: finishing write tid 5 to smithi07685715-3 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: oid 3 updating version 0 to 204 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: oid 3 updating version 204 to 207 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: oid 3 version 207 is already newer than 205 2024-09-21T09:50:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:2832: oid 3 version 207 is already newer than 206 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 207 (ObjNum 1067 snap 357 seq_num 1067) dirty exists 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2832: left oid 3 (ObjNum 1067 snap 357 seq_num 1067) 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: finishing write tid 1 to smithi07685715-37 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: finishing write tid 2 to smithi07685715-37 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: finishing write tid 3 to smithi07685715-37 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: finishing write tid 4 to smithi07685715-37 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: finishing write tid 5 to smithi07685715-37 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: oid 37 updating version 0 to 859 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: oid 37 version 859 is already newer than 856 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: oid 37 version 859 is already newer than 858 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: oid 37 version 859 is already newer than 857 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 859 (ObjNum 1068 snap 357 seq_num 1068) dirty exists 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2833: left oid 37 (ObjNum 1068 snap 357 seq_num 1068) 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2834: finishing write tid 1 to smithi07685715-35 2024-09-21T09:50:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:2834: finishing write tid 2 to smithi07685715-35 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: finishing write tid 3 to smithi07685715-35 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: finishing write tid 4 to smithi07685715-35 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: finishing write tid 5 to smithi07685715-35 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: oid 35 updating version 0 to 2160 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: oid 35 updating version 2160 to 2161 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: oid 35 updating version 2161 to 2162 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: oid 35 updating version 2162 to 2163 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2163 (ObjNum 1069 snap 357 seq_num 1069) dirty exists 2024-09-21T09:50:02.017 INFO:tasks.rados.rados.0.smithi076.stdout:2834: left oid 35 (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:01 smithi076 ceph-mon[106482]: pgmap v649: 89 pgs: 4 peering, 5 stale+active+clean, 80 active+clean; 188 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 755 KiB/s wr, 5 op/s 2024-09-21T09:50:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:01 smithi076 ceph-mon[106482]: osdmap e787: 8 total, 7 up, 8 in 2024-09-21T09:50:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:01 smithi076 ceph-mon[110963]: pgmap v649: 89 pgs: 4 peering, 5 stale+active+clean, 80 active+clean; 188 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 755 KiB/s wr, 5 op/s 2024-09-21T09:50:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:01 smithi076 ceph-mon[110963]: osdmap e787: 8 total, 7 up, 8 in 2024-09-21T09:50:02.123 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:01 smithi076 podman[132509]: 2024-09-21 09:50:01.879967791 +0000 UTC m=+0.146861094 container create 19ab4c8f84b027449860400580bbb4c364f90c9ae7a909ef70a5d5e3cc15ac01 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T09:50:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:01 smithi191 ceph-mon[102206]: pgmap v649: 89 pgs: 4 peering, 5 stale+active+clean, 80 active+clean; 188 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 755 KiB/s wr, 5 op/s 2024-09-21T09:50:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:01 smithi191 ceph-mon[102206]: osdmap e787: 8 total, 7 up, 8 in 2024-09-21T09:50:02.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 podman[132509]: 2024-09-21 09:50:02.124980983 +0000 UTC m=+0.391874289 container init 19ab4c8f84b027449860400580bbb4c364f90c9ae7a909ef70a5d5e3cc15ac01 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T09:50:02.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 podman[132509]: 2024-09-21 09:50:02.129971025 +0000 UTC m=+0.396864328 container start 19ab4c8f84b027449860400580bbb4c364f90c9ae7a909ef70a5d5e3cc15ac01 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:50:02.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 podman[132509]: 2024-09-21 09:50:02.216627776 +0000 UTC m=+0.483521096 container attach 19ab4c8f84b027449860400580bbb4c364f90c9ae7a909ef70a5d5e3cc15ac01 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T09:50:02.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:02.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 bash[132509]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:02.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:02.624 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 bash[132509]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:02.763 INFO:tasks.rados.rados.0.smithi076.stdout:2828: done (7 left) 2024-09-21T09:50:02.763 INFO:tasks.rados.rados.0.smithi076.stdout:2829: done (6 left) 2024-09-21T09:50:02.763 INFO:tasks.rados.rados.0.smithi076.stdout:2830: done (5 left) 2024-09-21T09:50:02.763 INFO:tasks.rados.rados.0.smithi076.stdout:2831: done (4 left) 2024-09-21T09:50:02.763 INFO:tasks.rados.rados.0.smithi076.stdout:2832: done (3 left) 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2833: done (2 left) 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2834: done (1 left) 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2835: done (0 left) 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2836: read oid 45 snap -1 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2836: expect (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2837: write oid 8 current snap is 358 2024-09-21T09:50:02.764 INFO:tasks.rados.rados.0.smithi076.stdout:2837: seq_num 1070 ranges {783490=405850,1980674=727030,3184014=487642} 2024-09-21T09:50:02.769 INFO:tasks.rados.rados.0.smithi076.stdout:2837: writing smithi07685715-8 from 783490 to 1189340 tid 1 2024-09-21T09:50:02.778 INFO:tasks.rados.rados.0.smithi076.stdout:2837: writing smithi07685715-8 from 1980674 to 2707704 tid 2 2024-09-21T09:50:02.784 INFO:tasks.rados.rados.0.smithi076.stdout:2837: writing smithi07685715-8 from 3184014 to 3671656 tid 3 2024-09-21T09:50:02.784 INFO:tasks.rados.rados.0.smithi076.stdout:2838: read oid 39 snap 355 2024-09-21T09:50:02.784 INFO:tasks.rados.rados.0.smithi076.stdout:2838: expect (ObjNum 1060 snap 354 seq_num 1060) 2024-09-21T09:50:02.784 INFO:tasks.rados.rados.0.smithi076.stdout:2839: rollback oid 40 current snap is 358 2024-09-21T09:50:02.784 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 351 2024-09-21T09:50:02.784 INFO:tasks.rados.rados.0.smithi076.stdout:2836: expect (ObjNum 970 snap 327 seq_num 970) 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:2837: finishing write tid 1 to smithi07685715-8 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:2837: finishing write tid 2 to smithi07685715-8 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:2839: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 0 (ObjNum 33 snap 0 seq_num 830379256) dirty dne 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:2837: finishing write tid 3 to smithi07685715-8 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:2837: finishing write tid 4 to smithi07685715-8 2024-09-21T09:50:02.882 INFO:tasks.rados.rados.0.smithi076.stdout:2837: finishing write tid 5 to smithi07685715-8 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:2837: oid 8 updating version 0 to 2134 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:2837: oid 8 version 2134 is already newer than 2132 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:2837: oid 8 version 2134 is already newer than 2133 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:2837: oid 8 version 2134 is already newer than 2131 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2134 (ObjNum 1070 snap 358 seq_num 1070) dirty exists 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:2837: left oid 8 (ObjNum 1070 snap 358 seq_num 1070) 2024-09-21T09:50:02.883 INFO:tasks.rados.rados.0.smithi076.stdout:2838: expect (ObjNum 1060 snap 354 seq_num 1060) 2024-09-21T09:50:03.019 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:50:03.019 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 bash[132509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:50:03.019 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:50:03.019 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:02 smithi076 bash[132509]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:50:03.019 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:50:03.019 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 bash[132509]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:50:03.168 INFO:tasks.rados.rados.0.smithi076.stdout:2836: done (3 left) 2024-09-21T09:50:03.168 INFO:tasks.rados.rados.0.smithi076.stdout:2837: done (2 left) 2024-09-21T09:50:03.168 INFO:tasks.rados.rados.0.smithi076.stdout:2838: done (1 left) 2024-09-21T09:50:03.168 INFO:tasks.rados.rados.0.smithi076.stdout:2839: done (0 left) 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2840: read oid 14 snap -1 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2840: expect (ObjNum 1050 snap 352 seq_num 1050) 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2841: read oid 44 snap -1 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2841: expect (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2842: rollback oid 34 current snap is 358 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 357 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2843: read oid 43 snap 336 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2843: expect (ObjNum 960 snap 326 seq_num 960) 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2844: read oid 11 snap -1 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2844: expect (ObjNum 1059 snap 354 seq_num 1059) 2024-09-21T09:50:03.169 INFO:tasks.rados.rados.0.smithi076.stdout:2845: snap_remove snap 347 2024-09-21T09:50:03.321 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:50:03] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:50:03.321 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:50:03.321 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 bash[132509]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:50:03.321 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:50:03.322 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 bash[132509]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:50:03.322 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:50:03.322 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 bash[132509]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:50:03.322 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate[132524]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:50:03.322 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 bash[132509]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:50:03.322 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 podman[132509]: 2024-09-21 09:50:03.050959954 +0000 UTC m=+1.317853258 container died 19ab4c8f84b027449860400580bbb4c364f90c9ae7a909ef70a5d5e3cc15ac01 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:50:03.756 INFO:tasks.rados.rados.0.smithi076.stdout:2845: done (5 left) 2024-09-21T09:50:03.756 INFO:tasks.rados.rados.0.smithi076.stdout:2846: write oid 37 current snap is 358 2024-09-21T09:50:03.756 INFO:tasks.rados.rados.0.smithi076.stdout:2846: seq_num 1071 ranges {400297=481099,1639156=361141} 2024-09-21T09:50:03.761 INFO:tasks.rados.rados.0.smithi076.stdout:2846: writing smithi07685715-37 from 400297 to 881396 tid 1 2024-09-21T09:50:03.768 INFO:tasks.rados.rados.0.smithi076.stdout:2846: writing smithi07685715-37 from 1639156 to 2000297 tid 2 2024-09-21T09:50:03.768 INFO:tasks.rados.rados.0.smithi076.stdout:2847: read oid 4 snap -1 2024-09-21T09:50:03.768 INFO:tasks.rados.rados.0.smithi076.stdout:2847: expect deleted 2024-09-21T09:50:03.768 INFO:tasks.rados.rados.0.smithi076.stdout:2848: snap_create 2024-09-21T09:50:03.769 INFO:tasks.rados.rados.0.smithi076.stdout:2840: expect (ObjNum 1050 snap 352 seq_num 1050) 2024-09-21T09:50:04.046 INFO:tasks.rados.rados.0.smithi076.stdout:2842: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:50:04.047 INFO:tasks.rados.rados.0.smithi076.stdout:2842: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:50:04.047 INFO:tasks.rados.rados.0.smithi076.stdout:2842: finishing rollback tid 2 to smithi07685715-34 2024-09-21T09:50:04.047 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 662 (ObjNum 975 snap 332 seq_num 975) dirty exists 2024-09-21T09:50:04.047 INFO:tasks.rados.rados.0.smithi076.stdout:2843: expect (ObjNum 960 snap 326 seq_num 960) 2024-09-21T09:50:04.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[106482]: pgmap v652: 89 pgs: 15 active+undersized, 4 peering, 6 active+undersized+degraded, 64 active+clean; 200 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 14 op/s; 22/390 objects degraded (5.641%) 2024-09-21T09:50:04.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[106482]: osdmap e788: 8 total, 7 up, 8 in 2024-09-21T09:50:04.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 22/390 objects degraded (5.641%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:04.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[106482]: osdmap e789: 8 total, 7 up, 8 in 2024-09-21T09:50:04.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[110963]: pgmap v652: 89 pgs: 15 active+undersized, 4 peering, 6 active+undersized+degraded, 64 active+clean; 200 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 14 op/s; 22/390 objects degraded (5.641%) 2024-09-21T09:50:04.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[110963]: osdmap e788: 8 total, 7 up, 8 in 2024-09-21T09:50:04.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 22/390 objects degraded (5.641%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:04.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:03 smithi076 ceph-mon[110963]: osdmap e789: 8 total, 7 up, 8 in 2024-09-21T09:50:04.075 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 podman[132669]: 2024-09-21 09:50:03.721598109 +0000 UTC m=+0.664364610 container remove 19ab4c8f84b027449860400580bbb4c364f90c9ae7a909ef70a5d5e3cc15ac01 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True) 2024-09-21T09:50:04.075 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 podman[132741]: 2024-09-21 09:50:03.842625946 +0000 UTC m=+0.016421676 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:04.075 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:03 smithi076 podman[132741]: 2024-09-21 09:50:03.947643884 +0000 UTC m=+0.121439607 container create d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:03 smithi191 ceph-mon[102206]: pgmap v652: 89 pgs: 15 active+undersized, 4 peering, 6 active+undersized+degraded, 64 active+clean; 200 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 14 op/s; 22/390 objects degraded (5.641%) 2024-09-21T09:50:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:03 smithi191 ceph-mon[102206]: osdmap e788: 8 total, 7 up, 8 in 2024-09-21T09:50:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:03 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 22/390 objects degraded (5.641%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:03 smithi191 ceph-mon[102206]: osdmap e789: 8 total, 7 up, 8 in 2024-09-21T09:50:04.135 INFO:tasks.rados.rados.0.smithi076.stdout:2841: expect (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:50:04.360 INFO:tasks.rados.rados.0.smithi076.stdout:2844: expect (ObjNum 1059 snap 354 seq_num 1059) 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: finishing write tid 1 to smithi07685715-37 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: finishing write tid 2 to smithi07685715-37 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: finishing write tid 3 to smithi07685715-37 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: finishing write tid 4 to smithi07685715-37 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: oid 37 updating version 0 to 865 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: oid 37 version 865 is already newer than 863 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: oid 37 version 865 is already newer than 864 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 865 (ObjNum 1071 snap 358 seq_num 1071) dirty exists 2024-09-21T09:50:04.537 INFO:tasks.rados.rados.0.smithi076.stdout:2846: left oid 37 (ObjNum 1071 snap 358 seq_num 1071) 2024-09-21T09:50:04.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:04 smithi076 podman[132741]: 2024-09-21 09:50:04.292999316 +0000 UTC m=+0.466795042 container init d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T09:50:04.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:04 smithi076 podman[132741]: 2024-09-21 09:50:04.299624466 +0000 UTC m=+0.473420188 container start d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:50:04.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:04 smithi076 ceph-osd[132780]: -- 172.21.15.76:0/2448896687 <== mon.0 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d65bab5a40 con 0x55d65c884000 2024-09-21T09:50:04.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:04 smithi076 bash[132741]: d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d 2024-09-21T09:50:04.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:04 smithi076 systemd[1]: Started Ceph osd.2 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:50:04.772 INFO:tasks.rados.rados.0.smithi076.stdout:2840: done (7 left) 2024-09-21T09:50:04.772 INFO:tasks.rados.rados.0.smithi076.stdout:2841: done (6 left) 2024-09-21T09:50:04.772 INFO:tasks.rados.rados.0.smithi076.stdout:2842: done (5 left) 2024-09-21T09:50:04.772 INFO:tasks.rados.rados.0.smithi076.stdout:2843: done (4 left) 2024-09-21T09:50:04.772 INFO:tasks.rados.rados.0.smithi076.stdout:2844: done (3 left) 2024-09-21T09:50:04.773 INFO:tasks.rados.rados.0.smithi076.stdout:2846: done (2 left) 2024-09-21T09:50:04.773 INFO:tasks.rados.rados.0.smithi076.stdout:2847: done (1 left) 2024-09-21T09:50:04.773 INFO:tasks.rados.rados.0.smithi076.stdout:2848: done (0 left) 2024-09-21T09:50:04.773 INFO:tasks.rados.rados.0.smithi076.stdout:2849: write oid 10 current snap is 359 2024-09-21T09:50:04.773 INFO:tasks.rados.rados.0.smithi076.stdout:2849: seq_num 1072 ranges {417104=672701,1813991=203113} 2024-09-21T09:50:04.780 INFO:tasks.rados.rados.0.smithi076.stdout:2849: writing smithi07685715-10 from 417104 to 1089805 tid 1 2024-09-21T09:50:04.785 INFO:tasks.rados.rados.0.smithi076.stdout:2849: writing smithi07685715-10 from 1813991 to 2017104 tid 2 2024-09-21T09:50:04.785 INFO:tasks.rados.rados.0.smithi076.stdout:2850: snap_create 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: finishing write tid 1 to smithi07685715-10 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: finishing write tid 2 to smithi07685715-10 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: finishing write tid 3 to smithi07685715-10 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: finishing write tid 4 to smithi07685715-10 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: oid 10 updating version 0 to 1722 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: oid 10 updating version 1722 to 1723 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: oid 10 version 1723 is already newer than 1721 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1723 (ObjNum 1072 snap 359 seq_num 1072) dirty exists 2024-09-21T09:50:04.789 INFO:tasks.rados.rados.0.smithi076.stdout:2849: left oid 10 (ObjNum 1072 snap 359 seq_num 1072) 2024-09-21T09:50:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[110963]: pgmap v655: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 201 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 34/393 objects degraded (8.651%) 2024-09-21T09:50:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[110963]: osdmap e790: 8 total, 7 up, 8 in 2024-09-21T09:50:05.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[106482]: pgmap v655: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 201 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 34/393 objects degraded (8.651%) 2024-09-21T09:50:05.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:05.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-mon[106482]: osdmap e790: 8 total, 7 up, 8 in 2024-09-21T09:50:05.772 INFO:tasks.rados.rados.0.smithi076.stdout:2849: done (1 left) 2024-09-21T09:50:05.772 INFO:tasks.rados.rados.0.smithi076.stdout:2850: done (0 left) 2024-09-21T09:50:05.773 INFO:tasks.rados.rados.0.smithi076.stdout:2851: write oid 12 current snap is 360 2024-09-21T09:50:05.773 INFO:tasks.rados.rados.0.smithi076.stdout:2851: seq_num 1073 ranges {433911=747950,1756120=277791} 2024-09-21T09:50:05.779 INFO:tasks.rados.rados.0.smithi076.stdout:2851: writing smithi07685715-12 from 433911 to 1181861 tid 1 2024-09-21T09:50:05.784 INFO:tasks.rados.rados.0.smithi076.stdout:2851: writing smithi07685715-12 from 1756120 to 2033911 tid 2 2024-09-21T09:50:05.784 INFO:tasks.rados.rados.0.smithi076.stdout:2852: rollback oid 13 current snap is 360 2024-09-21T09:50:05.784 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 358 2024-09-21T09:50:05.785 INFO:tasks.rados.rados.0.smithi076.stdout:2853: delete oid 41 current snap is 360 2024-09-21T09:50:05.785 INFO:tasks.rados.rados.0.smithi076.stdout:2851: finishing write tid 1 to smithi07685715-12 2024-09-21T09:50:05.786 INFO:tasks.rados.rados.0.smithi076.stdout:2851: finishing write tid 2 to smithi07685715-12 2024-09-21T09:50:05.786 INFO:tasks.rados.rados.0.smithi076.stdout:2851: finishing write tid 3 to smithi07685715-12 2024-09-21T09:50:05.787 INFO:tasks.rados.rados.0.smithi076.stdout:2851: finishing write tid 4 to smithi07685715-12 2024-09-21T09:50:05.787 INFO:tasks.rados.rados.0.smithi076.stdout:2851: oid 12 updating version 0 to 970 2024-09-21T09:50:05.787 INFO:tasks.rados.rados.0.smithi076.stdout:2851: oid 12 version 970 is already newer than 968 2024-09-21T09:50:05.787 INFO:tasks.rados.rados.0.smithi076.stdout:2851: oid 12 version 970 is already newer than 969 2024-09-21T09:50:05.787 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 970 (ObjNum 1073 snap 360 seq_num 1073) dirty exists 2024-09-21T09:50:05.787 INFO:tasks.rados.rados.0.smithi076.stdout:2851: left oid 12 (ObjNum 1073 snap 360 seq_num 1073) 2024-09-21T09:50:05.789 INFO:tasks.rados.rados.0.smithi076.stdout:2851: done (2 left) 2024-09-21T09:50:05.790 INFO:tasks.rados.rados.0.smithi076.stdout:2853: done (1 left) 2024-09-21T09:50:05.790 INFO:tasks.rados.rados.0.smithi076.stdout:2854: snap_create 2024-09-21T09:50:05.791 INFO:tasks.rados.rados.0.smithi076.stdout:2852: finishing rollback tid 0 to smithi07685715-13 2024-09-21T09:50:05.791 INFO:tasks.rados.rados.0.smithi076.stdout:2852: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:50:05.791 INFO:tasks.rados.rados.0.smithi076.stdout:2852: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:50:05.791 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 537 (ObjNum 1055 snap 353 seq_num 1055) dirty exists 2024-09-21T09:50:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:05 smithi191 ceph-mon[102206]: pgmap v655: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 201 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 34/393 objects degraded (8.651%) 2024-09-21T09:50:05.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:05.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:05.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:05 smithi191 ceph-mon[102206]: osdmap e790: 8 total, 7 up, 8 in 2024-09-21T09:50:06.035 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:05 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T09:50:05.675+0000 7f5c817af640 -1 Falling back to public interface 2024-09-21T09:50:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:2852: done (1 left) 2024-09-21T09:50:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:2854: done (0 left) 2024-09-21T09:50:06.781 INFO:tasks.rados.rados.0.smithi076.stdout:2855: read oid 48 snap -1 2024-09-21T09:50:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:2855: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:50:06.782 INFO:tasks.rados.rados.0.smithi076.stdout:2856: delete oid 23 current snap is 361 2024-09-21T09:50:06.793 INFO:tasks.rados.rados.0.smithi076.stdout:2855: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:50:06.986 INFO:tasks.rados.rados.0.smithi076.stdout:2855: done (1 left) 2024-09-21T09:50:06.986 INFO:tasks.rados.rados.0.smithi076.stdout:2856: done (0 left) 2024-09-21T09:50:06.987 INFO:tasks.rados.rados.0.smithi076.stdout:2857: read oid 47 snap -1 2024-09-21T09:50:06.987 INFO:tasks.rados.rados.0.smithi076.stdout:2857: expect (ObjNum 1020 snap 345 seq_num 1020) 2024-09-21T09:50:06.987 INFO:tasks.rados.rados.0.smithi076.stdout:2858: read oid 2 snap 351 2024-09-21T09:50:06.987 INFO:tasks.rados.rados.0.smithi076.stdout:2858: expect (ObjNum 1043 snap 351 seq_num 1043) 2024-09-21T09:50:06.987 INFO:tasks.rados.rados.0.smithi076.stdout:2859: snap_remove snap 336 2024-09-21T09:50:07.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:06 smithi191 ceph-mon[102206]: osdmap e791: 8 total, 7 up, 8 in 2024-09-21T09:50:07.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:06 smithi076 ceph-mon[106482]: osdmap e791: 8 total, 7 up, 8 in 2024-09-21T09:50:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:06 smithi076 ceph-mon[110963]: osdmap e791: 8 total, 7 up, 8 in 2024-09-21T09:50:07.786 INFO:tasks.rados.rados.0.smithi076.stdout:2859: done (2 left) 2024-09-21T09:50:07.786 INFO:tasks.rados.rados.0.smithi076.stdout:2860: write oid 50 current snap is 361 2024-09-21T09:50:07.786 INFO:tasks.rados.rados.0.smithi076.stdout:2860: seq_num 1076 ranges {484332=573697,1531566=552766} 2024-09-21T09:50:07.792 INFO:tasks.rados.rados.0.smithi076.stdout:2860: writing smithi07685715-50 from 484332 to 1058029 tid 1 2024-09-21T09:50:07.799 INFO:tasks.rados.rados.0.smithi076.stdout:2860: writing smithi07685715-50 from 1531566 to 2084332 tid 2 2024-09-21T09:50:07.799 INFO:tasks.rados.rados.0.smithi076.stdout:2861: snap_create 2024-09-21T09:50:07.799 INFO:tasks.rados.rados.0.smithi076.stdout:2857: expect (ObjNum 1020 snap 345 seq_num 1020) 2024-09-21T09:50:07.907 INFO:tasks.rados.rados.0.smithi076.stdout:2858: expect (ObjNum 1043 snap 351 seq_num 1043) 2024-09-21T09:50:08.027 INFO:tasks.rados.rados.0.smithi076.stdout:2860: finishing write tid 1 to smithi07685715-50 2024-09-21T09:50:08.027 INFO:tasks.rados.rados.0.smithi076.stdout:2860: finishing write tid 2 to smithi07685715-50 2024-09-21T09:50:08.027 INFO:tasks.rados.rados.0.smithi076.stdout:2860: finishing write tid 3 to smithi07685715-50 2024-09-21T09:50:08.027 INFO:tasks.rados.rados.0.smithi076.stdout:2860: finishing write tid 4 to smithi07685715-50 2024-09-21T09:50:08.027 INFO:tasks.rados.rados.0.smithi076.stdout:2860: oid 50 updating version 0 to 916 2024-09-21T09:50:08.027 INFO:tasks.rados.rados.0.smithi076.stdout:2860: oid 50 version 916 is already newer than 915 2024-09-21T09:50:08.028 INFO:tasks.rados.rados.0.smithi076.stdout:2860: oid 50 updating version 916 to 917 2024-09-21T09:50:08.028 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 917 (ObjNum 1076 snap 361 seq_num 1076) dirty exists 2024-09-21T09:50:08.028 INFO:tasks.rados.rados.0.smithi076.stdout:2860: left oid 50 (ObjNum 1076 snap 361 seq_num 1076) 2024-09-21T09:50:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:07 smithi191 ceph-mon[102206]: pgmap v658: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 217 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 35/405 objects degraded (8.642%) 2024-09-21T09:50:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:07 smithi191 ceph-mon[102206]: osdmap e792: 8 total, 7 up, 8 in 2024-09-21T09:50:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:07 smithi076 ceph-mon[106482]: pgmap v658: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 217 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 35/405 objects degraded (8.642%) 2024-09-21T09:50:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:07 smithi076 ceph-mon[106482]: osdmap e792: 8 total, 7 up, 8 in 2024-09-21T09:50:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:07 smithi076 ceph-mon[110963]: pgmap v658: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 217 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 35/405 objects degraded (8.642%) 2024-09-21T09:50:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:07 smithi076 ceph-mon[110963]: osdmap e792: 8 total, 7 up, 8 in 2024-09-21T09:50:08.787 INFO:tasks.rados.rados.0.smithi076.stdout:2857: done (3 left) 2024-09-21T09:50:08.787 INFO:tasks.rados.rados.0.smithi076.stdout:2858: done (2 left) 2024-09-21T09:50:08.787 INFO:tasks.rados.rados.0.smithi076.stdout:2860: done (1 left) 2024-09-21T09:50:08.787 INFO:tasks.rados.rados.0.smithi076.stdout:2861: done (0 left) 2024-09-21T09:50:08.788 INFO:tasks.rados.rados.0.smithi076.stdout:2862: snap_create 2024-09-21T09:50:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:08 smithi191 ceph-mon[102206]: osdmap e793: 8 total, 7 up, 8 in 2024-09-21T09:50:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:08 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 35/405 objects degraded (8.642%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[106482]: osdmap e793: 8 total, 7 up, 8 in 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 35/405 objects degraded (8.642%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[110963]: osdmap e793: 8 total, 7 up, 8 in 2024-09-21T09:50:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:08 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 35/405 objects degraded (8.642%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:09.623 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T09:50:09.301+0000 7f5c817af640 -1 osd.2 783 log_to_monitors true 2024-09-21T09:50:09.788 INFO:tasks.rados.rados.0.smithi076.stdout:2862: done (0 left) 2024-09-21T09:50:09.788 INFO:tasks.rados.rados.0.smithi076.stdout:2863: rollback oid 30 current snap is 363 2024-09-21T09:50:09.788 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 344 2024-09-21T09:50:09.788 INFO:tasks.rados.rados.0.smithi076.stdout:2864: read oid 44 snap -1 2024-09-21T09:50:09.788 INFO:tasks.rados.rados.0.smithi076.stdout:2864: expect (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:50:09.789 INFO:tasks.rados.rados.0.smithi076.stdout:2865: snap_remove snap 350 2024-09-21T09:50:10.021 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[106482]: pgmap v661: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 217 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 35/408 objects degraded (8.578%) 2024-09-21T09:50:10.021 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[106482]: osdmap e794: 8 total, 7 up, 8 in 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[106482]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[106482]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[106482]: osdmap e795: 8 total, 7 up, 8 in 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[106482]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[110963]: pgmap v661: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 217 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 35/408 objects degraded (8.578%) 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[110963]: osdmap e794: 8 total, 7 up, 8 in 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[110963]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[110963]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:50:10.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[110963]: osdmap e795: 8 total, 7 up, 8 in 2024-09-21T09:50:10.023 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-mon[110963]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:50:10.023 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:50:09 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T09:50:09.807+0000 7f5c79559640 -1 osd.2 783 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:50:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:09 smithi191 ceph-mon[102206]: pgmap v661: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 217 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 23 op/s; 35/408 objects degraded (8.578%) 2024-09-21T09:50:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:09 smithi191 ceph-mon[102206]: osdmap e794: 8 total, 7 up, 8 in 2024-09-21T09:50:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:09 smithi191 ceph-mon[102206]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:50:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:09 smithi191 ceph-mon[102206]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:50:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:09 smithi191 ceph-mon[102206]: osdmap e795: 8 total, 7 up, 8 in 2024-09-21T09:50:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:09 smithi191 ceph-mon[102206]: from='osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:50:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2865: done (2 left) 2024-09-21T09:50:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2866: read oid 7 snap -1 2024-09-21T09:50:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2866: expect deleted 2024-09-21T09:50:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2867: read oid 48 snap -1 2024-09-21T09:50:10.796 INFO:tasks.rados.rados.0.smithi076.stdout:2867: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:50:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2868: rollback oid 50 current snap is 363 2024-09-21T09:50:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 344 2024-09-21T09:50:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2869: read oid 34 snap -1 2024-09-21T09:50:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2869: expect (ObjNum 975 snap 332 seq_num 975) 2024-09-21T09:50:10.797 INFO:tasks.rados.rados.0.smithi076.stdout:2870: snap_remove snap 362 2024-09-21T09:50:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:2870: done (6 left) 2024-09-21T09:50:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:2871: read oid 18 snap -1 2024-09-21T09:50:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:2871: expect deleted 2024-09-21T09:50:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:2872: write oid 49 current snap is 363 2024-09-21T09:50:11.797 INFO:tasks.rados.rados.0.smithi076.stdout:2872: seq_num 1077 ranges {501139=648946,1873695=227444} 2024-09-21T09:50:11.803 INFO:tasks.rados.rados.0.smithi076.stdout:2872: writing smithi07685715-49 from 501139 to 1150085 tid 1 2024-09-21T09:50:11.809 INFO:tasks.rados.rados.0.smithi076.stdout:2872: writing smithi07685715-49 from 1873695 to 2101139 tid 2 2024-09-21T09:50:11.810 INFO:tasks.rados.rados.0.smithi076.stdout:2873: read oid 6 snap 356 2024-09-21T09:50:11.810 INFO:tasks.rados.rados.0.smithi076.stdout:2873: expect deleted 2024-09-21T09:50:11.810 INFO:tasks.rados.rados.0.smithi076.stdout:2863: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:50:11.810 INFO:tasks.rados.rados.0.smithi076.stdout:2863: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:50:11.810 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2152 (ObjNum 996 snap 339 seq_num 996) dirty exists 2024-09-21T09:50:11.811 INFO:tasks.rados.rados.0.smithi076.stdout:2864: expect (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:50:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: pgmap v664: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 207 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 465 KiB/s wr, 4 op/s; 36/396 objects degraded (9.091%) 2024-09-21T09:50:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:50:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517] boot 2024-09-21T09:50:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:50:11.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:50:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: pgmap v664: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 207 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 465 KiB/s wr, 4 op/s; 36/396 objects degraded (9.091%) 2024-09-21T09:50:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:50:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517] boot 2024-09-21T09:50:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:50:11.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:50:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: pgmap v664: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 207 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 465 KiB/s wr, 4 op/s; 36/396 objects degraded (9.091%) 2024-09-21T09:50:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:50:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: osd.2 [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517] boot 2024-09-21T09:50:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T09:50:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:50:12.041 INFO:tasks.rados.rados.0.smithi076.stdout:2869: expect (ObjNum 975 snap 332 seq_num 975) 2024-09-21T09:50:12.198 INFO:tasks.rados.rados.0.smithi076.stdout:2867: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:50:12.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T09:50:12.071+0000 7f86bc22c640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:50:12.390 INFO:tasks.rados.rados.0.smithi076.stdout:2868: finishing rollback tid 0 to smithi07685715-50 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2868: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2868: finishing rollback tid 2 to smithi07685715-50 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 921 (ObjNum 961 snap 326 seq_num 961) dirty exists 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2863: done (8 left) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2864: done (7 left) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2866: done (6 left) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2867: done (5 left) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2868: done (4 left) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2869: done (3 left) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2874: read oid 10 snap -1 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2874: expect (ObjNum 1072 snap 359 seq_num 1072) 2024-09-21T09:50:12.391 INFO:tasks.rados.rados.0.smithi076.stdout:2875: rollback oid 8 current snap is 363 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 358 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: finishing write tid 1 to smithi07685715-49 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: finishing write tid 2 to smithi07685715-49 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: finishing write tid 3 to smithi07685715-49 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: finishing write tid 4 to smithi07685715-49 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: oid 49 updating version 0 to 2157 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: oid 49 updating version 2157 to 2159 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: oid 49 version 2159 is already newer than 2158 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2159 (ObjNum 1077 snap 363 seq_num 1077) dirty exists 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: left oid 49 (ObjNum 1077 snap 363 seq_num 1077) 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2871: done (4 left) 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2872: done (3 left) 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2876: write oid 13 current snap is 363 2024-09-21T09:50:12.392 INFO:tasks.rados.rados.0.smithi076.stdout:2876: seq_num 1078 ranges {517946=724195,1932177=185769} 2024-09-21T09:50:12.397 INFO:tasks.rados.rados.0.smithi076.stdout:2876: writing smithi07685715-13 from 517946 to 1242141 tid 1 2024-09-21T09:50:12.402 INFO:tasks.rados.rados.0.smithi076.stdout:2876: writing smithi07685715-13 from 1932177 to 2117946 tid 2 2024-09-21T09:50:12.402 INFO:tasks.rados.rados.0.smithi076.stdout:2877: read oid 47 snap -1 2024-09-21T09:50:12.402 INFO:tasks.rados.rados.0.smithi076.stdout:2877: expect (ObjNum 1020 snap 345 seq_num 1020) 2024-09-21T09:50:12.403 INFO:tasks.rados.rados.0.smithi076.stdout:2878: delete oid 11 current snap is 363 2024-09-21T09:50:12.403 INFO:tasks.rados.rados.0.smithi076.stdout:2875: finishing rollback tid 0 to smithi07685715-8 2024-09-21T09:50:12.403 INFO:tasks.rados.rados.0.smithi076.stdout:2874: expect (ObjNum 1072 snap 359 seq_num 1072) 2024-09-21T09:50:12.493 INFO:tasks.rados.rados.0.smithi076.stdout:2875: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:50:12.493 INFO:tasks.rados.rados.0.smithi076.stdout:2875: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2163 (ObjNum 1070 snap 358 seq_num 1070) dirty exists 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: finishing write tid 1 to smithi07685715-13 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: finishing write tid 2 to smithi07685715-13 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: finishing write tid 3 to smithi07685715-13 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2873: done (5 left) 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2874: done (4 left) 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2875: done (3 left) 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2878: done (2 left) 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2879: rollback oid 44 current snap is 363 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 356 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: finishing write tid 4 to smithi07685715-13 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: oid 13 updating version 0 to 549 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: oid 13 version 549 is already newer than 548 2024-09-21T09:50:12.494 INFO:tasks.rados.rados.0.smithi076.stdout:2876: oid 13 version 549 is already newer than 547 2024-09-21T09:50:12.495 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 549 (ObjNum 1078 snap 363 seq_num 1078) dirty exists 2024-09-21T09:50:12.495 INFO:tasks.rados.rados.0.smithi076.stdout:2876: left oid 13 (ObjNum 1078 snap 363 seq_num 1078) 2024-09-21T09:50:12.495 INFO:tasks.rados.rados.0.smithi076.stdout:2877: expect (ObjNum 1020 snap 345 seq_num 1020) 2024-09-21T09:50:12.597 INFO:tasks.rados.rados.0.smithi076.stdout:2879: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:50:12.597 INFO:tasks.rados.rados.0.smithi076.stdout:2879: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:50:12.597 INFO:tasks.rados.rados.0.smithi076.stdout:2879: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:50:12.598 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 697 (ObjNum 1054 snap 353 seq_num 1054) dirty exists 2024-09-21T09:50:12.598 INFO:tasks.rados.rados.0.smithi076.stdout:2876: done (2 left) 2024-09-21T09:50:12.598 INFO:tasks.rados.rados.0.smithi076.stdout:2877: done (1 left) 2024-09-21T09:50:12.598 INFO:tasks.rados.rados.0.smithi076.stdout:2879: done (0 left) 2024-09-21T09:50:12.598 INFO:tasks.rados.rados.0.smithi076.stdout:2880: write oid 13 current snap is 363 2024-09-21T09:50:12.598 INFO:tasks.rados.rados.0.smithi076.stdout:2880: seq_num 1080 ranges {551560=591046,1765494=386066} 2024-09-21T09:50:12.603 INFO:tasks.rados.rados.0.smithi076.stdout:2880: writing smithi07685715-13 from 551560 to 1142606 tid 1 2024-09-21T09:50:12.609 INFO:tasks.rados.rados.0.smithi076.stdout:2880: writing smithi07685715-13 from 1765494 to 2151560 tid 2 2024-09-21T09:50:12.609 INFO:tasks.rados.rados.0.smithi076.stdout:2881: snap_remove snap 356 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2881: done (1 left) 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2882: delete oid 31 current snap is 363 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: finishing write tid 1 to smithi07685715-13 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: finishing write tid 2 to smithi07685715-13 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: finishing write tid 3 to smithi07685715-13 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: finishing write tid 4 to smithi07685715-13 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: oid 13 updating version 0 to 551 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: oid 13 version 551 is already newer than 550 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: oid 13 updating version 551 to 552 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 552 (ObjNum 1080 snap 363 seq_num 1080) dirty exists 2024-09-21T09:50:13.000 INFO:tasks.rados.rados.0.smithi076.stdout:2880: left oid 13 (ObjNum 1080 snap 363 seq_num 1080) 2024-09-21T09:50:13.005 INFO:tasks.rados.rados.0.smithi076.stdout:2880: done (1 left) 2024-09-21T09:50:13.005 INFO:tasks.rados.rados.0.smithi076.stdout:2882: done (0 left) 2024-09-21T09:50:13.005 INFO:tasks.rados.rados.0.smithi076.stdout:2883: snap_remove snap 357 2024-09-21T09:50:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:50:13.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:50:13.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:12 smithi191 ceph-mon[102206]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[110963]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:13.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:13.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:13.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:50:13.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:50:13.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:12 smithi076 ceph-mon[106482]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T09:50:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:50:13] "GET /metrics HTTP/1.1" 200 36114 "" "Prometheus/2.51.0" 2024-09-21T09:50:14.011 INFO:tasks.rados.rados.0.smithi076.stdout:2883: done (0 left) 2024-09-21T09:50:14.012 INFO:tasks.rados.rados.0.smithi076.stdout:2884: snap_remove snap 353 2024-09-21T09:50:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-mon[106482]: pgmap v667: 89 pgs: 12 peering, 3 active+undersized, 6 active+undersized+degraded, 68 active+clean; 191 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 740 KiB/s wr, 13 op/s; 28/369 objects degraded (7.588%) 2024-09-21T09:50:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-mon[106482]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:50:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 28/369 objects degraded (7.588%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-mon[110963]: pgmap v667: 89 pgs: 12 peering, 3 active+undersized, 6 active+undersized+degraded, 68 active+clean; 191 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 740 KiB/s wr, 13 op/s; 28/369 objects degraded (7.588%) 2024-09-21T09:50:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-mon[110963]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:50:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:13 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 28/369 objects degraded (7.588%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:13 smithi191 ceph-mon[102206]: pgmap v667: 89 pgs: 12 peering, 3 active+undersized, 6 active+undersized+degraded, 68 active+clean; 191 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 740 KiB/s wr, 13 op/s; 28/369 objects degraded (7.588%) 2024-09-21T09:50:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:13 smithi191 ceph-mon[102206]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T09:50:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:13 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 28/369 objects degraded (7.588%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:15.019 INFO:tasks.rados.rados.0.smithi076.stdout:2884: done (0 left) 2024-09-21T09:50:15.019 INFO:tasks.rados.rados.0.smithi076.stdout:2885: delete oid 43 current snap is 363 2024-09-21T09:50:15.026 INFO:tasks.rados.rados.0.smithi076.stdout:2885: done (0 left) 2024-09-21T09:50:15.026 INFO:tasks.rados.rados.0.smithi076.stdout:2886: write oid 30 current snap is 363 2024-09-21T09:50:15.026 INFO:tasks.rados.rados.0.smithi076.stdout:2886: seq_num 1083 ranges {601981=416793,1424587=492702} 2024-09-21T09:50:15.031 INFO:tasks.rados.rados.0.smithi076.stdout:2886: writing smithi07685715-30 from 601981 to 1018774 tid 1 2024-09-21T09:50:15.036 INFO:tasks.rados.rados.0.smithi076.stdout:2886: writing smithi07685715-30 from 1424587 to 1917289 tid 2 2024-09-21T09:50:15.036 INFO:tasks.rados.rados.0.smithi076.stdout:2887: write oid 26 current snap is 363 2024-09-21T09:50:15.036 INFO:tasks.rados.rados.0.smithi076.stdout:2887: seq_num 1084 ranges {618788=492042,1883069=335719} 2024-09-21T09:50:15.042 INFO:tasks.rados.rados.0.smithi076.stdout:2887: writing smithi07685715-26 from 618788 to 1110830 tid 1 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2887: writing smithi07685715-26 from 1883069 to 2218788 tid 2 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2888: snap_create 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: finishing write tid 1 to smithi07685715-30 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: finishing write tid 2 to smithi07685715-30 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: finishing write tid 3 to smithi07685715-30 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: finishing write tid 4 to smithi07685715-30 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: oid 30 updating version 0 to 2171 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: oid 30 version 2171 is already newer than 2170 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: oid 30 updating version 2171 to 2172 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2172 (ObjNum 1083 snap 363 seq_num 1083) dirty exists 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2886: left oid 30 (ObjNum 1083 snap 363 seq_num 1083) 2024-09-21T09:50:15.049 INFO:tasks.rados.rados.0.smithi076.stdout:2887: finishing write tid 1 to smithi07685715-26 2024-09-21T09:50:15.051 INFO:tasks.rados.rados.0.smithi076.stdout:2887: finishing write tid 2 to smithi07685715-26 2024-09-21T09:50:15.051 INFO:tasks.rados.rados.0.smithi076.stdout:2887: finishing write tid 3 to smithi07685715-26 2024-09-21T09:50:15.051 INFO:tasks.rados.rados.0.smithi076.stdout:2887: finishing write tid 4 to smithi07685715-26 2024-09-21T09:50:15.051 INFO:tasks.rados.rados.0.smithi076.stdout:2887: oid 26 updating version 0 to 992 2024-09-21T09:50:15.052 INFO:tasks.rados.rados.0.smithi076.stdout:2887: oid 26 version 992 is already newer than 990 2024-09-21T09:50:15.052 INFO:tasks.rados.rados.0.smithi076.stdout:2887: oid 26 version 992 is already newer than 991 2024-09-21T09:50:15.052 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 992 (ObjNum 1084 snap 363 seq_num 1084) dirty exists 2024-09-21T09:50:15.052 INFO:tasks.rados.rados.0.smithi076.stdout:2887: left oid 26 (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:15.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:15 smithi191 ceph-mon[102206]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:50:15.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:15 smithi076 ceph-mon[106482]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:50:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:15 smithi076 ceph-mon[110963]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T09:50:16.022 INFO:tasks.rados.rados.0.smithi076.stdout:2886: done (2 left) 2024-09-21T09:50:16.022 INFO:tasks.rados.rados.0.smithi076.stdout:2887: done (1 left) 2024-09-21T09:50:16.022 INFO:tasks.rados.rados.0.smithi076.stdout:2888: done (0 left) 2024-09-21T09:50:16.022 INFO:tasks.rados.rados.0.smithi076.stdout:2889: delete oid 34 current snap is 364 2024-09-21T09:50:16.026 INFO:tasks.rados.rados.0.smithi076.stdout:2889: done (0 left) 2024-09-21T09:50:16.026 INFO:tasks.rados.rados.0.smithi076.stdout:2890: read oid 43 snap -1 2024-09-21T09:50:16.026 INFO:tasks.rados.rados.0.smithi076.stdout:2890: expect deleted 2024-09-21T09:50:16.027 INFO:tasks.rados.rados.0.smithi076.stdout:2891: delete oid 36 current snap is 364 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:2890: done (1 left) 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:2891: done (0 left) 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:2892: rollback oid 17 current snap is 364 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 363 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:2893: read oid 28 snap 355 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:2893: expect (ObjNum 1045 snap 351 seq_num 1045) 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:2894: rollback oid 31 current snap is 364 2024-09-21T09:50:16.031 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 344 2024-09-21T09:50:16.032 INFO:tasks.rados.rados.0.smithi076.stdout:2895: rollback oid 30 current snap is 364 2024-09-21T09:50:16.032 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 358 2024-09-21T09:50:16.032 INFO:tasks.rados.rados.0.smithi076.stdout:2896: write oid 42 current snap is 364 2024-09-21T09:50:16.032 INFO:tasks.rados.rados.0.smithi076.stdout:2896: seq_num 1087 ranges {669209=717789,1942162=327047} 2024-09-21T09:50:16.039 INFO:tasks.rados.rados.0.smithi076.stdout:2896: writing smithi07685715-42 from 669209 to 1386998 tid 1 2024-09-21T09:50:16.044 INFO:tasks.rados.rados.0.smithi076.stdout:2896: writing smithi07685715-42 from 1942162 to 2269209 tid 2 2024-09-21T09:50:16.044 INFO:tasks.rados.rados.0.smithi076.stdout:2897: read oid 18 snap -1 2024-09-21T09:50:16.044 INFO:tasks.rados.rados.0.smithi076.stdout:2897: expect deleted 2024-09-21T09:50:16.045 INFO:tasks.rados.rados.0.smithi076.stdout:2898: write oid 38 current snap is 364 2024-09-21T09:50:16.045 INFO:tasks.rados.rados.0.smithi076.stdout:2892: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:50:16.045 INFO:tasks.rados.rados.0.smithi076.stdout:2898: seq_num 1088 ranges {686016=509391,1716997=569019} 2024-09-21T09:50:16.050 INFO:tasks.rados.rados.0.smithi076.stdout:2898: writing smithi07685715-38 from 686016 to 1195407 tid 1 2024-09-21T09:50:16.056 INFO:tasks.rados.rados.0.smithi076.stdout:2898: writing smithi07685715-38 from 1716997 to 2286016 tid 2 2024-09-21T09:50:16.056 INFO:tasks.rados.rados.0.smithi076.stdout:2899: delete oid 27 current snap is 364 2024-09-21T09:50:16.057 INFO:tasks.rados.rados.0.smithi076.stdout:2892: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:50:16.057 INFO:tasks.rados.rados.0.smithi076.stdout:2892: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:50:16.057 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1745 (ObjNum 1042 snap 351 seq_num 1042) dirty exists 2024-09-21T09:50:16.057 INFO:tasks.rados.rados.0.smithi076.stdout:2894: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:50:16.057 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 0 (ObjNum 257 snap 0 seq_num 864490624) dirty dne 2024-09-21T09:50:16.057 INFO:tasks.rados.rados.0.smithi076.stdout:2893: expect (ObjNum 1045 snap 351 seq_num 1045) 2024-09-21T09:50:16.173 INFO:tasks.rados.rados.0.smithi076.stdout:2895: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:50:16.173 INFO:tasks.rados.rados.0.smithi076.stdout:2895: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2175 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: finishing write tid 1 to smithi07685715-42 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: finishing write tid 2 to smithi07685715-42 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: finishing write tid 3 to smithi07685715-42 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: finishing write tid 4 to smithi07685715-42 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: oid 42 updating version 0 to 730 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: oid 42 version 730 is already newer than 729 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: oid 42 updating version 730 to 731 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 731 (ObjNum 1087 snap 364 seq_num 1087) dirty exists 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2896: left oid 42 (ObjNum 1087 snap 364 seq_num 1087) 2024-09-21T09:50:16.174 INFO:tasks.rados.rados.0.smithi076.stdout:2898: finishing write tid 1 to smithi07685715-38 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2898: finishing write tid 2 to smithi07685715-38 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2892: done (7 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2893: done (6 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2894: done (5 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2895: done (4 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2896: done (3 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2897: done (2 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2899: done (1 left) 2024-09-21T09:50:16.175 INFO:tasks.rados.rados.0.smithi076.stdout:2900: write oid 22 current snap is 364 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2898: finishing write tid 3 to smithi07685715-38 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2898: finishing write tid 4 to smithi07685715-38 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2898: oid 38 updating version 0 to 1748 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2898: oid 38 version 1748 is already newer than 1747 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2898: oid 38 version 1748 is already newer than 1746 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1748 (ObjNum 1088 snap 364 seq_num 1088) dirty exists 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2898: left oid 38 (ObjNum 1088 snap 364 seq_num 1088) 2024-09-21T09:50:16.176 INFO:tasks.rados.rados.0.smithi076.stdout:2900: seq_num 1090 ranges {719630=659889,2117608=202022} 2024-09-21T09:50:16.180 INFO:tasks.rados.rados.0.smithi076.stdout:2900: writing smithi07685715-22 from 719630 to 1379519 tid 1 2024-09-21T09:50:16.185 INFO:tasks.rados.rados.0.smithi076.stdout:2900: writing smithi07685715-22 from 2117608 to 2319630 tid 2 2024-09-21T09:50:16.186 INFO:tasks.rados.rados.0.smithi076.stdout:2898: done (1 left) 2024-09-21T09:50:16.186 INFO:tasks.rados.rados.0.smithi076.stdout:2901: read oid 35 snap -1 2024-09-21T09:50:16.186 INFO:tasks.rados.rados.0.smithi076.stdout:2901: expect (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:16.186 INFO:tasks.rados.rados.0.smithi076.stdout:2902: write oid 38 current snap is 364 2024-09-21T09:50:16.186 INFO:tasks.rados.rados.0.smithi076.stdout:2900: finishing write tid 1 to smithi07685715-22 2024-09-21T09:50:16.186 INFO:tasks.rados.rados.0.smithi076.stdout:2902: seq_num 1091 ranges {736437=735138,2176090=160347} 2024-09-21T09:50:16.194 INFO:tasks.rados.rados.0.smithi076.stdout:2902: writing smithi07685715-38 from 736437 to 1471575 tid 1 2024-09-21T09:50:16.199 INFO:tasks.rados.rados.0.smithi076.stdout:2902: writing smithi07685715-38 from 2176090 to 2336437 tid 2 2024-09-21T09:50:16.199 INFO:tasks.rados.rados.0.smithi076.stdout:2903: snap_remove snap 360 2024-09-21T09:50:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:16 smithi191 ceph-mon[102206]: pgmap v670: 89 pgs: 12 peering, 2 active+undersized, 5 active+undersized+degraded, 70 active+clean; 188 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 275 KiB/s wr, 13 op/s; 26/366 objects degraded (7.104%) 2024-09-21T09:50:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:16 smithi191 ceph-mon[102206]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:50:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:16 smithi076 ceph-mon[106482]: pgmap v670: 89 pgs: 12 peering, 2 active+undersized, 5 active+undersized+degraded, 70 active+clean; 188 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 275 KiB/s wr, 13 op/s; 26/366 objects degraded (7.104%) 2024-09-21T09:50:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:16 smithi076 ceph-mon[106482]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:50:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:16 smithi076 ceph-mon[110963]: pgmap v670: 89 pgs: 12 peering, 2 active+undersized, 5 active+undersized+degraded, 70 active+clean; 188 MiB data, 674 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 275 KiB/s wr, 13 op/s; 26/366 objects degraded (7.104%) 2024-09-21T09:50:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:16 smithi076 ceph-mon[110963]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T09:50:17.034 INFO:tasks.rados.rados.0.smithi076.stdout:2903: done (3 left) 2024-09-21T09:50:17.034 INFO:tasks.rados.rados.0.smithi076.stdout:2904: snap_remove snap 363 2024-09-21T09:50:17.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:17 smithi191 ceph-mon[102206]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:50:17.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:17 smithi076 ceph-mon[106482]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:50:17.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:17 smithi076 ceph-mon[110963]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2904: done (3 left) 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2905: delete oid 13 current snap is 364 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2900: finishing write tid 2 to smithi07685715-22 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2900: finishing write tid 3 to smithi07685715-22 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2900: finishing write tid 4 to smithi07685715-22 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2900: oid 22 updating version 0 to 1366 2024-09-21T09:50:18.036 INFO:tasks.rados.rados.0.smithi076.stdout:2900: oid 22 updating version 1366 to 1368 2024-09-21T09:50:18.037 INFO:tasks.rados.rados.0.smithi076.stdout:2900: oid 22 version 1368 is already newer than 1367 2024-09-21T09:50:18.037 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1368 (ObjNum 1090 snap 364 seq_num 1090) dirty exists 2024-09-21T09:50:18.037 INFO:tasks.rados.rados.0.smithi076.stdout:2900: left oid 22 (ObjNum 1090 snap 364 seq_num 1090) 2024-09-21T09:50:18.037 INFO:tasks.rados.rados.0.smithi076.stdout:2901: expect (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: finishing write tid 1 to smithi07685715-38 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: finishing write tid 2 to smithi07685715-38 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: finishing write tid 3 to smithi07685715-38 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: finishing write tid 4 to smithi07685715-38 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: oid 38 updating version 0 to 1751 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: oid 38 version 1751 is already newer than 1750 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:2902: oid 38 version 1751 is already newer than 1749 2024-09-21T09:50:18.164 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1751 (ObjNum 1091 snap 364 seq_num 1091) dirty exists 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2902: left oid 38 (ObjNum 1091 snap 364 seq_num 1091) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2900: done (3 left) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2901: done (2 left) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2902: done (1 left) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2905: done (0 left) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2906: read oid 32 snap -1 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2906: expect deleted 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2907: read oid 12 snap -1 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2907: expect (ObjNum 1073 snap 360 seq_num 1073) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2908: read oid 39 snap -1 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2908: expect (ObjNum 1060 snap 354 seq_num 1060) 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2909: write oid 50 current snap is 364 2024-09-21T09:50:18.165 INFO:tasks.rados.rados.0.smithi076.stdout:2909: seq_num 1093 ranges {770051=485636,1776701=511047} 2024-09-21T09:50:18.170 INFO:tasks.rados.rados.0.smithi076.stdout:2909: writing smithi07685715-50 from 770051 to 1255687 tid 1 2024-09-21T09:50:18.176 INFO:tasks.rados.rados.0.smithi076.stdout:2909: writing smithi07685715-50 from 1776701 to 2287748 tid 2 2024-09-21T09:50:18.176 INFO:tasks.rados.rados.0.smithi076.stdout:2910: delete oid 11 current snap is 364 2024-09-21T09:50:18.176 INFO:tasks.rados.rados.0.smithi076.stdout:2907: expect (ObjNum 1073 snap 360 seq_num 1073) 2024-09-21T09:50:18.266 INFO:tasks.rados.rados.0.smithi076.stdout:2909: finishing write tid 1 to smithi07685715-50 2024-09-21T09:50:18.266 INFO:tasks.rados.rados.0.smithi076.stdout:2908: expect (ObjNum 1060 snap 354 seq_num 1060) 2024-09-21T09:50:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:18 smithi191 ceph-mon[102206]: pgmap v673: 89 pgs: 89 active+clean; 177 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 2.6 MiB/s, 1 objects/s recovering 2024-09-21T09:50:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:18 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/366 objects degraded (7.104%), 5 pgs degraded) 2024-09-21T09:50:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:18 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T09:50:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:18 smithi191 ceph-mon[102206]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:50:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[106482]: pgmap v673: 89 pgs: 89 active+clean; 177 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 2.6 MiB/s, 1 objects/s recovering 2024-09-21T09:50:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/366 objects degraded (7.104%), 5 pgs degraded) 2024-09-21T09:50:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T09:50:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[106482]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:50:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[110963]: pgmap v673: 89 pgs: 89 active+clean; 177 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 2.6 MiB/s, 1 objects/s recovering 2024-09-21T09:50:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/366 objects degraded (7.104%), 5 pgs degraded) 2024-09-21T09:50:18.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T09:50:18.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:18 smithi076 ceph-mon[110963]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T09:50:18.545 INFO:tasks.rados.rados.0.smithi076.stdout:2909: finishing write tid 2 to smithi07685715-50 2024-09-21T09:50:18.545 INFO:tasks.rados.rados.0.smithi076.stdout:2909: finishing write tid 3 to smithi07685715-50 2024-09-21T09:50:18.545 INFO:tasks.rados.rados.0.smithi076.stdout:2909: finishing write tid 4 to smithi07685715-50 2024-09-21T09:50:18.545 INFO:tasks.rados.rados.0.smithi076.stdout:2909: oid 50 updating version 0 to 943 2024-09-21T09:50:18.545 INFO:tasks.rados.rados.0.smithi076.stdout:2909: oid 50 version 943 is already newer than 941 2024-09-21T09:50:18.545 INFO:tasks.rados.rados.0.smithi076.stdout:2909: oid 50 version 943 is already newer than 942 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 943 (ObjNum 1093 snap 364 seq_num 1093) dirty exists 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2909: left oid 50 (ObjNum 1093 snap 364 seq_num 1093) 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2906: done (4 left) 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2907: done (3 left) 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2908: done (2 left) 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2909: done (1 left) 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2910: done (0 left) 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2911: write oid 13 current snap is 364 2024-09-21T09:50:18.546 INFO:tasks.rados.rados.0.smithi076.stdout:2911: seq_num 1095 ranges {403665=752487,1610018=514716} 2024-09-21T09:50:18.550 INFO:tasks.rados.rados.0.smithi076.stdout:2911: writing smithi07685715-13 from 403665 to 1156152 tid 1 2024-09-21T09:50:18.556 INFO:tasks.rados.rados.0.smithi076.stdout:2911: writing smithi07685715-13 from 1610018 to 2124734 tid 2 2024-09-21T09:50:18.556 INFO:tasks.rados.rados.0.smithi076.stdout:2912: write oid 41 current snap is 364 2024-09-21T09:50:18.556 INFO:tasks.rados.rados.0.smithi076.stdout:2912: seq_num 1096 ranges {420472=427736,1268500=658374,2347953=72519} 2024-09-21T09:50:18.560 INFO:tasks.rados.rados.0.smithi076.stdout:2912: writing smithi07685715-41 from 420472 to 848208 tid 1 2024-09-21T09:50:18.566 INFO:tasks.rados.rados.0.smithi076.stdout:2912: writing smithi07685715-41 from 1268500 to 1926874 tid 2 2024-09-21T09:50:18.570 INFO:tasks.rados.rados.0.smithi076.stdout:2912: writing smithi07685715-41 from 2347953 to 2420472 tid 3 2024-09-21T09:50:18.570 INFO:tasks.rados.rados.0.smithi076.stdout:2913: rollback oid 12 current snap is 364 2024-09-21T09:50:18.570 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 358 2024-09-21T09:50:18.570 INFO:tasks.rados.rados.0.smithi076.stdout:2911: finishing write tid 1 to smithi07685715-13 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: finishing write tid 2 to smithi07685715-13 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: finishing write tid 3 to smithi07685715-13 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: finishing write tid 4 to smithi07685715-13 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: oid 13 updating version 0 to 564 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: oid 13 version 564 is already newer than 563 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: oid 13 version 564 is already newer than 562 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 564 (ObjNum 1095 snap 364 seq_num 1095) dirty exists 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2911: left oid 13 (ObjNum 1095 snap 364 seq_num 1095) 2024-09-21T09:50:18.571 INFO:tasks.rados.rados.0.smithi076.stdout:2912: finishing write tid 1 to smithi07685715-41 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:2911: done (2 left) 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:2914: read oid 37 snap -1 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:2914: expect (ObjNum 1071 snap 358 seq_num 1071) 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:2915: rollback oid 15 current snap is 364 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 358 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:2916: write oid 39 current snap is 364 2024-09-21T09:50:18.572 INFO:tasks.rados.rados.0.smithi076.stdout:2916: seq_num 1097 ranges {437279=502985,1610629=518385} 2024-09-21T09:50:18.574 INFO:tasks.rados.rados.0.smithi076.stdout:2916: writing smithi07685715-39 from 437279 to 940264 tid 1 2024-09-21T09:50:18.581 INFO:tasks.rados.rados.0.smithi076.stdout:2916: writing smithi07685715-39 from 1610629 to 2129014 tid 2 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2917: rollback oid 24 current snap is 364 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 24 to 354 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2912: finishing write tid 2 to smithi07685715-41 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2915: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 0 (ObjNum 609 snap 0 seq_num 1682286601) dirty dne 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2913: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2912: finishing write tid 3 to smithi07685715-41 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2915: done (5 left) 2024-09-21T09:50:18.582 INFO:tasks.rados.rados.0.smithi076.stdout:2918: write oid 43 current snap is 364 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: finishing write tid 4 to smithi07685715-41 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2913: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: finishing write tid 5 to smithi07685715-41 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: oid 41 updating version 0 to 1379 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: oid 41 version 1379 is already newer than 1378 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: oid 41 updating version 1379 to 1380 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: oid 41 version 1380 is already newer than 1377 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1380 (ObjNum 1096 snap 364 seq_num 1096) dirty exists 2024-09-21T09:50:18.583 INFO:tasks.rados.rados.0.smithi076.stdout:2912: left oid 41 (ObjNum 1096 snap 364 seq_num 1096) 2024-09-21T09:50:18.584 INFO:tasks.rados.rados.0.smithi076.stdout:2913: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:50:18.584 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 998 (ObjNum 981 snap 336 seq_num 981) dirty exists 2024-09-21T09:50:18.584 INFO:tasks.rados.rados.0.smithi076.stdout:2914: expect (ObjNum 1071 snap 358 seq_num 1071) 2024-09-21T09:50:18.663 INFO:tasks.rados.rados.0.smithi076.stdout:2916: finishing write tid 1 to smithi07685715-39 2024-09-21T09:50:18.663 INFO:tasks.rados.rados.0.smithi076.stdout:2917: finishing rollback tid 0 to smithi07685715-24 2024-09-21T09:50:18.663 INFO:tasks.rados.rados.0.smithi076.stdout:2917: finishing rollback tid 1 to smithi07685715-24 2024-09-21T09:50:18.663 INFO:tasks.rados.rados.0.smithi076.stdout:2917: finishing rollback tid 2 to smithi07685715-24 2024-09-21T09:50:18.663 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 295 (ObjNum 1062 snap 354 seq_num 1062) dirty exists 2024-09-21T09:50:18.663 INFO:tasks.rados.rados.0.smithi076.stdout:2918: seq_num 1098 ranges {454086=578234,1669111=662043} 2024-09-21T09:50:18.668 INFO:tasks.rados.rados.0.smithi076.stdout:2918: writing smithi07685715-43 from 454086 to 1032320 tid 1 2024-09-21T09:50:18.676 INFO:tasks.rados.rados.0.smithi076.stdout:2918: writing smithi07685715-43 from 1669111 to 2331154 tid 2 2024-09-21T09:50:18.676 INFO:tasks.rados.rados.0.smithi076.stdout:2912: done (5 left) 2024-09-21T09:50:18.676 INFO:tasks.rados.rados.0.smithi076.stdout:2913: done (4 left) 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2914: done (3 left) 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2917: done (2 left) 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2919: write oid 4 current snap is 364 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2916: finishing write tid 2 to smithi07685715-39 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2916: finishing write tid 3 to smithi07685715-39 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2916: finishing write tid 4 to smithi07685715-39 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2916: oid 39 updating version 0 to 2196 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2916: oid 39 updating version 2196 to 2198 2024-09-21T09:50:18.677 INFO:tasks.rados.rados.0.smithi076.stdout:2916: oid 39 version 2198 is already newer than 2197 2024-09-21T09:50:18.678 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2198 (ObjNum 1097 snap 364 seq_num 1097) dirty exists 2024-09-21T09:50:18.678 INFO:tasks.rados.rados.0.smithi076.stdout:2916: left oid 39 (ObjNum 1097 snap 364 seq_num 1097) 2024-09-21T09:50:18.678 INFO:tasks.rados.rados.0.smithi076.stdout:2919: seq_num 1099 ranges {470893=653483,1727593=522054} 2024-09-21T09:50:18.682 INFO:tasks.rados.rados.0.smithi076.stdout:2919: writing smithi07685715-4 from 470893 to 1124376 tid 1 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2919: writing smithi07685715-4 from 1727593 to 2249647 tid 2 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2916: done (2 left) 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2920: rollback oid 24 current snap is 364 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 24 to 351 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2918: finishing write tid 1 to smithi07685715-43 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2918: finishing write tid 2 to smithi07685715-43 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2918: finishing write tid 3 to smithi07685715-43 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2918: finishing write tid 4 to smithi07685715-43 2024-09-21T09:50:18.688 INFO:tasks.rados.rados.0.smithi076.stdout:2918: oid 43 updating version 0 to 690 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2918: oid 43 version 690 is already newer than 689 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2918: oid 43 updating version 690 to 691 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 691 (ObjNum 1098 snap 364 seq_num 1098) dirty exists 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2918: left oid 43 (ObjNum 1098 snap 364 seq_num 1098) 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2919: finishing write tid 1 to smithi07685715-4 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2918: done (2 left) 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2921: delete oid 5 current snap is 364 2024-09-21T09:50:18.689 INFO:tasks.rados.rados.0.smithi076.stdout:2920: finishing rollback tid 0 to smithi07685715-24 2024-09-21T09:50:18.690 INFO:tasks.rados.rados.0.smithi076.stdout:2920: finishing rollback tid 1 to smithi07685715-24 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2920: finishing rollback tid 2 to smithi07685715-24 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 298 (ObjNum 1028 snap 347 seq_num 1028) dirty exists 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2919: finishing write tid 2 to smithi07685715-4 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2919: finishing write tid 3 to smithi07685715-4 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2920: done (2 left) 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2921: done (1 left) 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2922: rollback oid 30 current snap is 364 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 354 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2923: read oid 26 snap -1 2024-09-21T09:50:18.692 INFO:tasks.rados.rados.0.smithi076.stdout:2923: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2924: delete oid 37 current snap is 364 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2919: finishing write tid 4 to smithi07685715-4 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2919: oid 4 updating version 0 to 944 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2919: oid 4 updating version 944 to 945 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2919: oid 4 updating version 945 to 946 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 946 (ObjNum 1099 snap 364 seq_num 1099) dirty exists 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2919: left oid 4 (ObjNum 1099 snap 364 seq_num 1099) 2024-09-21T09:50:18.693 INFO:tasks.rados.rados.0.smithi076.stdout:2919: done (3 left) 2024-09-21T09:50:18.694 INFO:tasks.rados.rados.0.smithi076.stdout:2924: done (2 left) 2024-09-21T09:50:18.694 INFO:tasks.rados.rados.0.smithi076.stdout:2925: snap_create 2024-09-21T09:50:18.694 INFO:tasks.rados.rados.0.smithi076.stdout:2922: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:50:18.694 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2175 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:18.695 INFO:tasks.rados.rados.0.smithi076.stdout:2923: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:18.914 INFO:tasks.rados.rados.0.smithi076.stdout:2922: done (2 left) 2024-09-21T09:50:18.914 INFO:tasks.rados.rados.0.smithi076.stdout:2923: done (1 left) 2024-09-21T09:50:18.914 INFO:tasks.rados.rados.0.smithi076.stdout:2925: done (0 left) 2024-09-21T09:50:18.914 INFO:tasks.rados.rados.0.smithi076.stdout:2926: write oid 3 current snap is 365 2024-09-21T09:50:18.914 INFO:tasks.rados.rados.0.smithi076.stdout:2926: seq_num 1102 ranges {521314=479230,1786686=669381} 2024-09-21T09:50:18.919 INFO:tasks.rados.rados.0.smithi076.stdout:2926: writing smithi07685715-3 from 521314 to 1000544 tid 1 2024-09-21T09:50:18.927 INFO:tasks.rados.rados.0.smithi076.stdout:2926: writing smithi07685715-3 from 1786686 to 2456067 tid 2 2024-09-21T09:50:18.927 INFO:tasks.rados.rados.0.smithi076.stdout:2927: snap_create 2024-09-21T09:50:18.927 INFO:tasks.rados.rados.0.smithi076.stdout:2926: finishing write tid 1 to smithi07685715-3 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: finishing write tid 2 to smithi07685715-3 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: finishing write tid 3 to smithi07685715-3 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: finishing write tid 4 to smithi07685715-3 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: oid 3 updating version 0 to 210 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: oid 3 version 210 is already newer than 209 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: oid 3 updating version 210 to 211 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 211 (ObjNum 1102 snap 365 seq_num 1102) dirty exists 2024-09-21T09:50:18.930 INFO:tasks.rados.rados.0.smithi076.stdout:2926: left oid 3 (ObjNum 1102 snap 365 seq_num 1102) 2024-09-21T09:50:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:19 smithi191 ceph-mon[102206]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:50:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:19 smithi191 ceph-mon[102206]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:50:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:19 smithi076 ceph-mon[106482]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:50:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:19 smithi076 ceph-mon[106482]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:50:19.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:19 smithi076 ceph-mon[110963]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T09:50:19.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:19 smithi076 ceph-mon[110963]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T09:50:19.760 INFO:tasks.rados.rados.0.smithi076.stdout:2926: done (1 left) 2024-09-21T09:50:19.760 INFO:tasks.rados.rados.0.smithi076.stdout:2927: done (0 left) 2024-09-21T09:50:19.760 INFO:tasks.rados.rados.0.smithi076.stdout:2928: delete oid 47 current snap is 366 2024-09-21T09:50:19.764 INFO:tasks.rados.rados.0.smithi076.stdout:2928: done (0 left) 2024-09-21T09:50:19.764 INFO:tasks.rados.rados.0.smithi076.stdout:2929: rollback oid 10 current snap is 366 2024-09-21T09:50:19.764 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 358 2024-09-21T09:50:19.764 INFO:tasks.rados.rados.0.smithi076.stdout:2930: snap_create 2024-09-21T09:50:19.768 INFO:tasks.rados.rados.0.smithi076.stdout:2929: finishing rollback tid 0 to smithi07685715-10 2024-09-21T09:50:19.768 INFO:tasks.rados.rados.0.smithi076.stdout:2929: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:50:19.768 INFO:tasks.rados.rados.0.smithi076.stdout:2929: finishing rollback tid 2 to smithi07685715-10 2024-09-21T09:50:19.768 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1759 (ObjNum 985 snap 337 seq_num 985) dirty exists 2024-09-21T09:50:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:20 smithi191 ceph-mon[102206]: pgmap v676: 89 pgs: 89 active+clean; 177 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 2.6 MiB/s, 1 objects/s recovering 2024-09-21T09:50:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:20 smithi191 ceph-mon[102206]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:50:20.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:20 smithi076 ceph-mon[106482]: pgmap v676: 89 pgs: 89 active+clean; 177 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 2.6 MiB/s, 1 objects/s recovering 2024-09-21T09:50:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:20 smithi076 ceph-mon[106482]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:50:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:20 smithi076 ceph-mon[110963]: pgmap v676: 89 pgs: 89 active+clean; 177 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 2.6 MiB/s, 1 objects/s recovering 2024-09-21T09:50:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:20 smithi076 ceph-mon[110963]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2929: done (1 left) 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2930: done (0 left) 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2931: read oid 35 snap -1 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2931: expect (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2932: read oid 48 snap 354 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2932: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2933: write oid 46 current snap is 367 2024-09-21T09:50:20.763 INFO:tasks.rados.rados.0.smithi076.stdout:2933: seq_num 1104 ranges {554928=746081,2020003=534925} 2024-09-21T09:50:20.770 INFO:tasks.rados.rados.0.smithi076.stdout:2933: writing smithi07685715-46 from 554928 to 1301009 tid 1 2024-09-21T09:50:20.778 INFO:tasks.rados.rados.0.smithi076.stdout:2933: writing smithi07685715-46 from 2020003 to 2554928 tid 2 2024-09-21T09:50:20.778 INFO:tasks.rados.rados.0.smithi076.stdout:2934: read oid 4 snap 354 2024-09-21T09:50:20.778 INFO:tasks.rados.rados.0.smithi076.stdout:2934: expect (ObjNum 1058 snap 354 seq_num 1058) 2024-09-21T09:50:20.778 INFO:tasks.rados.rados.0.smithi076.stdout:2935: write oid 22 current snap is 367 2024-09-21T09:50:20.778 INFO:tasks.rados.rados.0.smithi076.stdout:2935: seq_num 1105 ranges {571735=421330,1562132=416708} 2024-09-21T09:50:20.783 INFO:tasks.rados.rados.0.smithi076.stdout:2935: writing smithi07685715-22 from 571735 to 993065 tid 1 2024-09-21T09:50:20.789 INFO:tasks.rados.rados.0.smithi076.stdout:2935: writing smithi07685715-22 from 1562132 to 1978840 tid 2 2024-09-21T09:50:20.789 INFO:tasks.rados.rados.0.smithi076.stdout:2936: write oid 49 current snap is 367 2024-09-21T09:50:20.789 INFO:tasks.rados.rados.0.smithi076.stdout:2932: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:50:20.979 INFO:tasks.rados.rados.0.smithi076.stdout:2931: expect (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:21.103 INFO:tasks.rados.rados.0.smithi076.stdout:2933: finishing write tid 1 to smithi07685715-46 2024-09-21T09:50:21.103 INFO:tasks.rados.rados.0.smithi076.stdout:2933: finishing write tid 2 to smithi07685715-46 2024-09-21T09:50:21.103 INFO:tasks.rados.rados.0.smithi076.stdout:2933: finishing write tid 3 to smithi07685715-46 2024-09-21T09:50:21.103 INFO:tasks.rados.rados.0.smithi076.stdout:2933: finishing write tid 4 to smithi07685715-46 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2933: oid 46 updating version 0 to 2202 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2933: oid 46 version 2202 is already newer than 2201 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2933: oid 46 updating version 2202 to 2203 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2203 (ObjNum 1104 snap 367 seq_num 1104) dirty exists 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2933: left oid 46 (ObjNum 1104 snap 367 seq_num 1104) 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2935: finishing write tid 1 to smithi07685715-22 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2935: finishing write tid 2 to smithi07685715-22 2024-09-21T09:50:21.104 INFO:tasks.rados.rados.0.smithi076.stdout:2936: seq_num 1106 ranges {588542=496579,1620614=676719} 2024-09-21T09:50:21.109 INFO:tasks.rados.rados.0.smithi076.stdout:2936: writing smithi07685715-49 from 588542 to 1085121 tid 1 2024-09-21T09:50:21.116 INFO:tasks.rados.rados.0.smithi076.stdout:2936: writing smithi07685715-49 from 1620614 to 2297333 tid 2 2024-09-21T09:50:21.116 INFO:tasks.rados.rados.0.smithi076.stdout:2931: done (5 left) 2024-09-21T09:50:21.116 INFO:tasks.rados.rados.0.smithi076.stdout:2932: done (4 left) 2024-09-21T09:50:21.117 INFO:tasks.rados.rados.0.smithi076.stdout:2933: done (3 left) 2024-09-21T09:50:21.117 INFO:tasks.rados.rados.0.smithi076.stdout:2937: read oid 35 snap -1 2024-09-21T09:50:21.117 INFO:tasks.rados.rados.0.smithi076.stdout:2937: expect (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:21.117 INFO:tasks.rados.rados.0.smithi076.stdout:2938: snap_remove snap 361 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2938: done (4 left) 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2939: read oid 26 snap -1 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2939: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2940: read oid 17 snap -1 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2940: expect (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2941: delete oid 45 current snap is 367 2024-09-21T09:50:22.057 INFO:tasks.rados.rados.0.smithi076.stdout:2935: finishing write tid 3 to smithi07685715-22 2024-09-21T09:50:22.058 INFO:tasks.rados.rados.0.smithi076.stdout:2934: expect (ObjNum 1058 snap 354 seq_num 1058) 2024-09-21T09:50:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:21 smithi191 ceph-mon[102206]: pgmap v679: 89 pgs: 89 active+clean; 185 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:50:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:21 smithi191 ceph-mon[102206]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:50:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:21 smithi076 ceph-mon[106482]: pgmap v679: 89 pgs: 89 active+clean; 185 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:50:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:21 smithi076 ceph-mon[106482]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:50:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:21 smithi076 ceph-mon[110963]: pgmap v679: 89 pgs: 89 active+clean; 185 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:50:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:21 smithi076 ceph-mon[110963]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2935: finishing write tid 4 to smithi07685715-22 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2935: oid 22 updating version 0 to 1382 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2935: oid 22 updating version 1382 to 1384 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2935: oid 22 version 1384 is already newer than 1383 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1384 (ObjNum 1105 snap 367 seq_num 1105) dirty exists 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2935: left oid 22 (ObjNum 1105 snap 367 seq_num 1105) 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2936: finishing write tid 1 to smithi07685715-49 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2934: done (6 left) 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2935: done (5 left) 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2941: done (4 left) 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2942: read oid 46 snap -1 2024-09-21T09:50:22.263 INFO:tasks.rados.rados.0.smithi076.stdout:2942: expect (ObjNum 1104 snap 367 seq_num 1104) 2024-09-21T09:50:22.264 INFO:tasks.rados.rados.0.smithi076.stdout:2943: write oid 48 current snap is 367 2024-09-21T09:50:22.264 INFO:tasks.rados.rados.0.smithi076.stdout:2943: seq_num 1108 ranges {622156=647077,1737578=680388} 2024-09-21T09:50:22.270 INFO:tasks.rados.rados.0.smithi076.stdout:2943: writing smithi07685715-48 from 622156 to 1269233 tid 1 2024-09-21T09:50:22.278 INFO:tasks.rados.rados.0.smithi076.stdout:2943: writing smithi07685715-48 from 1737578 to 2417966 tid 2 2024-09-21T09:50:22.279 INFO:tasks.rados.rados.0.smithi076.stdout:2944: write oid 21 current snap is 367 2024-09-21T09:50:22.279 INFO:tasks.rados.rados.0.smithi076.stdout:2944: seq_num 1109 ranges {638963=722326,2079707=424046} 2024-09-21T09:50:22.287 INFO:tasks.rados.rados.0.smithi076.stdout:2944: writing smithi07685715-21 from 638963 to 1361289 tid 1 2024-09-21T09:50:22.295 INFO:tasks.rados.rados.0.smithi076.stdout:2944: writing smithi07685715-21 from 2079707 to 2503753 tid 2 2024-09-21T09:50:22.295 INFO:tasks.rados.rados.0.smithi076.stdout:2945: delete oid 40 current snap is 367 2024-09-21T09:50:22.295 INFO:tasks.rados.rados.0.smithi076.stdout:2936: finishing write tid 2 to smithi07685715-49 2024-09-21T09:50:22.295 INFO:tasks.rados.rados.0.smithi076.stdout:2936: finishing write tid 3 to smithi07685715-49 2024-09-21T09:50:22.295 INFO:tasks.rados.rados.0.smithi076.stdout:2936: finishing write tid 4 to smithi07685715-49 2024-09-21T09:50:22.296 INFO:tasks.rados.rados.0.smithi076.stdout:2936: oid 49 updating version 0 to 2188 2024-09-21T09:50:22.296 INFO:tasks.rados.rados.0.smithi076.stdout:2936: oid 49 version 2188 is already newer than 2187 2024-09-21T09:50:22.296 INFO:tasks.rados.rados.0.smithi076.stdout:2936: oid 49 version 2188 is already newer than 2186 2024-09-21T09:50:22.296 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2188 (ObjNum 1106 snap 367 seq_num 1106) dirty exists 2024-09-21T09:50:22.296 INFO:tasks.rados.rados.0.smithi076.stdout:2936: left oid 49 (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:50:22.296 INFO:tasks.rados.rados.0.smithi076.stdout:2937: expect (ObjNum 1069 snap 357 seq_num 1069) 2024-09-21T09:50:22.354 INFO:teuthology.orchestra.run.smithi076.stdout:true 2024-09-21T09:50:22.427 INFO:tasks.rados.rados.0.smithi076.stdout:2939: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:22.650 INFO:tasks.rados.rados.0.smithi076.stdout:2940: expect (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:50:22.782 INFO:tasks.rados.rados.0.smithi076.stdout:2942: expect (ObjNum 1104 snap 367 seq_num 1104) 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: finishing write tid 1 to smithi07685715-48 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: finishing write tid 2 to smithi07685715-48 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: finishing write tid 3 to smithi07685715-48 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: finishing write tid 4 to smithi07685715-48 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: oid 48 updating version 0 to 2196 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: oid 48 version 2196 is already newer than 2194 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: oid 48 version 2196 is already newer than 2195 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2196 (ObjNum 1108 snap 367 seq_num 1108) dirty exists 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2943: left oid 48 (ObjNum 1108 snap 367 seq_num 1108) 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2944: finishing write tid 1 to smithi07685715-21 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2936: done (7 left) 2024-09-21T09:50:22.898 INFO:tasks.rados.rados.0.smithi076.stdout:2937: done (6 left) 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2939: done (5 left) 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2940: done (4 left) 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2942: done (3 left) 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2943: done (2 left) 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2945: done (1 left) 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2946: write oid 4 current snap is 367 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2944: finishing write tid 2 to smithi07685715-21 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2944: finishing write tid 3 to smithi07685715-21 2024-09-21T09:50:22.899 INFO:tasks.rados.rados.0.smithi076.stdout:2946: seq_num 1111 ranges {672577=589177,1913024=427715} 2024-09-21T09:50:22.904 INFO:tasks.rados.rados.0.smithi076.stdout:2946: writing smithi07685715-4 from 672577 to 1261754 tid 1 2024-09-21T09:50:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2946: writing smithi07685715-4 from 1913024 to 2340739 tid 2 2024-09-21T09:50:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2947: snap_create 2024-09-21T09:50:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2944: finishing write tid 4 to smithi07685715-21 2024-09-21T09:50:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2944: oid 21 updating version 0 to 2212 2024-09-21T09:50:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2944: oid 21 version 2212 is already newer than 2211 2024-09-21T09:50:22.910 INFO:tasks.rados.rados.0.smithi076.stdout:2944: oid 21 updating version 2212 to 2213 2024-09-21T09:50:22.911 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 2213 (ObjNum 1109 snap 367 seq_num 1109) dirty exists 2024-09-21T09:50:22.911 INFO:tasks.rados.rados.0.smithi076.stdout:2944: left oid 21 (ObjNum 1109 snap 367 seq_num 1109) 2024-09-21T09:50:22.911 INFO:tasks.rados.rados.0.smithi076.stdout:2946: finishing write tid 1 to smithi07685715-4 2024-09-21T09:50:22.913 INFO:tasks.rados.rados.0.smithi076.stdout:2946: finishing write tid 2 to smithi07685715-4 2024-09-21T09:50:22.913 INFO:tasks.rados.rados.0.smithi076.stdout:2946: finishing write tid 3 to smithi07685715-4 2024-09-21T09:50:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:2946: finishing write tid 4 to smithi07685715-4 2024-09-21T09:50:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:2946: oid 4 updating version 0 to 950 2024-09-21T09:50:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:2946: oid 4 updating version 950 to 951 2024-09-21T09:50:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:2946: oid 4 updating version 951 to 952 2024-09-21T09:50:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 952 (ObjNum 1111 snap 367 seq_num 1111) dirty exists 2024-09-21T09:50:22.914 INFO:tasks.rados.rados.0.smithi076.stdout:2946: left oid 4 (ObjNum 1111 snap 367 seq_num 1111) 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:alertmanager.a smithi076 *:9093,9094 running (8m) 15s ago 14m 21.1M - 0.27.0 11f11916f8cd 1e9964bd966f 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:grafana.a smithi191 *:3000 running (8m) 47s ago 14m 79.7M - 10.4.8 5aad1d7cf004 61303ed53f7b 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi076.jymebx smithi076 running (13m) 15s ago 13m 23.4M - 18.2.4-871-g6a1bdf85 0adab276c97e bb876a2b7a52 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:mds.foo.smithi191.dhrpzx smithi191 running (13m) 47s ago 13m 18.3M - 18.2.4-871-g6a1bdf85 0adab276c97e 558c91d3577c 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.x smithi191 *:8443,9283,8765 running (9m) 47s ago 18m 458M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 141a29e6e005 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:mgr.y smithi076 *:8443,9283,8765 running (10m) 15s ago 20m 518M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 151afa33d0ee 2024-09-21T09:50:22.950 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a smithi076 running (7m) 15s ago 20m 77.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 034b7218a76b 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b smithi191 running (5m) 47s ago 19m 46.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 bb6de71151ea 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c smithi076 running (7m) 15s ago 19m 58.4M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 a49012ff5abf 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.a smithi076 *:9100 running (9m) 15s ago 14m 12.1M - 1.7.0 72c9c2088986 a75033fa919d 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:node-exporter.b smithi191 *:9100 running (9m) 47s ago 14m 11.7M - 1.7.0 72c9c2088986 703c0a6bfe5f 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 smithi076 running (85s) 15s ago 18m 79.0M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 f0b24f5d8966 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 smithi076 running (52s) 15s ago 18m 58.8M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 c1218f57993c 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 smithi076 running (19s) 15s ago 17m 11.5M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 d1afb0dbaa23 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 smithi076 running (17m) 15s ago 17m 96.4M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 7095b98dd4df 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 smithi191 running (16m) 47s ago 16m 118M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 01f1dafb6a21 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 smithi191 running (16m) 47s ago 16m 134M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 19b2c345ce82 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 smithi191 running (16m) 47s ago 16m 95.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e 6a3f1d2d2637 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 smithi191 running (15m) 47s ago 15m 94.5M 2480M 18.2.4-871-g6a1bdf85 0adab276c97e b84d3545a305 2024-09-21T09:50:22.951 INFO:teuthology.orchestra.run.smithi076.stdout:prometheus.a smithi191 *:9095 running (9m) 47s ago 15m 48.1M - 2.51.0 1d3b7f56885b eda26012a522 2024-09-21T09:50:23.062 INFO:tasks.rados.rados.0.smithi076.stdout:2944: done (2 left) 2024-09-21T09:50:23.062 INFO:tasks.rados.rados.0.smithi076.stdout:2946: done (1 left) 2024-09-21T09:50:23.063 INFO:tasks.rados.rados.0.smithi076.stdout:2947: done (0 left) 2024-09-21T09:50:23.063 INFO:tasks.rados.rados.0.smithi076.stdout:2948: write oid 23 current snap is 368 2024-09-21T09:50:23.063 INFO:tasks.rados.rados.0.smithi076.stdout:2948: seq_num 1112 ranges {689384=664426,1971506=687726} 2024-09-21T09:50:23.071 INFO:tasks.rados.rados.0.smithi076.stdout:2948: writing smithi07685715-23 from 689384 to 1353810 tid 1 2024-09-21T09:50:23.080 INFO:tasks.rados.rados.0.smithi076.stdout:2948: writing smithi07685715-23 from 1971506 to 2659232 tid 2 2024-09-21T09:50:23.081 INFO:tasks.rados.rados.0.smithi076.stdout:2949: snap_remove snap 344 2024-09-21T09:50:23.250 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:23 smithi076 ceph-mon[106482]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:50:23.250 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:23 smithi076 ceph-mon[110963]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:50:23.303 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:50:23.303 INFO:teuthology.orchestra.run.smithi076.stdout: "mon": { 2024-09-21T09:50:23.303 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:50:23.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:50:23.303 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr": { 2024-09-21T09:50:23.303 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "osd": { 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 5, 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "mds": { 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 2 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "overall": { 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 18.2.4-871-g6a1bdf85 (6a1bdf854bb5803efd58957907442aa01f41a8fc) reef (stable)": 7, 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:50:23.304 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:50:23.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:23 smithi191 ceph-mon[102206]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T09:50:23.590 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "in_progress": true, 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "services_complete": [], 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "progress": "3/8 daemons upgraded", 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "is_paused": false 2024-09-21T09:50:23.591 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:50:23.591 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:50:23] "GET /metrics HTTP/1.1" 200 36114 "" "Prometheus/2.51.0" 2024-09-21T09:50:23.769 INFO:tasks.rados.rados.0.smithi076.stdout:2949: done (1 left) 2024-09-21T09:50:23.769 INFO:tasks.rados.rados.0.smithi076.stdout:2950: snap_create 2024-09-21T09:50:23.769 INFO:tasks.rados.rados.0.smithi076.stdout:2948: finishing write tid 1 to smithi07685715-23 2024-09-21T09:50:23.769 INFO:tasks.rados.rados.0.smithi076.stdout:2948: finishing write tid 2 to smithi07685715-23 2024-09-21T09:50:23.769 INFO:tasks.rados.rados.0.smithi076.stdout:2948: finishing write tid 3 to smithi07685715-23 2024-09-21T09:50:23.769 INFO:tasks.rados.rados.0.smithi076.stdout:2948: finishing write tid 4 to smithi07685715-23 2024-09-21T09:50:23.770 INFO:tasks.rados.rados.0.smithi076.stdout:2948: oid 23 updating version 0 to 2197 2024-09-21T09:50:23.770 INFO:tasks.rados.rados.0.smithi076.stdout:2948: oid 23 updating version 2197 to 2198 2024-09-21T09:50:23.770 INFO:tasks.rados.rados.0.smithi076.stdout:2948: oid 23 updating version 2198 to 2199 2024-09-21T09:50:23.770 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2199 (ObjNum 1112 snap 368 seq_num 1112) dirty exists 2024-09-21T09:50:23.770 INFO:tasks.rados.rados.0.smithi076.stdout:2948: left oid 23 (ObjNum 1112 snap 368 seq_num 1112) 2024-09-21T09:50:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: pgmap v682: 89 pgs: 89 active+clean; 196 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-21T09:50:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:50:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2008458928' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:24 smithi191 ceph-mon[102206]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:50:24.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: pgmap v682: 89 pgs: 89 active+clean; 196 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2008458928' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[106482]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: pgmap v682: 89 pgs: 89 active+clean; 196 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T09:50:24.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2008458928' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:24.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:24 smithi076 ceph-mon[110963]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T09:50:24.762 INFO:tasks.rados.rados.0.smithi076.stdout:2948: done (1 left) 2024-09-21T09:50:24.762 INFO:tasks.rados.rados.0.smithi076.stdout:2950: done (0 left) 2024-09-21T09:50:24.763 INFO:tasks.rados.rados.0.smithi076.stdout:2951: rollback oid 14 current snap is 369 2024-09-21T09:50:24.763 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 359 2024-09-21T09:50:24.763 INFO:tasks.rados.rados.0.smithi076.stdout:2952: read oid 17 snap -1 2024-09-21T09:50:24.763 INFO:tasks.rados.rados.0.smithi076.stdout:2952: expect (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:50:24.763 INFO:tasks.rados.rados.0.smithi076.stdout:2953: snap_remove snap 364 2024-09-21T09:50:25.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:25 smithi191 ceph-mon[102206]: from='client.34431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:25 smithi191 ceph-mon[102206]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:50:25.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:25 smithi076 ceph-mon[110963]: from='client.34431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:25.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:25 smithi076 ceph-mon[110963]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:50:25.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:25 smithi076 ceph-mon[106482]: from='client.34431 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:25.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:25 smithi076 ceph-mon[106482]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2953: done (2 left) 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2954: delete oid 46 current snap is 369 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2951: finishing rollback tid 0 to smithi07685715-14 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2951: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2951: finishing rollback tid 2 to smithi07685715-14 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1769 (ObjNum 1050 snap 352 seq_num 1050) dirty exists 2024-09-21T09:50:25.766 INFO:tasks.rados.rados.0.smithi076.stdout:2952: expect (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:50:25.879 INFO:tasks.rados.rados.0.smithi076.stdout:2951: done (2 left) 2024-09-21T09:50:25.880 INFO:tasks.rados.rados.0.smithi076.stdout:2952: done (1 left) 2024-09-21T09:50:25.880 INFO:tasks.rados.rados.0.smithi076.stdout:2954: done (0 left) 2024-09-21T09:50:25.880 INFO:tasks.rados.rados.0.smithi076.stdout:2955: delete oid 24 current snap is 369 2024-09-21T09:50:25.881 INFO:tasks.rados.rados.0.smithi076.stdout:2955: done (0 left) 2024-09-21T09:50:25.881 INFO:tasks.rados.rados.0.smithi076.stdout:2956: delete oid 15 current snap is 369 2024-09-21T09:50:25.883 INFO:tasks.rados.rados.0.smithi076.stdout:2956: done (0 left) 2024-09-21T09:50:25.883 INFO:tasks.rados.rados.0.smithi076.stdout:2957: rollback oid 32 current snap is 369 2024-09-21T09:50:25.883 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 359 2024-09-21T09:50:25.883 INFO:tasks.rados.rados.0.smithi076.stdout:2958: write oid 48 current snap is 369 2024-09-21T09:50:25.883 INFO:tasks.rados.rados.0.smithi076.stdout:2958: seq_num 1116 ranges {756612=565422,2089081=578711} 2024-09-21T09:50:25.890 INFO:tasks.rados.rados.0.smithi076.stdout:2958: writing smithi07685715-48 from 756612 to 1322034 tid 1 2024-09-21T09:50:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:2958: writing smithi07685715-48 from 2089081 to 2667792 tid 2 2024-09-21T09:50:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:2959: write oid 28 current snap is 369 2024-09-21T09:50:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:2957: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:50:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 0 (ObjNum 129 snap 0 seq_num 864114224) dirty dne 2024-09-21T09:50:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:2958: finishing write tid 1 to smithi07685715-48 2024-09-21T09:50:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:2959: seq_num 1117 ranges {773419=640671,2147563=438722} 2024-09-21T09:50:25.905 INFO:tasks.rados.rados.0.smithi076.stdout:2959: writing smithi07685715-28 from 773419 to 1414090 tid 1 2024-09-21T09:50:25.912 INFO:tasks.rados.rados.0.smithi076.stdout:2959: writing smithi07685715-28 from 2147563 to 2586285 tid 2 2024-09-21T09:50:25.912 INFO:tasks.rados.rados.0.smithi076.stdout:2957: done (2 left) 2024-09-21T09:50:25.912 INFO:tasks.rados.rados.0.smithi076.stdout:2960: read oid 47 snap -1 2024-09-21T09:50:25.912 INFO:tasks.rados.rados.0.smithi076.stdout:2960: expect deleted 2024-09-21T09:50:25.912 INFO:tasks.rados.rados.0.smithi076.stdout:2961: snap_create 2024-09-21T09:50:25.912 INFO:tasks.rados.rados.0.smithi076.stdout:2958: finishing write tid 2 to smithi07685715-48 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2958: finishing write tid 3 to smithi07685715-48 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2958: finishing write tid 4 to smithi07685715-48 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2958: oid 48 updating version 0 to 2215 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2958: oid 48 updating version 2215 to 2216 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2958: oid 48 updating version 2216 to 2217 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2217 (ObjNum 1116 snap 369 seq_num 1116) dirty exists 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2958: left oid 48 (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:50:25.913 INFO:tasks.rados.rados.0.smithi076.stdout:2959: finishing write tid 1 to smithi07685715-28 2024-09-21T09:50:25.915 INFO:tasks.rados.rados.0.smithi076.stdout:2959: finishing write tid 2 to smithi07685715-28 2024-09-21T09:50:25.915 INFO:tasks.rados.rados.0.smithi076.stdout:2959: finishing write tid 3 to smithi07685715-28 2024-09-21T09:50:25.915 INFO:tasks.rados.rados.0.smithi076.stdout:2959: finishing write tid 4 to smithi07685715-28 2024-09-21T09:50:25.916 INFO:tasks.rados.rados.0.smithi076.stdout:2959: oid 28 updating version 0 to 2221 2024-09-21T09:50:25.916 INFO:tasks.rados.rados.0.smithi076.stdout:2959: oid 28 version 2221 is already newer than 2220 2024-09-21T09:50:25.916 INFO:tasks.rados.rados.0.smithi076.stdout:2959: oid 28 version 2221 is already newer than 2219 2024-09-21T09:50:25.916 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 2221 (ObjNum 1117 snap 369 seq_num 1117) dirty exists 2024-09-21T09:50:25.916 INFO:tasks.rados.rados.0.smithi076.stdout:2959: left oid 28 (ObjNum 1117 snap 369 seq_num 1117) 2024-09-21T09:50:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:26 smithi191 ceph-mon[102206]: pgmap v685: 89 pgs: 89 active+clean; 196 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-09-21T09:50:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:26 smithi191 ceph-mon[102206]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[106482]: pgmap v685: 89 pgs: 89 active+clean; 196 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[106482]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[110963]: pgmap v685: 89 pgs: 89 active+clean; 196 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:26 smithi076 ceph-mon[110963]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T09:50:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2958: done (3 left) 2024-09-21T09:50:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2959: done (2 left) 2024-09-21T09:50:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2960: done (1 left) 2024-09-21T09:50:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2961: done (0 left) 2024-09-21T09:50:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2962: write oid 45 current snap is 370 2024-09-21T09:50:26.766 INFO:tasks.rados.rados.0.smithi076.stdout:2962: seq_num 1118 ranges {790226=432273,1806045=582380} 2024-09-21T09:50:26.772 INFO:tasks.rados.rados.0.smithi076.stdout:2962: writing smithi07685715-45 from 790226 to 1222499 tid 1 2024-09-21T09:50:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2962: writing smithi07685715-45 from 1806045 to 2388425 tid 2 2024-09-21T09:50:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2963: write oid 4 current snap is 370 2024-09-21T09:50:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2962: finishing write tid 1 to smithi07685715-45 2024-09-21T09:50:26.780 INFO:tasks.rados.rados.0.smithi076.stdout:2963: seq_num 1119 ranges {407033=507522,1464527=442391,2345976=461057} 2024-09-21T09:50:26.784 INFO:tasks.rados.rados.0.smithi076.stdout:2963: writing smithi07685715-4 from 407033 to 914555 tid 1 2024-09-21T09:50:26.790 INFO:tasks.rados.rados.0.smithi076.stdout:2963: writing smithi07685715-4 from 1464527 to 1906918 tid 2 2024-09-21T09:50:26.796 INFO:tasks.rados.rados.0.smithi076.stdout:2963: writing smithi07685715-4 from 2345976 to 2807033 tid 3 2024-09-21T09:50:26.796 INFO:tasks.rados.rados.0.smithi076.stdout:2964: read oid 26 snap -1 2024-09-21T09:50:26.797 INFO:tasks.rados.rados.0.smithi076.stdout:2964: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2965: delete oid 13 current snap is 370 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: finishing write tid 2 to smithi07685715-45 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: finishing write tid 3 to smithi07685715-45 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: finishing write tid 4 to smithi07685715-45 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: oid 45 updating version 0 to 646 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: oid 45 updating version 646 to 647 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: oid 45 version 647 is already newer than 645 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 647 (ObjNum 1118 snap 370 seq_num 1118) dirty exists 2024-09-21T09:50:26.798 INFO:tasks.rados.rados.0.smithi076.stdout:2962: left oid 45 (ObjNum 1118 snap 370 seq_num 1118) 2024-09-21T09:50:26.799 INFO:tasks.rados.rados.0.smithi076.stdout:2963: finishing write tid 1 to smithi07685715-4 2024-09-21T09:50:26.799 INFO:tasks.rados.rados.0.smithi076.stdout:2963: finishing write tid 2 to smithi07685715-4 2024-09-21T09:50:26.799 INFO:tasks.rados.rados.0.smithi076.stdout:2964: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:50:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:2963: finishing write tid 3 to smithi07685715-4 2024-09-21T09:50:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:2963: finishing write tid 4 to smithi07685715-4 2024-09-21T09:50:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:2963: finishing write tid 5 to smithi07685715-4 2024-09-21T09:50:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:2963: oid 4 updating version 0 to 963 2024-09-21T09:50:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:2963: oid 4 version 963 is already newer than 961 2024-09-21T09:50:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:2963: oid 4 version 963 is already newer than 962 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2963: oid 4 version 963 is already newer than 960 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 963 (ObjNum 1119 snap 370 seq_num 1119) dirty exists 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2963: left oid 4 (ObjNum 1119 snap 370 seq_num 1119) 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2962: done (3 left) 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2963: done (2 left) 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2964: done (1 left) 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2965: done (0 left) 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2966: rollback oid 46 current snap is 370 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 351 2024-09-21T09:50:27.014 INFO:tasks.rados.rados.0.smithi076.stdout:2967: delete oid 7 current snap is 370 2024-09-21T09:50:27.017 INFO:tasks.rados.rados.0.smithi076.stdout:2966: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:50:27.017 INFO:tasks.rados.rados.0.smithi076.stdout:2966: finishing rollback tid 2 to smithi07685715-46 2024-09-21T09:50:27.017 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2229 (ObjNum 832 snap 289 seq_num 832) dirty exists 2024-09-21T09:50:27.017 INFO:tasks.rados.rados.0.smithi076.stdout:2966: done (1 left) 2024-09-21T09:50:27.017 INFO:tasks.rados.rados.0.smithi076.stdout:2967: done (0 left) 2024-09-21T09:50:27.017 INFO:tasks.rados.rados.0.smithi076.stdout:2968: delete oid 2 current snap is 370 2024-09-21T09:50:27.019 INFO:tasks.rados.rados.0.smithi076.stdout:2968: done (0 left) 2024-09-21T09:50:27.019 INFO:tasks.rados.rados.0.smithi076.stdout:2969: read oid 48 snap -1 2024-09-21T09:50:27.019 INFO:tasks.rados.rados.0.smithi076.stdout:2969: expect (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:50:27.019 INFO:tasks.rados.rados.0.smithi076.stdout:2970: write oid 22 current snap is 370 2024-09-21T09:50:27.019 INFO:tasks.rados.rados.0.smithi076.stdout:2970: seq_num 1123 ranges {474261=408518,1582102=449729,2739315=134946} 2024-09-21T09:50:27.023 INFO:tasks.rados.rados.0.smithi076.stdout:2970: writing smithi07685715-22 from 474261 to 882779 tid 1 2024-09-21T09:50:27.030 INFO:tasks.rados.rados.0.smithi076.stdout:2970: writing smithi07685715-22 from 1582102 to 2031831 tid 2 2024-09-21T09:50:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:2970: writing smithi07685715-22 from 2739315 to 2874261 tid 3 2024-09-21T09:50:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:2971: snap_create 2024-09-21T09:50:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:2970: finishing write tid 1 to smithi07685715-22 2024-09-21T09:50:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:2969: expect (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: finishing write tid 2 to smithi07685715-22 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: finishing write tid 3 to smithi07685715-22 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: finishing write tid 4 to smithi07685715-22 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: finishing write tid 5 to smithi07685715-22 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: oid 22 updating version 0 to 1403 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: oid 22 version 1403 is already newer than 1401 2024-09-21T09:50:27.219 INFO:tasks.rados.rados.0.smithi076.stdout:2970: oid 22 version 1403 is already newer than 1402 2024-09-21T09:50:27.220 INFO:tasks.rados.rados.0.smithi076.stdout:2970: oid 22 updating version 1403 to 1404 2024-09-21T09:50:27.220 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1404 (ObjNum 1123 snap 370 seq_num 1123) dirty exists 2024-09-21T09:50:27.220 INFO:tasks.rados.rados.0.smithi076.stdout:2970: left oid 22 (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:50:27.779 INFO:tasks.rados.rados.0.smithi076.stdout:2969: done (2 left) 2024-09-21T09:50:27.780 INFO:tasks.rados.rados.0.smithi076.stdout:2970: done (1 left) 2024-09-21T09:50:27.780 INFO:tasks.rados.rados.0.smithi076.stdout:2971: done (0 left) 2024-09-21T09:50:27.780 INFO:tasks.rados.rados.0.smithi076.stdout:2972: delete oid 36 current snap is 371 2024-09-21T09:50:27.783 INFO:tasks.rados.rados.0.smithi076.stdout:2972: done (0 left) 2024-09-21T09:50:27.783 INFO:tasks.rados.rados.0.smithi076.stdout:2973: delete oid 42 current snap is 371 2024-09-21T09:50:27.786 INFO:tasks.rados.rados.0.smithi076.stdout:2973: done (0 left) 2024-09-21T09:50:27.786 INFO:tasks.rados.rados.0.smithi076.stdout:2974: write oid 47 current snap is 371 2024-09-21T09:50:27.786 INFO:tasks.rados.rados.0.smithi076.stdout:2974: seq_num 1126 ranges {524682=750618,1757548=597056} 2024-09-21T09:50:27.793 INFO:tasks.rados.rados.0.smithi076.stdout:2974: writing smithi07685715-47 from 524682 to 1275300 tid 1 2024-09-21T09:50:27.800 INFO:tasks.rados.rados.0.smithi076.stdout:2974: writing smithi07685715-47 from 1757548 to 2354604 tid 2 2024-09-21T09:50:27.800 INFO:tasks.rados.rados.0.smithi076.stdout:2975: write oid 20 current snap is 371 2024-09-21T09:50:27.800 INFO:tasks.rados.rados.0.smithi076.stdout:2974: finishing write tid 1 to smithi07685715-47 2024-09-21T09:50:27.801 INFO:tasks.rados.rados.0.smithi076.stdout:2975: seq_num 1127 ranges {541489=425867,1416030=740714,2732654=208835} 2024-09-21T09:50:27.805 INFO:tasks.rados.rados.0.smithi076.stdout:2975: writing smithi07685715-20 from 541489 to 967356 tid 1 2024-09-21T09:50:27.812 INFO:tasks.rados.rados.0.smithi076.stdout:2975: writing smithi07685715-20 from 1416030 to 2156744 tid 2 2024-09-21T09:50:27.817 INFO:tasks.rados.rados.0.smithi076.stdout:2975: writing smithi07685715-20 from 2732654 to 2941489 tid 3 2024-09-21T09:50:27.817 INFO:tasks.rados.rados.0.smithi076.stdout:2976: read oid 50 snap -1 2024-09-21T09:50:27.817 INFO:tasks.rados.rados.0.smithi076.stdout:2976: expect (ObjNum 1093 snap 364 seq_num 1093) 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2977: write oid 46 current snap is 371 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: finishing write tid 2 to smithi07685715-47 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: finishing write tid 3 to smithi07685715-47 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: finishing write tid 4 to smithi07685715-47 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: oid 47 updating version 0 to 2223 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: oid 47 updating version 2223 to 2224 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: oid 47 version 2224 is already newer than 2222 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2224 (ObjNum 1126 snap 371 seq_num 1126) dirty exists 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2974: left oid 47 (ObjNum 1126 snap 371 seq_num 1126) 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2975: finishing write tid 1 to smithi07685715-20 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2975: finishing write tid 2 to smithi07685715-20 2024-09-21T09:50:27.818 INFO:tasks.rados.rados.0.smithi076.stdout:2977: seq_num 1128 ranges {558296=501116,1474512=600725,2760077=198219} 2024-09-21T09:50:27.823 INFO:tasks.rados.rados.0.smithi076.stdout:2977: writing smithi07685715-46 from 558296 to 1059412 tid 1 2024-09-21T09:50:27.829 INFO:tasks.rados.rados.0.smithi076.stdout:2977: writing smithi07685715-46 from 1474512 to 2075237 tid 2 2024-09-21T09:50:27.835 INFO:tasks.rados.rados.0.smithi076.stdout:2977: writing smithi07685715-46 from 2760077 to 2958296 tid 3 2024-09-21T09:50:27.835 INFO:tasks.rados.rados.0.smithi076.stdout:2974: done (3 left) 2024-09-21T09:50:27.835 INFO:tasks.rados.rados.0.smithi076.stdout:2978: snap_create 2024-09-21T09:50:27.835 INFO:tasks.rados.rados.0.smithi076.stdout:2975: finishing write tid 3 to smithi07685715-20 2024-09-21T09:50:27.835 INFO:tasks.rados.rados.0.smithi076.stdout:2975: finishing write tid 4 to smithi07685715-20 2024-09-21T09:50:27.835 INFO:tasks.rados.rados.0.smithi076.stdout:2975: finishing write tid 5 to smithi07685715-20 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:2975: oid 20 updating version 0 to 880 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:2975: oid 20 updating version 880 to 881 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:2975: oid 20 version 881 is already newer than 879 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:2975: oid 20 updating version 881 to 882 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 882 (ObjNum 1127 snap 371 seq_num 1127) dirty exists 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:2975: left oid 20 (ObjNum 1127 snap 371 seq_num 1127) 2024-09-21T09:50:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:2976: expect (ObjNum 1093 snap 364 seq_num 1093) 2024-09-21T09:50:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2977: finishing write tid 1 to smithi07685715-46 2024-09-21T09:50:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2977: finishing write tid 2 to smithi07685715-46 2024-09-21T09:50:27.945 INFO:tasks.rados.rados.0.smithi076.stdout:2977: finishing write tid 3 to smithi07685715-46 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: finishing write tid 4 to smithi07685715-46 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: finishing write tid 5 to smithi07685715-46 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: oid 46 updating version 0 to 2233 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: oid 46 version 2233 is already newer than 2232 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: oid 46 version 2233 is already newer than 2231 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: oid 46 updating version 2233 to 2234 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2234 (ObjNum 1128 snap 371 seq_num 1128) dirty exists 2024-09-21T09:50:27.946 INFO:tasks.rados.rados.0.smithi076.stdout:2977: left oid 46 (ObjNum 1128 snap 371 seq_num 1128) 2024-09-21T09:50:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:27 smithi191 ceph-mon[102206]: pgmap v688: 89 pgs: 89 active+clean; 181 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:50:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:27 smithi191 ceph-mon[102206]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:50:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:27 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:50:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:27 smithi191 ceph-mon[102206]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:50:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:27 smithi191 ceph-mon[102206]: Upgrade: osd.3 is safe to restart 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[110963]: pgmap v688: 89 pgs: 89 active+clean; 181 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[110963]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[110963]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[110963]: Upgrade: osd.3 is safe to restart 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[106482]: pgmap v688: 89 pgs: 89 active+clean; 181 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[106482]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[106482]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:50:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:27 smithi076 ceph-mon[106482]: Upgrade: osd.3 is safe to restart 2024-09-21T09:50:28.781 INFO:tasks.rados.rados.0.smithi076.stdout:2975: done (3 left) 2024-09-21T09:50:28.781 INFO:tasks.rados.rados.0.smithi076.stdout:2976: done (2 left) 2024-09-21T09:50:28.781 INFO:tasks.rados.rados.0.smithi076.stdout:2977: done (1 left) 2024-09-21T09:50:28.782 INFO:tasks.rados.rados.0.smithi076.stdout:2978: done (0 left) 2024-09-21T09:50:28.782 INFO:tasks.rados.rados.0.smithi076.stdout:2979: snap_remove snap 366 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[110963]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[106482]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:50:29.069 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:28 smithi191 ceph-mon[102206]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T09:50:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:50:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:29.783 INFO:tasks.rados.rados.0.smithi076.stdout:2979: done (0 left) 2024-09-21T09:50:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:2980: read oid 11 snap -1 2024-09-21T09:50:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:2980: expect deleted 2024-09-21T09:50:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:2981: rollback oid 23 current snap is 372 2024-09-21T09:50:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 367 2024-09-21T09:50:29.784 INFO:tasks.rados.rados.0.smithi076.stdout:2982: snap_create 2024-09-21T09:50:29.794 INFO:tasks.rados.rados.0.smithi076.stdout:2981: finishing rollback tid 0 to smithi07685715-23 2024-09-21T09:50:29.795 INFO:tasks.rados.rados.0.smithi076.stdout:2981: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:50:29.795 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2227 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:50:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:29 smithi191 ceph-mon[102206]: pgmap v691: 89 pgs: 89 active+clean; 178 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:50:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:29 smithi191 ceph-mon[102206]: Upgrade: Updating osd.3 2024-09-21T09:50:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:29 smithi191 ceph-mon[102206]: Deploying daemon osd.3 on smithi076 2024-09-21T09:50:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:29 smithi191 ceph-mon[102206]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:50:30.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[106482]: pgmap v691: 89 pgs: 89 active+clean; 178 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:50:30.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[106482]: Upgrade: Updating osd.3 2024-09-21T09:50:30.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[106482]: Deploying daemon osd.3 on smithi076 2024-09-21T09:50:30.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[106482]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:50:30.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[110963]: pgmap v691: 89 pgs: 89 active+clean; 178 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 1.7 MiB/s wr, 12 op/s 2024-09-21T09:50:30.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[110963]: Upgrade: Updating osd.3 2024-09-21T09:50:30.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[110963]: Deploying daemon osd.3 on smithi076 2024-09-21T09:50:30.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:29 smithi076 ceph-mon[110963]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2980: done (2 left) 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2981: done (1 left) 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2982: done (0 left) 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2983: rollback oid 9 current snap is 373 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 9 to 369 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2984: write oid 29 current snap is 373 2024-09-21T09:50:30.788 INFO:tasks.rados.rados.0.smithi076.stdout:2984: seq_num 1129 ranges {575103=576365,1932994=744383} 2024-09-21T09:50:30.794 INFO:tasks.rados.rados.0.smithi076.stdout:2984: writing smithi07685715-29 from 575103 to 1151468 tid 1 2024-09-21T09:50:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2984: writing smithi07685715-29 from 1932994 to 2677377 tid 2 2024-09-21T09:50:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2985: write oid 44 current snap is 373 2024-09-21T09:50:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2983: finishing rollback tid 1 to smithi07685715-9 2024-09-21T09:50:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 0 (ObjNum 113 snap 0 seq_num 864120000) dirty dne 2024-09-21T09:50:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2984: finishing write tid 1 to smithi07685715-29 2024-09-21T09:50:30.803 INFO:tasks.rados.rados.0.smithi076.stdout:2985: seq_num 1130 ranges {591910=651614,1875123=604394} 2024-09-21T09:50:30.809 INFO:tasks.rados.rados.0.smithi076.stdout:2985: writing smithi07685715-44 from 591910 to 1243524 tid 1 2024-09-21T09:50:30.816 INFO:tasks.rados.rados.0.smithi076.stdout:2985: writing smithi07685715-44 from 1875123 to 2479517 tid 2 2024-09-21T09:50:30.816 INFO:tasks.rados.rados.0.smithi076.stdout:2983: done (2 left) 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2986: snap_create 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: finishing write tid 2 to smithi07685715-29 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: finishing write tid 3 to smithi07685715-29 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: finishing write tid 4 to smithi07685715-29 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: oid 29 updating version 0 to 887 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: oid 29 updating version 887 to 888 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: oid 29 version 888 is already newer than 886 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 888 (ObjNum 1129 snap 373 seq_num 1129) dirty exists 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2984: left oid 29 (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:30.817 INFO:tasks.rados.rados.0.smithi076.stdout:2985: finishing write tid 1 to smithi07685715-44 2024-09-21T09:50:30.819 INFO:tasks.rados.rados.0.smithi076.stdout:2985: finishing write tid 2 to smithi07685715-44 2024-09-21T09:50:30.819 INFO:tasks.rados.rados.0.smithi076.stdout:2985: finishing write tid 3 to smithi07685715-44 2024-09-21T09:50:30.819 INFO:tasks.rados.rados.0.smithi076.stdout:2985: finishing write tid 4 to smithi07685715-44 2024-09-21T09:50:30.819 INFO:tasks.rados.rados.0.smithi076.stdout:2985: oid 44 updating version 0 to 712 2024-09-21T09:50:30.819 INFO:tasks.rados.rados.0.smithi076.stdout:2985: oid 44 updating version 712 to 713 2024-09-21T09:50:30.819 INFO:tasks.rados.rados.0.smithi076.stdout:2985: oid 44 version 713 is already newer than 711 2024-09-21T09:50:30.820 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 713 (ObjNum 1130 snap 373 seq_num 1130) dirty exists 2024-09-21T09:50:30.820 INFO:tasks.rados.rados.0.smithi076.stdout:2985: left oid 44 (ObjNum 1130 snap 373 seq_num 1130) 2024-09-21T09:50:30.999 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:30 smithi076 ceph-mon[106482]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:50:30.999 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:30 smithi076 ceph-mon[110963]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:50:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:30 smithi191 ceph-mon[102206]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T09:50:31.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:31 smithi076 systemd[1]: Stopping Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:50:31.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[64169]: 2024-09-21T09:50:31.587+0000 7f91f8dfb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:50:31.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[64169]: 2024-09-21T09:50:31.587+0000 7f91f8dfb640 -1 osd.3 816 *** Got signal Terminated *** 2024-09-21T09:50:31.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[64169]: 2024-09-21T09:50:31.587+0000 7f91f8dfb640 -1 osd.3 816 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:50:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2984: done (2 left) 2024-09-21T09:50:31.806 INFO:tasks.rados.rados.0.smithi076.stdout:2985: done (1 left) 2024-09-21T09:50:31.807 INFO:tasks.rados.rados.0.smithi076.stdout:2986: done (0 left) 2024-09-21T09:50:31.807 INFO:tasks.rados.rados.0.smithi076.stdout:2987: snap_remove snap 369 2024-09-21T09:50:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:31 smithi191 ceph-mon[102206]: pgmap v694: 89 pgs: 89 active+clean; 181 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 248 KiB/s wr, 6 op/s 2024-09-21T09:50:32.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:31 smithi191 ceph-mon[102206]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:50:32.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:31 smithi191 ceph-mon[102206]: osd.3 marked itself down and dead 2024-09-21T09:50:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-mon[106482]: pgmap v694: 89 pgs: 89 active+clean; 181 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 248 KiB/s wr, 6 op/s 2024-09-21T09:50:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-mon[106482]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:50:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-mon[106482]: osd.3 marked itself down and dead 2024-09-21T09:50:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-mon[110963]: pgmap v694: 89 pgs: 89 active+clean; 181 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 248 KiB/s wr, 6 op/s 2024-09-21T09:50:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-mon[110963]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T09:50:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:31 smithi076 ceph-mon[110963]: osd.3 marked itself down and dead 2024-09-21T09:50:32.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:32 smithi076 podman[137298]: 2024-09-21 09:50:32.343782726 +0000 UTC m=+0.857706850 container died 7095b98dd4dfc1e0ff56c53c9c428bbba6c2c96967795d0692d612f442913df6 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:50:32.811 INFO:tasks.rados.rados.0.smithi076.stdout:2987: done (0 left) 2024-09-21T09:50:32.811 INFO:tasks.rados.rados.0.smithi076.stdout:2988: snap_create 2024-09-21T09:50:33.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:32 smithi076 ceph-mon[106482]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:50:33.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:32 smithi076 ceph-mon[106482]: osdmap e817: 8 total, 7 up, 8 in 2024-09-21T09:50:33.073 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:32 smithi076 ceph-mon[110963]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:50:33.073 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:32 smithi076 ceph-mon[110963]: osdmap e817: 8 total, 7 up, 8 in 2024-09-21T09:50:33.073 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:32 smithi076 podman[137298]: 2024-09-21 09:50:32.794482211 +0000 UTC m=+1.308406332 container cleanup 7095b98dd4dfc1e0ff56c53c9c428bbba6c2c96967795d0692d612f442913df6 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:50:33.073 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:32 smithi076 bash[137298]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3 2024-09-21T09:50:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:32 smithi191 ceph-mon[102206]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:50:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:32 smithi191 ceph-mon[102206]: osdmap e817: 8 total, 7 up, 8 in 2024-09-21T09:50:33.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:50:33] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:50:33.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137320]: 2024-09-21 09:50:33.071093849 +0000 UTC m=+0.727608744 container remove 7095b98dd4dfc1e0ff56c53c9c428bbba6c2c96967795d0692d612f442913df6 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:50:33.668 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137412]: 2024-09-21 09:50:33.310960191 +0000 UTC m=+0.015713511 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:33.668 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137412]: 2024-09-21 09:50:33.472956617 +0000 UTC m=+0.177709923 container create 040cd81937084af17b01a57cab29a602478c3ce7c169275ce307f33fb9be9599 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T09:50:33.762 INFO:tasks.rados.rados.0.smithi076.stdout:2988: done (0 left) 2024-09-21T09:50:33.762 INFO:tasks.rados.rados.0.smithi076.stdout:2989: rollback oid 7 current snap is 375 2024-09-21T09:50:33.762 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 367 2024-09-21T09:50:33.762 INFO:tasks.rados.rados.0.smithi076.stdout:2990: write oid 14 current snap is 375 2024-09-21T09:50:33.762 INFO:tasks.rados.rados.0.smithi076.stdout:2990: seq_num 1131 ranges {608717=726863,1933605=748052} 2024-09-21T09:50:33.769 INFO:tasks.rados.rados.0.smithi076.stdout:2990: writing smithi07685715-14 from 608717 to 1335580 tid 1 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:2990: writing smithi07685715-14 from 1933605 to 2681657 tid 2 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:2991: read oid 39 snap -1 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:2991: expect (ObjNum 1097 snap 364 seq_num 1097) 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:2992: delete oid 41 current snap is 375 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:2989: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 0 (ObjNum 129 snap 0 seq_num 865205512) dirty dne 2024-09-21T09:50:33.777 INFO:tasks.rados.rados.0.smithi076.stdout:2990: finishing write tid 1 to smithi07685715-14 2024-09-21T09:50:33.780 INFO:tasks.rados.rados.0.smithi076.stdout:2989: done (3 left) 2024-09-21T09:50:33.781 INFO:tasks.rados.rados.0.smithi076.stdout:2992: done (2 left) 2024-09-21T09:50:33.781 INFO:tasks.rados.rados.0.smithi076.stdout:2993: write oid 7 current snap is 375 2024-09-21T09:50:33.781 INFO:tasks.rados.rados.0.smithi076.stdout:2993: seq_num 1133 ranges {642331=593714,1766922=751721} 2024-09-21T09:50:33.788 INFO:tasks.rados.rados.0.smithi076.stdout:2993: writing smithi07685715-7 from 642331 to 1236045 tid 1 2024-09-21T09:50:33.795 INFO:tasks.rados.rados.0.smithi076.stdout:2993: writing smithi07685715-7 from 1766922 to 2518643 tid 2 2024-09-21T09:50:33.796 INFO:tasks.rados.rados.0.smithi076.stdout:2994: write oid 37 current snap is 375 2024-09-21T09:50:33.796 INFO:tasks.rados.rados.0.smithi076.stdout:2990: finishing write tid 2 to smithi07685715-14 2024-09-21T09:50:33.796 INFO:tasks.rados.rados.0.smithi076.stdout:2994: seq_num 1134 ranges {659138=668963,2109051=611732} 2024-09-21T09:50:33.803 INFO:tasks.rados.rados.0.smithi076.stdout:2994: writing smithi07685715-37 from 659138 to 1328101 tid 1 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2994: writing smithi07685715-37 from 2109051 to 2720783 tid 2 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2995: snap_create 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2990: finishing write tid 3 to smithi07685715-14 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2990: finishing write tid 4 to smithi07685715-14 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2990: oid 14 updating version 0 to 1780 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2990: oid 14 version 1780 is already newer than 1778 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2990: oid 14 version 1780 is already newer than 1779 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1780 (ObjNum 1131 snap 375 seq_num 1131) dirty exists 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2990: left oid 14 (ObjNum 1131 snap 375 seq_num 1131) 2024-09-21T09:50:33.812 INFO:tasks.rados.rados.0.smithi076.stdout:2991: expect (ObjNum 1097 snap 364 seq_num 1097) 2024-09-21T09:50:34.037 INFO:tasks.rados.rados.0.smithi076.stdout:2993: finishing write tid 1 to smithi07685715-7 2024-09-21T09:50:34.037 INFO:tasks.rados.rados.0.smithi076.stdout:2993: finishing write tid 2 to smithi07685715-7 2024-09-21T09:50:34.037 INFO:tasks.rados.rados.0.smithi076.stdout:2993: finishing write tid 3 to smithi07685715-7 2024-09-21T09:50:34.037 INFO:tasks.rados.rados.0.smithi076.stdout:2993: finishing write tid 4 to smithi07685715-7 2024-09-21T09:50:34.037 INFO:tasks.rados.rados.0.smithi076.stdout:2993: oid 7 updating version 0 to 1781 2024-09-21T09:50:34.037 INFO:tasks.rados.rados.0.smithi076.stdout:2993: oid 7 updating version 1781 to 1782 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2993: oid 7 updating version 1782 to 1783 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1783 (ObjNum 1133 snap 375 seq_num 1133) dirty exists 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2993: left oid 7 (ObjNum 1133 snap 375 seq_num 1133) 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: finishing write tid 1 to smithi07685715-37 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: finishing write tid 2 to smithi07685715-37 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: finishing write tid 3 to smithi07685715-37 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: finishing write tid 4 to smithi07685715-37 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: oid 37 updating version 0 to 895 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: oid 37 version 895 is already newer than 893 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: oid 37 version 895 is already newer than 894 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 895 (ObjNum 1134 snap 375 seq_num 1134) dirty exists 2024-09-21T09:50:34.038 INFO:tasks.rados.rados.0.smithi076.stdout:2994: left oid 37 (ObjNum 1134 snap 375 seq_num 1134) 2024-09-21T09:50:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:33 smithi191 ceph-mon[102206]: pgmap v697: 89 pgs: 8 peering, 9 stale+active+clean, 72 active+clean; 195 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-09-21T09:50:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:33 smithi191 ceph-mon[102206]: osdmap e818: 8 total, 7 up, 8 in 2024-09-21T09:50:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:33 smithi191 ceph-mon[102206]: osdmap e819: 8 total, 7 up, 8 in 2024-09-21T09:50:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-mon[110963]: pgmap v697: 89 pgs: 8 peering, 9 stale+active+clean, 72 active+clean; 195 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-09-21T09:50:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-mon[110963]: osdmap e818: 8 total, 7 up, 8 in 2024-09-21T09:50:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-mon[110963]: osdmap e819: 8 total, 7 up, 8 in 2024-09-21T09:50:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-mon[106482]: pgmap v697: 89 pgs: 8 peering, 9 stale+active+clean, 72 active+clean; 195 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 7.5 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-09-21T09:50:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-mon[106482]: osdmap e818: 8 total, 7 up, 8 in 2024-09-21T09:50:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:33 smithi076 ceph-mon[106482]: osdmap e819: 8 total, 7 up, 8 in 2024-09-21T09:50:34.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137412]: 2024-09-21 09:50:33.667290097 +0000 UTC m=+0.372043413 container init 040cd81937084af17b01a57cab29a602478c3ce7c169275ce307f33fb9be9599 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b) 2024-09-21T09:50:34.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137412]: 2024-09-21 09:50:33.672019802 +0000 UTC m=+0.376773108 container start 040cd81937084af17b01a57cab29a602478c3ce7c169275ce307f33fb9be9599 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T09:50:34.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137412]: 2024-09-21 09:50:33.750460486 +0000 UTC m=+0.455213806 container attach 040cd81937084af17b01a57cab29a602478c3ce7c169275ce307f33fb9be9599 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T09:50:34.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:33 smithi076 podman[137412]: 2024-09-21 09:50:33.880529414 +0000 UTC m=+0.585282731 container died 040cd81937084af17b01a57cab29a602478c3ce7c169275ce307f33fb9be9599 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:50:34.755 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:34 smithi076 podman[137455]: 2024-09-21 09:50:34.463141856 +0000 UTC m=+0.573203678 container remove 040cd81937084af17b01a57cab29a602478c3ce7c169275ce307f33fb9be9599 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T09:50:34.755 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:34 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T09:50:34.755 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:34 smithi076 systemd[1]: Stopped Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:50:34.755 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:34 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service: Consumed 14.139s CPU time. 2024-09-21T09:50:34.755 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:34 smithi076 systemd[1]: Starting Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T09:50:34.818 INFO:tasks.rados.rados.0.smithi076.stdout:2990: done (4 left) 2024-09-21T09:50:34.818 INFO:tasks.rados.rados.0.smithi076.stdout:2991: done (3 left) 2024-09-21T09:50:34.818 INFO:tasks.rados.rados.0.smithi076.stdout:2993: done (2 left) 2024-09-21T09:50:34.818 INFO:tasks.rados.rados.0.smithi076.stdout:2994: done (1 left) 2024-09-21T09:50:34.818 INFO:tasks.rados.rados.0.smithi076.stdout:2995: done (0 left) 2024-09-21T09:50:34.818 INFO:tasks.rados.rados.0.smithi076.stdout:2996: rollback oid 7 current snap is 376 2024-09-21T09:50:34.819 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 374 2024-09-21T09:50:34.819 INFO:tasks.rados.rados.0.smithi076.stdout:2997: write oid 26 current snap is 376 2024-09-21T09:50:34.819 INFO:tasks.rados.rados.0.smithi076.stdout:2997: seq_num 1135 ranges {675945=744212,2167533=755390} 2024-09-21T09:50:34.827 INFO:tasks.rados.rados.0.smithi076.stdout:2997: writing smithi07685715-26 from 675945 to 1420157 tid 1 2024-09-21T09:50:34.837 INFO:tasks.rados.rados.0.smithi076.stdout:2997: writing smithi07685715-26 from 2167533 to 2922923 tid 2 2024-09-21T09:50:34.837 INFO:tasks.rados.rados.0.smithi076.stdout:2998: snap_create 2024-09-21T09:50:34.837 INFO:tasks.rados.rados.0.smithi076.stdout:2996: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:50:34.837 INFO:tasks.rados.rados.0.smithi076.stdout:2996: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:50:34.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1786 (ObjNum 129 snap 0 seq_num 864361024) dirty dne 2024-09-21T09:50:34.837 INFO:tasks.rados.rados.0.smithi076.stdout:2997: finishing write tid 1 to smithi07685715-26 2024-09-21T09:50:34.840 INFO:tasks.rados.rados.0.smithi076.stdout:2997: finishing write tid 2 to smithi07685715-26 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:2997: finishing write tid 3 to smithi07685715-26 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:2997: finishing write tid 4 to smithi07685715-26 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:2997: oid 26 updating version 0 to 1010 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:2997: oid 26 updating version 1010 to 1011 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:2997: oid 26 updating version 1011 to 1012 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1012 (ObjNum 1135 snap 376 seq_num 1135) dirty exists 2024-09-21T09:50:34.841 INFO:tasks.rados.rados.0.smithi076.stdout:2997: left oid 26 (ObjNum 1135 snap 376 seq_num 1135) 2024-09-21T09:50:35.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:34 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 1/393 objects degraded (0.254%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:50:35.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:34 smithi076 ceph-mon[106482]: osdmap e820: 8 total, 7 up, 8 in 2024-09-21T09:50:35.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:34 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 1/393 objects degraded (0.254%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:50:35.034 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:34 smithi076 ceph-mon[110963]: osdmap e820: 8 total, 7 up, 8 in 2024-09-21T09:50:35.034 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:34 smithi076 podman[137561]: 2024-09-21 09:50:34.816658032 +0000 UTC m=+0.018219966 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:34 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 1/393 objects degraded (0.254%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:50:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:34 smithi191 ceph-mon[102206]: osdmap e820: 8 total, 7 up, 8 in 2024-09-21T09:50:35.353 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 podman[137561]: 2024-09-21 09:50:35.032230307 +0000 UTC m=+0.233792233 container create 3ec8d841782dab1c71360e0f2c607f4234385e759399739e177743a3ad3aa826 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:50:35.353 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 podman[137561]: 2024-09-21 09:50:35.276771545 +0000 UTC m=+0.478333481 container init 3ec8d841782dab1c71360e0f2c607f4234385e759399739e177743a3ad3aa826 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:50:35.353 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 podman[137561]: 2024-09-21 09:50:35.282033958 +0000 UTC m=+0.483595894 container start 3ec8d841782dab1c71360e0f2c607f4234385e759399739e177743a3ad3aa826 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T09:50:35.353 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 podman[137561]: 2024-09-21 09:50:35.351698377 +0000 UTC m=+0.553260320 container attach 3ec8d841782dab1c71360e0f2c607f4234385e759399739e177743a3ad3aa826 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T09:50:35.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:35.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 bash[137561]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:35.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:35.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 bash[137561]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:50:35.818 INFO:tasks.rados.rados.0.smithi076.stdout:2996: done (2 left) 2024-09-21T09:50:35.818 INFO:tasks.rados.rados.0.smithi076.stdout:2997: done (1 left) 2024-09-21T09:50:35.818 INFO:tasks.rados.rados.0.smithi076.stdout:2998: done (0 left) 2024-09-21T09:50:35.818 INFO:tasks.rados.rados.0.smithi076.stdout:2999: rollback oid 2 current snap is 377 2024-09-21T09:50:35.818 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 375 2024-09-21T09:50:35.818 INFO:tasks.rados.rados.0.smithi076.stdout:3000: snap_create 2024-09-21T09:50:35.821 INFO:tasks.rados.rados.0.smithi076.stdout:2999: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:50:35.821 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 574 (ObjNum 801 snap 0 seq_num 1283432432) dirty dne 2024-09-21T09:50:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:35 smithi191 ceph-mon[102206]: pgmap v700: 89 pgs: 6 active+undersized, 8 peering, 7 stale+active+clean, 1 active+undersized+degraded, 67 active+clean; 199 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.8 MiB/s wr, 16 op/s; 1/393 objects degraded (0.254%) 2024-09-21T09:50:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:35 smithi191 ceph-mon[102206]: osdmap e821: 8 total, 7 up, 8 in 2024-09-21T09:50:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-mon[106482]: pgmap v700: 89 pgs: 6 active+undersized, 8 peering, 7 stale+active+clean, 1 active+undersized+degraded, 67 active+clean; 199 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.8 MiB/s wr, 16 op/s; 1/393 objects degraded (0.254%) 2024-09-21T09:50:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-mon[106482]: osdmap e821: 8 total, 7 up, 8 in 2024-09-21T09:50:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-mon[110963]: pgmap v700: 89 pgs: 6 active+undersized, 8 peering, 7 stale+active+clean, 1 active+undersized+degraded, 67 active+clean; 199 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.8 MiB/s wr, 16 op/s; 1/393 objects degraded (0.254%) 2024-09-21T09:50:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-mon[110963]: osdmap e821: 8 total, 7 up, 8 in 2024-09-21T09:50:36.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:50:36.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 bash[137561]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:50:36.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:50:36.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:35 smithi076 bash[137561]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:50:36.541 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:50:36.541 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 bash[137561]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:50:36.541 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:50:36.541 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 bash[137561]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 bash[137561]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 bash[137561]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[137584]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 bash[137561]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:50:36.542 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 podman[137561]: 2024-09-21 09:50:36.196484513 +0000 UTC m=+1.398046441 container died 3ec8d841782dab1c71360e0f2c607f4234385e759399739e177743a3ad3aa826 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T09:50:36.821 INFO:tasks.rados.rados.0.smithi076.stdout:2999: done (1 left) 2024-09-21T09:50:36.821 INFO:tasks.rados.rados.0.smithi076.stdout:3000: done (0 left) 2024-09-21T09:50:36.821 INFO:tasks.rados.rados.0.smithi076.stdout:3001: write oid 18 current snap is 378 2024-09-21T09:50:36.821 INFO:tasks.rados.rados.0.smithi076.stdout:3001: seq_num 1136 ranges {692752=419461,1826015=615401,2979461=113291} 2024-09-21T09:50:36.827 INFO:tasks.rados.rados.0.smithi076.stdout:3001: writing smithi07685715-18 from 692752 to 1112213 tid 1 2024-09-21T09:50:36.835 INFO:tasks.rados.rados.0.smithi076.stdout:3001: writing smithi07685715-18 from 1826015 to 2441416 tid 2 2024-09-21T09:50:36.839 INFO:tasks.rados.rados.0.smithi076.stdout:3001: writing smithi07685715-18 from 2979461 to 3092752 tid 3 2024-09-21T09:50:36.840 INFO:tasks.rados.rados.0.smithi076.stdout:3002: delete oid 49 current snap is 378 2024-09-21T09:50:36.840 INFO:tasks.rados.rados.0.smithi076.stdout:3001: finishing write tid 1 to smithi07685715-18 2024-09-21T09:50:36.840 INFO:tasks.rados.rados.0.smithi076.stdout:3001: finishing write tid 2 to smithi07685715-18 2024-09-21T09:50:36.841 INFO:tasks.rados.rados.0.smithi076.stdout:3001: finishing write tid 3 to smithi07685715-18 2024-09-21T09:50:36.842 INFO:tasks.rados.rados.0.smithi076.stdout:3001: finishing write tid 4 to smithi07685715-18 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: finishing write tid 5 to smithi07685715-18 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: oid 18 updating version 0 to 748 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: oid 18 updating version 748 to 749 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: oid 18 version 749 is already newer than 747 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: oid 18 version 749 is already newer than 746 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 749 (ObjNum 1136 snap 378 seq_num 1136) dirty exists 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: left oid 18 (ObjNum 1136 snap 378 seq_num 1136) 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3001: done (1 left) 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3002: done (0 left) 2024-09-21T09:50:36.843 INFO:tasks.rados.rados.0.smithi076.stdout:3003: snap_remove snap 371 2024-09-21T09:50:37.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:36 smithi076 podman[137732]: 2024-09-21 09:50:36.806632129 +0000 UTC m=+0.603912628 container remove 3ec8d841782dab1c71360e0f2c607f4234385e759399739e177743a3ad3aa826 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T09:50:37.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 podman[137796]: 2024-09-21 09:50:36.923469263 +0000 UTC m=+0.016348526 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:50:37.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 podman[137796]: 2024-09-21 09:50:37.024347058 +0000 UTC m=+0.117226311 container create 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=main-62da54b) 2024-09-21T09:50:37.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 podman[137796]: 2024-09-21 09:50:37.218478091 +0000 UTC m=+0.311357357 container init 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T09:50:37.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 podman[137796]: 2024-09-21 09:50:37.22292435 +0000 UTC m=+0.315803604 container start 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:50:37.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-osd[137833]: -- 172.21.15.76:0/844692128 <== mon.0 v2:172.21.15.76:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555db6063a40 con 0x555db6e32000 2024-09-21T09:50:37.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 bash[137796]: 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 2024-09-21T09:50:37.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:37 smithi076 systemd[1]: Started Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T09:50:37.821 INFO:tasks.rados.rados.0.smithi076.stdout:3003: done (0 left) 2024-09-21T09:50:37.821 INFO:tasks.rados.rados.0.smithi076.stdout:3004: write oid 42 current snap is 378 2024-09-21T09:50:37.821 INFO:tasks.rados.rados.0.smithi076.stdout:3004: seq_num 1138 ranges {726366=569959,1826626=502717,2801601=324765} 2024-09-21T09:50:37.828 INFO:tasks.rados.rados.0.smithi076.stdout:3004: writing smithi07685715-42 from 726366 to 1296325 tid 1 2024-09-21T09:50:37.835 INFO:tasks.rados.rados.0.smithi076.stdout:3004: writing smithi07685715-42 from 1826626 to 2329343 tid 2 2024-09-21T09:50:37.840 INFO:tasks.rados.rados.0.smithi076.stdout:3004: writing smithi07685715-42 from 2801601 to 3126366 tid 3 2024-09-21T09:50:37.840 INFO:tasks.rados.rados.0.smithi076.stdout:3005: rollback oid 32 current snap is 378 2024-09-21T09:50:37.840 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 372 2024-09-21T09:50:37.840 INFO:tasks.rados.rados.0.smithi076.stdout:3004: finishing write tid 1 to smithi07685715-42 2024-09-21T09:50:37.840 INFO:tasks.rados.rados.0.smithi076.stdout:3004: finishing write tid 2 to smithi07685715-42 2024-09-21T09:50:37.840 INFO:tasks.rados.rados.0.smithi076.stdout:3006: delete oid 41 current snap is 378 2024-09-21T09:50:37.842 INFO:tasks.rados.rados.0.smithi076.stdout:3005: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:50:37.842 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: finishing write tid 3 to smithi07685715-42 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: finishing write tid 4 to smithi07685715-42 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: finishing write tid 5 to smithi07685715-42 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: oid 42 updating version 0 to 750 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: oid 42 updating version 750 to 751 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: oid 42 updating version 751 to 752 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: oid 42 updating version 752 to 753 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 753 (ObjNum 1138 snap 378 seq_num 1138) dirty exists 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: left oid 42 (ObjNum 1138 snap 378 seq_num 1138) 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3004: done (2 left) 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3005: done (1 left) 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3006: done (0 left) 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3007: read oid 9 snap -1 2024-09-21T09:50:37.843 INFO:tasks.rados.rados.0.smithi076.stdout:3007: expect deleted 2024-09-21T09:50:37.844 INFO:tasks.rados.rados.0.smithi076.stdout:3008: write oid 35 current snap is 378 2024-09-21T09:50:37.844 INFO:tasks.rados.rados.0.smithi076.stdout:3008: seq_num 1140 ranges {759980=720457,1943590=506386,2856447=303533} 2024-09-21T09:50:37.850 INFO:tasks.rados.rados.0.smithi076.stdout:3008: writing smithi07685715-35 from 759980 to 1480437 tid 1 2024-09-21T09:50:37.855 INFO:tasks.rados.rados.0.smithi076.stdout:3008: writing smithi07685715-35 from 1943590 to 2449976 tid 2 2024-09-21T09:50:37.859 INFO:tasks.rados.rados.0.smithi076.stdout:3008: writing smithi07685715-35 from 2856447 to 3159980 tid 3 2024-09-21T09:50:37.859 INFO:tasks.rados.rados.0.smithi076.stdout:3009: write oid 14 current snap is 378 2024-09-21T09:50:37.859 INFO:tasks.rados.rados.0.smithi076.stdout:3009: seq_num 1141 ranges {776787=512059,1718425=766397,3000223=176564} 2024-09-21T09:50:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3009: writing smithi07685715-14 from 776787 to 1288846 tid 1 2024-09-21T09:50:37.873 INFO:tasks.rados.rados.0.smithi076.stdout:3009: writing smithi07685715-14 from 1718425 to 2484822 tid 2 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3009: writing smithi07685715-14 from 3000223 to 3176787 tid 3 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3010: rollback oid 15 current snap is 378 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 377 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: finishing write tid 1 to smithi07685715-35 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: finishing write tid 2 to smithi07685715-35 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: finishing write tid 3 to smithi07685715-35 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: finishing write tid 4 to smithi07685715-35 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: finishing write tid 5 to smithi07685715-35 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: oid 35 updating version 0 to 2240 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: oid 35 updating version 2240 to 2241 2024-09-21T09:50:37.878 INFO:tasks.rados.rados.0.smithi076.stdout:3008: oid 35 updating version 2241 to 2243 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3008: oid 35 version 2243 is already newer than 2242 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2243 (ObjNum 1140 snap 378 seq_num 1140) dirty exists 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3008: left oid 35 (ObjNum 1140 snap 378 seq_num 1140) 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3009: finishing write tid 1 to smithi07685715-14 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3007: done (3 left) 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3008: done (2 left) 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3011: rollback oid 41 current snap is 378 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 370 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3012: read oid 29 snap -1 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3012: expect (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:37.879 INFO:tasks.rados.rados.0.smithi076.stdout:3013: snap_remove snap 354 2024-09-21T09:50:38.092 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:37 smithi191 ceph-mon[102206]: pgmap v703: 89 pgs: 21 active+undersized, 8 peering, 5 active+undersized+degraded, 55 active+clean; 202 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1002 KiB/s wr, 5 op/s; 21/396 objects degraded (5.303%) 2024-09-21T09:50:38.092 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:37 smithi191 ceph-mon[102206]: osdmap e822: 8 total, 7 up, 8 in 2024-09-21T09:50:38.092 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.092 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.092 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.092 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[106482]: pgmap v703: 89 pgs: 21 active+undersized, 8 peering, 5 active+undersized+degraded, 55 active+clean; 202 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1002 KiB/s wr, 5 op/s; 21/396 objects degraded (5.303%) 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[106482]: osdmap e822: 8 total, 7 up, 8 in 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[110963]: pgmap v703: 89 pgs: 21 active+undersized, 8 peering, 5 active+undersized+degraded, 55 active+clean; 202 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 1002 KiB/s wr, 5 op/s; 21/396 objects degraded (5.303%) 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[110963]: osdmap e822: 8 total, 7 up, 8 in 2024-09-21T09:50:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:38.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3013: done (4 left) 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3014: read oid 10 snap -1 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3014: expect (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3015: read oid 7 snap -1 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3015: expect deleted 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3016: read oid 26 snap -1 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3016: expect (ObjNum 1135 snap 376 seq_num 1135) 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3017: delete oid 28 current snap is 378 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3009: finishing write tid 2 to smithi07685715-14 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3010: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 0 (ObjNum 33 snap 0 seq_num 864303776) dirty dne 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3011: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:50:38.765 INFO:tasks.rados.rados.0.smithi076.stdout:3011: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1415 (ObjNum 1096 snap 364 seq_num 1096) dirty exists 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: finishing write tid 3 to smithi07685715-14 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: finishing write tid 4 to smithi07685715-14 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: finishing write tid 5 to smithi07685715-14 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: oid 14 updating version 0 to 1791 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: oid 14 updating version 1791 to 1793 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: oid 14 version 1793 is already newer than 1790 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: oid 14 version 1793 is already newer than 1792 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1793 (ObjNum 1141 snap 378 seq_num 1141) dirty exists 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3009: left oid 14 (ObjNum 1141 snap 378 seq_num 1141) 2024-09-21T09:50:38.766 INFO:tasks.rados.rados.0.smithi076.stdout:3012: expect (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:38.904 INFO:tasks.rados.rados.0.smithi076.stdout:3014: expect (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:50:38.982 INFO:tasks.rados.rados.0.smithi076.stdout:3016: expect (ObjNum 1135 snap 376 seq_num 1135) 2024-09-21T09:50:39.094 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:38 smithi076 ceph-mon[106482]: osdmap e823: 8 total, 7 up, 8 in 2024-09-21T09:50:39.094 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:38 smithi076 ceph-mon[106482]: osdmap e824: 8 total, 7 up, 8 in 2024-09-21T09:50:39.095 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:38 smithi076 ceph-mon[110963]: osdmap e823: 8 total, 7 up, 8 in 2024-09-21T09:50:39.095 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:38 smithi076 ceph-mon[110963]: osdmap e824: 8 total, 7 up, 8 in 2024-09-21T09:50:39.095 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[137810]: 2024-09-21T09:50:38.865+0000 7fd7ed537640 -1 Falling back to public interface 2024-09-21T09:50:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:38 smithi191 ceph-mon[102206]: osdmap e823: 8 total, 7 up, 8 in 2024-09-21T09:50:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:38 smithi191 ceph-mon[102206]: osdmap e824: 8 total, 7 up, 8 in 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3009: done (7 left) 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3010: done (6 left) 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3011: done (5 left) 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3012: done (4 left) 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3014: done (3 left) 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3015: done (2 left) 2024-09-21T09:50:39.275 INFO:tasks.rados.rados.0.smithi076.stdout:3016: done (1 left) 2024-09-21T09:50:39.276 INFO:tasks.rados.rados.0.smithi076.stdout:3017: done (0 left) 2024-09-21T09:50:39.276 INFO:tasks.rados.rados.0.smithi076.stdout:3018: snap_create 2024-09-21T09:50:39.835 INFO:tasks.rados.rados.0.smithi076.stdout:3018: done (0 left) 2024-09-21T09:50:39.835 INFO:tasks.rados.rados.0.smithi076.stdout:3019: rollback oid 13 current snap is 379 2024-09-21T09:50:39.835 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 365 2024-09-21T09:50:39.835 INFO:tasks.rados.rados.0.smithi076.stdout:3020: snap_create 2024-09-21T09:50:39.841 INFO:tasks.rados.rados.0.smithi076.stdout:3019: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:50:39.841 INFO:tasks.rados.rados.0.smithi076.stdout:3019: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:50:39.841 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 589 (ObjNum 1095 snap 364 seq_num 1095) dirty exists 2024-09-21T09:50:40.067 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:39 smithi076 ceph-mon[106482]: pgmap v706: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 202 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 313 KiB/s wr, 3 op/s; 24/396 objects degraded (6.061%) 2024-09-21T09:50:40.067 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:39 smithi076 ceph-mon[110963]: pgmap v706: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 202 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 313 KiB/s wr, 3 op/s; 24/396 objects degraded (6.061%) 2024-09-21T09:50:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:39 smithi191 ceph-mon[102206]: pgmap v706: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 202 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 313 KiB/s wr, 3 op/s; 24/396 objects degraded (6.061%) 2024-09-21T09:50:40.837 INFO:tasks.rados.rados.0.smithi076.stdout:3019: done (1 left) 2024-09-21T09:50:40.837 INFO:tasks.rados.rados.0.smithi076.stdout:3020: done (0 left) 2024-09-21T09:50:40.837 INFO:tasks.rados.rados.0.smithi076.stdout:3021: read oid 9 snap -1 2024-09-21T09:50:40.837 INFO:tasks.rados.rados.0.smithi076.stdout:3021: expect deleted 2024-09-21T09:50:40.837 INFO:tasks.rados.rados.0.smithi076.stdout:3022: write oid 11 current snap is 380 2024-09-21T09:50:40.837 INFO:tasks.rados.rados.0.smithi076.stdout:3022: seq_num 1143 ranges {410401=662557,1719036=770066,2938716=271685} 2024-09-21T09:50:40.843 INFO:tasks.rados.rados.0.smithi076.stdout:3022: writing smithi07685715-11 from 410401 to 1072958 tid 1 2024-09-21T09:50:40.851 INFO:tasks.rados.rados.0.smithi076.stdout:3022: writing smithi07685715-11 from 1719036 to 2489102 tid 2 2024-09-21T09:50:40.855 INFO:tasks.rados.rados.0.smithi076.stdout:3022: writing smithi07685715-11 from 2938716 to 3210401 tid 3 2024-09-21T09:50:40.855 INFO:tasks.rados.rados.0.smithi076.stdout:3023: snap_remove snap 374 2024-09-21T09:50:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:40 smithi076 ceph-mon[110963]: osdmap e825: 8 total, 7 up, 8 in 2024-09-21T09:50:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:40 smithi076 ceph-mon[106482]: osdmap e825: 8 total, 7 up, 8 in 2024-09-21T09:50:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:40 smithi191 ceph-mon[102206]: osdmap e825: 8 total, 7 up, 8 in 2024-09-21T09:50:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:41.849 INFO:tasks.rados.rados.0.smithi076.stdout:3023: done (2 left) 2024-09-21T09:50:41.849 INFO:tasks.rados.rados.0.smithi076.stdout:3024: write oid 27 current snap is 380 2024-09-21T09:50:41.849 INFO:tasks.rados.rados.0.smithi076.stdout:3024: seq_num 1144 ranges {427208=737806,1777518=513724,2966139=261069} 2024-09-21T09:50:41.855 INFO:tasks.rados.rados.0.smithi076.stdout:3024: writing smithi07685715-27 from 427208 to 1165014 tid 1 2024-09-21T09:50:41.861 INFO:tasks.rados.rados.0.smithi076.stdout:3024: writing smithi07685715-27 from 1777518 to 2291242 tid 2 2024-09-21T09:50:41.867 INFO:tasks.rados.rados.0.smithi076.stdout:3024: writing smithi07685715-27 from 2966139 to 3227208 tid 3 2024-09-21T09:50:41.867 INFO:tasks.rados.rados.0.smithi076.stdout:3025: write oid 5 current snap is 380 2024-09-21T09:50:41.867 INFO:tasks.rados.rados.0.smithi076.stdout:3025: seq_num 1145 ranges {444015=413055,1436000=773735,2709915=534100} 2024-09-21T09:50:41.872 INFO:tasks.rados.rados.0.smithi076.stdout:3025: writing smithi07685715-5 from 444015 to 857070 tid 1 2024-09-21T09:50:41.879 INFO:tasks.rados.rados.0.smithi076.stdout:3025: writing smithi07685715-5 from 1436000 to 2209735 tid 2 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3025: writing smithi07685715-5 from 2709915 to 3244015 tid 3 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3022: finishing write tid 1 to smithi07685715-11 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3022: finishing write tid 2 to smithi07685715-11 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3026: read oid 1 snap -1 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3026: expect (ObjNum 1013 snap 342 seq_num 1013) 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3021: done (4 left) 2024-09-21T09:50:41.885 INFO:tasks.rados.rados.0.smithi076.stdout:3027: snap_remove snap 377 2024-09-21T09:50:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: pgmap v709: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 211 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s; 24/408 objects degraded (5.882%) 2024-09-21T09:50:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: osdmap e826: 8 total, 7 up, 8 in 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 24/408 objects degraded (5.882%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: pgmap v709: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 211 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s; 24/408 objects degraded (5.882%) 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: osdmap e826: 8 total, 7 up, 8 in 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 24/408 objects degraded (5.882%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: pgmap v709: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 211 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 10 op/s; 24/408 objects degraded (5.882%) 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: osdmap e826: 8 total, 7 up, 8 in 2024-09-21T09:50:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 24/408 objects degraded (5.882%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:42.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:42.580 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[137810]: 2024-09-21T09:50:42.499+0000 7fd7ed537640 -1 osd.3 816 log_to_monitors true 2024-09-21T09:50:42.861 INFO:tasks.rados.rados.0.smithi076.stdout:3027: done (4 left) 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3028: snap_create 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3022: finishing write tid 3 to smithi07685715-11 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3022: finishing write tid 4 to smithi07685715-11 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3022: finishing write tid 5 to smithi07685715-11 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3022: oid 11 updating version 0 to 2246 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3022: oid 11 updating version 2246 to 2247 2024-09-21T09:50:42.862 INFO:tasks.rados.rados.0.smithi076.stdout:3022: oid 11 version 2247 is already newer than 2245 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:3022: oid 11 version 2247 is already newer than 2244 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2247 (ObjNum 1143 snap 380 seq_num 1143) dirty exists 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:3022: left oid 11 (ObjNum 1143 snap 380 seq_num 1143) 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:3024: finishing write tid 1 to smithi07685715-27 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:3024: finishing write tid 2 to smithi07685715-27 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:3024: finishing write tid 3 to smithi07685715-27 2024-09-21T09:50:42.863 INFO:tasks.rados.rados.0.smithi076.stdout:3024: finishing write tid 4 to smithi07685715-27 2024-09-21T09:50:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3024: finishing write tid 5 to smithi07685715-27 2024-09-21T09:50:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3024: oid 27 updating version 0 to 1425 2024-09-21T09:50:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3024: oid 27 updating version 1425 to 1426 2024-09-21T09:50:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3024: oid 27 version 1426 is already newer than 1424 2024-09-21T09:50:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3024: oid 27 updating version 1426 to 1427 2024-09-21T09:50:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1427 (ObjNum 1144 snap 380 seq_num 1144) dirty exists 2024-09-21T09:50:42.865 INFO:tasks.rados.rados.0.smithi076.stdout:3024: left oid 27 (ObjNum 1144 snap 380 seq_num 1144) 2024-09-21T09:50:42.865 INFO:tasks.rados.rados.0.smithi076.stdout:3025: finishing write tid 1 to smithi07685715-5 2024-09-21T09:50:42.865 INFO:tasks.rados.rados.0.smithi076.stdout:3025: finishing write tid 2 to smithi07685715-5 2024-09-21T09:50:42.865 INFO:tasks.rados.rados.0.smithi076.stdout:3025: finishing write tid 3 to smithi07685715-5 2024-09-21T09:50:42.865 INFO:tasks.rados.rados.0.smithi076.stdout:3026: expect (ObjNum 1013 snap 342 seq_num 1013) 2024-09-21T09:50:42.963 INFO:tasks.rados.rados.0.smithi076.stdout:3025: finishing write tid 4 to smithi07685715-5 2024-09-21T09:50:42.963 INFO:tasks.rados.rados.0.smithi076.stdout:3025: finishing write tid 5 to smithi07685715-5 2024-09-21T09:50:42.963 INFO:tasks.rados.rados.0.smithi076.stdout:3025: oid 5 updating version 0 to 2249 2024-09-21T09:50:42.963 INFO:tasks.rados.rados.0.smithi076.stdout:3025: oid 5 updating version 2249 to 2250 2024-09-21T09:50:42.964 INFO:tasks.rados.rados.0.smithi076.stdout:3025: oid 5 version 2250 is already newer than 2248 2024-09-21T09:50:42.964 INFO:tasks.rados.rados.0.smithi076.stdout:3025: oid 5 updating version 2250 to 2251 2024-09-21T09:50:42.964 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2251 (ObjNum 1145 snap 380 seq_num 1145) dirty exists 2024-09-21T09:50:42.964 INFO:tasks.rados.rados.0.smithi076.stdout:3025: left oid 5 (ObjNum 1145 snap 380 seq_num 1145) 2024-09-21T09:50:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:42 smithi191 ceph-mon[102206]: osdmap e827: 8 total, 7 up, 8 in 2024-09-21T09:50:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:42 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:50:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:42 smithi076 ceph-mon[110963]: osdmap e827: 8 total, 7 up, 8 in 2024-09-21T09:50:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:42 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:50:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:42 smithi076 ceph-mon[106482]: osdmap e827: 8 total, 7 up, 8 in 2024-09-21T09:50:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:42 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:50:43.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 09:50:42 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[137810]: 2024-09-21T09:50:42.877+0000 7fd7e12d9640 -1 osd.3 816 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:50:43.588 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:50:43] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-21T09:50:43.861 INFO:tasks.rados.rados.0.smithi076.stdout:3022: done (4 left) 2024-09-21T09:50:43.862 INFO:tasks.rados.rados.0.smithi076.stdout:3024: done (3 left) 2024-09-21T09:50:43.862 INFO:tasks.rados.rados.0.smithi076.stdout:3025: done (2 left) 2024-09-21T09:50:43.862 INFO:tasks.rados.rados.0.smithi076.stdout:3026: done (1 left) 2024-09-21T09:50:43.862 INFO:tasks.rados.rados.0.smithi076.stdout:3028: done (0 left) 2024-09-21T09:50:43.862 INFO:tasks.rados.rados.0.smithi076.stdout:3029: write oid 49 current snap is 381 2024-09-21T09:50:43.862 INFO:tasks.rados.rados.0.smithi076.stdout:3029: seq_num 1146 ranges {460822=488304,1378129=517393,2504632=405962} 2024-09-21T09:50:43.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[106482]: pgmap v712: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 216 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 25/414 objects degraded (6.039%) 2024-09-21T09:50:43.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:50:43.865 INFO:tasks.rados.rados.0.smithi076.stdout:3029: writing smithi07685715-49 from 460822 to 949126 tid 1 2024-09-21T09:50:43.871 INFO:tasks.rados.rados.0.smithi076.stdout:3029: writing smithi07685715-49 from 1378129 to 1895522 tid 2 2024-09-21T09:50:43.878 INFO:tasks.rados.rados.0.smithi076.stdout:3029: writing smithi07685715-49 from 2504632 to 2910594 tid 3 2024-09-21T09:50:43.878 INFO:tasks.rados.rados.0.smithi076.stdout:3029: finishing write tid 1 to smithi07685715-49 2024-09-21T09:50:43.879 INFO:tasks.rados.rados.0.smithi076.stdout:3029: finishing write tid 2 to smithi07685715-49 2024-09-21T09:50:43.879 INFO:tasks.rados.rados.0.smithi076.stdout:3030: read oid 38 snap -1 2024-09-21T09:50:43.879 INFO:tasks.rados.rados.0.smithi076.stdout:3030: expect (ObjNum 1091 snap 364 seq_num 1091) 2024-09-21T09:50:43.879 INFO:tasks.rados.rados.0.smithi076.stdout:3031: rollback oid 21 current snap is 381 2024-09-21T09:50:43.879 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 368 2024-09-21T09:50:43.879 INFO:tasks.rados.rados.0.smithi076.stdout:3032: snap_create 2024-09-21T09:50:43.882 INFO:tasks.rados.rados.0.smithi076.stdout:3029: finishing write tid 3 to smithi07685715-49 2024-09-21T09:50:43.882 INFO:tasks.rados.rados.0.smithi076.stdout:3029: finishing write tid 4 to smithi07685715-49 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3029: finishing write tid 5 to smithi07685715-49 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3029: oid 49 updating version 0 to 2261 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3029: oid 49 version 2261 is already newer than 2260 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3029: oid 49 updating version 2261 to 2263 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3029: oid 49 version 2263 is already newer than 2262 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2263 (ObjNum 1146 snap 381 seq_num 1146) dirty exists 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3029: left oid 49 (ObjNum 1146 snap 381 seq_num 1146) 2024-09-21T09:50:43.883 INFO:tasks.rados.rados.0.smithi076.stdout:3030: expect (ObjNum 1091 snap 364 seq_num 1091) 2024-09-21T09:50:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[106482]: osdmap e828: 8 total, 7 up, 8 in 2024-09-21T09:50:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:50:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[110963]: pgmap v712: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 216 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 25/414 objects degraded (6.039%) 2024-09-21T09:50:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:50:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[110963]: osdmap e828: 8 total, 7 up, 8 in 2024-09-21T09:50:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:43 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:50:44.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:43 smithi191 ceph-mon[102206]: pgmap v712: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 216 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 7.6 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 25/414 objects degraded (6.039%) 2024-09-21T09:50:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:43 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:50:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:43 smithi191 ceph-mon[102206]: osdmap e828: 8 total, 7 up, 8 in 2024-09-21T09:50:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:43 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T09:50:44.858 INFO:tasks.rados.rados.0.smithi076.stdout:3029: done (3 left) 2024-09-21T09:50:44.858 INFO:tasks.rados.rados.0.smithi076.stdout:3030: done (2 left) 2024-09-21T09:50:44.858 INFO:tasks.rados.rados.0.smithi076.stdout:3032: done (1 left) 2024-09-21T09:50:44.858 INFO:tasks.rados.rados.0.smithi076.stdout:3033: rollback oid 41 current snap is 382 2024-09-21T09:50:44.858 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 379 2024-09-21T09:50:44.858 INFO:tasks.rados.rados.0.smithi076.stdout:3034: snap_remove snap 376 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[106482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[106482]: osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128] boot 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[106482]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[106482]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[110963]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:50:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[110963]: osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128] boot 2024-09-21T09:50:45.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[110963]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:50:45.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:50:45.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:44 smithi076 ceph-mon[110963]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:50:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:44 smithi191 ceph-mon[102206]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:50:45.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:44 smithi191 ceph-mon[102206]: osd.3 [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128] boot 2024-09-21T09:50:45.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:44 smithi191 ceph-mon[102206]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T09:50:45.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:50:45.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:44 smithi191 ceph-mon[102206]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3034: done (2 left) 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3035: read oid 3 snap -1 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3035: expect (ObjNum 1102 snap 365 seq_num 1102) 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3036: read oid 37 snap 351 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3036: expect deleted 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3037: write oid 16 current snap is 382 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3033: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:50:45.862 INFO:tasks.rados.rados.0.smithi076.stdout:3037: seq_num 1147 ranges {477629=563553,1836611=777404,3048408=229221} 2024-09-21T09:50:45.868 INFO:tasks.rados.rados.0.smithi076.stdout:3037: writing smithi07685715-16 from 477629 to 1041182 tid 1 2024-09-21T09:50:45.877 INFO:tasks.rados.rados.0.smithi076.stdout:3037: writing smithi07685715-16 from 1836611 to 2614015 tid 2 2024-09-21T09:50:45.882 INFO:tasks.rados.rados.0.smithi076.stdout:3037: writing smithi07685715-16 from 3048408 to 3277629 tid 3 2024-09-21T09:50:45.882 INFO:tasks.rados.rados.0.smithi076.stdout:3038: rollback oid 22 current snap is 382 2024-09-21T09:50:45.882 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 373 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3033: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3033: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1431 (ObjNum 1096 snap 364 seq_num 1096) dirty exists 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3031: finishing rollback tid 0 to smithi07685715-21 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3031: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3033: done (5 left) 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3039: rollback oid 5 current snap is 382 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 359 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3031: finishing rollback tid 2 to smithi07685715-21 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 2255 (ObjNum 1109 snap 367 seq_num 1109) dirty exists 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3037: finishing write tid 1 to smithi07685715-16 2024-09-21T09:50:45.883 INFO:tasks.rados.rados.0.smithi076.stdout:3035: expect (ObjNum 1102 snap 365 seq_num 1102) 2024-09-21T09:50:45.997 INFO:tasks.rados.rados.0.smithi076.stdout:3037: finishing write tid 2 to smithi07685715-16 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3039: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3039: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2272 (ObjNum 129 snap 0 seq_num 865206072) dirty dne 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3031: done (5 left) 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3035: done (4 left) 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3036: done (3 left) 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3039: done (2 left) 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3040: delete oid 50 current snap is 382 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3037: finishing write tid 3 to smithi07685715-16 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3037: finishing write tid 4 to smithi07685715-16 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3037: finishing write tid 5 to smithi07685715-16 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3037: oid 16 updating version 0 to 1435 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3037: oid 16 version 1435 is already newer than 1434 2024-09-21T09:50:45.998 INFO:tasks.rados.rados.0.smithi076.stdout:3037: oid 16 updating version 1435 to 1436 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:3037: oid 16 version 1436 is already newer than 1433 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1436 (ObjNum 1147 snap 382 seq_num 1147) dirty exists 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:3037: left oid 16 (ObjNum 1147 snap 382 seq_num 1147) 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:3038: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:3038: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:3038: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:50:45.999 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1440 (ObjNum 1123 snap 370 seq_num 1123) dirty exists 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3037: done (2 left) 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3038: done (1 left) 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3040: done (0 left) 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3041: read oid 29 snap -1 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3041: expect (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3042: write oid 23 current snap is 382 2024-09-21T09:50:46.000 INFO:tasks.rados.rados.0.smithi076.stdout:3042: seq_num 1149 ranges {511243=430404,1669928=664720,3103254=207989} 2024-09-21T09:50:46.005 INFO:tasks.rados.rados.0.smithi076.stdout:3042: writing smithi07685715-23 from 511243 to 941647 tid 1 2024-09-21T09:50:46.012 INFO:tasks.rados.rados.0.smithi076.stdout:3042: writing smithi07685715-23 from 1669928 to 2334648 tid 2 2024-09-21T09:50:46.018 INFO:tasks.rados.rados.0.smithi076.stdout:3042: writing smithi07685715-23 from 3103254 to 3311243 tid 3 2024-09-21T09:50:46.018 INFO:tasks.rados.rados.0.smithi076.stdout:3043: read oid 15 snap -1 2024-09-21T09:50:46.018 INFO:tasks.rados.rados.0.smithi076.stdout:3043: expect deleted 2024-09-21T09:50:46.019 INFO:tasks.rados.rados.0.smithi076.stdout:3044: snap_create 2024-09-21T09:50:46.019 INFO:tasks.rados.rados.0.smithi076.stdout:3041: expect (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: pgmap v715: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 216 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 791 KiB/s wr, 5 op/s; 25/414 objects degraded (6.039%) 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:50:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: pgmap v715: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 216 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 791 KiB/s wr, 5 op/s; 25/414 objects degraded (6.039%) 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:50:46.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:50:46.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.128 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[110963]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.129 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:45 smithi076 ceph-mon[106482]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: finishing write tid 1 to smithi07685715-23 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: finishing write tid 2 to smithi07685715-23 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: finishing write tid 3 to smithi07685715-23 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: finishing write tid 4 to smithi07685715-23 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: finishing write tid 5 to smithi07685715-23 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: oid 23 updating version 0 to 2270 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: oid 23 updating version 2270 to 2271 2024-09-21T09:50:46.144 INFO:tasks.rados.rados.0.smithi076.stdout:3042: oid 23 version 2271 is already newer than 2268 2024-09-21T09:50:46.145 INFO:tasks.rados.rados.0.smithi076.stdout:3042: oid 23 version 2271 is already newer than 2269 2024-09-21T09:50:46.145 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2271 (ObjNum 1149 snap 382 seq_num 1149) dirty exists 2024-09-21T09:50:46.145 INFO:tasks.rados.rados.0.smithi076.stdout:3042: left oid 23 (ObjNum 1149 snap 382 seq_num 1149) 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: pgmap v715: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 216 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 791 KiB/s wr, 5 op/s; 25/414 objects degraded (6.039%) 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T09:50:46.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:46.366 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:45 smithi191 ceph-mon[102206]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T09:50:46.884 INFO:tasks.rados.rados.0.smithi076.stdout:3041: done (3 left) 2024-09-21T09:50:46.884 INFO:tasks.rados.rados.0.smithi076.stdout:3042: done (2 left) 2024-09-21T09:50:46.884 INFO:tasks.rados.rados.0.smithi076.stdout:3043: done (1 left) 2024-09-21T09:50:46.885 INFO:tasks.rados.rados.0.smithi076.stdout:3044: done (0 left) 2024-09-21T09:50:46.885 INFO:tasks.rados.rados.0.smithi076.stdout:3045: snap_create 2024-09-21T09:50:47.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:46 smithi191 ceph-mon[102206]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:50:47.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:46 smithi191 ceph-mon[102206]: Upgrade: Finalizing container_image settings 2024-09-21T09:50:47.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:46 smithi191 ceph-mon[102206]: Upgrade: Complete! 2024-09-21T09:50:47.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:46 smithi076 ceph-mon[106482]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:50:47.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:46 smithi076 ceph-mon[106482]: Upgrade: Finalizing container_image settings 2024-09-21T09:50:47.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:46 smithi076 ceph-mon[106482]: Upgrade: Complete! 2024-09-21T09:50:47.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:46 smithi076 ceph-mon[110963]: Upgrade: Setting container_image for all nvmeof 2024-09-21T09:50:47.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:46 smithi076 ceph-mon[110963]: Upgrade: Finalizing container_image settings 2024-09-21T09:50:47.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:46 smithi076 ceph-mon[110963]: Upgrade: Complete! 2024-09-21T09:50:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:3045: done (0 left) 2024-09-21T09:50:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:3046: read oid 22 snap -1 2024-09-21T09:50:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:3046: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:50:47.885 INFO:tasks.rados.rados.0.smithi076.stdout:3047: snap_create 2024-09-21T09:50:47.903 INFO:tasks.rados.rados.0.smithi076.stdout:3046: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:50:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:47 smithi191 ceph-mon[102206]: pgmap v718: 89 pgs: 3 peering, 19 active+undersized, 3 active+undersized+degraded, 64 active+clean; 233 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s; 5/435 objects degraded (1.149%); 1.7 MiB/s, 0 objects/s recovering 2024-09-21T09:50:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:47 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 5/435 objects degraded (1.149%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:47 smithi191 ceph-mon[102206]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:50:48.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:47 smithi076 ceph-mon[106482]: pgmap v718: 89 pgs: 3 peering, 19 active+undersized, 3 active+undersized+degraded, 64 active+clean; 233 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s; 5/435 objects degraded (1.149%); 1.7 MiB/s, 0 objects/s recovering 2024-09-21T09:50:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:47 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 5/435 objects degraded (1.149%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:47 smithi076 ceph-mon[106482]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:50:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:47 smithi076 ceph-mon[110963]: pgmap v718: 89 pgs: 3 peering, 19 active+undersized, 3 active+undersized+degraded, 64 active+clean; 233 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s; 5/435 objects degraded (1.149%); 1.7 MiB/s, 0 objects/s recovering 2024-09-21T09:50:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:47 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 5/435 objects degraded (1.149%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:50:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:47 smithi076 ceph-mon[110963]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T09:50:48.886 INFO:tasks.rados.rados.0.smithi076.stdout:3046: done (1 left) 2024-09-21T09:50:48.886 INFO:tasks.rados.rados.0.smithi076.stdout:3047: done (0 left) 2024-09-21T09:50:48.887 INFO:tasks.rados.rados.0.smithi076.stdout:3048: read oid 29 snap -1 2024-09-21T09:50:48.887 INFO:tasks.rados.rados.0.smithi076.stdout:3048: expect (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:48.887 INFO:tasks.rados.rados.0.smithi076.stdout:3049: snap_create 2024-09-21T09:50:48.899 INFO:tasks.rados.rados.0.smithi076.stdout:3048: expect (ObjNum 1129 snap 373 seq_num 1129) 2024-09-21T09:50:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:48 smithi191 ceph-mon[102206]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:50:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:48 smithi076 ceph-mon[106482]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:50:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:48 smithi076 ceph-mon[110963]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T09:50:49.888 INFO:tasks.rados.rados.0.smithi076.stdout:3048: done (1 left) 2024-09-21T09:50:49.888 INFO:tasks.rados.rados.0.smithi076.stdout:3049: done (0 left) 2024-09-21T09:50:49.889 INFO:tasks.rados.rados.0.smithi076.stdout:3050: write oid 19 current snap is 386 2024-09-21T09:50:49.889 INFO:tasks.rados.rados.0.smithi076.stdout:3050: seq_num 1150 ranges {528050=505653,1612057=524731,2614324=567403} 2024-09-21T09:50:49.894 INFO:tasks.rados.rados.0.smithi076.stdout:3050: writing smithi07685715-19 from 528050 to 1033703 tid 1 2024-09-21T09:50:49.900 INFO:tasks.rados.rados.0.smithi076.stdout:3050: writing smithi07685715-19 from 1612057 to 2136788 tid 2 2024-09-21T09:50:49.906 INFO:tasks.rados.rados.0.smithi076.stdout:3050: writing smithi07685715-19 from 2614324 to 3181727 tid 3 2024-09-21T09:50:49.906 INFO:tasks.rados.rados.0.smithi076.stdout:3051: delete oid 35 current snap is 386 2024-09-21T09:50:49.906 INFO:tasks.rados.rados.0.smithi076.stdout:3050: finishing write tid 1 to smithi07685715-19 2024-09-21T09:50:49.906 INFO:tasks.rados.rados.0.smithi076.stdout:3050: finishing write tid 2 to smithi07685715-19 2024-09-21T09:50:49.910 INFO:tasks.rados.rados.0.smithi076.stdout:3050: finishing write tid 3 to smithi07685715-19 2024-09-21T09:50:49.910 INFO:tasks.rados.rados.0.smithi076.stdout:3050: finishing write tid 4 to smithi07685715-19 2024-09-21T09:50:49.910 INFO:tasks.rados.rados.0.smithi076.stdout:3051: done (1 left) 2024-09-21T09:50:49.910 INFO:tasks.rados.rados.0.smithi076.stdout:3052: write oid 45 current snap is 386 2024-09-21T09:50:49.910 INFO:tasks.rados.rados.0.smithi076.stdout:3052: seq_num 1152 ranges {561664=656151,1729021=528400,2669170=692494} 2024-09-21T09:50:49.916 INFO:tasks.rados.rados.0.smithi076.stdout:3052: writing smithi07685715-45 from 561664 to 1217815 tid 1 2024-09-21T09:50:49.922 INFO:tasks.rados.rados.0.smithi076.stdout:3052: writing smithi07685715-45 from 1729021 to 2257421 tid 2 2024-09-21T09:50:49.928 INFO:tasks.rados.rados.0.smithi076.stdout:3052: writing smithi07685715-45 from 2669170 to 3361664 tid 3 2024-09-21T09:50:49.928 INFO:tasks.rados.rados.0.smithi076.stdout:3053: write oid 46 current snap is 386 2024-09-21T09:50:49.928 INFO:tasks.rados.rados.0.smithi076.stdout:3050: finishing write tid 5 to smithi07685715-19 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:3050: oid 19 updating version 0 to 880 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:3050: oid 19 version 880 is already newer than 879 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:3050: oid 19 version 880 is already newer than 878 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:3050: oid 19 version 880 is already newer than 877 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 880 (ObjNum 1150 snap 386 seq_num 1150) dirty exists 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:3050: left oid 19 (ObjNum 1150 snap 386 seq_num 1150) 2024-09-21T09:50:49.929 INFO:tasks.rados.rados.0.smithi076.stdout:3053: seq_num 1153 ranges {578471=731400,1787503=672058,3096593=281878} 2024-09-21T09:50:49.935 INFO:tasks.rados.rados.0.smithi076.stdout:3053: writing smithi07685715-46 from 578471 to 1309871 tid 1 2024-09-21T09:50:49.941 INFO:tasks.rados.rados.0.smithi076.stdout:3053: writing smithi07685715-46 from 1787503 to 2459561 tid 2 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3053: writing smithi07685715-46 from 3096593 to 3378471 tid 3 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3050: done (2 left) 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3054: snap_create 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: finishing write tid 1 to smithi07685715-45 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: finishing write tid 2 to smithi07685715-45 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: finishing write tid 3 to smithi07685715-45 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: finishing write tid 4 to smithi07685715-45 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: finishing write tid 5 to smithi07685715-45 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: oid 45 updating version 0 to 654 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: oid 45 updating version 654 to 656 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: oid 45 version 656 is already newer than 655 2024-09-21T09:50:49.947 INFO:tasks.rados.rados.0.smithi076.stdout:3052: oid 45 updating version 656 to 657 2024-09-21T09:50:49.948 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 657 (ObjNum 1152 snap 386 seq_num 1152) dirty exists 2024-09-21T09:50:49.948 INFO:tasks.rados.rados.0.smithi076.stdout:3052: left oid 45 (ObjNum 1152 snap 386 seq_num 1152) 2024-09-21T09:50:49.948 INFO:tasks.rados.rados.0.smithi076.stdout:3053: finishing write tid 1 to smithi07685715-46 2024-09-21T09:50:49.948 INFO:tasks.rados.rados.0.smithi076.stdout:3053: finishing write tid 2 to smithi07685715-46 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: finishing write tid 3 to smithi07685715-46 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: finishing write tid 4 to smithi07685715-46 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: finishing write tid 5 to smithi07685715-46 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: oid 46 updating version 0 to 2276 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: oid 46 updating version 2276 to 2277 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: oid 46 updating version 2277 to 2278 2024-09-21T09:50:49.950 INFO:tasks.rados.rados.0.smithi076.stdout:3053: oid 46 updating version 2278 to 2279 2024-09-21T09:50:49.951 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2279 (ObjNum 1153 snap 386 seq_num 1153) dirty exists 2024-09-21T09:50:49.951 INFO:tasks.rados.rados.0.smithi076.stdout:3053: left oid 46 (ObjNum 1153 snap 386 seq_num 1153) 2024-09-21T09:50:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:49 smithi191 ceph-mon[102206]: pgmap v721: 89 pgs: 3 peering, 5 active+undersized, 81 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s; 1.7 MiB/s, 0 objects/s recovering 2024-09-21T09:50:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:49 smithi191 ceph-mon[102206]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:50:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:49 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/435 objects degraded (1.149%), 3 pgs degraded) 2024-09-21T09:50:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:49 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T09:50:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:49 smithi191 ceph-mon[102206]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[110963]: pgmap v721: 89 pgs: 3 peering, 5 active+undersized, 81 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s; 1.7 MiB/s, 0 objects/s recovering 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[110963]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/435 objects degraded (1.149%), 3 pgs degraded) 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[110963]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[106482]: pgmap v721: 89 pgs: 3 peering, 5 active+undersized, 81 active+clean; 233 MiB data, 769 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.8 MiB/s wr, 16 op/s; 1.7 MiB/s, 0 objects/s recovering 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[106482]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/435 objects degraded (1.149%), 3 pgs degraded) 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T09:50:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:49 smithi076 ceph-mon[106482]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T09:50:50.903 INFO:tasks.rados.rados.0.smithi076.stdout:3052: done (2 left) 2024-09-21T09:50:50.903 INFO:tasks.rados.rados.0.smithi076.stdout:3053: done (1 left) 2024-09-21T09:50:50.903 INFO:tasks.rados.rados.0.smithi076.stdout:3054: done (0 left) 2024-09-21T09:50:50.903 INFO:tasks.rados.rados.0.smithi076.stdout:3055: snap_create 2024-09-21T09:50:51.906 INFO:tasks.rados.rados.0.smithi076.stdout:3055: done (0 left) 2024-09-21T09:50:51.906 INFO:tasks.rados.rados.0.smithi076.stdout:3056: snap_create 2024-09-21T09:50:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:51 smithi191 ceph-mon[102206]: pgmap v724: 89 pgs: 89 active+clean; 233 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 1 op/s 2024-09-21T09:50:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:51 smithi191 ceph-mon[102206]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:50:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:51 smithi076 ceph-mon[106482]: pgmap v724: 89 pgs: 89 active+clean; 233 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 1 op/s 2024-09-21T09:50:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:51 smithi076 ceph-mon[106482]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:50:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:51 smithi076 ceph-mon[110963]: pgmap v724: 89 pgs: 89 active+clean; 233 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 1 op/s 2024-09-21T09:50:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:51 smithi076 ceph-mon[110963]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T09:50:52.907 INFO:tasks.rados.rados.0.smithi076.stdout:3056: done (0 left) 2024-09-21T09:50:52.907 INFO:tasks.rados.rados.0.smithi076.stdout:3057: delete oid 36 current snap is 389 2024-09-21T09:50:52.912 INFO:tasks.rados.rados.0.smithi076.stdout:3057: done (0 left) 2024-09-21T09:50:52.912 INFO:tasks.rados.rados.0.smithi076.stdout:3058: read oid 20 snap -1 2024-09-21T09:50:52.912 INFO:tasks.rados.rados.0.smithi076.stdout:3058: expect (ObjNum 1127 snap 371 seq_num 1127) 2024-09-21T09:50:52.912 INFO:tasks.rados.rados.0.smithi076.stdout:3059: snap_remove snap 382 2024-09-21T09:50:53.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:52 smithi076 ceph-mon[106482]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:50:53.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:52 smithi076 ceph-mon[106482]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:50:53.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:52 smithi076 ceph-mon[110963]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:50:53.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:52 smithi076 ceph-mon[110963]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:50:53.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:52 smithi191 ceph-mon[102206]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T09:50:53.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:52 smithi191 ceph-mon[102206]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T09:50:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:50:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:50:53] "GET /metrics HTTP/1.1" 200 36265 "" "Prometheus/2.51.0" 2024-09-21T09:50:53.770 INFO:tasks.rados.rados.0.smithi076.stdout:3059: done (1 left) 2024-09-21T09:50:53.770 INFO:tasks.rados.rados.0.smithi076.stdout:3060: snap_remove snap 383 2024-09-21T09:50:54.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:53 smithi076 ceph-mon[106482]: pgmap v727: 89 pgs: 89 active+clean; 235 MiB data, 783 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 412 KiB/s wr, 7 op/s 2024-09-21T09:50:54.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:53 smithi076 ceph-mon[106482]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:50:54.255 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:53 smithi076 ceph-mon[110963]: pgmap v727: 89 pgs: 89 active+clean; 235 MiB data, 783 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 412 KiB/s wr, 7 op/s 2024-09-21T09:50:54.255 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:53 smithi076 ceph-mon[110963]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:50:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:53 smithi191 ceph-mon[102206]: pgmap v727: 89 pgs: 89 active+clean; 235 MiB data, 783 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 412 KiB/s wr, 7 op/s 2024-09-21T09:50:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:53 smithi191 ceph-mon[102206]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T09:50:54.686 DEBUG:teuthology.parallel:result is None 2024-09-21T09:50:54.771 INFO:tasks.rados.rados.0.smithi076.stdout:3060: done (1 left) 2024-09-21T09:50:54.772 INFO:tasks.rados.rados.0.smithi076.stdout:3061: read oid 48 snap -1 2024-09-21T09:50:54.772 INFO:tasks.rados.rados.0.smithi076.stdout:3061: expect (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:50:54.772 INFO:tasks.rados.rados.0.smithi076.stdout:3062: write oid 11 current snap is 389 2024-09-21T09:50:54.772 INFO:tasks.rados.rados.0.smithi076.stdout:3062: seq_num 1155 ranges {612085=481898,1788114=675727,3035086=376999} 2024-09-21T09:50:54.778 INFO:tasks.rados.rados.0.smithi076.stdout:3062: writing smithi07685715-11 from 612085 to 1093983 tid 1 2024-09-21T09:50:54.785 INFO:tasks.rados.rados.0.smithi076.stdout:3062: writing smithi07685715-11 from 1788114 to 2463841 tid 2 2024-09-21T09:50:54.791 INFO:tasks.rados.rados.0.smithi076.stdout:3062: writing smithi07685715-11 from 3035086 to 3412085 tid 3 2024-09-21T09:50:54.791 INFO:tasks.rados.rados.0.smithi076.stdout:3063: read oid 50 snap -1 2024-09-21T09:50:54.791 INFO:tasks.rados.rados.0.smithi076.stdout:3063: expect deleted 2024-09-21T09:50:54.791 INFO:tasks.rados.rados.0.smithi076.stdout:3064: delete oid 29 current snap is 389 2024-09-21T09:50:54.791 INFO:tasks.rados.rados.0.smithi076.stdout:3058: expect (ObjNum 1127 snap 371 seq_num 1127) 2024-09-21T09:50:54.951 INFO:tasks.rados.rados.0.smithi076.stdout:3061: expect (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: finishing write tid 1 to smithi07685715-11 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: finishing write tid 2 to smithi07685715-11 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: finishing write tid 3 to smithi07685715-11 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: finishing write tid 4 to smithi07685715-11 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: finishing write tid 5 to smithi07685715-11 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: oid 11 updating version 0 to 2290 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: oid 11 updating version 2290 to 2291 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: oid 11 updating version 2291 to 2292 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: oid 11 version 2292 is already newer than 2289 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2292 (ObjNum 1155 snap 389 seq_num 1155) dirty exists 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3062: left oid 11 (ObjNum 1155 snap 389 seq_num 1155) 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3058: done (4 left) 2024-09-21T09:50:55.132 INFO:tasks.rados.rados.0.smithi076.stdout:3061: done (3 left) 2024-09-21T09:50:55.133 INFO:tasks.rados.rados.0.smithi076.stdout:3062: done (2 left) 2024-09-21T09:50:55.133 INFO:tasks.rados.rados.0.smithi076.stdout:3063: done (1 left) 2024-09-21T09:50:55.133 INFO:tasks.rados.rados.0.smithi076.stdout:3064: done (0 left) 2024-09-21T09:50:55.133 INFO:tasks.rados.rados.0.smithi076.stdout:3065: read oid 25 snap -1 2024-09-21T09:50:55.133 INFO:tasks.rados.rados.0.smithi076.stdout:3065: expect deleted 2024-09-21T09:50:55.133 INFO:tasks.rados.rados.0.smithi076.stdout:3066: snap_create 2024-09-21T09:50:55.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:54 smithi191 ceph-mon[102206]: from='client.44372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:54 smithi191 ceph-mon[102206]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:50:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:54 smithi076 ceph-mon[106482]: from='client.44372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:54 smithi076 ceph-mon[106482]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:50:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:54 smithi076 ceph-mon[110963]: from='client.44372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:50:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:54 smithi076 ceph-mon[110963]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T09:50:55.933 INFO:tasks.rados.rados.0.smithi076.stdout:3065: done (1 left) 2024-09-21T09:50:55.933 INFO:tasks.rados.rados.0.smithi076.stdout:3066: done (0 left) 2024-09-21T09:50:55.933 INFO:tasks.rados.rados.0.smithi076.stdout:3067: rollback oid 39 current snap is 390 2024-09-21T09:50:55.933 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 367 2024-09-21T09:50:55.934 INFO:tasks.rados.rados.0.smithi076.stdout:3068: delete oid 40 current snap is 390 2024-09-21T09:50:55.938 INFO:tasks.rados.rados.0.smithi076.stdout:3067: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:50:55.939 INFO:tasks.rados.rados.0.smithi076.stdout:3068: done (1 left) 2024-09-21T09:50:55.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: write oid 45 current snap is 390 2024-09-21T09:50:55.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: seq_num 1158 ranges {662506=423998,1563560=539407,2717355=490285} 2024-09-21T09:50:55.944 INFO:tasks.rados.rados.0.smithi076.stdout:3069: writing smithi07685715-45 from 662506 to 1086504 tid 1 2024-09-21T09:50:55.950 INFO:tasks.rados.rados.0.smithi076.stdout:3069: writing smithi07685715-45 from 1563560 to 2102967 tid 2 2024-09-21T09:50:55.959 INFO:tasks.rados.rados.0.smithi076.stdout:3069: writing smithi07685715-45 from 2717355 to 3207640 tid 3 2024-09-21T09:50:55.959 INFO:tasks.rados.rados.0.smithi076.stdout:3070: read oid 47 snap -1 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3070: expect (ObjNum 1126 snap 371 seq_num 1126) 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3071: read oid 36 snap -1 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3071: expect deleted 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3072: write oid 8 current snap is 390 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3067: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3067: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2296 (ObjNum 1097 snap 364 seq_num 1097) dirty exists 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3069: finishing write tid 1 to smithi07685715-45 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3069: finishing write tid 2 to smithi07685715-45 2024-09-21T09:50:55.960 INFO:tasks.rados.rados.0.smithi076.stdout:3072: seq_num 1159 ranges {679313=499247,1622042=683065,2744778=701557} 2024-09-21T09:50:55.962 INFO:tasks.rados.rados.0.smithi076.stdout:3072: writing smithi07685715-8 from 679313 to 1178560 tid 1 2024-09-21T09:50:55.969 INFO:tasks.rados.rados.0.smithi076.stdout:3072: writing smithi07685715-8 from 1622042 to 2305107 tid 2 2024-09-21T09:50:55.976 INFO:tasks.rados.rados.0.smithi076.stdout:3072: writing smithi07685715-8 from 2744778 to 3446335 tid 3 2024-09-21T09:50:55.976 INFO:tasks.rados.rados.0.smithi076.stdout:3067: done (4 left) 2024-09-21T09:50:55.976 INFO:tasks.rados.rados.0.smithi076.stdout:3073: write oid 5 current snap is 390 2024-09-21T09:50:55.976 INFO:tasks.rados.rados.0.smithi076.stdout:3069: finishing write tid 3 to smithi07685715-45 2024-09-21T09:50:55.976 INFO:tasks.rados.rados.0.smithi076.stdout:3073: seq_num 1160 ranges {696120=574496,1680524=543076,2888554=512829} 2024-09-21T09:50:55.983 INFO:tasks.rados.rados.0.smithi076.stdout:3073: writing smithi07685715-5 from 696120 to 1270616 tid 1 2024-09-21T09:50:55.988 INFO:tasks.rados.rados.0.smithi076.stdout:3073: writing smithi07685715-5 from 1680524 to 2223600 tid 2 2024-09-21T09:50:55.995 INFO:tasks.rados.rados.0.smithi076.stdout:3073: writing smithi07685715-5 from 2888554 to 3401383 tid 3 2024-09-21T09:50:55.995 INFO:tasks.rados.rados.0.smithi076.stdout:3074: snap_remove snap 386 2024-09-21T09:50:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:55 smithi191 ceph-mon[102206]: pgmap v730: 89 pgs: 89 active+clean; 235 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 412 KiB/s wr, 6 op/s 2024-09-21T09:50:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:56.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[110963]: pgmap v730: 89 pgs: 89 active+clean; 235 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 412 KiB/s wr, 6 op/s 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[106482]: pgmap v730: 89 pgs: 89 active+clean; 235 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 412 KiB/s wr, 6 op/s 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:50:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:50:56.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: finishing write tid 4 to smithi07685715-45 2024-09-21T09:50:56.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: finishing write tid 5 to smithi07685715-45 2024-09-21T09:50:56.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: oid 45 updating version 0 to 663 2024-09-21T09:50:56.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: oid 45 updating version 663 to 665 2024-09-21T09:50:56.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: oid 45 updating version 665 to 666 2024-09-21T09:50:56.939 INFO:tasks.rados.rados.0.smithi076.stdout:3069: oid 45 version 666 is already newer than 664 2024-09-21T09:50:56.940 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 666 (ObjNum 1158 snap 390 seq_num 1158) dirty exists 2024-09-21T09:50:56.940 INFO:tasks.rados.rados.0.smithi076.stdout:3069: left oid 45 (ObjNum 1158 snap 390 seq_num 1158) 2024-09-21T09:50:56.940 INFO:tasks.rados.rados.0.smithi076.stdout:3070: expect (ObjNum 1126 snap 371 seq_num 1126) 2024-09-21T09:50:57.034 INFO:tasks.rados.rados.0.smithi076.stdout:3072: finishing write tid 1 to smithi07685715-8 2024-09-21T09:50:57.034 INFO:tasks.rados.rados.0.smithi076.stdout:3072: finishing write tid 2 to smithi07685715-8 2024-09-21T09:50:57.034 INFO:tasks.rados.rados.0.smithi076.stdout:3072: finishing write tid 3 to smithi07685715-8 2024-09-21T09:50:57.034 INFO:tasks.rados.rados.0.smithi076.stdout:3072: finishing write tid 4 to smithi07685715-8 2024-09-21T09:50:57.034 INFO:tasks.rados.rados.0.smithi076.stdout:3072: finishing write tid 5 to smithi07685715-8 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3072: oid 8 updating version 0 to 2276 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3072: oid 8 version 2276 is already newer than 2274 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3072: oid 8 version 2276 is already newer than 2275 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3072: oid 8 version 2276 is already newer than 2273 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2276 (ObjNum 1159 snap 390 seq_num 1159) dirty exists 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3072: left oid 8 (ObjNum 1159 snap 390 seq_num 1159) 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3073: finishing write tid 1 to smithi07685715-5 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3069: done (5 left) 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3070: done (4 left) 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3071: done (3 left) 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3072: done (2 left) 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3074: done (1 left) 2024-09-21T09:50:57.035 INFO:tasks.rados.rados.0.smithi076.stdout:3075: snap_create 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: finishing write tid 2 to smithi07685715-5 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: finishing write tid 3 to smithi07685715-5 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: finishing write tid 4 to smithi07685715-5 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: finishing write tid 5 to smithi07685715-5 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: oid 5 updating version 0 to 2299 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: oid 5 updating version 2299 to 2300 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: oid 5 version 2300 is already newer than 2297 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: oid 5 version 2300 is already newer than 2298 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2300 (ObjNum 1160 snap 390 seq_num 1160) dirty exists 2024-09-21T09:50:57.036 INFO:tasks.rados.rados.0.smithi076.stdout:3073: left oid 5 (ObjNum 1160 snap 390 seq_num 1160) 2024-09-21T09:50:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:56 smithi191 ceph-mon[102206]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:50:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:56 smithi076 ceph-mon[106482]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:50:57.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:56 smithi076 ceph-mon[110963]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T09:50:57.941 INFO:tasks.rados.rados.0.smithi076.stdout:3073: done (1 left) 2024-09-21T09:50:57.941 INFO:tasks.rados.rados.0.smithi076.stdout:3075: done (0 left) 2024-09-21T09:50:57.941 INFO:tasks.rados.rados.0.smithi076.stdout:3076: write oid 7 current snap is 391 2024-09-21T09:50:57.941 INFO:tasks.rados.rados.0.smithi076.stdout:3076: seq_num 1161 ranges {712927=649745,2139006=686734} 2024-09-21T09:50:57.948 INFO:tasks.rados.rados.0.smithi076.stdout:3076: writing smithi07685715-7 from 712927 to 1362672 tid 1 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3076: writing smithi07685715-7 from 2139006 to 2825740 tid 2 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3077: read oid 21 snap -1 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3077: expect (ObjNum 1109 snap 367 seq_num 1109) 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3078: rollback oid 17 current snap is 391 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 370 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3076: finishing write tid 1 to smithi07685715-7 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3079: read oid 44 snap -1 2024-09-21T09:50:57.957 INFO:tasks.rados.rados.0.smithi076.stdout:3079: expect (ObjNum 1130 snap 373 seq_num 1130) 2024-09-21T09:50:57.958 INFO:tasks.rados.rados.0.smithi076.stdout:3080: snap_remove snap 381 2024-09-21T09:50:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:57 smithi191 ceph-mon[102206]: pgmap v733: 89 pgs: 89 active+clean; 245 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2024-09-21T09:50:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:57 smithi191 ceph-mon[102206]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:50:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:57 smithi076 ceph-mon[106482]: pgmap v733: 89 pgs: 89 active+clean; 245 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2024-09-21T09:50:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:57 smithi076 ceph-mon[106482]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:50:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:57 smithi076 ceph-mon[110963]: pgmap v733: 89 pgs: 89 active+clean; 245 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2024-09-21T09:50:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:57 smithi076 ceph-mon[110963]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T09:50:58.953 INFO:tasks.rados.rados.0.smithi076.stdout:3080: done (4 left) 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3081: read oid 9 snap -1 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3081: expect deleted 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3082: rollback oid 6 current snap is 391 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 388 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: finishing write tid 2 to smithi07685715-7 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: finishing write tid 3 to smithi07685715-7 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: finishing write tid 4 to smithi07685715-7 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: oid 7 updating version 0 to 1807 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: oid 7 version 1807 is already newer than 1806 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: oid 7 updating version 1807 to 1808 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1808 (ObjNum 1161 snap 391 seq_num 1161) dirty exists 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3076: left oid 7 (ObjNum 1161 snap 391 seq_num 1161) 2024-09-21T09:50:58.954 INFO:tasks.rados.rados.0.smithi076.stdout:3078: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:50:58.955 INFO:tasks.rados.rados.0.smithi076.stdout:3078: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:50:58.955 INFO:tasks.rados.rados.0.smithi076.stdout:3078: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:50:58.955 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1812 (ObjNum 1042 snap 351 seq_num 1042) dirty exists 2024-09-21T09:50:58.955 INFO:tasks.rados.rados.0.smithi076.stdout:3079: expect (ObjNum 1130 snap 373 seq_num 1130) 2024-09-21T09:50:59.164 INFO:tasks.rados.rados.0.smithi076.stdout:3077: expect (ObjNum 1109 snap 367 seq_num 1109) 2024-09-21T09:50:59.345 INFO:tasks.rados.rados.0.smithi076.stdout:3082: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 0 (ObjNum 129 snap 0 seq_num 864027792) dirty dne 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3076: done (5 left) 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3077: done (4 left) 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3078: done (3 left) 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3079: done (2 left) 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3081: done (1 left) 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3082: done (0 left) 2024-09-21T09:50:59.346 INFO:tasks.rados.rados.0.smithi076.stdout:3083: snap_create 2024-09-21T09:50:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:58 smithi191 ceph-mon[102206]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:50:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:58 smithi076 ceph-mon[106482]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:50:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:58 smithi076 ceph-mon[110963]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T09:50:59.953 INFO:tasks.rados.rados.0.smithi076.stdout:3083: done (0 left) 2024-09-21T09:50:59.953 INFO:tasks.rados.rados.0.smithi076.stdout:3084: delete oid 11 current snap is 392 2024-09-21T09:50:59.959 INFO:tasks.rados.rados.0.smithi076.stdout:3084: done (0 left) 2024-09-21T09:50:59.959 INFO:tasks.rados.rados.0.smithi076.stdout:3085: snap_remove snap 370 2024-09-21T09:51:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:59 smithi191 ceph-mon[102206]: pgmap v736: 89 pgs: 89 active+clean; 245 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2024-09-21T09:51:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:50:59 smithi191 ceph-mon[102206]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:51:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:59 smithi076 ceph-mon[106482]: pgmap v736: 89 pgs: 89 active+clean; 245 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2024-09-21T09:51:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:50:59 smithi076 ceph-mon[106482]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:51:00.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:59 smithi076 ceph-mon[110963]: pgmap v736: 89 pgs: 89 active+clean; 245 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2024-09-21T09:51:00.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:50:59 smithi076 ceph-mon[110963]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T09:51:00.965 INFO:tasks.rados.rados.0.smithi076.stdout:3085: done (0 left) 2024-09-21T09:51:00.965 INFO:tasks.rados.rados.0.smithi076.stdout:3086: rollback oid 30 current snap is 392 2024-09-21T09:51:00.965 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 355 2024-09-21T09:51:00.965 INFO:tasks.rados.rados.0.smithi076.stdout:3087: delete oid 35 current snap is 392 2024-09-21T09:51:00.978 INFO:tasks.rados.rados.0.smithi076.stdout:3087: done (1 left) 2024-09-21T09:51:00.978 INFO:tasks.rados.rados.0.smithi076.stdout:3088: snap_create 2024-09-21T09:51:00.980 INFO:tasks.rados.rados.0.smithi076.stdout:3086: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:51:00.980 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2175 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:00 smithi191 ceph-mon[102206]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:51:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:00 smithi076 ceph-mon[106482]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:51:01.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:00 smithi076 ceph-mon[110963]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T09:51:01.975 INFO:tasks.rados.rados.0.smithi076.stdout:3086: done (1 left) 2024-09-21T09:51:01.975 INFO:tasks.rados.rados.0.smithi076.stdout:3088: done (0 left) 2024-09-21T09:51:01.975 INFO:tasks.rados.rados.0.smithi076.stdout:3089: read oid 34 snap -1 2024-09-21T09:51:01.975 INFO:tasks.rados.rados.0.smithi076.stdout:3089: expect deleted 2024-09-21T09:51:01.976 INFO:tasks.rados.rados.0.smithi076.stdout:3090: write oid 10 current snap is 393 2024-09-21T09:51:01.976 INFO:tasks.rados.rados.0.smithi076.stdout:3090: seq_num 1164 ranges {763348=591845,1914452=434061,2881893=674270} 2024-09-21T09:51:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3090: writing smithi07685715-10 from 763348 to 1355193 tid 1 2024-09-21T09:51:01.988 INFO:tasks.rados.rados.0.smithi076.stdout:3090: writing smithi07685715-10 from 1914452 to 2348513 tid 2 2024-09-21T09:51:01.995 INFO:tasks.rados.rados.0.smithi076.stdout:3090: writing smithi07685715-10 from 2881893 to 3556163 tid 3 2024-09-21T09:51:01.995 INFO:tasks.rados.rados.0.smithi076.stdout:3091: rollback oid 26 current snap is 393 2024-09-21T09:51:01.995 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 365 2024-09-21T09:51:01.995 INFO:tasks.rados.rados.0.smithi076.stdout:3090: finishing write tid 1 to smithi07685715-10 2024-09-21T09:51:01.995 INFO:tasks.rados.rados.0.smithi076.stdout:3090: finishing write tid 2 to smithi07685715-10 2024-09-21T09:51:01.996 INFO:tasks.rados.rados.0.smithi076.stdout:3089: done (2 left) 2024-09-21T09:51:01.996 INFO:tasks.rados.rados.0.smithi076.stdout:3092: read oid 50 snap -1 2024-09-21T09:51:01.996 INFO:tasks.rados.rados.0.smithi076.stdout:3092: expect deleted 2024-09-21T09:51:01.996 INFO:tasks.rados.rados.0.smithi076.stdout:3093: write oid 25 current snap is 393 2024-09-21T09:51:01.996 INFO:tasks.rados.rados.0.smithi076.stdout:3093: seq_num 1165 ranges {780155=667094,1972934=694072,3309316=270839} 2024-09-21T09:51:02.003 INFO:tasks.rados.rados.0.smithi076.stdout:3093: writing smithi07685715-25 from 780155 to 1447249 tid 1 2024-09-21T09:51:02.010 INFO:tasks.rados.rados.0.smithi076.stdout:3093: writing smithi07685715-25 from 1972934 to 2667006 tid 2 2024-09-21T09:51:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:3093: writing smithi07685715-25 from 3309316 to 3580155 tid 3 2024-09-21T09:51:02.015 INFO:tasks.rados.rados.0.smithi076.stdout:3094: read oid 43 snap -1 2024-09-21T09:51:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:3094: expect (ObjNum 1098 snap 364 seq_num 1098) 2024-09-21T09:51:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:3095: write oid 9 current snap is 393 2024-09-21T09:51:02.016 INFO:tasks.rados.rados.0.smithi076.stdout:3095: seq_num 1166 ranges {796962=742343,2031416=437730,2936739=413167} 2024-09-21T09:51:02.024 INFO:tasks.rados.rados.0.smithi076.stdout:3095: writing smithi07685715-9 from 796962 to 1539305 tid 1 2024-09-21T09:51:02.029 INFO:tasks.rados.rados.0.smithi076.stdout:3095: writing smithi07685715-9 from 2031416 to 2469146 tid 2 2024-09-21T09:51:02.034 INFO:tasks.rados.rados.0.smithi076.stdout:3095: writing smithi07685715-9 from 2936739 to 3349906 tid 3 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3096: read oid 17 snap 358 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3096: expect (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3097: rollback oid 49 current snap is 393 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 359 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: finishing write tid 3 to smithi07685715-10 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: finishing write tid 4 to smithi07685715-10 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: finishing write tid 5 to smithi07685715-10 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: oid 10 updating version 0 to 1822 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: oid 10 version 1822 is already newer than 1820 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: oid 10 version 1822 is already newer than 1821 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: oid 10 updating version 1822 to 1823 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 1823 (ObjNum 1164 snap 393 seq_num 1164) dirty exists 2024-09-21T09:51:02.035 INFO:tasks.rados.rados.0.smithi076.stdout:3090: left oid 10 (ObjNum 1164 snap 393 seq_num 1164) 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3091: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3091: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3091: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1026 (ObjNum 1084 snap 363 seq_num 1084) dirty exists 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3093: finishing write tid 1 to smithi07685715-25 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3090: done (7 left) 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3091: done (6 left) 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3092: done (5 left) 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3098: read oid 10 snap 380 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3098: expect (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3099: write oid 50 current snap is 393 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3093: finishing write tid 2 to smithi07685715-25 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3093: finishing write tid 3 to smithi07685715-25 2024-09-21T09:51:02.036 INFO:tasks.rados.rados.0.smithi076.stdout:3093: finishing write tid 4 to smithi07685715-25 2024-09-21T09:51:02.037 INFO:tasks.rados.rados.0.smithi076.stdout:3094: expect (ObjNum 1098 snap 364 seq_num 1098) 2024-09-21T09:51:02.113 INFO:tasks.rados.rados.0.smithi076.stdout:3093: finishing write tid 5 to smithi07685715-25 2024-09-21T09:51:02.113 INFO:tasks.rados.rados.0.smithi076.stdout:3093: oid 25 updating version 0 to 679 2024-09-21T09:51:02.113 INFO:tasks.rados.rados.0.smithi076.stdout:3093: oid 25 updating version 679 to 681 2024-09-21T09:51:02.113 INFO:tasks.rados.rados.0.smithi076.stdout:3093: oid 25 updating version 681 to 682 2024-09-21T09:51:02.113 INFO:tasks.rados.rados.0.smithi076.stdout:3093: oid 25 version 682 is already newer than 680 2024-09-21T09:51:02.114 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 682 (ObjNum 1165 snap 393 seq_num 1165) dirty exists 2024-09-21T09:51:02.114 INFO:tasks.rados.rados.0.smithi076.stdout:3093: left oid 25 (ObjNum 1165 snap 393 seq_num 1165) 2024-09-21T09:51:02.114 INFO:tasks.rados.rados.0.smithi076.stdout:3095: finishing write tid 1 to smithi07685715-9 2024-09-21T09:51:02.114 INFO:tasks.rados.rados.0.smithi076.stdout:3099: seq_num 1167 ranges {413769=417592,1573545=697741,2847809=624439} 2024-09-21T09:51:02.117 INFO:tasks.rados.rados.0.smithi076.stdout:3099: writing smithi07685715-50 from 413769 to 831361 tid 1 2024-09-21T09:51:02.125 INFO:tasks.rados.rados.0.smithi076.stdout:3099: writing smithi07685715-50 from 1573545 to 2271286 tid 2 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3099: writing smithi07685715-50 from 2847809 to 3472248 tid 3 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3093: done (6 left) 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3094: done (5 left) 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3100: rollback oid 43 current snap is 393 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 359 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3095: finishing write tid 2 to smithi07685715-9 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3095: finishing write tid 3 to smithi07685715-9 2024-09-21T09:51:02.132 INFO:tasks.rados.rados.0.smithi076.stdout:3095: finishing write tid 4 to smithi07685715-9 2024-09-21T09:51:02.133 INFO:tasks.rados.rados.0.smithi076.stdout:3097: finishing rollback tid 0 to smithi07685715-49 2024-09-21T09:51:02.133 INFO:tasks.rados.rados.0.smithi076.stdout:3097: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:51:02.133 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2292 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:51:02.133 INFO:tasks.rados.rados.0.smithi076.stdout:3097: done (5 left) 2024-09-21T09:51:02.133 INFO:tasks.rados.rados.0.smithi076.stdout:3101: snap_remove snap 368 2024-09-21T09:51:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:01 smithi191 ceph-mon[102206]: pgmap v739: 89 pgs: 89 active+clean; 249 MiB data, 805 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 682 KiB/s wr, 3 op/s 2024-09-21T09:51:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:01 smithi191 ceph-mon[102206]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:51:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:01 smithi076 ceph-mon[106482]: pgmap v739: 89 pgs: 89 active+clean; 249 MiB data, 805 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 682 KiB/s wr, 3 op/s 2024-09-21T09:51:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:01 smithi076 ceph-mon[106482]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:51:02.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:01 smithi076 ceph-mon[110963]: pgmap v739: 89 pgs: 89 active+clean; 249 MiB data, 805 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 682 KiB/s wr, 3 op/s 2024-09-21T09:51:02.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:01 smithi076 ceph-mon[110963]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T09:51:02.980 INFO:tasks.rados.rados.0.smithi076.stdout:3101: done (5 left) 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3102: write oid 38 current snap is 393 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3095: finishing write tid 5 to smithi07685715-9 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3095: oid 9 updating version 0 to 1029 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3095: oid 9 version 1029 is already newer than 1028 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3095: oid 9 version 1029 is already newer than 1027 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3095: oid 9 updating version 1029 to 1030 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 1030 (ObjNum 1166 snap 393 seq_num 1166) dirty exists 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3095: left oid 9 (ObjNum 1166 snap 393 seq_num 1166) 2024-09-21T09:51:02.981 INFO:tasks.rados.rados.0.smithi076.stdout:3102: seq_num 1168 ranges {430576=492841,1632027=441399,2475232=435711,3365397=265179} 2024-09-21T09:51:02.986 INFO:tasks.rados.rados.0.smithi076.stdout:3102: writing smithi07685715-38 from 430576 to 923417 tid 1 2024-09-21T09:51:02.993 INFO:tasks.rados.rados.0.smithi076.stdout:3102: writing smithi07685715-38 from 1632027 to 2073426 tid 2 2024-09-21T09:51:02.998 INFO:tasks.rados.rados.0.smithi076.stdout:3102: writing smithi07685715-38 from 2475232 to 2910943 tid 3 2024-09-21T09:51:03.002 INFO:tasks.rados.rados.0.smithi076.stdout:3102: writing smithi07685715-38 from 3365397 to 3630576 tid 4 2024-09-21T09:51:03.002 INFO:tasks.rados.rados.0.smithi076.stdout:3095: done (5 left) 2024-09-21T09:51:03.002 INFO:tasks.rados.rados.0.smithi076.stdout:3103: write oid 31 current snap is 393 2024-09-21T09:51:03.002 INFO:tasks.rados.rados.0.smithi076.stdout:3096: expect (ObjNum 1042 snap 351 seq_num 1042) 2024-09-21T09:51:03.113 INFO:tasks.rados.rados.0.smithi076.stdout:3098: expect (ObjNum 985 snap 337 seq_num 985) 2024-09-21T09:51:03.189 INFO:tasks.rados.rados.0.smithi076.stdout:3099: finishing write tid 1 to smithi07685715-50 2024-09-21T09:51:03.189 INFO:tasks.rados.rados.0.smithi076.stdout:3099: finishing write tid 2 to smithi07685715-50 2024-09-21T09:51:03.189 INFO:tasks.rados.rados.0.smithi076.stdout:3100: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:51:03.189 INFO:tasks.rados.rados.0.smithi076.stdout:3103: seq_num 1169 ranges {447383=568090,1690509=701410,2902655=646983} 2024-09-21T09:51:03.194 INFO:tasks.rados.rados.0.smithi076.stdout:3103: writing smithi07685715-31 from 447383 to 1015473 tid 1 2024-09-21T09:51:03.201 INFO:tasks.rados.rados.0.smithi076.stdout:3103: writing smithi07685715-31 from 1690509 to 2391919 tid 2 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3103: writing smithi07685715-31 from 2902655 to 3549638 tid 3 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3096: done (5 left) 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3098: done (4 left) 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3104: read oid 23 snap -1 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3104: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3105: rollback oid 34 current snap is 393 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 392 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3100: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:3100: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:51:03.209 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 707 (ObjNum 1033 snap 349 seq_num 1033) dirty exists 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: finishing write tid 3 to smithi07685715-50 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: finishing write tid 4 to smithi07685715-50 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: finishing write tid 5 to smithi07685715-50 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: oid 50 updating version 0 to 981 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: oid 50 version 981 is already newer than 980 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: oid 50 updating version 981 to 982 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: oid 50 updating version 982 to 983 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 983 (ObjNum 1167 snap 393 seq_num 1167) dirty exists 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: left oid 50 (ObjNum 1167 snap 393 seq_num 1167) 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3102: finishing write tid 1 to smithi07685715-38 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3102: finishing write tid 2 to smithi07685715-38 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3099: done (5 left) 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3100: done (4 left) 2024-09-21T09:51:03.210 INFO:tasks.rados.rados.0.smithi076.stdout:3106: write oid 43 current snap is 393 2024-09-21T09:51:03.211 INFO:tasks.rados.rados.0.smithi076.stdout:3102: finishing write tid 3 to smithi07685715-38 2024-09-21T09:51:03.211 INFO:tasks.rados.rados.0.smithi076.stdout:3102: finishing write tid 4 to smithi07685715-38 2024-09-21T09:51:03.211 INFO:tasks.rados.rados.0.smithi076.stdout:3102: finishing write tid 5 to smithi07685715-38 2024-09-21T09:51:03.211 INFO:tasks.rados.rados.0.smithi076.stdout:3106: seq_num 1170 ranges {464190=643339,1748991=445068,2930078=574608} 2024-09-21T09:51:03.214 INFO:tasks.rados.rados.0.smithi076.stdout:3106: writing smithi07685715-43 from 464190 to 1107529 tid 1 2024-09-21T09:51:03.220 INFO:tasks.rados.rados.0.smithi076.stdout:3106: writing smithi07685715-43 from 1748991 to 2194059 tid 2 2024-09-21T09:51:03.227 INFO:tasks.rados.rados.0.smithi076.stdout:3106: writing smithi07685715-43 from 2930078 to 3504686 tid 3 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3107: write oid 46 current snap is 393 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: finishing write tid 6 to smithi07685715-38 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: oid 38 updating version 0 to 1830 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: oid 38 updating version 1830 to 1833 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: oid 38 updating version 1833 to 1834 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: oid 38 version 1834 is already newer than 1829 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: oid 38 updating version 1834 to 1835 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1835 (ObjNum 1168 snap 393 seq_num 1168) dirty exists 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3102: left oid 38 (ObjNum 1168 snap 393 seq_num 1168) 2024-09-21T09:51:03.228 INFO:tasks.rados.rados.0.smithi076.stdout:3107: seq_num 1171 ranges {480997=434941,1407473=705079,2557501=785880} 2024-09-21T09:51:03.232 INFO:tasks.rados.rados.0.smithi076.stdout:3107: writing smithi07685715-46 from 480997 to 915938 tid 1 2024-09-21T09:51:03.239 INFO:tasks.rados.rados.0.smithi076.stdout:3107: writing smithi07685715-46 from 1407473 to 2112552 tid 2 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3107: writing smithi07685715-46 from 2557501 to 3343381 tid 3 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3102: done (5 left) 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3108: read oid 22 snap -1 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3108: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3109: rollback oid 37 current snap is 393 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 378 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3103: finishing write tid 1 to smithi07685715-31 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3103: finishing write tid 2 to smithi07685715-31 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3105: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 682 (ObjNum 33 snap 0 seq_num 830379256) dirty dne 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3103: finishing write tid 3 to smithi07685715-31 2024-09-21T09:51:03.247 INFO:tasks.rados.rados.0.smithi076.stdout:3103: finishing write tid 4 to smithi07685715-31 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3105: done (6 left) 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3110: rollback oid 11 current snap is 393 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 365 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3103: finishing write tid 5 to smithi07685715-31 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3103: oid 31 updating version 0 to 892 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3103: oid 31 version 892 is already newer than 891 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3103: oid 31 updating version 892 to 893 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3103: oid 31 updating version 893 to 894 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 894 (ObjNum 1169 snap 393 seq_num 1169) dirty exists 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3103: left oid 31 (ObjNum 1169 snap 393 seq_num 1169) 2024-09-21T09:51:03.248 INFO:tasks.rados.rados.0.smithi076.stdout:3104: expect (ObjNum 1149 snap 382 seq_num 1149) 2024-09-21T09:51:03.250 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:02 smithi076 ceph-mon[106482]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:51:03.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:02 smithi076 ceph-mon[110963]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:51:03.355 INFO:tasks.rados.rados.0.smithi076.stdout:3106: finishing write tid 1 to smithi07685715-43 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: finishing write tid 2 to smithi07685715-43 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3103: done (6 left) 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3104: done (5 left) 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3111: snap_create 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: finishing write tid 3 to smithi07685715-43 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: finishing write tid 4 to smithi07685715-43 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: finishing write tid 5 to smithi07685715-43 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: oid 43 updating version 0 to 714 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: oid 43 version 714 is already newer than 712 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: oid 43 version 714 is already newer than 713 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: oid 43 version 714 is already newer than 711 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 714 (ObjNum 1170 snap 393 seq_num 1170) dirty exists 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3106: left oid 43 (ObjNum 1170 snap 393 seq_num 1170) 2024-09-21T09:51:03.356 INFO:tasks.rados.rados.0.smithi076.stdout:3107: finishing write tid 1 to smithi07685715-46 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: finishing write tid 2 to smithi07685715-46 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3109: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3109: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3109: finishing rollback tid 2 to smithi07685715-37 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 917 (ObjNum 1134 snap 375 seq_num 1134) dirty exists 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: finishing write tid 3 to smithi07685715-46 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: finishing write tid 4 to smithi07685715-46 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: finishing write tid 5 to smithi07685715-46 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: oid 46 updating version 0 to 2335 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: oid 46 updating version 2335 to 2338 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: oid 46 version 2338 is already newer than 2336 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: oid 46 version 2338 is already newer than 2337 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2338 (ObjNum 1171 snap 393 seq_num 1171) dirty exists 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3107: left oid 46 (ObjNum 1171 snap 393 seq_num 1171) 2024-09-21T09:51:03.357 INFO:tasks.rados.rados.0.smithi076.stdout:3108: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:51:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:02 smithi191 ceph-mon[102206]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T09:51:03.580 INFO:tasks.rados.rados.0.smithi076.stdout:3110: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:51:03.580 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2316 (ObjNum 65 snap 0 seq_num 1682031199) dirty dne 2024-09-21T09:51:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:51:03] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3106: done (5 left) 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3107: done (4 left) 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3108: done (3 left) 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3109: done (2 left) 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3110: done (1 left) 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3111: done (0 left) 2024-09-21T09:51:03.771 INFO:tasks.rados.rados.0.smithi076.stdout:3112: snap_remove snap 358 2024-09-21T09:51:04.266 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-mon[106482]: pgmap v742: 89 pgs: 89 active+clean; 247 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 682 KiB/s wr, 7 op/s 2024-09-21T09:51:04.266 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-mon[106482]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:51:04.266 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-mon[106482]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:51:04.266 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-mon[110963]: pgmap v742: 89 pgs: 89 active+clean; 247 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 682 KiB/s wr, 7 op/s 2024-09-21T09:51:04.267 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-mon[110963]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:51:04.267 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:03 smithi076 ceph-mon[110963]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:51:04.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:03 smithi191 ceph-mon[102206]: pgmap v742: 89 pgs: 89 active+clean; 247 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 682 KiB/s wr, 7 op/s 2024-09-21T09:51:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:03 smithi191 ceph-mon[102206]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T09:51:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:03 smithi191 ceph-mon[102206]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T09:51:04.773 INFO:tasks.rados.rados.0.smithi076.stdout:3112: done (0 left) 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3113: read oid 35 snap -1 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3113: expect deleted 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3114: read oid 15 snap 378 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3114: expect deleted 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3115: rollback oid 13 current snap is 394 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 385 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3116: rollback oid 37 current snap is 394 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 37 to 355 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3117: read oid 46 snap -1 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3117: expect (ObjNum 1171 snap 393 seq_num 1171) 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:3118: rollback oid 8 current snap is 394 2024-09-21T09:51:04.774 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 367 2024-09-21T09:51:04.775 INFO:tasks.rados.rados.0.smithi076.stdout:3119: write oid 12 current snap is 394 2024-09-21T09:51:04.775 INFO:tasks.rados.rados.0.smithi076.stdout:3119: seq_num 1172 ranges {497804=510190,1465955=448737,2584924=597152} 2024-09-21T09:51:04.779 INFO:tasks.rados.rados.0.smithi076.stdout:3119: writing smithi07685715-12 from 497804 to 1007994 tid 1 2024-09-21T09:51:04.785 INFO:tasks.rados.rados.0.smithi076.stdout:3119: writing smithi07685715-12 from 1465955 to 1914692 tid 2 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3119: writing smithi07685715-12 from 2584924 to 3182076 tid 3 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3120: delete oid 3 current snap is 394 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3116: finishing rollback tid 0 to smithi07685715-37 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3115: finishing rollback tid 0 to smithi07685715-13 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3115: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3115: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:51:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 597 (ObjNum 1095 snap 364 seq_num 1095) dirty exists 2024-09-21T09:51:04.794 INFO:tasks.rados.rados.0.smithi076.stdout:3119: finishing write tid 1 to smithi07685715-12 2024-09-21T09:51:04.794 INFO:tasks.rados.rados.0.smithi076.stdout:3119: finishing write tid 2 to smithi07685715-12 2024-09-21T09:51:04.794 INFO:tasks.rados.rados.0.smithi076.stdout:3116: finishing rollback tid 1 to smithi07685715-37 2024-09-21T09:51:04.794 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 920 (ObjNum 65 snap 0 seq_num 864144352) dirty dne 2024-09-21T09:51:04.796 INFO:tasks.rados.rados.0.smithi076.stdout:3117: expect (ObjNum 1171 snap 393 seq_num 1171) 2024-09-21T09:51:05.001 INFO:tasks.rados.rados.0.smithi076.stdout:3118: finishing rollback tid 0 to smithi07685715-8 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3118: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3118: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2306 (ObjNum 1070 snap 358 seq_num 1070) dirty exists 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3119: finishing write tid 3 to smithi07685715-12 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3119: finishing write tid 4 to smithi07685715-12 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3113: done (7 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3114: done (6 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3115: done (5 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3116: done (4 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3117: done (3 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3118: done (2 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3120: done (1 left) 2024-09-21T09:51:05.002 INFO:tasks.rados.rados.0.smithi076.stdout:3121: snap_remove snap 365 2024-09-21T09:51:05.984 INFO:tasks.rados.rados.0.smithi076.stdout:3121: done (1 left) 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3122: read oid 48 snap 355 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3122: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3123: write oid 32 current snap is 394 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3119: finishing write tid 5 to smithi07685715-12 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3119: oid 12 updating version 0 to 1041 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3119: oid 12 version 1041 is already newer than 1040 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3119: oid 12 version 1041 is already newer than 1038 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3119: oid 12 version 1041 is already newer than 1039 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1041 (ObjNum 1172 snap 394 seq_num 1172) dirty exists 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3119: left oid 12 (ObjNum 1172 snap 394 seq_num 1172) 2024-09-21T09:51:05.985 INFO:tasks.rados.rados.0.smithi076.stdout:3123: seq_num 1174 ranges {531418=660688,1982919=452406,3039770=691648} 2024-09-21T09:51:05.992 INFO:tasks.rados.rados.0.smithi076.stdout:3123: writing smithi07685715-32 from 531418 to 1192106 tid 1 2024-09-21T09:51:06.000 INFO:tasks.rados.rados.0.smithi076.stdout:3123: writing smithi07685715-32 from 1982919 to 2435325 tid 2 2024-09-21T09:51:06.007 INFO:tasks.rados.rados.0.smithi076.stdout:3123: writing smithi07685715-32 from 3039770 to 3731418 tid 3 2024-09-21T09:51:06.007 INFO:tasks.rados.rados.0.smithi076.stdout:3119: done (2 left) 2024-09-21T09:51:06.007 INFO:tasks.rados.rados.0.smithi076.stdout:3124: rollback oid 5 current snap is 394 2024-09-21T09:51:06.008 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 379 2024-09-21T09:51:06.008 INFO:tasks.rados.rados.0.smithi076.stdout:3122: expect (ObjNum 1023 snap 345 seq_num 1023) 2024-09-21T09:51:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:05 smithi191 ceph-mon[102206]: pgmap v745: 89 pgs: 89 active+clean; 255 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:51:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:05 smithi191 ceph-mon[102206]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:51:06.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:05 smithi076 ceph-mon[106482]: pgmap v745: 89 pgs: 89 active+clean; 255 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:51:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:05 smithi076 ceph-mon[106482]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:51:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:05 smithi076 ceph-mon[110963]: pgmap v745: 89 pgs: 89 active+clean; 255 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-09-21T09:51:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:05 smithi076 ceph-mon[110963]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T09:51:06.200 INFO:tasks.rados.rados.0.smithi076.stdout:3123: finishing write tid 1 to smithi07685715-32 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: finishing write tid 2 to smithi07685715-32 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: finishing write tid 3 to smithi07685715-32 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: finishing write tid 4 to smithi07685715-32 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3124: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: finishing write tid 5 to smithi07685715-32 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: oid 32 updating version 0 to 897 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: oid 32 updating version 897 to 902 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: oid 32 version 902 is already newer than 900 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: oid 32 version 902 is already newer than 901 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 902 (ObjNum 1174 snap 394 seq_num 1174) dirty exists 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3123: left oid 32 (ObjNum 1174 snap 394 seq_num 1174) 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:3124: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:51:06.201 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2360 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:06.202 INFO:tasks.rados.rados.0.smithi076.stdout:3122: done (2 left) 2024-09-21T09:51:06.202 INFO:tasks.rados.rados.0.smithi076.stdout:3123: done (1 left) 2024-09-21T09:51:06.202 INFO:tasks.rados.rados.0.smithi076.stdout:3124: done (0 left) 2024-09-21T09:51:06.202 INFO:tasks.rados.rados.0.smithi076.stdout:3125: snap_create 2024-09-21T09:51:06.991 INFO:tasks.rados.rados.0.smithi076.stdout:3125: done (0 left) 2024-09-21T09:51:06.991 INFO:tasks.rados.rados.0.smithi076.stdout:3126: read oid 8 snap -1 2024-09-21T09:51:06.991 INFO:tasks.rados.rados.0.smithi076.stdout:3126: expect (ObjNum 1070 snap 358 seq_num 1070) 2024-09-21T09:51:06.991 INFO:tasks.rados.rados.0.smithi076.stdout:3127: write oid 31 current snap is 395 2024-09-21T09:51:06.991 INFO:tasks.rados.rados.0.smithi076.stdout:3127: seq_num 1175 ranges {548225=735937,1925048=596064,2950840=547321} 2024-09-21T09:51:06.997 INFO:tasks.rados.rados.0.smithi076.stdout:3127: writing smithi07685715-31 from 548225 to 1284162 tid 1 2024-09-21T09:51:07.005 INFO:tasks.rados.rados.0.smithi076.stdout:3127: writing smithi07685715-31 from 1925048 to 2521112 tid 2 2024-09-21T09:51:07.010 INFO:tasks.rados.rados.0.smithi076.stdout:3127: writing smithi07685715-31 from 2950840 to 3498161 tid 3 2024-09-21T09:51:07.010 INFO:tasks.rados.rados.0.smithi076.stdout:3128: snap_create 2024-09-21T09:51:07.010 INFO:tasks.rados.rados.0.smithi076.stdout:3126: expect (ObjNum 1070 snap 358 seq_num 1070) 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: finishing write tid 1 to smithi07685715-31 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: finishing write tid 2 to smithi07685715-31 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: finishing write tid 3 to smithi07685715-31 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: finishing write tid 4 to smithi07685715-31 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: finishing write tid 5 to smithi07685715-31 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: oid 31 updating version 0 to 907 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: oid 31 version 907 is already newer than 906 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: oid 31 version 907 is already newer than 905 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:3127: oid 31 version 907 is already newer than 904 2024-09-21T09:51:07.141 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 907 (ObjNum 1175 snap 395 seq_num 1175) dirty exists 2024-09-21T09:51:07.142 INFO:tasks.rados.rados.0.smithi076.stdout:3127: left oid 31 (ObjNum 1175 snap 395 seq_num 1175) 2024-09-21T09:51:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:06 smithi191 ceph-mon[102206]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:51:07.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:06 smithi076 ceph-mon[106482]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:51:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:06 smithi076 ceph-mon[110963]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T09:51:07.995 INFO:tasks.rados.rados.0.smithi076.stdout:3126: done (2 left) 2024-09-21T09:51:07.995 INFO:tasks.rados.rados.0.smithi076.stdout:3127: done (1 left) 2024-09-21T09:51:07.995 INFO:tasks.rados.rados.0.smithi076.stdout:3128: done (0 left) 2024-09-21T09:51:07.995 INFO:tasks.rados.rados.0.smithi076.stdout:3129: snap_remove snap 390 2024-09-21T09:51:08.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:07 smithi191 ceph-mon[102206]: pgmap v748: 89 pgs: 89 active+clean; 267 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-09-21T09:51:08.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:07 smithi191 ceph-mon[102206]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:51:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:07 smithi076 ceph-mon[106482]: pgmap v748: 89 pgs: 89 active+clean; 267 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-09-21T09:51:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:07 smithi076 ceph-mon[106482]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:51:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:07 smithi076 ceph-mon[110963]: pgmap v748: 89 pgs: 89 active+clean; 267 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-09-21T09:51:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:07 smithi076 ceph-mon[110963]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T09:51:08.772 INFO:tasks.rados.rados.0.smithi076.stdout:3129: done (0 left) 2024-09-21T09:51:08.772 INFO:tasks.rados.rados.0.smithi076.stdout:3130: read oid 35 snap -1 2024-09-21T09:51:08.772 INFO:tasks.rados.rados.0.smithi076.stdout:3130: expect deleted 2024-09-21T09:51:08.772 INFO:tasks.rados.rados.0.smithi076.stdout:3131: snap_remove snap 385 2024-09-21T09:51:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:08 smithi191 ceph-mon[102206]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:51:09.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:08 smithi191 ceph-mon[102206]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:51:09.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:08 smithi076 ceph-mon[106482]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:51:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:08 smithi076 ceph-mon[106482]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:51:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:08 smithi076 ceph-mon[110963]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T09:51:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:08 smithi076 ceph-mon[110963]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T09:51:09.775 INFO:tasks.rados.rados.0.smithi076.stdout:3131: done (1 left) 2024-09-21T09:51:09.775 INFO:tasks.rados.rados.0.smithi076.stdout:3132: write oid 42 current snap is 396 2024-09-21T09:51:09.775 INFO:tasks.rados.rados.0.smithi076.stdout:3132: seq_num 1176 ranges {565032=411186,1583530=456075,2578263=758593} 2024-09-21T09:51:09.781 INFO:tasks.rados.rados.0.smithi076.stdout:3132: writing smithi07685715-42 from 565032 to 976218 tid 1 2024-09-21T09:51:09.787 INFO:tasks.rados.rados.0.smithi076.stdout:3132: writing smithi07685715-42 from 1583530 to 2039605 tid 2 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3132: writing smithi07685715-42 from 2578263 to 3336856 tid 3 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3133: read oid 37 snap -1 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3133: expect deleted 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3134: read oid 27 snap -1 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3134: expect (ObjNum 1144 snap 380 seq_num 1144) 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3135: write oid 24 current snap is 396 2024-09-21T09:51:09.795 INFO:tasks.rados.rados.0.smithi076.stdout:3135: seq_num 1177 ranges {581839=486435,1642012=599733,3005686=569865} 2024-09-21T09:51:09.801 INFO:tasks.rados.rados.0.smithi076.stdout:3135: writing smithi07685715-24 from 581839 to 1068274 tid 1 2024-09-21T09:51:09.808 INFO:tasks.rados.rados.0.smithi076.stdout:3135: writing smithi07685715-24 from 1642012 to 2241745 tid 2 2024-09-21T09:51:09.816 INFO:tasks.rados.rados.0.smithi076.stdout:3135: writing smithi07685715-24 from 3005686 to 3575551 tid 3 2024-09-21T09:51:09.816 INFO:tasks.rados.rados.0.smithi076.stdout:3136: read oid 13 snap 355 2024-09-21T09:51:09.816 INFO:tasks.rados.rados.0.smithi076.stdout:3136: expect (ObjNum 1055 snap 353 seq_num 1055) 2024-09-21T09:51:09.816 INFO:tasks.rados.rados.0.smithi076.stdout:3137: rollback oid 38 current snap is 396 2024-09-21T09:51:09.816 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 392 2024-09-21T09:51:09.816 INFO:tasks.rados.rados.0.smithi076.stdout:3132: finishing write tid 1 to smithi07685715-42 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3132: finishing write tid 2 to smithi07685715-42 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3130: done (6 left) 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3138: read oid 28 snap 388 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3138: expect deleted 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3139: rollback oid 30 current snap is 396 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 375 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3133: done (7 left) 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3140: read oid 45 snap -1 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3140: expect (ObjNum 1158 snap 390 seq_num 1158) 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3141: delete oid 46 current snap is 396 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3132: finishing write tid 3 to smithi07685715-42 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3132: finishing write tid 4 to smithi07685715-42 2024-09-21T09:51:09.817 INFO:tasks.rados.rados.0.smithi076.stdout:3132: finishing write tid 5 to smithi07685715-42 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3132: oid 42 updating version 0 to 768 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3132: oid 42 version 768 is already newer than 767 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3132: oid 42 version 768 is already newer than 766 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3132: oid 42 version 768 is already newer than 765 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 768 (ObjNum 1176 snap 396 seq_num 1176) dirty exists 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3132: left oid 42 (ObjNum 1176 snap 396 seq_num 1176) 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3135: finishing write tid 1 to smithi07685715-24 2024-09-21T09:51:09.818 INFO:tasks.rados.rados.0.smithi076.stdout:3134: expect (ObjNum 1144 snap 380 seq_num 1144) 2024-09-21T09:51:09.921 INFO:tasks.rados.rados.0.smithi076.stdout:3135: finishing write tid 2 to smithi07685715-24 2024-09-21T09:51:09.921 INFO:tasks.rados.rados.0.smithi076.stdout:3132: done (8 left) 2024-09-21T09:51:09.921 INFO:tasks.rados.rados.0.smithi076.stdout:3134: done (7 left) 2024-09-21T09:51:09.921 INFO:tasks.rados.rados.0.smithi076.stdout:3141: done (6 left) 2024-09-21T09:51:09.921 INFO:tasks.rados.rados.0.smithi076.stdout:3142: snap_remove snap 389 2024-09-21T09:51:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:09 smithi191 ceph-mon[102206]: pgmap v751: 89 pgs: 89 active+clean; 271 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.5 MiB/s wr, 23 op/s 2024-09-21T09:51:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:09 smithi191 ceph-mon[102206]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:51:10.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:09 smithi076 ceph-mon[106482]: pgmap v751: 89 pgs: 89 active+clean; 271 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.5 MiB/s wr, 23 op/s 2024-09-21T09:51:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:09 smithi076 ceph-mon[106482]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:51:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:09 smithi076 ceph-mon[110963]: pgmap v751: 89 pgs: 89 active+clean; 271 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.5 MiB/s wr, 23 op/s 2024-09-21T09:51:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:09 smithi076 ceph-mon[110963]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T09:51:10.774 INFO:tasks.rados.rados.0.smithi076.stdout:3142: done (6 left) 2024-09-21T09:51:10.774 INFO:tasks.rados.rados.0.smithi076.stdout:3143: snap_remove snap 380 2024-09-21T09:51:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:11 smithi191 ceph-mon[102206]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:51:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:11 smithi076 ceph-mon[110963]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:51:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:11 smithi076 ceph-mon[106482]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T09:51:11.778 INFO:tasks.rados.rados.0.smithi076.stdout:3143: done (6 left) 2024-09-21T09:51:11.778 INFO:tasks.rados.rados.0.smithi076.stdout:3144: write oid 14 current snap is 396 2024-09-21T09:51:11.778 INFO:tasks.rados.rados.0.smithi076.stdout:3144: seq_num 1179 ranges {615453=753286,2158976=603402,3460532=354921} 2024-09-21T09:51:11.785 INFO:tasks.rados.rados.0.smithi076.stdout:3144: writing smithi07685715-14 from 615453 to 1368739 tid 1 2024-09-21T09:51:11.793 INFO:tasks.rados.rados.0.smithi076.stdout:3144: writing smithi07685715-14 from 2158976 to 2762378 tid 2 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3144: writing smithi07685715-14 from 3460532 to 3815453 tid 3 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3138: done (6 left) 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3145: delete oid 18 current snap is 396 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3135: finishing write tid 3 to smithi07685715-24 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3139: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2175 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3135: finishing write tid 4 to smithi07685715-24 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3135: finishing write tid 5 to smithi07685715-24 2024-09-21T09:51:11.800 INFO:tasks.rados.rados.0.smithi076.stdout:3135: oid 24 updating version 0 to 315 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3135: oid 24 updating version 315 to 316 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3135: oid 24 updating version 316 to 318 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3135: oid 24 version 318 is already newer than 317 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 318 (ObjNum 1177 snap 396 seq_num 1177) dirty exists 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3135: left oid 24 (ObjNum 1177 snap 396 seq_num 1177) 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3137: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3137: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3137: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1865 (ObjNum 1091 snap 364 seq_num 1091) dirty exists 2024-09-21T09:51:11.801 INFO:tasks.rados.rados.0.smithi076.stdout:3136: expect (ObjNum 1055 snap 353 seq_num 1055) 2024-09-21T09:51:12.091 INFO:tasks.rados.rados.0.smithi076.stdout:3140: expect (ObjNum 1158 snap 390 seq_num 1158) 2024-09-21T09:51:12.283 INFO:tasks.rados.rados.0.smithi076.stdout:3144: finishing write tid 1 to smithi07685715-14 2024-09-21T09:51:12.283 INFO:tasks.rados.rados.0.smithi076.stdout:3144: finishing write tid 2 to smithi07685715-14 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: finishing write tid 3 to smithi07685715-14 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: finishing write tid 4 to smithi07685715-14 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: finishing write tid 5 to smithi07685715-14 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: oid 14 updating version 0 to 1879 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: oid 14 updating version 1879 to 1880 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: oid 14 updating version 1880 to 1881 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: oid 14 updating version 1881 to 1882 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 1882 (ObjNum 1179 snap 396 seq_num 1179) dirty exists 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3144: left oid 14 (ObjNum 1179 snap 396 seq_num 1179) 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3135: done (6 left) 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3136: done (5 left) 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3137: done (4 left) 2024-09-21T09:51:12.284 INFO:tasks.rados.rados.0.smithi076.stdout:3139: done (3 left) 2024-09-21T09:51:12.285 INFO:tasks.rados.rados.0.smithi076.stdout:3140: done (2 left) 2024-09-21T09:51:12.285 INFO:tasks.rados.rados.0.smithi076.stdout:3144: done (1 left) 2024-09-21T09:51:12.285 INFO:tasks.rados.rados.0.smithi076.stdout:3145: done (0 left) 2024-09-21T09:51:12.285 INFO:tasks.rados.rados.0.smithi076.stdout:3146: delete oid 32 current snap is 396 2024-09-21T09:51:12.285 INFO:tasks.rados.rados.0.smithi076.stdout:3146: done (0 left) 2024-09-21T09:51:12.286 INFO:tasks.rados.rados.0.smithi076.stdout:3147: snap_create 2024-09-21T09:51:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:12 smithi191 ceph-mon[102206]: pgmap v754: 89 pgs: 89 active+clean; 278 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:51:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:12 smithi191 ceph-mon[102206]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:51:12.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:12 smithi076 ceph-mon[106482]: pgmap v754: 89 pgs: 89 active+clean; 278 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:51:12.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:12 smithi076 ceph-mon[106482]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:51:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:12 smithi076 ceph-mon[110963]: pgmap v754: 89 pgs: 89 active+clean; 278 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 8 op/s 2024-09-21T09:51:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:12 smithi076 ceph-mon[110963]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T09:51:13.017 INFO:tasks.rados.rados.0.smithi076.stdout:3147: done (0 left) 2024-09-21T09:51:13.017 INFO:tasks.rados.rados.0.smithi076.stdout:3148: rollback oid 4 current snap is 397 2024-09-21T09:51:13.017 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 395 2024-09-21T09:51:13.017 INFO:tasks.rados.rados.0.smithi076.stdout:3149: snap_create 2024-09-21T09:51:13.023 INFO:tasks.rados.rados.0.smithi076.stdout:3148: finishing rollback tid 0 to smithi07685715-4 2024-09-21T09:51:13.023 INFO:tasks.rados.rados.0.smithi076.stdout:3148: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:51:13.023 INFO:tasks.rados.rados.0.smithi076.stdout:3148: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:51:13.023 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 999 (ObjNum 1119 snap 370 seq_num 1119) dirty exists 2024-09-21T09:51:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:51:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:51:13] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:51:13.774 INFO:tasks.rados.rados.0.smithi076.stdout:3148: done (1 left) 2024-09-21T09:51:13.775 INFO:tasks.rados.rados.0.smithi076.stdout:3149: done (0 left) 2024-09-21T09:51:13.775 INFO:tasks.rados.rados.0.smithi076.stdout:3150: rollback oid 18 current snap is 398 2024-09-21T09:51:13.775 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 18 to 384 2024-09-21T09:51:13.775 INFO:tasks.rados.rados.0.smithi076.stdout:3151: read oid 29 snap 396 2024-09-21T09:51:13.775 INFO:tasks.rados.rados.0.smithi076.stdout:3151: expect deleted 2024-09-21T09:51:13.775 INFO:tasks.rados.rados.0.smithi076.stdout:3152: snap_create 2024-09-21T09:51:13.777 INFO:tasks.rados.rados.0.smithi076.stdout:3150: finishing rollback tid 1 to smithi07685715-18 2024-09-21T09:51:13.778 INFO:tasks.rados.rados.0.smithi076.stdout:3150: finishing rollback tid 2 to smithi07685715-18 2024-09-21T09:51:13.778 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 776 (ObjNum 1136 snap 378 seq_num 1136) dirty exists 2024-09-21T09:51:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:14 smithi191 ceph-mon[102206]: pgmap v757: 89 pgs: 89 active+clean; 278 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:51:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:14 smithi191 ceph-mon[102206]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:51:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:14 smithi191 ceph-mon[102206]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:51:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:14 smithi076 ceph-mon[110963]: pgmap v757: 89 pgs: 89 active+clean; 278 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:51:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:14 smithi076 ceph-mon[110963]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:51:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:14 smithi076 ceph-mon[110963]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:51:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:14 smithi076 ceph-mon[106482]: pgmap v757: 89 pgs: 89 active+clean; 278 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-09-21T09:51:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:14 smithi076 ceph-mon[106482]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T09:51:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:14 smithi076 ceph-mon[106482]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T09:51:14.776 INFO:tasks.rados.rados.0.smithi076.stdout:3150: done (2 left) 2024-09-21T09:51:14.776 INFO:tasks.rados.rados.0.smithi076.stdout:3151: done (1 left) 2024-09-21T09:51:14.776 INFO:tasks.rados.rados.0.smithi076.stdout:3152: done (0 left) 2024-09-21T09:51:14.776 INFO:tasks.rados.rados.0.smithi076.stdout:3153: read oid 26 snap -1 2024-09-21T09:51:14.776 INFO:tasks.rados.rados.0.smithi076.stdout:3153: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:51:14.776 INFO:tasks.rados.rados.0.smithi076.stdout:3154: snap_remove snap 393 2024-09-21T09:51:15.780 INFO:tasks.rados.rados.0.smithi076.stdout:3154: done (1 left) 2024-09-21T09:51:15.780 INFO:tasks.rados.rados.0.smithi076.stdout:3155: write oid 47 current snap is 399 2024-09-21T09:51:15.780 INFO:tasks.rados.rados.0.smithi076.stdout:3153: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:51:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:15 smithi076 ceph-mon[106482]: pgmap v760: 89 pgs: 89 active+clean; 285 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 823 KiB/s wr, 7 op/s 2024-09-21T09:51:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:15 smithi076 ceph-mon[106482]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:51:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:15 smithi076 ceph-mon[110963]: pgmap v760: 89 pgs: 89 active+clean; 285 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 823 KiB/s wr, 7 op/s 2024-09-21T09:51:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:15 smithi076 ceph-mon[110963]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:51:15.998 INFO:tasks.rados.rados.0.smithi076.stdout:3155: seq_num 1182 ranges {665874=579033,1934422=467082,3142801=542578} 2024-09-21T09:51:16.005 INFO:tasks.rados.rados.0.smithi076.stdout:3155: writing smithi07685715-47 from 665874 to 1244907 tid 1 2024-09-21T09:51:16.011 INFO:tasks.rados.rados.0.smithi076.stdout:3155: writing smithi07685715-47 from 1934422 to 2401504 tid 2 2024-09-21T09:51:16.019 INFO:tasks.rados.rados.0.smithi076.stdout:3155: writing smithi07685715-47 from 3142801 to 3685379 tid 3 2024-09-21T09:51:16.019 INFO:tasks.rados.rados.0.smithi076.stdout:3153: done (1 left) 2024-09-21T09:51:16.019 INFO:tasks.rados.rados.0.smithi076.stdout:3156: snap_remove snap 392 2024-09-21T09:51:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:15 smithi191 ceph-mon[102206]: pgmap v760: 89 pgs: 89 active+clean; 285 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 823 KiB/s wr, 7 op/s 2024-09-21T09:51:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:15 smithi191 ceph-mon[102206]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T09:51:16.788 INFO:tasks.rados.rados.0.smithi076.stdout:3155: finishing write tid 1 to smithi07685715-47 2024-09-21T09:51:16.788 INFO:tasks.rados.rados.0.smithi076.stdout:3155: finishing write tid 2 to smithi07685715-47 2024-09-21T09:51:16.788 INFO:tasks.rados.rados.0.smithi076.stdout:3155: finishing write tid 3 to smithi07685715-47 2024-09-21T09:51:16.788 INFO:tasks.rados.rados.0.smithi076.stdout:3155: finishing write tid 4 to smithi07685715-47 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: finishing write tid 5 to smithi07685715-47 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: oid 47 updating version 0 to 2346 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: oid 47 updating version 2346 to 2347 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: oid 47 updating version 2347 to 2348 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: oid 47 version 2348 is already newer than 2345 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2348 (ObjNum 1182 snap 399 seq_num 1182) dirty exists 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: left oid 47 (ObjNum 1182 snap 399 seq_num 1182) 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3155: done (1 left) 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3156: done (0 left) 2024-09-21T09:51:16.789 INFO:tasks.rados.rados.0.smithi076.stdout:3157: snap_remove snap 396 2024-09-21T09:51:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:16 smithi191 ceph-mon[102206]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:51:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:16 smithi076 ceph-mon[106482]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:51:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:16 smithi076 ceph-mon[110963]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T09:51:17.787 INFO:tasks.rados.rados.0.smithi076.stdout:3157: done (0 left) 2024-09-21T09:51:17.787 INFO:tasks.rados.rados.0.smithi076.stdout:3158: delete oid 32 current snap is 399 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:3158: done (0 left) 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:3159: rollback oid 49 current snap is 399 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 373 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:3160: rollback oid 18 current snap is 399 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 18 to 378 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:3161: rollback oid 41 current snap is 399 2024-09-21T09:51:17.793 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 379 2024-09-21T09:51:17.794 INFO:tasks.rados.rados.0.smithi076.stdout:3162: read oid 30 snap -1 2024-09-21T09:51:17.794 INFO:tasks.rados.rados.0.smithi076.stdout:3162: expect deleted 2024-09-21T09:51:17.794 INFO:tasks.rados.rados.0.smithi076.stdout:3163: read oid 25 snap -1 2024-09-21T09:51:17.794 INFO:tasks.rados.rados.0.smithi076.stdout:3163: expect (ObjNum 1165 snap 393 seq_num 1165) 2024-09-21T09:51:17.794 INFO:tasks.rados.rados.0.smithi076.stdout:3164: write oid 37 current snap is 399 2024-09-21T09:51:17.794 INFO:tasks.rados.rados.0.smithi076.stdout:3164: seq_num 1184 ranges {699488=729531,1935033=470751,3081294=681475} 2024-09-21T09:51:17.801 INFO:tasks.rados.rados.0.smithi076.stdout:3164: writing smithi07685715-37 from 699488 to 1429019 tid 1 2024-09-21T09:51:17.807 INFO:tasks.rados.rados.0.smithi076.stdout:3164: writing smithi07685715-37 from 1935033 to 2405784 tid 2 2024-09-21T09:51:17.814 INFO:tasks.rados.rados.0.smithi076.stdout:3164: writing smithi07685715-37 from 3081294 to 3762769 tid 3 2024-09-21T09:51:17.815 INFO:tasks.rados.rados.0.smithi076.stdout:3165: write oid 23 current snap is 399 2024-09-21T09:51:17.815 INFO:tasks.rados.rados.0.smithi076.stdout:3160: finishing rollback tid 0 to smithi07685715-18 2024-09-21T09:51:17.815 INFO:tasks.rados.rados.0.smithi076.stdout:3165: seq_num 1185 ranges {716295=404780,1593515=614409,2708717=492747,3640519=275776} 2024-09-21T09:51:17.820 INFO:tasks.rados.rados.0.smithi076.stdout:3165: writing smithi07685715-23 from 716295 to 1121075 tid 1 2024-09-21T09:51:17.826 INFO:tasks.rados.rados.0.smithi076.stdout:3165: writing smithi07685715-23 from 1593515 to 2207924 tid 2 2024-09-21T09:51:17.832 INFO:tasks.rados.rados.0.smithi076.stdout:3165: writing smithi07685715-23 from 2708717 to 3201464 tid 3 2024-09-21T09:51:17.836 INFO:tasks.rados.rados.0.smithi076.stdout:3165: writing smithi07685715-23 from 3640519 to 3916295 tid 4 2024-09-21T09:51:17.836 INFO:tasks.rados.rados.0.smithi076.stdout:3166: write oid 7 current snap is 399 2024-09-21T09:51:17.836 INFO:tasks.rados.rados.0.smithi076.stdout:3160: finishing rollback tid 1 to smithi07685715-18 2024-09-21T09:51:17.836 INFO:tasks.rados.rados.0.smithi076.stdout:3166: seq_num 1186 ranges {733102=596382,1768350=758067,3136140=704019} 2024-09-21T09:51:17.843 INFO:tasks.rados.rados.0.smithi076.stdout:3166: writing smithi07685715-7 from 733102 to 1329484 tid 1 2024-09-21T09:51:17.850 INFO:tasks.rados.rados.0.smithi076.stdout:3166: writing smithi07685715-7 from 1768350 to 2526417 tid 2 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:3166: writing smithi07685715-7 from 3136140 to 3840159 tid 3 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:3167: read oid 38 snap 388 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:3167: expect (ObjNum 1091 snap 364 seq_num 1091) 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:3168: delete oid 2 current snap is 399 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:3160: finishing rollback tid 2 to smithi07685715-18 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 788 (ObjNum 1136 snap 378 seq_num 1136) dirty exists 2024-09-21T09:51:17.858 INFO:tasks.rados.rados.0.smithi076.stdout:3159: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:3159: finishing rollback tid 2 to smithi07685715-49 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2356 (ObjNum 1106 snap 367 seq_num 1106) dirty exists 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:3161: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:3161: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:3161: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1483 (ObjNum 1096 snap 364 seq_num 1096) dirty exists 2024-09-21T09:51:17.859 INFO:tasks.rados.rados.0.smithi076.stdout:3163: expect (ObjNum 1165 snap 393 seq_num 1165) 2024-09-21T09:51:17.973 INFO:tasks.rados.rados.0.smithi076.stdout:3164: finishing write tid 1 to smithi07685715-37 2024-09-21T09:51:17.973 INFO:tasks.rados.rados.0.smithi076.stdout:3164: finishing write tid 2 to smithi07685715-37 2024-09-21T09:51:17.973 INFO:tasks.rados.rados.0.smithi076.stdout:3159: done (9 left) 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3160: done (8 left) 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3161: done (7 left) 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3162: done (6 left) 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3163: done (5 left) 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3168: done (4 left) 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3169: write oid 33 current snap is 399 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3164: finishing write tid 3 to smithi07685715-37 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3164: finishing write tid 4 to smithi07685715-37 2024-09-21T09:51:17.974 INFO:tasks.rados.rados.0.smithi076.stdout:3169: seq_num 1188 ranges {766716=746880,2168961=761736,3474633=442916} 2024-09-21T09:51:17.981 INFO:tasks.rados.rados.0.smithi076.stdout:3169: writing smithi07685715-33 from 766716 to 1513596 tid 1 2024-09-21T09:51:17.988 INFO:tasks.rados.rados.0.smithi076.stdout:3169: writing smithi07685715-33 from 2168961 to 2930697 tid 2 2024-09-21T09:51:17.995 INFO:tasks.rados.rados.0.smithi076.stdout:3169: writing smithi07685715-33 from 3474633 to 3917549 tid 3 2024-09-21T09:51:17.995 INFO:tasks.rados.rados.0.smithi076.stdout:3170: read oid 40 snap 397 2024-09-21T09:51:17.995 INFO:tasks.rados.rados.0.smithi076.stdout:3170: expect deleted 2024-09-21T09:51:17.995 INFO:tasks.rados.rados.0.smithi076.stdout:3171: delete oid 4 current snap is 399 2024-09-21T09:51:17.995 INFO:tasks.rados.rados.0.smithi076.stdout:3164: finishing write tid 5 to smithi07685715-37 2024-09-21T09:51:17.995 INFO:tasks.rados.rados.0.smithi076.stdout:3164: oid 37 updating version 0 to 947 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3164: oid 37 updating version 947 to 950 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3164: oid 37 version 950 is already newer than 948 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3164: oid 37 version 950 is already newer than 949 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 950 (ObjNum 1184 snap 399 seq_num 1184) dirty exists 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3164: left oid 37 (ObjNum 1184 snap 399 seq_num 1184) 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: finishing write tid 1 to smithi07685715-23 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: finishing write tid 2 to smithi07685715-23 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: finishing write tid 3 to smithi07685715-23 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: finishing write tid 4 to smithi07685715-23 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: finishing write tid 5 to smithi07685715-23 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: finishing write tid 6 to smithi07685715-23 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: oid 23 updating version 0 to 2361 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: oid 23 updating version 2361 to 2364 2024-09-21T09:51:17.996 INFO:tasks.rados.rados.0.smithi076.stdout:3165: oid 23 version 2364 is already newer than 2360 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3165: oid 23 version 2364 is already newer than 2363 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3165: oid 23 version 2364 is already newer than 2362 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2364 (ObjNum 1185 snap 399 seq_num 1185) dirty exists 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3165: left oid 23 (ObjNum 1185 snap 399 seq_num 1185) 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: finishing write tid 1 to smithi07685715-7 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: finishing write tid 2 to smithi07685715-7 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: finishing write tid 3 to smithi07685715-7 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: finishing write tid 4 to smithi07685715-7 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: finishing write tid 5 to smithi07685715-7 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: oid 7 updating version 0 to 1895 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: oid 7 updating version 1895 to 1896 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: oid 7 version 1896 is already newer than 1894 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: oid 7 updating version 1896 to 1897 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 1897 (ObjNum 1186 snap 399 seq_num 1186) dirty exists 2024-09-21T09:51:17.997 INFO:tasks.rados.rados.0.smithi076.stdout:3166: left oid 7 (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:51:17.998 INFO:tasks.rados.rados.0.smithi076.stdout:3167: expect (ObjNum 1091 snap 364 seq_num 1091) 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: finishing write tid 1 to smithi07685715-33 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: finishing write tid 2 to smithi07685715-33 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: finishing write tid 3 to smithi07685715-33 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: finishing write tid 4 to smithi07685715-33 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: finishing write tid 5 to smithi07685715-33 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: oid 33 updating version 0 to 686 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: oid 33 updating version 686 to 687 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: oid 33 version 687 is already newer than 684 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: oid 33 version 687 is already newer than 685 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 687 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3169: left oid 33 (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3164: done (6 left) 2024-09-21T09:51:18.101 INFO:tasks.rados.rados.0.smithi076.stdout:3165: done (5 left) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3166: done (4 left) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3167: done (3 left) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3169: done (2 left) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3170: done (1 left) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3171: done (0 left) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3172: read oid 7 snap -1 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3172: expect (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3173: write oid 4 current snap is 399 2024-09-21T09:51:18.102 INFO:tasks.rados.rados.0.smithi076.stdout:3173: seq_num 1190 ranges {400330=497378,1485925=514405} 2024-09-21T09:51:18.105 INFO:tasks.rados.rados.0.smithi076.stdout:3173: writing smithi07685715-4 from 400330 to 897708 tid 1 2024-09-21T09:51:18.111 INFO:tasks.rados.rados.0.smithi076.stdout:3173: writing smithi07685715-4 from 1485925 to 2000330 tid 2 2024-09-21T09:51:18.111 INFO:tasks.rados.rados.0.smithi076.stdout:3174: write oid 39 current snap is 399 2024-09-21T09:51:18.111 INFO:tasks.rados.rados.0.smithi076.stdout:3174: seq_num 1191 ranges {417137=572627,1428054=589083} 2024-09-21T09:51:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:17 smithi191 ceph-mon[102206]: pgmap v763: 89 pgs: 89 active+clean; 287 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:51:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:17 smithi191 ceph-mon[102206]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:51:18.116 INFO:tasks.rados.rados.0.smithi076.stdout:3174: writing smithi07685715-39 from 417137 to 989764 tid 1 2024-09-21T09:51:18.122 INFO:tasks.rados.rados.0.smithi076.stdout:3174: writing smithi07685715-39 from 1428054 to 2017137 tid 2 2024-09-21T09:51:18.122 INFO:tasks.rados.rados.0.smithi076.stdout:3175: read oid 26 snap -1 2024-09-21T09:51:18.122 INFO:tasks.rados.rados.0.smithi076.stdout:3175: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:51:18.122 INFO:tasks.rados.rados.0.smithi076.stdout:3176: write oid 2 current snap is 399 2024-09-21T09:51:18.122 INFO:tasks.rados.rados.0.smithi076.stdout:3176: seq_num 1192 ranges {433944=647876,1486536=547408} 2024-09-21T09:51:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:17 smithi076 ceph-mon[110963]: pgmap v763: 89 pgs: 89 active+clean; 287 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:51:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:17 smithi076 ceph-mon[110963]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:51:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:17 smithi076 ceph-mon[106482]: pgmap v763: 89 pgs: 89 active+clean; 287 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T09:51:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:17 smithi076 ceph-mon[106482]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T09:51:18.128 INFO:tasks.rados.rados.0.smithi076.stdout:3176: writing smithi07685715-2 from 433944 to 1081820 tid 1 2024-09-21T09:51:18.133 INFO:tasks.rados.rados.0.smithi076.stdout:3176: writing smithi07685715-2 from 1486536 to 2033944 tid 2 2024-09-21T09:51:18.134 INFO:tasks.rados.rados.0.smithi076.stdout:3177: read oid 18 snap 395 2024-09-21T09:51:18.134 INFO:tasks.rados.rados.0.smithi076.stdout:3177: expect (ObjNum 1136 snap 378 seq_num 1136) 2024-09-21T09:51:18.134 INFO:tasks.rados.rados.0.smithi076.stdout:3178: snap_create 2024-09-21T09:51:18.134 INFO:tasks.rados.rados.0.smithi076.stdout:3172: expect (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:51:18.311 INFO:tasks.rados.rados.0.smithi076.stdout:3173: finishing write tid 1 to smithi07685715-4 2024-09-21T09:51:18.311 INFO:tasks.rados.rados.0.smithi076.stdout:3173: finishing write tid 2 to smithi07685715-4 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3173: finishing write tid 3 to smithi07685715-4 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3173: finishing write tid 4 to smithi07685715-4 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3173: oid 4 updating version 0 to 1008 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3173: oid 4 updating version 1008 to 1010 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3173: oid 4 version 1010 is already newer than 1009 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 1010 (ObjNum 1190 snap 399 seq_num 1190) dirty exists 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3173: left oid 4 (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3174: finishing write tid 1 to smithi07685715-39 2024-09-21T09:51:18.312 INFO:tasks.rados.rados.0.smithi076.stdout:3175: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:51:18.525 INFO:tasks.rados.rados.0.smithi076.stdout:3174: finishing write tid 2 to smithi07685715-39 2024-09-21T09:51:18.525 INFO:tasks.rados.rados.0.smithi076.stdout:3174: finishing write tid 3 to smithi07685715-39 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3174: finishing write tid 4 to smithi07685715-39 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3174: oid 39 updating version 0 to 2405 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3174: oid 39 version 2405 is already newer than 2404 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3174: oid 39 updating version 2405 to 2406 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2406 (ObjNum 1191 snap 399 seq_num 1191) dirty exists 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3174: left oid 39 (ObjNum 1191 snap 399 seq_num 1191) 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3176: finishing write tid 1 to smithi07685715-2 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3176: finishing write tid 2 to smithi07685715-2 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3176: finishing write tid 3 to smithi07685715-2 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3176: finishing write tid 4 to smithi07685715-2 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3176: oid 2 updating version 0 to 621 2024-09-21T09:51:18.526 INFO:tasks.rados.rados.0.smithi076.stdout:3176: oid 2 version 621 is already newer than 619 2024-09-21T09:51:18.527 INFO:tasks.rados.rados.0.smithi076.stdout:3176: oid 2 version 621 is already newer than 620 2024-09-21T09:51:18.527 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 621 (ObjNum 1192 snap 399 seq_num 1192) dirty exists 2024-09-21T09:51:18.527 INFO:tasks.rados.rados.0.smithi076.stdout:3176: left oid 2 (ObjNum 1192 snap 399 seq_num 1192) 2024-09-21T09:51:18.527 INFO:tasks.rados.rados.0.smithi076.stdout:3177: expect (ObjNum 1136 snap 378 seq_num 1136) 2024-09-21T09:51:18.774 INFO:tasks.rados.rados.0.smithi076.stdout:3172: done (6 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3173: done (5 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3174: done (4 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3175: done (3 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3176: done (2 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3177: done (1 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3178: done (0 left) 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3179: write oid 1 current snap is 400 2024-09-21T09:51:18.775 INFO:tasks.rados.rados.0.smithi076.stdout:3179: seq_num 1193 ranges {450751=723125,1945018=105733} 2024-09-21T09:51:18.780 INFO:tasks.rados.rados.0.smithi076.stdout:3179: writing smithi07685715-1 from 450751 to 1173876 tid 1 2024-09-21T09:51:18.785 INFO:tasks.rados.rados.0.smithi076.stdout:3179: writing smithi07685715-1 from 1945018 to 2050751 tid 2 2024-09-21T09:51:18.785 INFO:tasks.rados.rados.0.smithi076.stdout:3179: finishing write tid 1 to smithi07685715-1 2024-09-21T09:51:18.786 INFO:tasks.rados.rados.0.smithi076.stdout:3180: write oid 38 current snap is 400 2024-09-21T09:51:18.786 INFO:tasks.rados.rados.0.smithi076.stdout:3180: seq_num 1194 ranges {467558=514727,1719853=347705} 2024-09-21T09:51:18.791 INFO:tasks.rados.rados.0.smithi076.stdout:3180: writing smithi07685715-38 from 467558 to 982285 tid 1 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3180: writing smithi07685715-38 from 1719853 to 2067558 tid 2 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3181: read oid 48 snap -1 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3181: expect (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3182: write oid 13 current snap is 400 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: finishing write tid 2 to smithi07685715-1 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: finishing write tid 3 to smithi07685715-1 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: finishing write tid 4 to smithi07685715-1 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: oid 1 updating version 0 to 2409 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: oid 1 updating version 2409 to 2410 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: oid 1 version 2410 is already newer than 2408 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2410 (ObjNum 1193 snap 400 seq_num 1193) dirty exists 2024-09-21T09:51:18.798 INFO:tasks.rados.rados.0.smithi076.stdout:3179: left oid 1 (ObjNum 1193 snap 400 seq_num 1193) 2024-09-21T09:51:18.799 INFO:tasks.rados.rados.0.smithi076.stdout:3182: seq_num 1195 ranges {484365=589976,1661982=422383} 2024-09-21T09:51:18.803 INFO:tasks.rados.rados.0.smithi076.stdout:3182: writing smithi07685715-13 from 484365 to 1074341 tid 1 2024-09-21T09:51:18.810 INFO:tasks.rados.rados.0.smithi076.stdout:3182: writing smithi07685715-13 from 1661982 to 2084365 tid 2 2024-09-21T09:51:18.810 INFO:tasks.rados.rados.0.smithi076.stdout:3179: done (3 left) 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3183: read oid 4 snap -1 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3183: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3184: snap_create 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: finishing write tid 1 to smithi07685715-38 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: finishing write tid 2 to smithi07685715-38 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: finishing write tid 3 to smithi07685715-38 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: finishing write tid 4 to smithi07685715-38 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: oid 38 updating version 0 to 1900 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: oid 38 updating version 1900 to 1901 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: oid 38 version 1901 is already newer than 1899 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1901 (ObjNum 1194 snap 400 seq_num 1194) dirty exists 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3180: left oid 38 (ObjNum 1194 snap 400 seq_num 1194) 2024-09-21T09:51:18.811 INFO:tasks.rados.rados.0.smithi076.stdout:3182: finishing write tid 1 to smithi07685715-13 2024-09-21T09:51:18.812 INFO:tasks.rados.rados.0.smithi076.stdout:3181: expect (ObjNum 1116 snap 369 seq_num 1116) 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: finishing write tid 2 to smithi07685715-13 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: finishing write tid 3 to smithi07685715-13 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: finishing write tid 4 to smithi07685715-13 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: oid 13 updating version 0 to 625 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: oid 13 version 625 is already newer than 624 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: oid 13 version 625 is already newer than 623 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 625 (ObjNum 1195 snap 400 seq_num 1195) dirty exists 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3182: left oid 13 (ObjNum 1195 snap 400 seq_num 1195) 2024-09-21T09:51:18.990 INFO:tasks.rados.rados.0.smithi076.stdout:3183: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:51:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:18 smithi191 ceph-mon[102206]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:51:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:18 smithi191 ceph-mon[102206]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:51:19.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:18 smithi076 ceph-mon[106482]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:51:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:18 smithi076 ceph-mon[106482]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:51:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:18 smithi076 ceph-mon[110963]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T09:51:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:18 smithi076 ceph-mon[110963]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T09:51:19.807 INFO:tasks.rados.rados.0.smithi076.stdout:3180: done (4 left) 2024-09-21T09:51:19.807 INFO:tasks.rados.rados.0.smithi076.stdout:3181: done (3 left) 2024-09-21T09:51:19.807 INFO:tasks.rados.rados.0.smithi076.stdout:3182: done (2 left) 2024-09-21T09:51:19.807 INFO:tasks.rados.rados.0.smithi076.stdout:3183: done (1 left) 2024-09-21T09:51:19.807 INFO:tasks.rados.rados.0.smithi076.stdout:3184: done (0 left) 2024-09-21T09:51:19.807 INFO:tasks.rados.rados.0.smithi076.stdout:3185: write oid 19 current snap is 401 2024-09-21T09:51:19.808 INFO:tasks.rados.rados.0.smithi076.stdout:3185: seq_num 1196 ranges {501172=665225,1720464=380708} 2024-09-21T09:51:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3185: writing smithi07685715-19 from 501172 to 1166397 tid 1 2024-09-21T09:51:19.819 INFO:tasks.rados.rados.0.smithi076.stdout:3185: writing smithi07685715-19 from 1720464 to 2101172 tid 2 2024-09-21T09:51:19.819 INFO:tasks.rados.rados.0.smithi076.stdout:3186: rollback oid 2 current snap is 401 2024-09-21T09:51:19.819 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 400 2024-09-21T09:51:19.819 INFO:tasks.rados.rados.0.smithi076.stdout:3185: finishing write tid 1 to smithi07685715-19 2024-09-21T09:51:19.819 INFO:tasks.rados.rados.0.smithi076.stdout:3187: delete oid 15 current snap is 401 2024-09-21T09:51:19.821 INFO:tasks.rados.rados.0.smithi076.stdout:3187: done (2 left) 2024-09-21T09:51:19.821 INFO:tasks.rados.rados.0.smithi076.stdout:3188: delete oid 8 current snap is 401 2024-09-21T09:51:19.822 INFO:tasks.rados.rados.0.smithi076.stdout:3186: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:51:19.822 INFO:tasks.rados.rados.0.smithi076.stdout:3185: finishing write tid 2 to smithi07685715-19 2024-09-21T09:51:19.823 INFO:tasks.rados.rados.0.smithi076.stdout:3185: finishing write tid 3 to smithi07685715-19 2024-09-21T09:51:19.823 INFO:tasks.rados.rados.0.smithi076.stdout:3186: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:51:19.823 INFO:tasks.rados.rados.0.smithi076.stdout:3185: finishing write tid 4 to smithi07685715-19 2024-09-21T09:51:19.823 INFO:tasks.rados.rados.0.smithi076.stdout:3185: oid 19 updating version 0 to 920 2024-09-21T09:51:19.823 INFO:tasks.rados.rados.0.smithi076.stdout:3185: oid 19 version 920 is already newer than 919 2024-09-21T09:51:19.824 INFO:tasks.rados.rados.0.smithi076.stdout:3185: oid 19 version 920 is already newer than 918 2024-09-21T09:51:19.824 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 920 (ObjNum 1196 snap 401 seq_num 1196) dirty exists 2024-09-21T09:51:19.824 INFO:tasks.rados.rados.0.smithi076.stdout:3185: left oid 19 (ObjNum 1196 snap 401 seq_num 1196) 2024-09-21T09:51:19.824 INFO:tasks.rados.rados.0.smithi076.stdout:3186: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:51:19.824 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 629 (ObjNum 1192 snap 399 seq_num 1192) dirty exists 2024-09-21T09:51:19.825 INFO:tasks.rados.rados.0.smithi076.stdout:3185: done (2 left) 2024-09-21T09:51:19.825 INFO:tasks.rados.rados.0.smithi076.stdout:3186: done (1 left) 2024-09-21T09:51:19.825 INFO:tasks.rados.rados.0.smithi076.stdout:3188: done (0 left) 2024-09-21T09:51:19.825 INFO:tasks.rados.rados.0.smithi076.stdout:3189: write oid 32 current snap is 401 2024-09-21T09:51:19.825 INFO:tasks.rados.rados.0.smithi076.stdout:3189: seq_num 1199 ranges {551593=490972,1779557=372036} 2024-09-21T09:51:19.830 INFO:tasks.rados.rados.0.smithi076.stdout:3189: writing smithi07685715-32 from 551593 to 1042565 tid 1 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:3189: writing smithi07685715-32 from 1779557 to 2151593 tid 2 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:3190: rollback oid 35 current snap is 401 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 384 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:3189: finishing write tid 1 to smithi07685715-32 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:3191: rollback oid 38 current snap is 401 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 400 2024-09-21T09:51:19.837 INFO:tasks.rados.rados.0.smithi076.stdout:3192: snap_remove snap 367 2024-09-21T09:51:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:19 smithi191 ceph-mon[102206]: pgmap v766: 89 pgs: 89 active+clean; 287 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 388 KiB/s wr, 11 op/s 2024-09-21T09:51:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:19 smithi076 ceph-mon[106482]: pgmap v766: 89 pgs: 89 active+clean; 287 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 388 KiB/s wr, 11 op/s 2024-09-21T09:51:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:19 smithi076 ceph-mon[110963]: pgmap v766: 89 pgs: 89 active+clean; 287 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 388 KiB/s wr, 11 op/s 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3191: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3189: finishing write tid 2 to smithi07685715-32 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3189: finishing write tid 3 to smithi07685715-32 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3190: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3189: finishing write tid 4 to smithi07685715-32 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3189: oid 32 updating version 0 to 921 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3189: oid 32 updating version 921 to 922 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:3189: oid 32 updating version 922 to 923 2024-09-21T09:51:20.815 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 923 (ObjNum 1199 snap 401 seq_num 1199) dirty exists 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3189: left oid 32 (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3190: finishing rollback tid 2 to smithi07685715-35 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2412 (ObjNum 1140 snap 378 seq_num 1140) dirty exists 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3191: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3191: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1905 (ObjNum 1194 snap 400 seq_num 1194) dirty exists 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3189: done (3 left) 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3190: done (2 left) 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3191: done (1 left) 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3192: done (0 left) 2024-09-21T09:51:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3193: write oid 1 current snap is 401 2024-09-21T09:51:20.817 INFO:tasks.rados.rados.0.smithi076.stdout:3193: seq_num 1200 ranges {568400=566221,1838039=330361} 2024-09-21T09:51:20.821 INFO:tasks.rados.rados.0.smithi076.stdout:3193: writing smithi07685715-1 from 568400 to 1134621 tid 1 2024-09-21T09:51:20.828 INFO:tasks.rados.rados.0.smithi076.stdout:3193: writing smithi07685715-1 from 1838039 to 2168400 tid 2 2024-09-21T09:51:20.828 INFO:tasks.rados.rados.0.smithi076.stdout:3194: snap_remove snap 384 2024-09-21T09:51:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:20 smithi191 ceph-mon[102206]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:51:21.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:20 smithi076 ceph-mon[106482]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:51:21.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:20 smithi076 ceph-mon[110963]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T09:51:21.817 INFO:tasks.rados.rados.0.smithi076.stdout:3193: finishing write tid 1 to smithi07685715-1 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: finishing write tid 2 to smithi07685715-1 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: finishing write tid 3 to smithi07685715-1 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: finishing write tid 4 to smithi07685715-1 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: oid 1 updating version 0 to 2425 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: oid 1 version 2425 is already newer than 2422 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: oid 1 updating version 2425 to 2426 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2426 (ObjNum 1200 snap 401 seq_num 1200) dirty exists 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: left oid 1 (ObjNum 1200 snap 401 seq_num 1200) 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3193: done (1 left) 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3194: done (0 left) 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3195: read oid 9 snap -1 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3195: expect (ObjNum 1166 snap 393 seq_num 1166) 2024-09-21T09:51:21.818 INFO:tasks.rados.rados.0.smithi076.stdout:3196: snap_create 2024-09-21T09:51:21.832 INFO:tasks.rados.rados.0.smithi076.stdout:3195: expect (ObjNum 1166 snap 393 seq_num 1166) 2024-09-21T09:51:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:21 smithi191 ceph-mon[102206]: pgmap v769: 89 pgs: 89 active+clean; 295 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:51:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:21 smithi191 ceph-mon[102206]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:51:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:21 smithi076 ceph-mon[106482]: pgmap v769: 89 pgs: 89 active+clean; 295 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:51:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:21 smithi076 ceph-mon[106482]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:51:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:21 smithi076 ceph-mon[110963]: pgmap v769: 89 pgs: 89 active+clean; 295 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-09-21T09:51:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:21 smithi076 ceph-mon[110963]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T09:51:22.824 INFO:tasks.rados.rados.0.smithi076.stdout:3195: done (1 left) 2024-09-21T09:51:22.824 INFO:tasks.rados.rados.0.smithi076.stdout:3196: done (0 left) 2024-09-21T09:51:22.824 INFO:tasks.rados.rados.0.smithi076.stdout:3197: write oid 31 current snap is 402 2024-09-21T09:51:22.824 INFO:tasks.rados.rados.0.smithi076.stdout:3197: seq_num 1201 ranges {585207=641470,1896521=288686} 2024-09-21T09:51:22.830 INFO:tasks.rados.rados.0.smithi076.stdout:3197: writing smithi07685715-31 from 585207 to 1226677 tid 1 2024-09-21T09:51:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3197: writing smithi07685715-31 from 1896521 to 2185207 tid 2 2024-09-21T09:51:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3198: delete oid 2 current snap is 402 2024-09-21T09:51:22.839 INFO:tasks.rados.rados.0.smithi076.stdout:3197: finishing write tid 1 to smithi07685715-31 2024-09-21T09:51:22.840 INFO:tasks.rados.rados.0.smithi076.stdout:3198: done (1 left) 2024-09-21T09:51:22.840 INFO:tasks.rados.rados.0.smithi076.stdout:3199: snap_create 2024-09-21T09:51:22.840 INFO:tasks.rados.rados.0.smithi076.stdout:3197: finishing write tid 2 to smithi07685715-31 2024-09-21T09:51:22.840 INFO:tasks.rados.rados.0.smithi076.stdout:3197: finishing write tid 3 to smithi07685715-31 2024-09-21T09:51:22.840 INFO:tasks.rados.rados.0.smithi076.stdout:3197: finishing write tid 4 to smithi07685715-31 2024-09-21T09:51:22.841 INFO:tasks.rados.rados.0.smithi076.stdout:3197: oid 31 updating version 0 to 930 2024-09-21T09:51:22.841 INFO:tasks.rados.rados.0.smithi076.stdout:3197: oid 31 updating version 930 to 931 2024-09-21T09:51:22.841 INFO:tasks.rados.rados.0.smithi076.stdout:3197: oid 31 version 931 is already newer than 929 2024-09-21T09:51:22.841 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 931 (ObjNum 1201 snap 402 seq_num 1201) dirty exists 2024-09-21T09:51:22.841 INFO:tasks.rados.rados.0.smithi076.stdout:3197: left oid 31 (ObjNum 1201 snap 402 seq_num 1201) 2024-09-21T09:51:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:22 smithi191 ceph-mon[102206]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:51:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:22 smithi076 ceph-mon[106482]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:51:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:22 smithi076 ceph-mon[110963]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T09:51:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:51:23] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:51:23.776 INFO:tasks.rados.rados.0.smithi076.stdout:3197: done (1 left) 2024-09-21T09:51:23.777 INFO:tasks.rados.rados.0.smithi076.stdout:3199: done (0 left) 2024-09-21T09:51:23.777 INFO:tasks.rados.rados.0.smithi076.stdout:3200: write oid 13 current snap is 403 2024-09-21T09:51:23.777 INFO:tasks.rados.rados.0.smithi076.stdout:3200: seq_num 1203 ranges {618821=508321,1613485=531077} 2024-09-21T09:51:23.782 INFO:tasks.rados.rados.0.smithi076.stdout:3200: writing smithi07685715-13 from 618821 to 1127142 tid 1 2024-09-21T09:51:23.788 INFO:tasks.rados.rados.0.smithi076.stdout:3200: writing smithi07685715-13 from 1613485 to 2144562 tid 2 2024-09-21T09:51:23.789 INFO:tasks.rados.rados.0.smithi076.stdout:3201: snap_create 2024-09-21T09:51:23.789 INFO:tasks.rados.rados.0.smithi076.stdout:3200: finishing write tid 1 to smithi07685715-13 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: finishing write tid 2 to smithi07685715-13 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: finishing write tid 3 to smithi07685715-13 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: finishing write tid 4 to smithi07685715-13 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: oid 13 updating version 0 to 641 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: oid 13 version 641 is already newer than 639 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: oid 13 version 641 is already newer than 640 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 641 (ObjNum 1203 snap 403 seq_num 1203) dirty exists 2024-09-21T09:51:23.792 INFO:tasks.rados.rados.0.smithi076.stdout:3200: left oid 13 (ObjNum 1203 snap 403 seq_num 1203) 2024-09-21T09:51:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:23 smithi191 ceph-mon[102206]: pgmap v772: 89 pgs: 89 active+clean; 307 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-09-21T09:51:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:23 smithi191 ceph-mon[102206]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:51:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:23 smithi191 ceph-mon[102206]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:51:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-mon[106482]: pgmap v772: 89 pgs: 89 active+clean; 307 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-09-21T09:51:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-mon[106482]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:51:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-mon[106482]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:51:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-mon[110963]: pgmap v772: 89 pgs: 89 active+clean; 307 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-09-21T09:51:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-mon[110963]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T09:51:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:23 smithi076 ceph-mon[110963]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T09:51:24.780 INFO:tasks.rados.rados.0.smithi076.stdout:3200: done (1 left) 2024-09-21T09:51:24.780 INFO:tasks.rados.rados.0.smithi076.stdout:3201: done (0 left) 2024-09-21T09:51:24.780 INFO:tasks.rados.rados.0.smithi076.stdout:3202: write oid 38 current snap is 404 2024-09-21T09:51:24.780 INFO:tasks.rados.rados.0.smithi076.stdout:3202: seq_num 1204 ranges {635628=583570,1671967=563661} 2024-09-21T09:51:24.786 INFO:tasks.rados.rados.0.smithi076.stdout:3202: writing smithi07685715-38 from 635628 to 1219198 tid 1 2024-09-21T09:51:24.793 INFO:tasks.rados.rados.0.smithi076.stdout:3202: writing smithi07685715-38 from 1671967 to 2235628 tid 2 2024-09-21T09:51:24.793 INFO:tasks.rados.rados.0.smithi076.stdout:3203: write oid 18 current snap is 404 2024-09-21T09:51:24.793 INFO:tasks.rados.rados.0.smithi076.stdout:3203: seq_num 1205 ranges {652435=658819,1730449=521986} 2024-09-21T09:51:24.800 INFO:tasks.rados.rados.0.smithi076.stdout:3203: writing smithi07685715-18 from 652435 to 1311254 tid 1 2024-09-21T09:51:24.806 INFO:tasks.rados.rados.0.smithi076.stdout:3203: writing smithi07685715-18 from 1730449 to 2252435 tid 2 2024-09-21T09:51:24.806 INFO:tasks.rados.rados.0.smithi076.stdout:3204: rollback oid 40 current snap is 404 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 372 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: finishing write tid 1 to smithi07685715-38 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: finishing write tid 2 to smithi07685715-38 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: finishing write tid 3 to smithi07685715-38 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3205: read oid 22 snap 399 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3205: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3206: read oid 12 snap -1 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3206: expect (ObjNum 1172 snap 394 seq_num 1172) 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3207: rollback oid 41 current snap is 404 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 359 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: finishing write tid 4 to smithi07685715-38 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: oid 38 updating version 0 to 1923 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: oid 38 updating version 1923 to 1925 2024-09-21T09:51:24.807 INFO:tasks.rados.rados.0.smithi076.stdout:3202: oid 38 version 1925 is already newer than 1924 2024-09-21T09:51:24.808 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 1925 (ObjNum 1204 snap 404 seq_num 1204) dirty exists 2024-09-21T09:51:24.808 INFO:tasks.rados.rados.0.smithi076.stdout:3202: left oid 38 (ObjNum 1204 snap 404 seq_num 1204) 2024-09-21T09:51:24.808 INFO:tasks.rados.rados.0.smithi076.stdout:3202: done (5 left) 2024-09-21T09:51:24.808 INFO:tasks.rados.rados.0.smithi076.stdout:3208: write oid 1 current snap is 404 2024-09-21T09:51:24.808 INFO:tasks.rados.rados.0.smithi076.stdout:3208: seq_num 1206 ranges {669242=734068,2188931=80311} 2024-09-21T09:51:24.814 INFO:tasks.rados.rados.0.smithi076.stdout:3208: writing smithi07685715-1 from 669242 to 1403310 tid 1 2024-09-21T09:51:24.819 INFO:tasks.rados.rados.0.smithi076.stdout:3208: writing smithi07685715-1 from 2188931 to 2269242 tid 2 2024-09-21T09:51:24.819 INFO:tasks.rados.rados.0.smithi076.stdout:3209: delete oid 17 current snap is 404 2024-09-21T09:51:24.819 INFO:tasks.rados.rados.0.smithi076.stdout:3203: finishing write tid 1 to smithi07685715-18 2024-09-21T09:51:24.819 INFO:tasks.rados.rados.0.smithi076.stdout:3206: expect (ObjNum 1172 snap 394 seq_num 1172) 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: finishing write tid 2 to smithi07685715-18 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: finishing write tid 3 to smithi07685715-18 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: finishing write tid 4 to smithi07685715-18 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: oid 18 updating version 0 to 797 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: oid 18 version 797 is already newer than 796 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: oid 18 updating version 797 to 798 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 798 (ObjNum 1205 snap 404 seq_num 1205) dirty exists 2024-09-21T09:51:24.985 INFO:tasks.rados.rados.0.smithi076.stdout:3203: left oid 18 (ObjNum 1205 snap 404 seq_num 1205) 2024-09-21T09:51:24.986 INFO:tasks.rados.rados.0.smithi076.stdout:3204: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:51:24.986 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 0 (ObjNum 145 snap 0 seq_num 864183432) dirty dne 2024-09-21T09:51:24.986 INFO:tasks.rados.rados.0.smithi076.stdout:3205: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3207: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3207: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3207: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1496 (ObjNum 1041 snap 351 seq_num 1041) dirty exists 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: finishing write tid 1 to smithi07685715-1 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: finishing write tid 2 to smithi07685715-1 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: finishing write tid 3 to smithi07685715-1 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: finishing write tid 4 to smithi07685715-1 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: oid 1 updating version 0 to 2439 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: oid 1 updating version 2439 to 2440 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:3208: oid 1 version 2440 is already newer than 2438 2024-09-21T09:51:25.214 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2440 (ObjNum 1206 snap 404 seq_num 1206) dirty exists 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3208: left oid 1 (ObjNum 1206 snap 404 seq_num 1206) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3203: done (6 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3204: done (5 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3205: done (4 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3206: done (3 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3207: done (2 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3208: done (1 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3209: done (0 left) 2024-09-21T09:51:25.215 INFO:tasks.rados.rados.0.smithi076.stdout:3210: snap_remove snap 391 2024-09-21T09:51:25.827 INFO:tasks.rados.rados.0.smithi076.stdout:3210: done (0 left) 2024-09-21T09:51:25.827 INFO:tasks.rados.rados.0.smithi076.stdout:3211: rollback oid 8 current snap is 404 2024-09-21T09:51:25.827 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 395 2024-09-21T09:51:25.827 INFO:tasks.rados.rados.0.smithi076.stdout:3212: read oid 20 snap -1 2024-09-21T09:51:25.828 INFO:tasks.rados.rados.0.smithi076.stdout:3212: expect (ObjNum 1127 snap 371 seq_num 1127) 2024-09-21T09:51:25.828 INFO:tasks.rados.rados.0.smithi076.stdout:3213: delete oid 9 current snap is 404 2024-09-21T09:51:25.835 INFO:tasks.rados.rados.0.smithi076.stdout:3213: done (2 left) 2024-09-21T09:51:25.835 INFO:tasks.rados.rados.0.smithi076.stdout:3214: snap_remove snap 398 2024-09-21T09:51:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:25 smithi191 ceph-mon[102206]: pgmap v775: 89 pgs: 89 active+clean; 308 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:51:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:25 smithi191 ceph-mon[102206]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:51:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:25 smithi076 ceph-mon[106482]: pgmap v775: 89 pgs: 89 active+clean; 308 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:51:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:25 smithi076 ceph-mon[106482]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:51:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:25 smithi076 ceph-mon[110963]: pgmap v775: 89 pgs: 89 active+clean; 308 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:51:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:25 smithi076 ceph-mon[110963]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T09:51:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:26.838 INFO:tasks.rados.rados.0.smithi076.stdout:3214: done (2 left) 2024-09-21T09:51:26.838 INFO:tasks.rados.rados.0.smithi076.stdout:3215: read oid 2 snap -1 2024-09-21T09:51:26.838 INFO:tasks.rados.rados.0.smithi076.stdout:3215: expect deleted 2024-09-21T09:51:26.838 INFO:tasks.rados.rados.0.smithi076.stdout:3216: snap_remove snap 388 2024-09-21T09:51:27.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:26 smithi191 ceph-mon[102206]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:51:27.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:26 smithi076 ceph-mon[106482]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:51:27.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:26 smithi076 ceph-mon[110963]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T09:51:27.846 INFO:tasks.rados.rados.0.smithi076.stdout:3216: done (3 left) 2024-09-21T09:51:27.846 INFO:tasks.rados.rados.0.smithi076.stdout:3217: write oid 31 current snap is 404 2024-09-21T09:51:27.846 INFO:tasks.rados.rados.0.smithi076.stdout:3217: seq_num 1209 ranges {719663=676168,1964377=355286} 2024-09-21T09:51:27.854 INFO:tasks.rados.rados.0.smithi076.stdout:3217: writing smithi07685715-31 from 719663 to 1395831 tid 1 2024-09-21T09:51:27.859 INFO:tasks.rados.rados.0.smithi076.stdout:3217: writing smithi07685715-31 from 1964377 to 2319663 tid 2 2024-09-21T09:51:27.861 INFO:tasks.rados.rados.0.smithi076.stdout:3212: expect (ObjNum 1127 snap 371 seq_num 1127) 2024-09-21T09:51:28.018 INFO:tasks.rados.rados.0.smithi076.stdout:3211: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:51:28.018 INFO:tasks.rados.rados.0.smithi076.stdout:3211: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:51:28.018 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2384 (ObjNum 1070 snap 358 seq_num 1070) dirty exists 2024-09-21T09:51:28.018 INFO:tasks.rados.rados.0.smithi076.stdout:3217: finishing write tid 1 to smithi07685715-31 2024-09-21T09:51:28.018 INFO:tasks.rados.rados.0.smithi076.stdout:3211: done (3 left) 2024-09-21T09:51:28.018 INFO:tasks.rados.rados.0.smithi076.stdout:3212: done (2 left) 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3215: done (1 left) 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3218: delete oid 9 current snap is 404 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: finishing write tid 2 to smithi07685715-31 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: finishing write tid 3 to smithi07685715-31 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: finishing write tid 4 to smithi07685715-31 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: oid 31 updating version 0 to 940 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: oid 31 updating version 940 to 941 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: oid 31 version 941 is already newer than 939 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 941 (ObjNum 1209 snap 404 seq_num 1209) dirty exists 2024-09-21T09:51:28.019 INFO:tasks.rados.rados.0.smithi076.stdout:3217: left oid 31 (ObjNum 1209 snap 404 seq_num 1209) 2024-09-21T09:51:28.020 INFO:tasks.rados.rados.0.smithi076.stdout:3217: done (1 left) 2024-09-21T09:51:28.020 INFO:tasks.rados.rados.0.smithi076.stdout:3218: done (0 left) 2024-09-21T09:51:28.020 INFO:tasks.rados.rados.0.smithi076.stdout:3219: write oid 9 current snap is 404 2024-09-21T09:51:28.020 INFO:tasks.rados.rados.0.smithi076.stdout:3219: seq_num 1211 ranges {753277=426666,1681341=545753} 2024-09-21T09:51:28.026 INFO:tasks.rados.rados.0.smithi076.stdout:3219: writing smithi07685715-9 from 753277 to 1179943 tid 1 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:3219: writing smithi07685715-9 from 1681341 to 2227094 tid 2 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:3220: rollback oid 19 current snap is 404 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 373 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:3219: finishing write tid 1 to smithi07685715-9 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:3221: read oid 7 snap -1 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:3221: expect (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:51:28.032 INFO:tasks.rados.rados.0.smithi076.stdout:3222: write oid 31 current snap is 404 2024-09-21T09:51:28.033 INFO:tasks.rados.rados.0.smithi076.stdout:3222: seq_num 1212 ranges {770084=501915,2023470=346614} 2024-09-21T09:51:28.039 INFO:tasks.rados.rados.0.smithi076.stdout:3222: writing smithi07685715-31 from 770084 to 1271999 tid 1 2024-09-21T09:51:28.045 INFO:tasks.rados.rados.0.smithi076.stdout:3222: writing smithi07685715-31 from 2023470 to 2370084 tid 2 2024-09-21T09:51:28.045 INFO:tasks.rados.rados.0.smithi076.stdout:3223: read oid 40 snap -1 2024-09-21T09:51:28.045 INFO:tasks.rados.rados.0.smithi076.stdout:3223: expect deleted 2024-09-21T09:51:28.045 INFO:tasks.rados.rados.0.smithi076.stdout:3224: snap_remove snap 355 2024-09-21T09:51:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:27 smithi191 ceph-mon[102206]: pgmap v778: 89 pgs: 89 active+clean; 311 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T09:51:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:27 smithi191 ceph-mon[102206]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:51:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:27 smithi076 ceph-mon[106482]: pgmap v778: 89 pgs: 89 active+clean; 311 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T09:51:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:27 smithi076 ceph-mon[106482]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:51:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:27 smithi076 ceph-mon[110963]: pgmap v778: 89 pgs: 89 active+clean; 311 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T09:51:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:27 smithi076 ceph-mon[110963]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T09:51:28.779 INFO:tasks.rados.rados.0.smithi076.stdout:3224: done (5 left) 2024-09-21T09:51:28.779 INFO:tasks.rados.rados.0.smithi076.stdout:3225: delete oid 3 current snap is 404 2024-09-21T09:51:28.779 INFO:tasks.rados.rados.0.smithi076.stdout:3221: expect (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3220: finishing rollback tid 0 to smithi07685715-19 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3219: finishing write tid 2 to smithi07685715-9 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3220: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3219: finishing write tid 3 to smithi07685715-9 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3220: finishing rollback tid 2 to smithi07685715-19 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 947 (ObjNum 1032 snap 349 seq_num 1032) dirty exists 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3219: finishing write tid 4 to smithi07685715-9 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3219: oid 9 updating version 0 to 1091 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3219: oid 9 version 1091 is already newer than 1089 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:3219: oid 9 version 1091 is already newer than 1090 2024-09-21T09:51:28.962 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 1091 (ObjNum 1211 snap 404 seq_num 1211) dirty exists 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3219: left oid 9 (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3222: finishing write tid 1 to smithi07685715-31 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3219: done (5 left) 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3220: done (4 left) 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3221: done (3 left) 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3225: done (2 left) 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3226: write oid 23 current snap is 404 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3222: finishing write tid 2 to smithi07685715-31 2024-09-21T09:51:28.963 INFO:tasks.rados.rados.0.smithi076.stdout:3226: seq_num 1214 ranges {403698=652413,1740434=663264} 2024-09-21T09:51:28.967 INFO:tasks.rados.rados.0.smithi076.stdout:3226: writing smithi07685715-23 from 403698 to 1056111 tid 1 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3226: writing smithi07685715-23 from 1740434 to 2403698 tid 2 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3223: done (2 left) 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3227: snap_create 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3222: finishing write tid 3 to smithi07685715-31 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3222: finishing write tid 4 to smithi07685715-31 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3222: oid 31 updating version 0 to 948 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3222: oid 31 updating version 948 to 950 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3222: oid 31 version 950 is already newer than 949 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 950 (ObjNum 1212 snap 404 seq_num 1212) dirty exists 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3222: left oid 31 (ObjNum 1212 snap 404 seq_num 1212) 2024-09-21T09:51:28.975 INFO:tasks.rados.rados.0.smithi076.stdout:3226: finishing write tid 1 to smithi07685715-23 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: finishing write tid 2 to smithi07685715-23 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: finishing write tid 3 to smithi07685715-23 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: finishing write tid 4 to smithi07685715-23 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: oid 23 updating version 0 to 2418 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: oid 23 version 2418 is already newer than 2417 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: oid 23 version 2418 is already newer than 2416 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2418 (ObjNum 1214 snap 404 seq_num 1214) dirty exists 2024-09-21T09:51:28.979 INFO:tasks.rados.rados.0.smithi076.stdout:3226: left oid 23 (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:51:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:28 smithi191 ceph-mon[102206]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:51:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:28 smithi191 ceph-mon[102206]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:51:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:28 smithi076 ceph-mon[106482]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:51:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:28 smithi076 ceph-mon[106482]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:51:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:28 smithi076 ceph-mon[110963]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T09:51:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:28 smithi076 ceph-mon[110963]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:3222: done (2 left) 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:3226: done (1 left) 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:3227: done (0 left) 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:3228: rollback oid 11 current snap is 405 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 395 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:3229: read oid 22 snap 399 2024-09-21T09:51:29.883 INFO:tasks.rados.rados.0.smithi076.stdout:3229: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:51:29.884 INFO:tasks.rados.rados.0.smithi076.stdout:3230: snap_remove snap 379 2024-09-21T09:51:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:29 smithi191 ceph-mon[102206]: pgmap v781: 89 pgs: 89 active+clean; 311 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-21T09:51:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:29 smithi076 ceph-mon[106482]: pgmap v781: 89 pgs: 89 active+clean; 311 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-21T09:51:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:29 smithi076 ceph-mon[110963]: pgmap v781: 89 pgs: 89 active+clean; 311 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-21T09:51:30.893 INFO:tasks.rados.rados.0.smithi076.stdout:3230: done (2 left) 2024-09-21T09:51:30.893 INFO:tasks.rados.rados.0.smithi076.stdout:3231: snap_remove snap 397 2024-09-21T09:51:30.893 INFO:tasks.rados.rados.0.smithi076.stdout:3228: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:51:30.893 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2316 (ObjNum 129 snap 0 seq_num 864555816) dirty dne 2024-09-21T09:51:30.893 INFO:tasks.rados.rados.0.smithi076.stdout:3229: expect (ObjNum 1123 snap 370 seq_num 1123) 2024-09-21T09:51:31.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:30 smithi191 ceph-mon[102206]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:51:31.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:30 smithi076 ceph-mon[106482]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:51:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:30 smithi076 ceph-mon[110963]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3228: done (2 left) 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3229: done (1 left) 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3231: done (0 left) 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3232: read oid 37 snap -1 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3232: expect (ObjNum 1184 snap 399 seq_num 1184) 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3233: rollback oid 12 current snap is 405 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 351 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3234: rollback oid 13 current snap is 405 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 13 to 403 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3235: read oid 27 snap -1 2024-09-21T09:51:31.890 INFO:tasks.rados.rados.0.smithi076.stdout:3235: expect (ObjNum 1144 snap 380 seq_num 1144) 2024-09-21T09:51:31.891 INFO:tasks.rados.rados.0.smithi076.stdout:3236: delete oid 14 current snap is 405 2024-09-21T09:51:31.900 INFO:tasks.rados.rados.0.smithi076.stdout:3234: finishing rollback tid 0 to smithi07685715-13 2024-09-21T09:51:31.900 INFO:tasks.rados.rados.0.smithi076.stdout:3233: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:51:31.900 INFO:tasks.rados.rados.0.smithi076.stdout:3234: finishing rollback tid 1 to smithi07685715-13 2024-09-21T09:51:31.900 INFO:tasks.rados.rados.0.smithi076.stdout:3234: finishing rollback tid 2 to smithi07685715-13 2024-09-21T09:51:31.900 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 657 (ObjNum 1203 snap 403 seq_num 1203) dirty exists 2024-09-21T09:51:31.901 INFO:tasks.rados.rados.0.smithi076.stdout:3233: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:51:31.901 INFO:tasks.rados.rados.0.smithi076.stdout:3233: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:51:31.901 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1103 (ObjNum 981 snap 336 seq_num 981) dirty exists 2024-09-21T09:51:31.904 INFO:tasks.rados.rados.0.smithi076.stdout:3233: done (4 left) 2024-09-21T09:51:31.905 INFO:tasks.rados.rados.0.smithi076.stdout:3234: done (3 left) 2024-09-21T09:51:31.905 INFO:tasks.rados.rados.0.smithi076.stdout:3236: done (2 left) 2024-09-21T09:51:31.905 INFO:tasks.rados.rados.0.smithi076.stdout:3237: delete oid 17 current snap is 405 2024-09-21T09:51:31.905 INFO:tasks.rados.rados.0.smithi076.stdout:3235: expect (ObjNum 1144 snap 380 seq_num 1144) 2024-09-21T09:51:32.008 INFO:tasks.rados.rados.0.smithi076.stdout:3232: expect (ObjNum 1184 snap 399 seq_num 1184) 2024-09-21T09:51:32.130 INFO:tasks.rados.rados.0.smithi076.stdout:3232: done (2 left) 2024-09-21T09:51:32.130 INFO:tasks.rados.rados.0.smithi076.stdout:3235: done (1 left) 2024-09-21T09:51:32.130 INFO:tasks.rados.rados.0.smithi076.stdout:3237: done (0 left) 2024-09-21T09:51:32.130 INFO:tasks.rados.rados.0.smithi076.stdout:3238: snap_remove snap 402 2024-09-21T09:51:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:31 smithi191 ceph-mon[102206]: pgmap v784: 89 pgs: 89 active+clean; 316 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:51:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:31 smithi191 ceph-mon[102206]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:51:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:31 smithi076 ceph-mon[106482]: pgmap v784: 89 pgs: 89 active+clean; 316 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:51:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:31 smithi076 ceph-mon[106482]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:51:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:31 smithi076 ceph-mon[110963]: pgmap v784: 89 pgs: 89 active+clean; 316 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:51:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:31 smithi076 ceph-mon[110963]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T09:51:32.902 INFO:tasks.rados.rados.0.smithi076.stdout:3238: done (0 left) 2024-09-21T09:51:32.902 INFO:tasks.rados.rados.0.smithi076.stdout:3239: snap_create 2024-09-21T09:51:33.250 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:32 smithi076 ceph-mon[106482]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:51:33.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:32 smithi076 ceph-mon[110963]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:51:33.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:32 smithi191 ceph-mon[102206]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T09:51:33.511 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:51:33] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.51.0" 2024-09-21T09:51:33.779 INFO:tasks.rados.rados.0.smithi076.stdout:3239: done (0 left) 2024-09-21T09:51:33.779 INFO:tasks.rados.rados.0.smithi076.stdout:3240: write oid 30 current snap is 406 2024-09-21T09:51:33.779 INFO:tasks.rados.rados.0.smithi076.stdout:3240: seq_num 1217 ranges {454119=594513,1515880=556760} 2024-09-21T09:51:33.784 INFO:tasks.rados.rados.0.smithi076.stdout:3240: writing smithi07685715-30 from 454119 to 1048632 tid 1 2024-09-21T09:51:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:3240: writing smithi07685715-30 from 1515880 to 2072640 tid 2 2024-09-21T09:51:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:3241: read oid 44 snap 359 2024-09-21T09:51:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:3241: expect (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:51:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:3242: write oid 24 current snap is 406 2024-09-21T09:51:33.791 INFO:tasks.rados.rados.0.smithi076.stdout:3242: seq_num 1218 ranges {470926=669762,1574362=700418} 2024-09-21T09:51:33.797 INFO:tasks.rados.rados.0.smithi076.stdout:3242: writing smithi07685715-24 from 470926 to 1140688 tid 1 2024-09-21T09:51:33.804 INFO:tasks.rados.rados.0.smithi076.stdout:3242: writing smithi07685715-24 from 1574362 to 2274780 tid 2 2024-09-21T09:51:33.804 INFO:tasks.rados.rados.0.smithi076.stdout:3243: read oid 36 snap -1 2024-09-21T09:51:33.804 INFO:tasks.rados.rados.0.smithi076.stdout:3243: expect deleted 2024-09-21T09:51:33.804 INFO:tasks.rados.rados.0.smithi076.stdout:3244: delete oid 16 current snap is 406 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: finishing write tid 1 to smithi07685715-30 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: finishing write tid 2 to smithi07685715-30 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: finishing write tid 3 to smithi07685715-30 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: finishing write tid 4 to smithi07685715-30 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: oid 30 updating version 0 to 2432 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: oid 30 updating version 2432 to 2433 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: oid 30 version 2433 is already newer than 2431 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2433 (ObjNum 1217 snap 406 seq_num 1217) dirty exists 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3240: left oid 30 (ObjNum 1217 snap 406 seq_num 1217) 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3242: finishing write tid 1 to smithi07685715-24 2024-09-21T09:51:33.805 INFO:tasks.rados.rados.0.smithi076.stdout:3241: expect (ObjNum 1054 snap 353 seq_num 1054) 2024-09-21T09:51:34.030 INFO:tasks.rados.rados.0.smithi076.stdout:3242: finishing write tid 2 to smithi07685715-24 2024-09-21T09:51:34.030 INFO:tasks.rados.rados.0.smithi076.stdout:3240: done (4 left) 2024-09-21T09:51:34.030 INFO:tasks.rados.rados.0.smithi076.stdout:3241: done (3 left) 2024-09-21T09:51:34.030 INFO:tasks.rados.rados.0.smithi076.stdout:3243: done (2 left) 2024-09-21T09:51:34.030 INFO:tasks.rados.rados.0.smithi076.stdout:3244: done (1 left) 2024-09-21T09:51:34.031 INFO:tasks.rados.rados.0.smithi076.stdout:3245: snap_remove snap 359 2024-09-21T09:51:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:33 smithi191 ceph-mon[102206]: pgmap v787: 89 pgs: 89 active+clean; 317 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:51:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:33 smithi191 ceph-mon[102206]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:51:34.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:33 smithi191 ceph-mon[102206]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:51:34.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-mon[106482]: pgmap v787: 89 pgs: 89 active+clean; 317 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:51:34.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-mon[106482]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:51:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-mon[106482]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:51:34.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-mon[110963]: pgmap v787: 89 pgs: 89 active+clean; 317 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2024-09-21T09:51:34.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-mon[110963]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T09:51:34.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:33 smithi076 ceph-mon[110963]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T09:51:34.905 INFO:tasks.rados.rados.0.smithi076.stdout:3245: done (1 left) 2024-09-21T09:51:34.905 INFO:tasks.rados.rados.0.smithi076.stdout:3246: write oid 45 current snap is 406 2024-09-21T09:51:34.905 INFO:tasks.rados.rados.0.smithi076.stdout:3242: finishing write tid 3 to smithi07685715-24 2024-09-21T09:51:34.905 INFO:tasks.rados.rados.0.smithi076.stdout:3246: seq_num 1220 ranges {504540=420260,1574973=704087} 2024-09-21T09:51:34.910 INFO:tasks.rados.rados.0.smithi076.stdout:3246: writing smithi07685715-45 from 504540 to 924800 tid 1 2024-09-21T09:51:34.919 INFO:tasks.rados.rados.0.smithi076.stdout:3246: writing smithi07685715-45 from 1574973 to 2279060 tid 2 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3247: write oid 12 current snap is 406 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3242: finishing write tid 4 to smithi07685715-24 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3242: oid 24 updating version 0 to 324 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3242: oid 24 updating version 324 to 326 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3242: oid 24 version 326 is already newer than 325 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 326 (ObjNum 1218 snap 406 seq_num 1218) dirty exists 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3242: left oid 24 (ObjNum 1218 snap 406 seq_num 1218) 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3246: finishing write tid 1 to smithi07685715-45 2024-09-21T09:51:34.920 INFO:tasks.rados.rados.0.smithi076.stdout:3247: seq_num 1221 ranges {521347=495509,1633455=564098} 2024-09-21T09:51:34.925 INFO:tasks.rados.rados.0.smithi076.stdout:3247: writing smithi07685715-12 from 521347 to 1016856 tid 1 2024-09-21T09:51:34.932 INFO:tasks.rados.rados.0.smithi076.stdout:3247: writing smithi07685715-12 from 1633455 to 2197553 tid 2 2024-09-21T09:51:34.932 INFO:tasks.rados.rados.0.smithi076.stdout:3242: done (2 left) 2024-09-21T09:51:34.932 INFO:tasks.rados.rados.0.smithi076.stdout:3248: read oid 28 snap -1 2024-09-21T09:51:34.933 INFO:tasks.rados.rados.0.smithi076.stdout:3248: expect deleted 2024-09-21T09:51:34.933 INFO:tasks.rados.rados.0.smithi076.stdout:3249: write oid 21 current snap is 406 2024-09-21T09:51:34.933 INFO:tasks.rados.rados.0.smithi076.stdout:3246: finishing write tid 2 to smithi07685715-45 2024-09-21T09:51:34.933 INFO:tasks.rados.rados.0.smithi076.stdout:3249: seq_num 1222 ranges {538154=570758,1691937=707756} 2024-09-21T09:51:34.938 INFO:tasks.rados.rados.0.smithi076.stdout:3249: writing smithi07685715-21 from 538154 to 1108912 tid 1 2024-09-21T09:51:34.946 INFO:tasks.rados.rados.0.smithi076.stdout:3249: writing smithi07685715-21 from 1691937 to 2399693 tid 2 2024-09-21T09:51:34.946 INFO:tasks.rados.rados.0.smithi076.stdout:3250: write oid 8 current snap is 406 2024-09-21T09:51:34.946 INFO:tasks.rados.rados.0.smithi076.stdout:3250: seq_num 1223 ranges {554961=646007,1750419=451414} 2024-09-21T09:51:34.952 INFO:tasks.rados.rados.0.smithi076.stdout:3250: writing smithi07685715-8 from 554961 to 1200968 tid 1 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3250: writing smithi07685715-8 from 1750419 to 2201833 tid 2 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3251: delete oid 26 current snap is 406 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3246: finishing write tid 3 to smithi07685715-45 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3246: finishing write tid 4 to smithi07685715-45 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3246: oid 45 updating version 0 to 695 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3246: oid 45 version 695 is already newer than 694 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3246: oid 45 version 695 is already newer than 693 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 695 (ObjNum 1220 snap 406 seq_num 1220) dirty exists 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3246: left oid 45 (ObjNum 1220 snap 406 seq_num 1220) 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3247: finishing write tid 1 to smithi07685715-12 2024-09-21T09:51:34.959 INFO:tasks.rados.rados.0.smithi076.stdout:3247: finishing write tid 2 to smithi07685715-12 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3247: finishing write tid 3 to smithi07685715-12 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3247: finishing write tid 4 to smithi07685715-12 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3247: oid 12 updating version 0 to 1117 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3247: oid 12 version 1117 is already newer than 1116 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3247: oid 12 version 1117 is already newer than 1115 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1117 (ObjNum 1221 snap 406 seq_num 1221) dirty exists 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3247: left oid 12 (ObjNum 1221 snap 406 seq_num 1221) 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: finishing write tid 1 to smithi07685715-21 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: finishing write tid 2 to smithi07685715-21 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: finishing write tid 3 to smithi07685715-21 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: finishing write tid 4 to smithi07685715-21 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: oid 21 updating version 0 to 2504 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: oid 21 version 2504 is already newer than 2503 2024-09-21T09:51:34.960 INFO:tasks.rados.rados.0.smithi076.stdout:3249: oid 21 updating version 2504 to 2505 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 2505 (ObjNum 1222 snap 406 seq_num 1222) dirty exists 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:3249: left oid 21 (ObjNum 1222 snap 406 seq_num 1222) 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:3250: finishing write tid 1 to smithi07685715-8 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:3246: done (5 left) 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:3247: done (4 left) 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:3248: done (3 left) 2024-09-21T09:51:34.961 INFO:tasks.rados.rados.0.smithi076.stdout:3249: done (2 left) 2024-09-21T09:51:34.962 INFO:tasks.rados.rados.0.smithi076.stdout:3251: done (1 left) 2024-09-21T09:51:34.962 INFO:tasks.rados.rados.0.smithi076.stdout:3252: write oid 44 current snap is 406 2024-09-21T09:51:34.962 INFO:tasks.rados.rados.0.smithi076.stdout:3252: seq_num 1225 ranges {588575=512858,1867383=455083} 2024-09-21T09:51:34.966 INFO:tasks.rados.rados.0.smithi076.stdout:3252: writing smithi07685715-44 from 588575 to 1101433 tid 1 2024-09-21T09:51:34.973 INFO:tasks.rados.rados.0.smithi076.stdout:3252: writing smithi07685715-44 from 1867383 to 2322466 tid 2 2024-09-21T09:51:34.973 INFO:tasks.rados.rados.0.smithi076.stdout:3253: snap_remove snap 378 2024-09-21T09:51:35.373 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 09:51:34 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T09:51:34.922+0000 7f5c62d12640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 1ed after trimming was started 2024-09-21T09:51:35.913 INFO:tasks.rados.rados.0.smithi076.stdout:3253: done (2 left) 2024-09-21T09:51:35.913 INFO:tasks.rados.rados.0.smithi076.stdout:3254: write oid 40 current snap is 406 2024-09-21T09:51:35.914 INFO:tasks.rados.rados.0.smithi076.stdout:3254: seq_num 1226 ranges {605382=588107,1925865=679517} 2024-09-21T09:51:35.921 INFO:tasks.rados.rados.0.smithi076.stdout:3254: writing smithi07685715-40 from 605382 to 1193489 tid 1 2024-09-21T09:51:35.929 INFO:tasks.rados.rados.0.smithi076.stdout:3254: writing smithi07685715-40 from 1925865 to 2605382 tid 2 2024-09-21T09:51:35.930 INFO:tasks.rados.rados.0.smithi076.stdout:3255: snap_remove snap 403 2024-09-21T09:51:36.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:35 smithi191 ceph-mon[102206]: pgmap v790: 89 pgs: 89 active+clean; 318 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 656 KiB/s wr, 4 op/s 2024-09-21T09:51:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:35 smithi191 ceph-mon[102206]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:51:36.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:35 smithi076 ceph-mon[106482]: pgmap v790: 89 pgs: 89 active+clean; 318 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 656 KiB/s wr, 4 op/s 2024-09-21T09:51:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:35 smithi076 ceph-mon[106482]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:51:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:35 smithi076 ceph-mon[110963]: pgmap v790: 89 pgs: 89 active+clean; 318 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 656 KiB/s wr, 4 op/s 2024-09-21T09:51:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:35 smithi076 ceph-mon[110963]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T09:51:36.921 INFO:tasks.rados.rados.0.smithi076.stdout:3255: done (3 left) 2024-09-21T09:51:36.922 INFO:tasks.rados.rados.0.smithi076.stdout:3256: read oid 37 snap -1 2024-09-21T09:51:36.922 INFO:tasks.rados.rados.0.smithi076.stdout:3256: expect (ObjNum 1184 snap 399 seq_num 1184) 2024-09-21T09:51:36.922 INFO:tasks.rados.rados.0.smithi076.stdout:3257: write oid 47 current snap is 406 2024-09-21T09:51:36.922 INFO:tasks.rados.rados.0.smithi076.stdout:3250: finishing write tid 2 to smithi07685715-8 2024-09-21T09:51:36.922 INFO:tasks.rados.rados.0.smithi076.stdout:3257: seq_num 1227 ranges {622189=663356,1984347=458752} 2024-09-21T09:51:36.928 INFO:tasks.rados.rados.0.smithi076.stdout:3257: writing smithi07685715-47 from 622189 to 1285545 tid 1 2024-09-21T09:51:36.936 INFO:tasks.rados.rados.0.smithi076.stdout:3257: writing smithi07685715-47 from 1984347 to 2443099 tid 2 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3258: rollback oid 12 current snap is 406 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 372 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3259: read oid 32 snap -1 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3259: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3260: snap_create 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3250: finishing write tid 3 to smithi07685715-8 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3250: finishing write tid 4 to smithi07685715-8 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3250: oid 8 updating version 0 to 2446 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3250: oid 8 version 2446 is already newer than 2445 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3250: oid 8 updating version 2446 to 2447 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2447 (ObjNum 1223 snap 406 seq_num 1223) dirty exists 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3250: left oid 8 (ObjNum 1223 snap 406 seq_num 1223) 2024-09-21T09:51:36.937 INFO:tasks.rados.rados.0.smithi076.stdout:3252: finishing write tid 1 to smithi07685715-44 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: finishing write tid 2 to smithi07685715-44 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: finishing write tid 3 to smithi07685715-44 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: finishing write tid 4 to smithi07685715-44 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: oid 44 updating version 0 to 737 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: oid 44 version 737 is already newer than 736 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: oid 44 version 737 is already newer than 735 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 737 (ObjNum 1225 snap 406 seq_num 1225) dirty exists 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3252: left oid 44 (ObjNum 1225 snap 406 seq_num 1225) 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3254: finishing write tid 1 to smithi07685715-40 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3254: finishing write tid 2 to smithi07685715-40 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3254: finishing write tid 3 to smithi07685715-40 2024-09-21T09:51:36.938 INFO:tasks.rados.rados.0.smithi076.stdout:3254: finishing write tid 4 to smithi07685715-40 2024-09-21T09:51:36.939 INFO:tasks.rados.rados.0.smithi076.stdout:3254: oid 40 updating version 0 to 1539 2024-09-21T09:51:36.939 INFO:tasks.rados.rados.0.smithi076.stdout:3254: oid 40 version 1539 is already newer than 1535 2024-09-21T09:51:36.939 INFO:tasks.rados.rados.0.smithi076.stdout:3254: oid 40 version 1539 is already newer than 1538 2024-09-21T09:51:36.939 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1539 (ObjNum 1226 snap 406 seq_num 1226) dirty exists 2024-09-21T09:51:36.939 INFO:tasks.rados.rados.0.smithi076.stdout:3254: left oid 40 (ObjNum 1226 snap 406 seq_num 1226) 2024-09-21T09:51:36.939 INFO:tasks.rados.rados.0.smithi076.stdout:3256: expect (ObjNum 1184 snap 399 seq_num 1184) 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: finishing write tid 1 to smithi07685715-47 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: finishing write tid 2 to smithi07685715-47 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: finishing write tid 3 to smithi07685715-47 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: finishing write tid 4 to smithi07685715-47 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: oid 47 updating version 0 to 2456 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: oid 47 version 2456 is already newer than 2455 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: oid 47 updating version 2456 to 2457 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2457 (ObjNum 1227 snap 406 seq_num 1227) dirty exists 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3257: left oid 47 (ObjNum 1227 snap 406 seq_num 1227) 2024-09-21T09:51:37.062 INFO:tasks.rados.rados.0.smithi076.stdout:3258: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:51:37.063 INFO:tasks.rados.rados.0.smithi076.stdout:3259: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:51:37.131 INFO:tasks.rados.rados.0.smithi076.stdout:3258: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:51:37.131 INFO:tasks.rados.rados.0.smithi076.stdout:3258: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:51:37.131 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1132 (ObjNum 981 snap 336 seq_num 981) dirty exists 2024-09-21T09:51:37.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:36 smithi191 ceph-mon[102206]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:51:37.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:36 smithi076 ceph-mon[106482]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:51:37.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:36 smithi076 ceph-mon[110963]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T09:51:37.924 INFO:tasks.rados.rados.0.smithi076.stdout:3250: done (7 left) 2024-09-21T09:51:37.924 INFO:tasks.rados.rados.0.smithi076.stdout:3252: done (6 left) 2024-09-21T09:51:37.924 INFO:tasks.rados.rados.0.smithi076.stdout:3254: done (5 left) 2024-09-21T09:51:37.924 INFO:tasks.rados.rados.0.smithi076.stdout:3256: done (4 left) 2024-09-21T09:51:37.924 INFO:tasks.rados.rados.0.smithi076.stdout:3257: done (3 left) 2024-09-21T09:51:37.925 INFO:tasks.rados.rados.0.smithi076.stdout:3258: done (2 left) 2024-09-21T09:51:37.925 INFO:tasks.rados.rados.0.smithi076.stdout:3259: done (1 left) 2024-09-21T09:51:37.925 INFO:tasks.rados.rados.0.smithi076.stdout:3260: done (0 left) 2024-09-21T09:51:37.925 INFO:tasks.rados.rados.0.smithi076.stdout:3261: snap_create 2024-09-21T09:51:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:37 smithi191 ceph-mon[102206]: pgmap v793: 89 pgs: 89 active+clean; 315 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:51:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:37 smithi191 ceph-mon[102206]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:51:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:37 smithi076 ceph-mon[106482]: pgmap v793: 89 pgs: 89 active+clean; 315 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:51:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:37 smithi076 ceph-mon[106482]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:51:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:37 smithi076 ceph-mon[110963]: pgmap v793: 89 pgs: 89 active+clean; 315 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-09-21T09:51:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:37 smithi076 ceph-mon[110963]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi076.stdout:3261: done (0 left) 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi076.stdout:3262: read oid 12 snap -1 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi076.stdout:3262: expect (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi076.stdout:3263: write oid 27 current snap is 408 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi076.stdout:3263: seq_num 1228 ranges {638996=738605,1926476=712520} 2024-09-21T09:51:38.789 INFO:tasks.rados.rados.0.smithi076.stdout:3263: writing smithi07685715-27 from 638996 to 1377601 tid 1 2024-09-21T09:51:38.797 INFO:tasks.rados.rados.0.smithi076.stdout:3263: writing smithi07685715-27 from 1926476 to 2638996 tid 2 2024-09-21T09:51:38.797 INFO:tasks.rados.rados.0.smithi076.stdout:3264: rollback oid 3 current snap is 408 2024-09-21T09:51:38.797 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 407 2024-09-21T09:51:38.797 INFO:tasks.rados.rados.0.smithi076.stdout:3262: expect (ObjNum 981 snap 336 seq_num 981) 2024-09-21T09:51:38.877 INFO:tasks.rados.rados.0.smithi076.stdout:3263: finishing write tid 1 to smithi07685715-27 2024-09-21T09:51:38.877 INFO:tasks.rados.rados.0.smithi076.stdout:3264: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:51:38.877 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 219 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:38.877 INFO:tasks.rados.rados.0.smithi076.stdout:3263: finishing write tid 2 to smithi07685715-27 2024-09-21T09:51:38.877 INFO:tasks.rados.rados.0.smithi076.stdout:3263: finishing write tid 3 to smithi07685715-27 2024-09-21T09:51:38.877 INFO:tasks.rados.rados.0.smithi076.stdout:3263: finishing write tid 4 to smithi07685715-27 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3263: oid 27 updating version 0 to 1546 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3263: oid 27 version 1546 is already newer than 1545 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3263: oid 27 updating version 1546 to 1547 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1547 (ObjNum 1228 snap 408 seq_num 1228) dirty exists 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3263: left oid 27 (ObjNum 1228 snap 408 seq_num 1228) 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3262: done (2 left) 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3263: done (1 left) 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3264: done (0 left) 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3265: rollback oid 39 current snap is 408 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 351 2024-09-21T09:51:38.878 INFO:tasks.rados.rados.0.smithi076.stdout:3266: snap_remove snap 401 2024-09-21T09:51:39.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:38 smithi191 ceph-mon[102206]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:51:39.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:38 smithi191 ceph-mon[102206]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:51:39.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:38 smithi076 ceph-mon[106482]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:51:39.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:38 smithi076 ceph-mon[106482]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:51:39.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:38 smithi076 ceph-mon[110963]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T09:51:39.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:38 smithi076 ceph-mon[110963]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T09:51:39.785 INFO:tasks.rados.rados.0.smithi076.stdout:3266: done (1 left) 2024-09-21T09:51:39.785 INFO:tasks.rados.rados.0.smithi076.stdout:3267: read oid 49 snap -1 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3267: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3265: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3265: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3265: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2523 (ObjNum 984 snap 336 seq_num 984) dirty exists 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3265: done (1 left) 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3268: read oid 41 snap -1 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3268: expect (ObjNum 1041 snap 351 seq_num 1041) 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3269: rollback oid 26 current snap is 408 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 405 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3270: read oid 45 snap 404 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3270: expect (ObjNum 1158 snap 390 seq_num 1158) 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3271: read oid 29 snap -1 2024-09-21T09:51:39.786 INFO:tasks.rados.rados.0.smithi076.stdout:3271: expect deleted 2024-09-21T09:51:39.787 INFO:tasks.rados.rados.0.smithi076.stdout:3272: snap_create 2024-09-21T09:51:39.793 INFO:tasks.rados.rados.0.smithi076.stdout:3269: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:51:39.793 INFO:tasks.rados.rados.0.smithi076.stdout:3269: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:51:39.793 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1138 (ObjNum 1084 snap 363 seq_num 1084) dirty exists 2024-09-21T09:51:39.797 INFO:tasks.rados.rados.0.smithi076.stdout:3270: expect (ObjNum 1158 snap 390 seq_num 1158) 2024-09-21T09:51:39.997 INFO:tasks.rados.rados.0.smithi076.stdout:3267: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:51:40.113 INFO:tasks.rados.rados.0.smithi076.stdout:3268: expect (ObjNum 1041 snap 351 seq_num 1041) 2024-09-21T09:51:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:39 smithi191 ceph-mon[102206]: pgmap v796: 89 pgs: 89 active+clean; 315 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:51:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:39 smithi191 ceph-mon[102206]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:51:40.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:39 smithi076 ceph-mon[110963]: pgmap v796: 89 pgs: 89 active+clean; 315 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:51:40.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:39 smithi076 ceph-mon[110963]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:51:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:39 smithi076 ceph-mon[106482]: pgmap v796: 89 pgs: 89 active+clean; 315 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:51:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:39 smithi076 ceph-mon[106482]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T09:51:40.782 INFO:tasks.rados.rados.0.smithi076.stdout:3267: done (5 left) 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi076.stdout:3268: done (4 left) 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi076.stdout:3269: done (3 left) 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi076.stdout:3270: done (2 left) 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi076.stdout:3271: done (1 left) 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi076.stdout:3272: done (0 left) 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi076.stdout:3273: delete oid 47 current snap is 409 2024-09-21T09:51:40.786 INFO:tasks.rados.rados.0.smithi076.stdout:3273: done (0 left) 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3274: rollback oid 17 current snap is 409 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 408 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3275: read oid 18 snap -1 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3275: expect (ObjNum 1205 snap 404 seq_num 1205) 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3276: read oid 19 snap -1 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3276: expect (ObjNum 1032 snap 349 seq_num 1032) 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3277: read oid 23 snap -1 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3277: expect (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3278: rollback oid 40 current snap is 409 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 40 to 400 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi076.stdout:3279: snap_create 2024-09-21T09:51:40.789 INFO:tasks.rados.rados.0.smithi076.stdout:3274: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:51:40.789 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 1927 (ObjNum 129 snap 0 seq_num 864599232) dirty dne 2024-09-21T09:51:40.792 INFO:tasks.rados.rados.0.smithi076.stdout:3278: finishing rollback tid 0 to smithi07685715-40 2024-09-21T09:51:40.792 INFO:tasks.rados.rados.0.smithi076.stdout:3278: finishing rollback tid 1 to smithi07685715-40 2024-09-21T09:51:40.792 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1556 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:51:40.794 INFO:tasks.rados.rados.0.smithi076.stdout:3276: expect (ObjNum 1032 snap 349 seq_num 1032) 2024-09-21T09:51:40.948 INFO:tasks.rados.rados.0.smithi076.stdout:3275: expect (ObjNum 1205 snap 404 seq_num 1205) 2024-09-21T09:51:41.019 INFO:tasks.rados.rados.0.smithi076.stdout:3277: expect (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:51:41.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:41.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:40 smithi191 ceph-mon[102206]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:51:41.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:40 smithi076 ceph-mon[106482]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:51:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:40 smithi076 ceph-mon[110963]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T09:51:41.787 INFO:tasks.rados.rados.0.smithi076.stdout:3274: done (5 left) 2024-09-21T09:51:41.787 INFO:tasks.rados.rados.0.smithi076.stdout:3275: done (4 left) 2024-09-21T09:51:41.787 INFO:tasks.rados.rados.0.smithi076.stdout:3276: done (3 left) 2024-09-21T09:51:41.787 INFO:tasks.rados.rados.0.smithi076.stdout:3277: done (2 left) 2024-09-21T09:51:41.788 INFO:tasks.rados.rados.0.smithi076.stdout:3278: done (1 left) 2024-09-21T09:51:41.788 INFO:tasks.rados.rados.0.smithi076.stdout:3279: done (0 left) 2024-09-21T09:51:41.788 INFO:tasks.rados.rados.0.smithi076.stdout:3280: write oid 17 current snap is 410 2024-09-21T09:51:41.788 INFO:tasks.rados.rados.0.smithi076.stdout:3280: seq_num 1230 ranges {672610=489103,1643440=722432} 2024-09-21T09:51:41.793 INFO:tasks.rados.rados.0.smithi076.stdout:3280: writing smithi07685715-17 from 672610 to 1161713 tid 1 2024-09-21T09:51:41.800 INFO:tasks.rados.rados.0.smithi076.stdout:3280: writing smithi07685715-17 from 1643440 to 2365872 tid 2 2024-09-21T09:51:41.800 INFO:tasks.rados.rados.0.smithi076.stdout:3280: finishing write tid 1 to smithi07685715-17 2024-09-21T09:51:41.800 INFO:tasks.rados.rados.0.smithi076.stdout:3281: write oid 21 current snap is 410 2024-09-21T09:51:41.800 INFO:tasks.rados.rados.0.smithi076.stdout:3281: seq_num 1231 ranges {689417=564352,1701922=466090} 2024-09-21T09:51:41.806 INFO:tasks.rados.rados.0.smithi076.stdout:3281: writing smithi07685715-21 from 689417 to 1253769 tid 1 2024-09-21T09:51:41.812 INFO:tasks.rados.rados.0.smithi076.stdout:3281: writing smithi07685715-21 from 1701922 to 2168012 tid 2 2024-09-21T09:51:41.812 INFO:tasks.rados.rados.0.smithi076.stdout:3282: write oid 47 current snap is 410 2024-09-21T09:51:41.812 INFO:tasks.rados.rados.0.smithi076.stdout:3280: finishing write tid 2 to smithi07685715-17 2024-09-21T09:51:41.812 INFO:tasks.rados.rados.0.smithi076.stdout:3280: finishing write tid 3 to smithi07685715-17 2024-09-21T09:51:41.812 INFO:tasks.rados.rados.0.smithi076.stdout:3280: finishing write tid 4 to smithi07685715-17 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:3280: oid 17 updating version 0 to 2009 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:3280: oid 17 updating version 2009 to 2010 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:3280: oid 17 version 2010 is already newer than 2008 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2010 (ObjNum 1230 snap 410 seq_num 1230) dirty exists 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:3280: left oid 17 (ObjNum 1230 snap 410 seq_num 1230) 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:3281: finishing write tid 1 to smithi07685715-21 2024-09-21T09:51:41.813 INFO:tasks.rados.rados.0.smithi076.stdout:3282: seq_num 1232 ranges {706224=755954,2160404=545820} 2024-09-21T09:51:41.819 INFO:tasks.rados.rados.0.smithi076.stdout:3282: writing smithi07685715-47 from 706224 to 1462178 tid 1 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3282: writing smithi07685715-47 from 2160404 to 2706224 tid 2 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3280: done (2 left) 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3283: delete oid 10 current snap is 410 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3281: finishing write tid 2 to smithi07685715-21 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3281: finishing write tid 3 to smithi07685715-21 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3281: finishing write tid 4 to smithi07685715-21 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3281: oid 21 updating version 0 to 2533 2024-09-21T09:51:41.827 INFO:tasks.rados.rados.0.smithi076.stdout:3281: oid 21 version 2533 is already newer than 2532 2024-09-21T09:51:41.828 INFO:tasks.rados.rados.0.smithi076.stdout:3281: oid 21 version 2533 is already newer than 2531 2024-09-21T09:51:41.828 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 2533 (ObjNum 1231 snap 410 seq_num 1231) dirty exists 2024-09-21T09:51:41.828 INFO:tasks.rados.rados.0.smithi076.stdout:3281: left oid 21 (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:51:41.828 INFO:tasks.rados.rados.0.smithi076.stdout:3282: finishing write tid 1 to smithi07685715-47 2024-09-21T09:51:41.829 INFO:tasks.rados.rados.0.smithi076.stdout:3281: done (2 left) 2024-09-21T09:51:41.829 INFO:tasks.rados.rados.0.smithi076.stdout:3283: done (1 left) 2024-09-21T09:51:41.829 INFO:tasks.rados.rados.0.smithi076.stdout:3284: delete oid 31 current snap is 410 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: finishing write tid 2 to smithi07685715-47 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: finishing write tid 3 to smithi07685715-47 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: finishing write tid 4 to smithi07685715-47 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: oid 47 updating version 0 to 2465 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: oid 47 updating version 2465 to 2466 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: oid 47 version 2466 is already newer than 2464 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2466 (ObjNum 1232 snap 410 seq_num 1232) dirty exists 2024-09-21T09:51:41.831 INFO:tasks.rados.rados.0.smithi076.stdout:3282: left oid 47 (ObjNum 1232 snap 410 seq_num 1232) 2024-09-21T09:51:41.833 INFO:tasks.rados.rados.0.smithi076.stdout:3282: done (1 left) 2024-09-21T09:51:41.833 INFO:tasks.rados.rados.0.smithi076.stdout:3284: done (0 left) 2024-09-21T09:51:41.833 INFO:tasks.rados.rados.0.smithi076.stdout:3285: snap_create 2024-09-21T09:51:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:41 smithi191 ceph-mon[102206]: pgmap v799: 89 pgs: 89 active+clean; 310 MiB data, 931 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 533 KiB/s wr, 10 op/s 2024-09-21T09:51:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:41 smithi191 ceph-mon[102206]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:51:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:41 smithi076 ceph-mon[110963]: pgmap v799: 89 pgs: 89 active+clean; 310 MiB data, 931 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 533 KiB/s wr, 10 op/s 2024-09-21T09:51:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:41 smithi076 ceph-mon[110963]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:51:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:41 smithi076 ceph-mon[106482]: pgmap v799: 89 pgs: 89 active+clean; 310 MiB data, 931 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 533 KiB/s wr, 10 op/s 2024-09-21T09:51:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:41 smithi076 ceph-mon[106482]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T09:51:42.786 INFO:tasks.rados.rados.0.smithi076.stdout:3285: done (0 left) 2024-09-21T09:51:42.786 INFO:tasks.rados.rados.0.smithi076.stdout:3286: snap_create 2024-09-21T09:51:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:51:43] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:51:43.783 INFO:tasks.rados.rados.0.smithi076.stdout:3286: done (0 left) 2024-09-21T09:51:43.783 INFO:tasks.rados.rados.0.smithi076.stdout:3287: read oid 26 snap -1 2024-09-21T09:51:43.783 INFO:tasks.rados.rados.0.smithi076.stdout:3287: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:51:43.783 INFO:tasks.rados.rados.0.smithi076.stdout:3288: snap_remove snap 372 2024-09-21T09:51:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:43 smithi191 ceph-mon[102206]: pgmap v802: 89 pgs: 89 active+clean; 313 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.1 MiB/s wr, 22 op/s 2024-09-21T09:51:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:43 smithi191 ceph-mon[102206]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:51:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:43 smithi191 ceph-mon[102206]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:51:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-mon[106482]: pgmap v802: 89 pgs: 89 active+clean; 313 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.1 MiB/s wr, 22 op/s 2024-09-21T09:51:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-mon[106482]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:51:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-mon[106482]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:51:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-mon[110963]: pgmap v802: 89 pgs: 89 active+clean; 313 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 1.1 MiB/s wr, 22 op/s 2024-09-21T09:51:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-mon[110963]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T09:51:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:43 smithi076 ceph-mon[110963]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T09:51:44.787 INFO:tasks.rados.rados.0.smithi076.stdout:3288: done (1 left) 2024-09-21T09:51:44.787 INFO:tasks.rados.rados.0.smithi076.stdout:3289: write oid 35 current snap is 412 2024-09-21T09:51:44.787 INFO:tasks.rados.rados.0.smithi076.stdout:3289: seq_num 1235 ranges {756645=581701,1935850=473428} 2024-09-21T09:51:44.795 INFO:tasks.rados.rados.0.smithi076.stdout:3289: writing smithi07685715-35 from 756645 to 1338346 tid 1 2024-09-21T09:51:44.802 INFO:tasks.rados.rados.0.smithi076.stdout:3289: writing smithi07685715-35 from 1935850 to 2409278 tid 2 2024-09-21T09:51:44.802 INFO:tasks.rados.rados.0.smithi076.stdout:3290: rollback oid 46 current snap is 412 2024-09-21T09:51:44.802 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 46 to 400 2024-09-21T09:51:44.803 INFO:tasks.rados.rados.0.smithi076.stdout:3287: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:51:45.018 INFO:tasks.rados.rados.0.smithi076.stdout:3289: finishing write tid 1 to smithi07685715-35 2024-09-21T09:51:45.018 INFO:tasks.rados.rados.0.smithi076.stdout:3289: finishing write tid 2 to smithi07685715-35 2024-09-21T09:51:45.018 INFO:tasks.rados.rados.0.smithi076.stdout:3289: finishing write tid 3 to smithi07685715-35 2024-09-21T09:51:45.018 INFO:tasks.rados.rados.0.smithi076.stdout:3287: done (2 left) 2024-09-21T09:51:45.018 INFO:tasks.rados.rados.0.smithi076.stdout:3291: rollback oid 22 current snap is 412 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 22 to 387 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3289: finishing write tid 4 to smithi07685715-35 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3289: oid 35 updating version 0 to 2543 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3289: oid 35 updating version 2543 to 2546 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3289: oid 35 updating version 2546 to 2547 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2547 (ObjNum 1235 snap 412 seq_num 1235) dirty exists 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3289: left oid 35 (ObjNum 1235 snap 412 seq_num 1235) 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3290: finishing rollback tid 1 to smithi07685715-46 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2376 (ObjNum 289 snap 0 seq_num 1683134868) dirty dne 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3289: done (2 left) 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3290: done (1 left) 2024-09-21T09:51:45.019 INFO:tasks.rados.rados.0.smithi076.stdout:3292: delete oid 8 current snap is 412 2024-09-21T09:51:45.020 INFO:tasks.rados.rados.0.smithi076.stdout:3292: done (1 left) 2024-09-21T09:51:45.021 INFO:tasks.rados.rados.0.smithi076.stdout:3293: snap_remove snap 394 2024-09-21T09:51:45.796 INFO:tasks.rados.rados.0.smithi076.stdout:3293: done (1 left) 2024-09-21T09:51:45.796 INFO:tasks.rados.rados.0.smithi076.stdout:3294: snap_remove snap 373 2024-09-21T09:51:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:45 smithi191 ceph-mon[102206]: pgmap v805: 89 pgs: 89 active+clean; 316 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 924 KiB/s wr, 13 op/s 2024-09-21T09:51:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:45 smithi191 ceph-mon[102206]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:51:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:51:46.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:45 smithi076 ceph-mon[106482]: pgmap v805: 89 pgs: 89 active+clean; 316 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 924 KiB/s wr, 13 op/s 2024-09-21T09:51:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:45 smithi076 ceph-mon[106482]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:51:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:51:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:45 smithi076 ceph-mon[110963]: pgmap v805: 89 pgs: 89 active+clean; 316 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 924 KiB/s wr, 13 op/s 2024-09-21T09:51:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:45 smithi076 ceph-mon[110963]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T09:51:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:51:46.801 INFO:tasks.rados.rados.0.smithi076.stdout:3294: done (1 left) 2024-09-21T09:51:46.802 INFO:tasks.rados.rados.0.smithi076.stdout:3295: read oid 9 snap -1 2024-09-21T09:51:46.802 INFO:tasks.rados.rados.0.smithi076.stdout:3295: expect (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:51:46.802 INFO:tasks.rados.rados.0.smithi076.stdout:3296: write oid 16 current snap is 412 2024-09-21T09:51:46.802 INFO:tasks.rados.rados.0.smithi076.stdout:3291: finishing rollback tid 0 to smithi07685715-22 2024-09-21T09:51:46.802 INFO:tasks.rados.rados.0.smithi076.stdout:3296: seq_num 1237 ranges {790259=732199,1936461=477097} 2024-09-21T09:51:46.817 INFO:tasks.rados.rados.0.smithi076.stdout:3296: writing smithi07685715-16 from 790259 to 1522458 tid 1 2024-09-21T09:51:46.823 INFO:tasks.rados.rados.0.smithi076.stdout:3296: writing smithi07685715-16 from 1936461 to 2413558 tid 2 2024-09-21T09:51:46.823 INFO:tasks.rados.rados.0.smithi076.stdout:3297: write oid 41 current snap is 412 2024-09-21T09:51:46.823 INFO:tasks.rados.rados.0.smithi076.stdout:3291: finishing rollback tid 1 to smithi07685715-22 2024-09-21T09:51:46.823 INFO:tasks.rados.rados.0.smithi076.stdout:3291: finishing rollback tid 2 to smithi07685715-22 2024-09-21T09:51:46.823 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1566 (ObjNum 1123 snap 370 seq_num 1123) dirty exists 2024-09-21T09:51:46.824 INFO:tasks.rados.rados.0.smithi076.stdout:3297: seq_num 1238 ranges {407066=407448,1594943=620755} 2024-09-21T09:51:46.827 INFO:tasks.rados.rados.0.smithi076.stdout:3297: writing smithi07685715-41 from 407066 to 814514 tid 1 2024-09-21T09:51:46.835 INFO:tasks.rados.rados.0.smithi076.stdout:3297: writing smithi07685715-41 from 1594943 to 2215698 tid 2 2024-09-21T09:51:46.835 INFO:tasks.rados.rados.0.smithi076.stdout:3291: done (3 left) 2024-09-21T09:51:46.835 INFO:tasks.rados.rados.0.smithi076.stdout:3298: snap_remove snap 351 2024-09-21T09:51:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:46 smithi191 ceph-mon[102206]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:51:47.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:46 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:46 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:46 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:46 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[106482]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[110963]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:46 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:47.815 INFO:tasks.rados.rados.0.smithi076.stdout:3298: done (3 left) 2024-09-21T09:51:47.831 INFO:tasks.rados.rados.0.smithi076.stdout:3299: delete oid 2 current snap is 412 2024-09-21T09:51:47.831 INFO:tasks.rados.rados.0.smithi076.stdout:3295: expect (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:51:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:3296: finishing write tid 1 to smithi07685715-16 2024-09-21T09:51:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:3296: finishing write tid 2 to smithi07685715-16 2024-09-21T09:51:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:3296: finishing write tid 3 to smithi07685715-16 2024-09-21T09:51:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:3295: done (3 left) 2024-09-21T09:51:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:3299: done (2 left) 2024-09-21T09:51:47.902 INFO:tasks.rados.rados.0.smithi076.stdout:3300: snap_remove snap 400 2024-09-21T09:51:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:47 smithi191 ceph-mon[102206]: pgmap v808: 89 pgs: 89 active+clean; 317 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-21T09:51:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:47 smithi191 ceph-mon[102206]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:51:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:47 smithi076 ceph-mon[106482]: pgmap v808: 89 pgs: 89 active+clean; 317 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-21T09:51:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:47 smithi076 ceph-mon[106482]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:51:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:47 smithi076 ceph-mon[110963]: pgmap v808: 89 pgs: 89 active+clean; 317 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-21T09:51:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:47 smithi076 ceph-mon[110963]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:3300: done (2 left) 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:3301: rollback oid 50 current snap is 412 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 405 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:3302: rollback oid 38 current snap is 412 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 38 to 409 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:3303: write oid 22 current snap is 412 2024-09-21T09:51:48.785 INFO:tasks.rados.rados.0.smithi076.stdout:3303: seq_num 1240 ranges {440680=674299,1711907=624424} 2024-09-21T09:51:48.791 INFO:tasks.rados.rados.0.smithi076.stdout:3303: writing smithi07685715-22 from 440680 to 1114979 tid 1 2024-09-21T09:51:48.799 INFO:tasks.rados.rados.0.smithi076.stdout:3303: writing smithi07685715-22 from 1711907 to 2336331 tid 2 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3304: delete oid 8 current snap is 412 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3296: finishing write tid 4 to smithi07685715-16 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3296: oid 16 updating version 0 to 1583 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3296: oid 16 version 1583 is already newer than 1581 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3296: oid 16 version 1583 is already newer than 1582 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1583 (ObjNum 1237 snap 412 seq_num 1237) dirty exists 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3296: left oid 16 (ObjNum 1237 snap 412 seq_num 1237) 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: finishing write tid 1 to smithi07685715-41 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: finishing write tid 2 to smithi07685715-41 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: finishing write tid 3 to smithi07685715-41 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: finishing write tid 4 to smithi07685715-41 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: oid 41 updating version 0 to 1585 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: oid 41 updating version 1585 to 1586 2024-09-21T09:51:48.800 INFO:tasks.rados.rados.0.smithi076.stdout:3297: oid 41 updating version 1586 to 1587 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1587 (ObjNum 1238 snap 412 seq_num 1238) dirty exists 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3297: left oid 41 (ObjNum 1238 snap 412 seq_num 1238) 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3301: finishing rollback tid 0 to smithi07685715-50 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3301: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3301: finishing rollback tid 2 to smithi07685715-50 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 1056 (ObjNum 1167 snap 393 seq_num 1167) dirty exists 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3303: finishing write tid 1 to smithi07685715-22 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3302: finishing rollback tid 0 to smithi07685715-38 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3302: finishing rollback tid 1 to smithi07685715-38 2024-09-21T09:51:48.801 INFO:tasks.rados.rados.0.smithi076.stdout:3302: finishing rollback tid 2 to smithi07685715-38 2024-09-21T09:51:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 2048 (ObjNum 1204 snap 404 seq_num 1204) dirty exists 2024-09-21T09:51:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:3296: done (5 left) 2024-09-21T09:51:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:3297: done (4 left) 2024-09-21T09:51:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:3301: done (3 left) 2024-09-21T09:51:48.802 INFO:tasks.rados.rados.0.smithi076.stdout:3302: done (2 left) 2024-09-21T09:51:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:3304: done (1 left) 2024-09-21T09:51:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:3305: write oid 28 current snap is 412 2024-09-21T09:51:48.803 INFO:tasks.rados.rados.0.smithi076.stdout:3305: seq_num 1242 ranges {474294=424797,1428871=628093,2657357=216937} 2024-09-21T09:51:48.807 INFO:tasks.rados.rados.0.smithi076.stdout:3305: writing smithi07685715-28 from 474294 to 899091 tid 1 2024-09-21T09:51:48.813 INFO:tasks.rados.rados.0.smithi076.stdout:3305: writing smithi07685715-28 from 1428871 to 2056964 tid 2 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3305: writing smithi07685715-28 from 2657357 to 2874294 tid 3 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3306: delete oid 7 current snap is 412 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: finishing write tid 2 to smithi07685715-22 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: finishing write tid 3 to smithi07685715-22 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: finishing write tid 4 to smithi07685715-22 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: oid 22 updating version 0 to 1606 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: oid 22 version 1606 is already newer than 1605 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: oid 22 version 1606 is already newer than 1600 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1606 (ObjNum 1240 snap 412 seq_num 1240) dirty exists 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3303: left oid 22 (ObjNum 1240 snap 412 seq_num 1240) 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3305: finishing write tid 1 to smithi07685715-28 2024-09-21T09:51:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3305: finishing write tid 2 to smithi07685715-28 2024-09-21T09:51:48.821 INFO:tasks.rados.rados.0.smithi076.stdout:3303: done (2 left) 2024-09-21T09:51:48.821 INFO:tasks.rados.rados.0.smithi076.stdout:3306: done (1 left) 2024-09-21T09:51:48.821 INFO:tasks.rados.rados.0.smithi076.stdout:3307: snap_remove snap 395 2024-09-21T09:51:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:48 smithi191 ceph-mon[102206]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:51:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:48 smithi191 ceph-mon[102206]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:51:49.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:48 smithi076 ceph-mon[106482]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:51:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:48 smithi076 ceph-mon[106482]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:51:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:48 smithi076 ceph-mon[110963]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T09:51:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:48 smithi076 ceph-mon[110963]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T09:51:49.821 INFO:tasks.rados.rados.0.smithi076.stdout:3307: done (1 left) 2024-09-21T09:51:49.822 INFO:tasks.rados.rados.0.smithi076.stdout:3308: snap_remove snap 407 2024-09-21T09:51:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:49 smithi191 ceph-mon[102206]: pgmap v811: 89 pgs: 89 active+clean; 314 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 828 KiB/s wr, 11 op/s 2024-09-21T09:51:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:49 smithi076 ceph-mon[106482]: pgmap v811: 89 pgs: 89 active+clean; 314 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 828 KiB/s wr, 11 op/s 2024-09-21T09:51:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:49 smithi076 ceph-mon[110963]: pgmap v811: 89 pgs: 89 active+clean; 314 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 828 KiB/s wr, 11 op/s 2024-09-21T09:51:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3308: done (1 left) 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3309: snap_create 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: finishing write tid 3 to smithi07685715-28 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: finishing write tid 4 to smithi07685715-28 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: finishing write tid 5 to smithi07685715-28 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: oid 28 updating version 0 to 2510 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: oid 28 version 2510 is already newer than 2509 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: oid 28 updating version 2510 to 2511 2024-09-21T09:51:50.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: oid 28 version 2511 is already newer than 2508 2024-09-21T09:51:50.826 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 2511 (ObjNum 1242 snap 412 seq_num 1242) dirty exists 2024-09-21T09:51:50.826 INFO:tasks.rados.rados.0.smithi076.stdout:3305: left oid 28 (ObjNum 1242 snap 412 seq_num 1242) 2024-09-21T09:51:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:50 smithi191 ceph-mon[102206]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:51:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:50 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:50 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:50 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:51:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:50 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:51:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:50 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[106482]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[110963]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T09:51:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:51:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:51:51.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:50 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:51:51.613 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:51:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=cleanup t=2024-09-21T09:51:51.324849251Z level=info msg="Completed cleanup jobs" duration=2.799796ms 2024-09-21T09:51:51.613 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 09:51:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugins.update.checker t=2024-09-21T09:51:51.474054608Z level=info msg="Update check succeeded" duration=65.877921ms 2024-09-21T09:51:51.825 INFO:tasks.rados.rados.0.smithi076.stdout:3305: done (1 left) 2024-09-21T09:51:51.825 INFO:tasks.rados.rados.0.smithi076.stdout:3309: done (0 left) 2024-09-21T09:51:51.825 INFO:tasks.rados.rados.0.smithi076.stdout:3310: snap_create 2024-09-21T09:51:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:51 smithi191 ceph-mon[102206]: pgmap v814: 89 pgs: 89 active+clean; 289 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:51:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:51 smithi191 ceph-mon[102206]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:51:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:51 smithi191 ceph-mon[102206]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:51:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:51 smithi076 ceph-mon[106482]: pgmap v814: 89 pgs: 89 active+clean; 289 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:51:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:51 smithi076 ceph-mon[106482]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:51:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:51 smithi076 ceph-mon[106482]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:51:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:51 smithi076 ceph-mon[110963]: pgmap v814: 89 pgs: 89 active+clean; 289 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 0 B/s wr, 2 op/s 2024-09-21T09:51:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:51 smithi076 ceph-mon[110963]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T09:51:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:51 smithi076 ceph-mon[110963]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:3310: done (0 left) 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:3311: rollback oid 7 current snap is 414 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 406 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:3312: read oid 21 snap -1 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:3312: expect (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:3313: read oid 27 snap -1 2024-09-21T09:51:52.828 INFO:tasks.rados.rados.0.smithi076.stdout:3313: expect (ObjNum 1228 snap 408 seq_num 1228) 2024-09-21T09:51:52.829 INFO:tasks.rados.rados.0.smithi076.stdout:3314: write oid 15 current snap is 414 2024-09-21T09:51:52.829 INFO:tasks.rados.rados.0.smithi076.stdout:3314: seq_num 1244 ranges {507908=575295,1829482=631762} 2024-09-21T09:51:52.834 INFO:tasks.rados.rados.0.smithi076.stdout:3314: writing smithi07685715-15 from 507908 to 1083203 tid 1 2024-09-21T09:51:52.843 INFO:tasks.rados.rados.0.smithi076.stdout:3314: writing smithi07685715-15 from 1829482 to 2461244 tid 2 2024-09-21T09:51:52.843 INFO:tasks.rados.rados.0.smithi076.stdout:3315: snap_create 2024-09-21T09:51:52.843 INFO:tasks.rados.rados.0.smithi076.stdout:3311: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:51:52.844 INFO:tasks.rados.rados.0.smithi076.stdout:3311: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:51:52.844 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2076 (ObjNum 1186 snap 399 seq_num 1186) dirty exists 2024-09-21T09:51:52.844 INFO:tasks.rados.rados.0.smithi076.stdout:3314: finishing write tid 1 to smithi07685715-15 2024-09-21T09:51:52.844 INFO:tasks.rados.rados.0.smithi076.stdout:3312: expect (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:51:53.074 INFO:tasks.rados.rados.0.smithi076.stdout:3314: finishing write tid 2 to smithi07685715-15 2024-09-21T09:51:53.075 INFO:tasks.rados.rados.0.smithi076.stdout:3314: finishing write tid 3 to smithi07685715-15 2024-09-21T09:51:53.075 INFO:tasks.rados.rados.0.smithi076.stdout:3313: expect (ObjNum 1228 snap 408 seq_num 1228) 2024-09-21T09:51:53.180 INFO:tasks.rados.rados.0.smithi076.stdout:3314: finishing write tid 4 to smithi07685715-15 2024-09-21T09:51:53.181 INFO:tasks.rados.rados.0.smithi076.stdout:3314: oid 15 updating version 0 to 716 2024-09-21T09:51:53.181 INFO:tasks.rados.rados.0.smithi076.stdout:3314: oid 15 updating version 716 to 718 2024-09-21T09:51:53.181 INFO:tasks.rados.rados.0.smithi076.stdout:3314: oid 15 version 718 is already newer than 717 2024-09-21T09:51:53.181 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 718 (ObjNum 1244 snap 414 seq_num 1244) dirty exists 2024-09-21T09:51:53.181 INFO:tasks.rados.rados.0.smithi076.stdout:3314: left oid 15 (ObjNum 1244 snap 414 seq_num 1244) 2024-09-21T09:51:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:51:53] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3311: done (4 left) 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3312: done (3 left) 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3313: done (2 left) 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3314: done (1 left) 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3315: done (0 left) 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3316: write oid 20 current snap is 415 2024-09-21T09:51:53.786 INFO:tasks.rados.rados.0.smithi076.stdout:3316: seq_num 1245 ranges {524715=650544,1887964=491773} 2024-09-21T09:51:53.793 INFO:tasks.rados.rados.0.smithi076.stdout:3316: writing smithi07685715-20 from 524715 to 1175259 tid 1 2024-09-21T09:51:53.800 INFO:tasks.rados.rados.0.smithi076.stdout:3316: writing smithi07685715-20 from 1887964 to 2379737 tid 2 2024-09-21T09:51:53.800 INFO:tasks.rados.rados.0.smithi076.stdout:3317: read oid 32 snap -1 2024-09-21T09:51:53.800 INFO:tasks.rados.rados.0.smithi076.stdout:3317: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:51:53.800 INFO:tasks.rados.rados.0.smithi076.stdout:3318: snap_create 2024-09-21T09:51:53.800 INFO:tasks.rados.rados.0.smithi076.stdout:3316: finishing write tid 1 to smithi07685715-20 2024-09-21T09:51:53.803 INFO:tasks.rados.rados.0.smithi076.stdout:3316: finishing write tid 2 to smithi07685715-20 2024-09-21T09:51:53.803 INFO:tasks.rados.rados.0.smithi076.stdout:3316: finishing write tid 3 to smithi07685715-20 2024-09-21T09:51:53.803 INFO:tasks.rados.rados.0.smithi076.stdout:3316: finishing write tid 4 to smithi07685715-20 2024-09-21T09:51:53.804 INFO:tasks.rados.rados.0.smithi076.stdout:3316: oid 20 updating version 0 to 990 2024-09-21T09:51:53.804 INFO:tasks.rados.rados.0.smithi076.stdout:3316: oid 20 updating version 990 to 991 2024-09-21T09:51:53.804 INFO:tasks.rados.rados.0.smithi076.stdout:3316: oid 20 updating version 991 to 992 2024-09-21T09:51:53.804 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 992 (ObjNum 1245 snap 415 seq_num 1245) dirty exists 2024-09-21T09:51:53.804 INFO:tasks.rados.rados.0.smithi076.stdout:3316: left oid 20 (ObjNum 1245 snap 415 seq_num 1245) 2024-09-21T09:51:53.804 INFO:tasks.rados.rados.0.smithi076.stdout:3317: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:51:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:53 smithi191 ceph-mon[102206]: pgmap v817: 89 pgs: 89 active+clean; 240 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:51:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:53 smithi191 ceph-mon[102206]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:51:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:53 smithi191 ceph-mon[102206]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:51:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-mon[106482]: pgmap v817: 89 pgs: 89 active+clean; 240 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:51:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-mon[106482]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:51:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-mon[106482]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:51:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-mon[110963]: pgmap v817: 89 pgs: 89 active+clean; 240 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.1 MiB/s wr, 8 op/s 2024-09-21T09:51:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-mon[110963]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T09:51:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:53 smithi076 ceph-mon[110963]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T09:51:54.794 INFO:tasks.rados.rados.0.smithi076.stdout:3316: done (2 left) 2024-09-21T09:51:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:3317: done (1 left) 2024-09-21T09:51:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:3318: done (0 left) 2024-09-21T09:51:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:3319: rollback oid 49 current snap is 416 2024-09-21T09:51:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 49 to 404 2024-09-21T09:51:54.795 INFO:tasks.rados.rados.0.smithi076.stdout:3320: delete oid 35 current snap is 416 2024-09-21T09:51:54.802 INFO:tasks.rados.rados.0.smithi076.stdout:3319: finishing rollback tid 0 to smithi07685715-49 2024-09-21T09:51:54.802 INFO:tasks.rados.rados.0.smithi076.stdout:3319: finishing rollback tid 1 to smithi07685715-49 2024-09-21T09:51:54.803 INFO:tasks.rados.rados.0.smithi076.stdout:3320: done (1 left) 2024-09-21T09:51:54.803 INFO:tasks.rados.rados.0.smithi076.stdout:3321: rollback oid 15 current snap is 416 2024-09-21T09:51:54.803 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 413 2024-09-21T09:51:54.803 INFO:tasks.rados.rados.0.smithi076.stdout:3322: snap_create 2024-09-21T09:51:54.803 INFO:tasks.rados.rados.0.smithi076.stdout:3319: finishing rollback tid 2 to smithi07685715-49 2024-09-21T09:51:54.803 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2525 (ObjNum 1106 snap 367 seq_num 1106) dirty exists 2024-09-21T09:51:54.804 INFO:tasks.rados.rados.0.smithi076.stdout:3321: finishing rollback tid 0 to smithi07685715-15 2024-09-21T09:51:54.804 INFO:tasks.rados.rados.0.smithi076.stdout:3321: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:51:54.804 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 721 (ObjNum 129 snap 0 seq_num 864519440) dirty dne 2024-09-21T09:51:55.796 INFO:tasks.rados.rados.0.smithi076.stdout:3319: done (2 left) 2024-09-21T09:51:55.796 INFO:tasks.rados.rados.0.smithi076.stdout:3321: done (1 left) 2024-09-21T09:51:55.796 INFO:tasks.rados.rados.0.smithi076.stdout:3322: done (0 left) 2024-09-21T09:51:55.796 INFO:tasks.rados.rados.0.smithi076.stdout:3323: snap_create 2024-09-21T09:51:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:55 smithi191 ceph-mon[102206]: pgmap v820: 89 pgs: 89 active+clean; 240 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:51:56.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:55 smithi191 ceph-mon[102206]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:51:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:55 smithi076 ceph-mon[106482]: pgmap v820: 89 pgs: 89 active+clean; 240 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:51:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:55 smithi076 ceph-mon[106482]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:51:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:55 smithi076 ceph-mon[110963]: pgmap v820: 89 pgs: 89 active+clean; 240 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.1 MiB/s wr, 6 op/s 2024-09-21T09:51:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:55 smithi076 ceph-mon[110963]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T09:51:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:51:56.802 INFO:tasks.rados.rados.0.smithi076.stdout:3323: done (0 left) 2024-09-21T09:51:56.802 INFO:tasks.rados.rados.0.smithi076.stdout:3324: snap_remove snap 410 2024-09-21T09:51:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:56 smithi191 ceph-mon[102206]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:51:57.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:56 smithi076 ceph-mon[106482]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:51:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:56 smithi076 ceph-mon[110963]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T09:51:57.800 INFO:tasks.rados.rados.0.smithi076.stdout:3324: done (0 left) 2024-09-21T09:51:57.800 INFO:tasks.rados.rados.0.smithi076.stdout:3325: snap_remove snap 409 2024-09-21T09:51:58.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:57 smithi191 ceph-mon[102206]: pgmap v823: 89 pgs: 89 active+clean; 241 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 574 KiB/s wr, 10 op/s 2024-09-21T09:51:58.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:57 smithi191 ceph-mon[102206]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:51:58.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:57 smithi191 ceph-mon[102206]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:51:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:57 smithi076 ceph-mon[106482]: pgmap v823: 89 pgs: 89 active+clean; 241 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 574 KiB/s wr, 10 op/s 2024-09-21T09:51:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:57 smithi076 ceph-mon[106482]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:51:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:57 smithi076 ceph-mon[106482]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:51:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:57 smithi076 ceph-mon[110963]: pgmap v823: 89 pgs: 89 active+clean; 241 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 574 KiB/s wr, 10 op/s 2024-09-21T09:51:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:57 smithi076 ceph-mon[110963]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T09:51:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:57 smithi076 ceph-mon[110963]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T09:51:58.789 INFO:tasks.rados.rados.0.smithi076.stdout:3325: done (0 left) 2024-09-21T09:51:58.789 INFO:tasks.rados.rados.0.smithi076.stdout:3326: rollback oid 50 current snap is 418 2024-09-21T09:51:58.790 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 387 2024-09-21T09:51:58.790 INFO:tasks.rados.rados.0.smithi076.stdout:3327: rollback oid 12 current snap is 418 2024-09-21T09:51:58.790 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 405 2024-09-21T09:51:58.790 INFO:tasks.rados.rados.0.smithi076.stdout:3328: delete oid 33 current snap is 418 2024-09-21T09:51:58.797 INFO:tasks.rados.rados.0.smithi076.stdout:3327: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:51:58.797 INFO:tasks.rados.rados.0.smithi076.stdout:3327: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:51:58.797 INFO:tasks.rados.rados.0.smithi076.stdout:3327: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:51:58.798 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1174 (ObjNum 981 snap 336 seq_num 981) dirty exists 2024-09-21T09:51:58.798 INFO:tasks.rados.rados.0.smithi076.stdout:3327: done (2 left) 2024-09-21T09:51:58.798 INFO:tasks.rados.rados.0.smithi076.stdout:3328: done (1 left) 2024-09-21T09:51:58.798 INFO:tasks.rados.rados.0.smithi076.stdout:3329: snap_create 2024-09-21T09:51:58.799 INFO:tasks.rados.rados.0.smithi076.stdout:3326: finishing rollback tid 0 to smithi07685715-50 2024-09-21T09:51:58.799 INFO:tasks.rados.rados.0.smithi076.stdout:3326: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:51:58.799 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 1067 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:51:59.794 INFO:tasks.rados.rados.0.smithi076.stdout:3326: done (1 left) 2024-09-21T09:51:59.795 INFO:tasks.rados.rados.0.smithi076.stdout:3329: done (0 left) 2024-09-21T09:51:59.795 INFO:tasks.rados.rados.0.smithi076.stdout:3330: snap_create 2024-09-21T09:52:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:59 smithi191 ceph-mon[102206]: pgmap v826: 89 pgs: 89 active+clean; 241 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 574 KiB/s wr, 10 op/s 2024-09-21T09:52:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:51:59 smithi191 ceph-mon[102206]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:52:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:59 smithi076 ceph-mon[106482]: pgmap v826: 89 pgs: 89 active+clean; 241 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 574 KiB/s wr, 10 op/s 2024-09-21T09:52:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:51:59 smithi076 ceph-mon[106482]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:52:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:59 smithi076 ceph-mon[110963]: pgmap v826: 89 pgs: 89 active+clean; 241 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 574 KiB/s wr, 10 op/s 2024-09-21T09:52:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:51:59 smithi076 ceph-mon[110963]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T09:52:00.799 INFO:tasks.rados.rados.0.smithi076.stdout:3330: done (0 left) 2024-09-21T09:52:00.799 INFO:tasks.rados.rados.0.smithi076.stdout:3331: snap_remove snap 415 2024-09-21T09:52:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:00 smithi191 ceph-mon[102206]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:52:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:00 smithi076 ceph-mon[106482]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:52:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:00 smithi076 ceph-mon[110963]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T09:52:01.805 INFO:tasks.rados.rados.0.smithi076.stdout:3331: done (0 left) 2024-09-21T09:52:01.805 INFO:tasks.rados.rados.0.smithi076.stdout:3332: snap_remove snap 413 2024-09-21T09:52:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:01 smithi191 ceph-mon[102206]: pgmap v829: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:01 smithi191 ceph-mon[102206]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:52:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:01 smithi076 ceph-mon[106482]: pgmap v829: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:01 smithi076 ceph-mon[106482]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:52:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:01 smithi076 ceph-mon[110963]: pgmap v829: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:01 smithi076 ceph-mon[110963]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T09:52:02.805 INFO:tasks.rados.rados.0.smithi076.stdout:3332: done (0 left) 2024-09-21T09:52:02.806 INFO:tasks.rados.rados.0.smithi076.stdout:3333: snap_create 2024-09-21T09:52:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:02 smithi191 ceph-mon[102206]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:52:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:02 smithi191 ceph-mon[102206]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:52:03.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:02 smithi076 ceph-mon[106482]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:52:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:02 smithi076 ceph-mon[106482]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:52:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:02 smithi076 ceph-mon[110963]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T09:52:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:02 smithi076 ceph-mon[110963]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T09:52:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:52:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:52:03] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-21T09:52:03.790 INFO:tasks.rados.rados.0.smithi076.stdout:3333: done (0 left) 2024-09-21T09:52:03.790 INFO:tasks.rados.rados.0.smithi076.stdout:3334: snap_remove snap 414 2024-09-21T09:52:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:03 smithi191 ceph-mon[102206]: pgmap v832: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:03 smithi191 ceph-mon[102206]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:52:04.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:03 smithi076 ceph-mon[106482]: pgmap v832: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:03 smithi076 ceph-mon[106482]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:52:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:03 smithi076 ceph-mon[110963]: pgmap v832: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:03 smithi076 ceph-mon[110963]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T09:52:04.792 INFO:tasks.rados.rados.0.smithi076.stdout:3334: done (0 left) 2024-09-21T09:52:04.792 INFO:tasks.rados.rados.0.smithi076.stdout:3335: read oid 42 snap -1 2024-09-21T09:52:04.792 INFO:tasks.rados.rados.0.smithi076.stdout:3335: expect (ObjNum 1176 snap 396 seq_num 1176) 2024-09-21T09:52:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3336: rollback oid 23 current snap is 421 2024-09-21T09:52:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 23 to 418 2024-09-21T09:52:04.793 INFO:tasks.rados.rados.0.smithi076.stdout:3337: delete oid 41 current snap is 421 2024-09-21T09:52:04.800 INFO:tasks.rados.rados.0.smithi076.stdout:3337: done (2 left) 2024-09-21T09:52:04.800 INFO:tasks.rados.rados.0.smithi076.stdout:3338: delete oid 22 current snap is 421 2024-09-21T09:52:04.800 INFO:tasks.rados.rados.0.smithi076.stdout:3336: finishing rollback tid 0 to smithi07685715-23 2024-09-21T09:52:04.800 INFO:tasks.rados.rados.0.smithi076.stdout:3336: finishing rollback tid 1 to smithi07685715-23 2024-09-21T09:52:04.801 INFO:tasks.rados.rados.0.smithi076.stdout:3336: finishing rollback tid 2 to smithi07685715-23 2024-09-21T09:52:04.802 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2541 (ObjNum 1214 snap 404 seq_num 1214) dirty exists 2024-09-21T09:52:04.802 INFO:tasks.rados.rados.0.smithi076.stdout:3335: expect (ObjNum 1176 snap 396 seq_num 1176) 2024-09-21T09:52:04.970 INFO:tasks.rados.rados.0.smithi076.stdout:3335: done (2 left) 2024-09-21T09:52:04.970 INFO:tasks.rados.rados.0.smithi076.stdout:3336: done (1 left) 2024-09-21T09:52:04.970 INFO:tasks.rados.rados.0.smithi076.stdout:3338: done (0 left) 2024-09-21T09:52:04.970 INFO:tasks.rados.rados.0.smithi076.stdout:3339: read oid 31 snap 404 2024-09-21T09:52:04.970 INFO:tasks.rados.rados.0.smithi076.stdout:3339: expect (ObjNum 1212 snap 404 seq_num 1212) 2024-09-21T09:52:04.970 INFO:tasks.rados.rados.0.smithi076.stdout:3340: delete oid 8 current snap is 421 2024-09-21T09:52:04.973 INFO:tasks.rados.rados.0.smithi076.stdout:3340: done (1 left) 2024-09-21T09:52:04.973 INFO:tasks.rados.rados.0.smithi076.stdout:3341: write oid 40 current snap is 421 2024-09-21T09:52:04.974 INFO:tasks.rados.rados.0.smithi076.stdout:3341: seq_num 1251 ranges {625557=418391,1838856=786427} 2024-09-21T09:52:04.979 INFO:tasks.rados.rados.0.smithi076.stdout:3341: writing smithi07685715-40 from 625557 to 1043948 tid 1 2024-09-21T09:52:04.988 INFO:tasks.rados.rados.0.smithi076.stdout:3341: writing smithi07685715-40 from 1838856 to 2625283 tid 2 2024-09-21T09:52:04.988 INFO:tasks.rados.rados.0.smithi076.stdout:3342: snap_create 2024-09-21T09:52:04.988 INFO:tasks.rados.rados.0.smithi076.stdout:3339: expect (ObjNum 1212 snap 404 seq_num 1212) 2024-09-21T09:52:05.157 INFO:tasks.rados.rados.0.smithi076.stdout:3341: finishing write tid 1 to smithi07685715-40 2024-09-21T09:52:05.157 INFO:tasks.rados.rados.0.smithi076.stdout:3341: finishing write tid 2 to smithi07685715-40 2024-09-21T09:52:05.157 INFO:tasks.rados.rados.0.smithi076.stdout:3341: finishing write tid 3 to smithi07685715-40 2024-09-21T09:52:05.157 INFO:tasks.rados.rados.0.smithi076.stdout:3341: finishing write tid 4 to smithi07685715-40 2024-09-21T09:52:05.157 INFO:tasks.rados.rados.0.smithi076.stdout:3341: oid 40 updating version 0 to 1635 2024-09-21T09:52:05.158 INFO:tasks.rados.rados.0.smithi076.stdout:3341: oid 40 updating version 1635 to 1637 2024-09-21T09:52:05.158 INFO:tasks.rados.rados.0.smithi076.stdout:3341: oid 40 version 1637 is already newer than 1636 2024-09-21T09:52:05.158 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1637 (ObjNum 1251 snap 421 seq_num 1251) dirty exists 2024-09-21T09:52:05.158 INFO:tasks.rados.rados.0.smithi076.stdout:3341: left oid 40 (ObjNum 1251 snap 421 seq_num 1251) 2024-09-21T09:52:05.822 INFO:tasks.rados.rados.0.smithi076.stdout:3339: done (2 left) 2024-09-21T09:52:05.822 INFO:tasks.rados.rados.0.smithi076.stdout:3341: done (1 left) 2024-09-21T09:52:05.822 INFO:tasks.rados.rados.0.smithi076.stdout:3342: done (0 left) 2024-09-21T09:52:05.822 INFO:tasks.rados.rados.0.smithi076.stdout:3343: read oid 33 snap 418 2024-09-21T09:52:05.822 INFO:tasks.rados.rados.0.smithi076.stdout:3343: expect deleted 2024-09-21T09:52:05.823 INFO:tasks.rados.rados.0.smithi076.stdout:3344: read oid 22 snap -1 2024-09-21T09:52:05.823 INFO:tasks.rados.rados.0.smithi076.stdout:3344: expect deleted 2024-09-21T09:52:05.823 INFO:tasks.rados.rados.0.smithi076.stdout:3345: write oid 8 current snap is 422 2024-09-21T09:52:05.823 INFO:tasks.rados.rados.0.smithi076.stdout:3345: seq_num 1252 ranges {642364=493640,1897338=646438} 2024-09-21T09:52:05.828 INFO:tasks.rados.rados.0.smithi076.stdout:3345: writing smithi07685715-8 from 642364 to 1136004 tid 1 2024-09-21T09:52:05.836 INFO:tasks.rados.rados.0.smithi076.stdout:3345: writing smithi07685715-8 from 1897338 to 2543776 tid 2 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3346: rollback oid 42 current snap is 422 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 411 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3345: finishing write tid 1 to smithi07685715-8 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3343: done (3 left) 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3344: done (2 left) 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3347: read oid 38 snap -1 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3347: expect (ObjNum 1204 snap 404 seq_num 1204) 2024-09-21T09:52:05.837 INFO:tasks.rados.rados.0.smithi076.stdout:3348: snap_remove snap 405 2024-09-21T09:52:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:05 smithi191 ceph-mon[102206]: pgmap v835: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:05 smithi191 ceph-mon[102206]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:52:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:05 smithi076 ceph-mon[106482]: pgmap v835: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:05 smithi076 ceph-mon[106482]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:52:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:05 smithi076 ceph-mon[110963]: pgmap v835: 89 pgs: 89 active+clean; 242 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:52:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:05 smithi076 ceph-mon[110963]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T09:52:06.833 INFO:tasks.rados.rados.0.smithi076.stdout:3348: done (3 left) 2024-09-21T09:52:06.833 INFO:tasks.rados.rados.0.smithi076.stdout:3349: rollback oid 11 current snap is 422 2024-09-21T09:52:06.833 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 406 2024-09-21T09:52:06.834 INFO:tasks.rados.rados.0.smithi076.stdout:3345: finishing write tid 2 to smithi07685715-8 2024-09-21T09:52:06.834 INFO:tasks.rados.rados.0.smithi076.stdout:3347: expect (ObjNum 1204 snap 404 seq_num 1204) 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3345: finishing write tid 3 to smithi07685715-8 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3345: finishing write tid 4 to smithi07685715-8 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3345: oid 8 updating version 0 to 2547 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3345: oid 8 version 2547 is already newer than 2546 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3345: oid 8 version 2547 is already newer than 2545 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2547 (ObjNum 1252 snap 422 seq_num 1252) dirty exists 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3345: left oid 8 (ObjNum 1252 snap 422 seq_num 1252) 2024-09-21T09:52:06.935 INFO:tasks.rados.rados.0.smithi076.stdout:3346: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3346: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3346: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 842 (ObjNum 1176 snap 396 seq_num 1176) dirty exists 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3345: done (3 left) 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3346: done (2 left) 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3347: done (1 left) 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3350: read oid 47 snap -1 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3350: expect (ObjNum 1232 snap 410 seq_num 1232) 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3351: rollback oid 34 current snap is 422 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 404 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3349: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2316 (ObjNum 129 snap 0 seq_num 864555816) dirty dne 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3349: done (2 left) 2024-09-21T09:52:06.936 INFO:tasks.rados.rados.0.smithi076.stdout:3352: read oid 44 snap -1 2024-09-21T09:52:06.937 INFO:tasks.rados.rados.0.smithi076.stdout:3352: expect (ObjNum 1225 snap 406 seq_num 1225) 2024-09-21T09:52:06.937 INFO:tasks.rados.rados.0.smithi076.stdout:3353: read oid 33 snap -1 2024-09-21T09:52:06.937 INFO:tasks.rados.rados.0.smithi076.stdout:3353: expect deleted 2024-09-21T09:52:06.937 INFO:tasks.rados.rados.0.smithi076.stdout:3354: delete oid 30 current snap is 422 2024-09-21T09:52:06.937 INFO:tasks.rados.rados.0.smithi076.stdout:3351: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:52:06.937 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:52:06.943 INFO:tasks.rados.rados.0.smithi076.stdout:3350: expect (ObjNum 1232 snap 410 seq_num 1232) 2024-09-21T09:52:07.029 INFO:tasks.rados.rados.0.smithi076.stdout:3352: expect (ObjNum 1225 snap 406 seq_num 1225) 2024-09-21T09:52:07.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:06 smithi191 ceph-mon[102206]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:52:07.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:06 smithi076 ceph-mon[106482]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:52:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:06 smithi076 ceph-mon[110963]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T09:52:07.261 INFO:tasks.rados.rados.0.smithi076.stdout:3350: done (4 left) 2024-09-21T09:52:07.262 INFO:tasks.rados.rados.0.smithi076.stdout:3351: done (3 left) 2024-09-21T09:52:07.262 INFO:tasks.rados.rados.0.smithi076.stdout:3352: done (2 left) 2024-09-21T09:52:07.262 INFO:tasks.rados.rados.0.smithi076.stdout:3353: done (1 left) 2024-09-21T09:52:07.262 INFO:tasks.rados.rados.0.smithi076.stdout:3354: done (0 left) 2024-09-21T09:52:07.262 INFO:tasks.rados.rados.0.smithi076.stdout:3355: delete oid 47 current snap is 422 2024-09-21T09:52:07.264 INFO:tasks.rados.rados.0.smithi076.stdout:3355: done (0 left) 2024-09-21T09:52:07.264 INFO:tasks.rados.rados.0.smithi076.stdout:3356: snap_remove snap 416 2024-09-21T09:52:07.828 INFO:tasks.rados.rados.0.smithi076.stdout:3356: done (0 left) 2024-09-21T09:52:07.828 INFO:tasks.rados.rados.0.smithi076.stdout:3357: snap_remove snap 399 2024-09-21T09:52:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:07 smithi191 ceph-mon[102206]: pgmap v838: 89 pgs: 89 active+clean; 245 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 573 KiB/s wr, 5 op/s 2024-09-21T09:52:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:07 smithi191 ceph-mon[102206]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:52:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:07 smithi076 ceph-mon[106482]: pgmap v838: 89 pgs: 89 active+clean; 245 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 573 KiB/s wr, 5 op/s 2024-09-21T09:52:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:07 smithi076 ceph-mon[106482]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:52:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:07 smithi076 ceph-mon[110963]: pgmap v838: 89 pgs: 89 active+clean; 245 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 573 KiB/s wr, 5 op/s 2024-09-21T09:52:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:07 smithi076 ceph-mon[110963]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T09:52:08.793 INFO:tasks.rados.rados.0.smithi076.stdout:3357: done (0 left) 2024-09-21T09:52:08.793 INFO:tasks.rados.rados.0.smithi076.stdout:3358: delete oid 3 current snap is 422 2024-09-21T09:52:08.802 INFO:tasks.rados.rados.0.smithi076.stdout:3358: done (0 left) 2024-09-21T09:52:08.802 INFO:tasks.rados.rados.0.smithi076.stdout:3359: write oid 22 current snap is 422 2024-09-21T09:52:08.802 INFO:tasks.rados.rados.0.smithi076.stdout:3359: seq_num 1256 ranges {709592=510989,1731266=653776,3041279=68313} 2024-09-21T09:52:08.807 INFO:tasks.rados.rados.0.smithi076.stdout:3359: writing smithi07685715-22 from 709592 to 1220581 tid 1 2024-09-21T09:52:08.814 INFO:tasks.rados.rados.0.smithi076.stdout:3359: writing smithi07685715-22 from 1731266 to 2385042 tid 2 2024-09-21T09:52:08.818 INFO:tasks.rados.rados.0.smithi076.stdout:3359: writing smithi07685715-22 from 3041279 to 3109592 tid 3 2024-09-21T09:52:08.818 INFO:tasks.rados.rados.0.smithi076.stdout:3360: write oid 45 current snap is 422 2024-09-21T09:52:08.819 INFO:tasks.rados.rados.0.smithi076.stdout:3360: seq_num 1257 ranges {726399=586238,2073395=797434} 2024-09-21T09:52:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:3360: writing smithi07685715-45 from 726399 to 1312637 tid 1 2024-09-21T09:52:08.834 INFO:tasks.rados.rados.0.smithi076.stdout:3360: writing smithi07685715-45 from 2073395 to 2870829 tid 2 2024-09-21T09:52:08.834 INFO:tasks.rados.rados.0.smithi076.stdout:3361: write oid 10 current snap is 422 2024-09-21T09:52:08.835 INFO:tasks.rados.rados.0.smithi076.stdout:3359: finishing write tid 1 to smithi07685715-22 2024-09-21T09:52:08.835 INFO:tasks.rados.rados.0.smithi076.stdout:3361: seq_num 1258 ranges {743206=661487,2131877=541092} 2024-09-21T09:52:08.842 INFO:tasks.rados.rados.0.smithi076.stdout:3361: writing smithi07685715-10 from 743206 to 1404693 tid 1 2024-09-21T09:52:08.849 INFO:tasks.rados.rados.0.smithi076.stdout:3361: writing smithi07685715-10 from 2131877 to 2672969 tid 2 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3362: write oid 36 current snap is 422 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: finishing write tid 2 to smithi07685715-22 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: finishing write tid 3 to smithi07685715-22 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: finishing write tid 4 to smithi07685715-22 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: finishing write tid 5 to smithi07685715-22 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: oid 22 updating version 0 to 1659 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: oid 22 updating version 1659 to 1660 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: oid 22 updating version 1660 to 1661 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: oid 22 version 1661 is already newer than 1658 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1661 (ObjNum 1256 snap 422 seq_num 1256) dirty exists 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3359: left oid 22 (ObjNum 1256 snap 422 seq_num 1256) 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3360: finishing write tid 1 to smithi07685715-45 2024-09-21T09:52:08.850 INFO:tasks.rados.rados.0.smithi076.stdout:3362: seq_num 1259 ranges {760013=736736,2190359=401103} 2024-09-21T09:52:08.857 INFO:tasks.rados.rados.0.smithi076.stdout:3362: writing smithi07685715-36 from 760013 to 1496749 tid 1 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3362: writing smithi07685715-36 from 2190359 to 2591462 tid 2 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3359: done (3 left) 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3363: read oid 34 snap 421 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3363: expect deleted 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3364: delete oid 28 current snap is 422 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3360: finishing write tid 2 to smithi07685715-45 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3360: finishing write tid 3 to smithi07685715-45 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3360: finishing write tid 4 to smithi07685715-45 2024-09-21T09:52:08.864 INFO:tasks.rados.rados.0.smithi076.stdout:3360: oid 45 updating version 0 to 733 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3360: oid 45 version 733 is already newer than 732 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3360: oid 45 version 733 is already newer than 731 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 733 (ObjNum 1257 snap 422 seq_num 1257) dirty exists 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3360: left oid 45 (ObjNum 1257 snap 422 seq_num 1257) 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: finishing write tid 1 to smithi07685715-10 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: finishing write tid 2 to smithi07685715-10 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: finishing write tid 3 to smithi07685715-10 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: finishing write tid 4 to smithi07685715-10 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: oid 10 updating version 0 to 2104 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: oid 10 updating version 2104 to 2105 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: oid 10 version 2105 is already newer than 2103 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 2105 (ObjNum 1258 snap 422 seq_num 1258) dirty exists 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3361: left oid 10 (ObjNum 1258 snap 422 seq_num 1258) 2024-09-21T09:52:08.865 INFO:tasks.rados.rados.0.smithi076.stdout:3362: finishing write tid 1 to smithi07685715-36 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:3362: finishing write tid 2 to smithi07685715-36 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:3362: finishing write tid 3 to smithi07685715-36 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:3362: finishing write tid 4 to smithi07685715-36 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:3362: oid 36 updating version 0 to 1083 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:3362: oid 36 updating version 1083 to 1084 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:3362: oid 36 version 1084 is already newer than 1082 2024-09-21T09:52:08.868 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1084 (ObjNum 1259 snap 422 seq_num 1259) dirty exists 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3362: left oid 36 (ObjNum 1259 snap 422 seq_num 1259) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3360: done (4 left) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3361: done (3 left) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3362: done (2 left) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3363: done (1 left) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3364: done (0 left) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3365: read oid 32 snap 406 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3365: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3366: write oid 3 current snap is 422 2024-09-21T09:52:08.869 INFO:tasks.rados.rados.0.smithi076.stdout:3366: seq_num 1261 ranges {793627=487234,1790970=404772,2829335=364292} 2024-09-21T09:52:08.875 INFO:tasks.rados.rados.0.smithi076.stdout:3366: writing smithi07685715-3 from 793627 to 1280861 tid 1 2024-09-21T09:52:08.879 INFO:tasks.rados.rados.0.smithi076.stdout:3366: writing smithi07685715-3 from 1790970 to 2195742 tid 2 2024-09-21T09:52:08.885 INFO:tasks.rados.rados.0.smithi076.stdout:3366: writing smithi07685715-3 from 2829335 to 3193627 tid 3 2024-09-21T09:52:08.885 INFO:tasks.rados.rados.0.smithi076.stdout:3367: snap_create 2024-09-21T09:52:08.885 INFO:tasks.rados.rados.0.smithi076.stdout:3365: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:52:08.956 INFO:tasks.rados.rados.0.smithi076.stdout:3366: finishing write tid 1 to smithi07685715-3 2024-09-21T09:52:08.956 INFO:tasks.rados.rados.0.smithi076.stdout:3366: finishing write tid 2 to smithi07685715-3 2024-09-21T09:52:08.956 INFO:tasks.rados.rados.0.smithi076.stdout:3366: finishing write tid 3 to smithi07685715-3 2024-09-21T09:52:08.956 INFO:tasks.rados.rados.0.smithi076.stdout:3366: finishing write tid 4 to smithi07685715-3 2024-09-21T09:52:08.956 INFO:tasks.rados.rados.0.smithi076.stdout:3366: finishing write tid 5 to smithi07685715-3 2024-09-21T09:52:08.956 INFO:tasks.rados.rados.0.smithi076.stdout:3366: oid 3 updating version 0 to 256 2024-09-21T09:52:08.957 INFO:tasks.rados.rados.0.smithi076.stdout:3366: oid 3 updating version 256 to 257 2024-09-21T09:52:08.957 INFO:tasks.rados.rados.0.smithi076.stdout:3366: oid 3 version 257 is already newer than 255 2024-09-21T09:52:08.957 INFO:tasks.rados.rados.0.smithi076.stdout:3366: oid 3 updating version 257 to 258 2024-09-21T09:52:08.957 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 258 (ObjNum 1261 snap 422 seq_num 1261) dirty exists 2024-09-21T09:52:08.957 INFO:tasks.rados.rados.0.smithi076.stdout:3366: left oid 3 (ObjNum 1261 snap 422 seq_num 1261) 2024-09-21T09:52:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:08 smithi191 ceph-mon[102206]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:52:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:08 smithi191 ceph-mon[102206]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:52:09.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:08 smithi076 ceph-mon[110963]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:52:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:08 smithi076 ceph-mon[110963]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:52:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:08 smithi076 ceph-mon[106482]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T09:52:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:08 smithi076 ceph-mon[106482]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T09:52:09.850 INFO:tasks.rados.rados.0.smithi076.stdout:3365: done (2 left) 2024-09-21T09:52:09.851 INFO:tasks.rados.rados.0.smithi076.stdout:3366: done (1 left) 2024-09-21T09:52:09.851 INFO:tasks.rados.rados.0.smithi076.stdout:3367: done (0 left) 2024-09-21T09:52:09.851 INFO:tasks.rados.rados.0.smithi076.stdout:3368: snap_remove snap 404 2024-09-21T09:52:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:09 smithi191 ceph-mon[102206]: pgmap v841: 89 pgs: 89 active+clean; 245 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 573 KiB/s wr, 5 op/s 2024-09-21T09:52:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:09 smithi076 ceph-mon[106482]: pgmap v841: 89 pgs: 89 active+clean; 245 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 573 KiB/s wr, 5 op/s 2024-09-21T09:52:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:09 smithi076 ceph-mon[110963]: pgmap v841: 89 pgs: 89 active+clean; 245 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 573 KiB/s wr, 5 op/s 2024-09-21T09:52:10.860 INFO:tasks.rados.rados.0.smithi076.stdout:3368: done (0 left) 2024-09-21T09:52:10.860 INFO:tasks.rados.rados.0.smithi076.stdout:3369: read oid 7 snap 418 2024-09-21T09:52:10.860 INFO:tasks.rados.rados.0.smithi076.stdout:3369: expect (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:52:10.860 INFO:tasks.rados.rados.0.smithi076.stdout:3370: snap_create 2024-09-21T09:52:10.875 INFO:tasks.rados.rados.0.smithi076.stdout:3369: expect (ObjNum 1186 snap 399 seq_num 1186) 2024-09-21T09:52:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:10 smithi191 ceph-mon[102206]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:52:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:10 smithi076 ceph-mon[106482]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:52:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:10 smithi076 ceph-mon[110963]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T09:52:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3369: done (1 left) 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3370: done (0 left) 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3371: read oid 18 snap -1 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3371: expect (ObjNum 1205 snap 404 seq_num 1205) 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3372: read oid 27 snap -1 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3372: expect (ObjNum 1228 snap 408 seq_num 1228) 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3373: read oid 38 snap -1 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3373: expect (ObjNum 1204 snap 404 seq_num 1204) 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3374: read oid 25 snap 408 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3374: expect (ObjNum 1165 snap 393 seq_num 1165) 2024-09-21T09:52:11.858 INFO:tasks.rados.rados.0.smithi076.stdout:3375: read oid 49 snap 408 2024-09-21T09:52:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:3375: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:52:11.859 INFO:tasks.rados.rados.0.smithi076.stdout:3376: snap_remove snap 419 2024-09-21T09:52:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:11 smithi191 ceph-mon[102206]: pgmap v844: 89 pgs: 89 active+clean; 234 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 338 KiB/s wr, 8 op/s 2024-09-21T09:52:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:11 smithi191 ceph-mon[102206]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:52:12.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:11 smithi076 ceph-mon[106482]: pgmap v844: 89 pgs: 89 active+clean; 234 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 338 KiB/s wr, 8 op/s 2024-09-21T09:52:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:11 smithi076 ceph-mon[106482]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:52:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:11 smithi076 ceph-mon[110963]: pgmap v844: 89 pgs: 89 active+clean; 234 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 338 KiB/s wr, 8 op/s 2024-09-21T09:52:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:11 smithi076 ceph-mon[110963]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T09:52:12.861 INFO:tasks.rados.rados.0.smithi076.stdout:3376: done (5 left) 2024-09-21T09:52:12.861 INFO:tasks.rados.rados.0.smithi076.stdout:3377: rollback oid 20 current snap is 424 2024-09-21T09:52:12.861 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 375 2024-09-21T09:52:12.861 INFO:tasks.rados.rados.0.smithi076.stdout:3374: expect (ObjNum 1165 snap 393 seq_num 1165) 2024-09-21T09:52:12.978 INFO:tasks.rados.rados.0.smithi076.stdout:3371: expect (ObjNum 1205 snap 404 seq_num 1205) 2024-09-21T09:52:13.050 INFO:tasks.rados.rados.0.smithi076.stdout:3373: expect (ObjNum 1204 snap 404 seq_num 1204) 2024-09-21T09:52:13.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:12 smithi076 ceph-mon[106482]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:52:13.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:12 smithi076 ceph-mon[110963]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:52:13.147 INFO:tasks.rados.rados.0.smithi076.stdout:3375: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:52:13.265 INFO:tasks.rados.rados.0.smithi076.stdout:3372: expect (ObjNum 1228 snap 408 seq_num 1228) 2024-09-21T09:52:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:12 smithi191 ceph-mon[102206]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T09:52:13.372 INFO:tasks.rados.rados.0.smithi076.stdout:3377: finishing rollback tid 0 to smithi07685715-20 2024-09-21T09:52:13.372 INFO:tasks.rados.rados.0.smithi076.stdout:3377: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:52:13.372 INFO:tasks.rados.rados.0.smithi076.stdout:3377: finishing rollback tid 2 to smithi07685715-20 2024-09-21T09:52:13.372 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 1010 (ObjNum 1127 snap 371 seq_num 1127) dirty exists 2024-09-21T09:52:13.372 INFO:tasks.rados.rados.0.smithi076.stdout:3371: done (5 left) 2024-09-21T09:52:13.372 INFO:tasks.rados.rados.0.smithi076.stdout:3372: done (4 left) 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:3373: done (3 left) 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:3374: done (2 left) 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:3375: done (1 left) 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:3377: done (0 left) 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:3378: rollback oid 31 current snap is 424 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 412 2024-09-21T09:52:13.373 INFO:tasks.rados.rados.0.smithi076.stdout:3379: snap_remove snap 418 2024-09-21T09:52:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:52:13] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-09-21T09:52:13.792 INFO:tasks.rados.rados.0.smithi076.stdout:3379: done (1 left) 2024-09-21T09:52:13.792 INFO:tasks.rados.rados.0.smithi076.stdout:3380: snap_remove snap 375 2024-09-21T09:52:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-mon[110963]: pgmap v847: 89 pgs: 89 active+clean; 235 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-21T09:52:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-mon[110963]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:52:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-mon[110963]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:52:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-mon[106482]: pgmap v847: 89 pgs: 89 active+clean; 235 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-21T09:52:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-mon[106482]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:52:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:13 smithi076 ceph-mon[106482]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:52:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:13 smithi191 ceph-mon[102206]: pgmap v847: 89 pgs: 89 active+clean; 235 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-09-21T09:52:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:13 smithi191 ceph-mon[102206]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T09:52:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:13 smithi191 ceph-mon[102206]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T09:52:14.796 INFO:tasks.rados.rados.0.smithi076.stdout:3380: done (1 left) 2024-09-21T09:52:14.797 INFO:tasks.rados.rados.0.smithi076.stdout:3381: read oid 28 snap -1 2024-09-21T09:52:14.797 INFO:tasks.rados.rados.0.smithi076.stdout:3381: expect deleted 2024-09-21T09:52:14.797 INFO:tasks.rados.rados.0.smithi076.stdout:3382: write oid 38 current snap is 424 2024-09-21T09:52:14.797 INFO:tasks.rados.rados.0.smithi076.stdout:3382: seq_num 1262 ranges {410434=678836,1565805=548430,2573111=637323} 2024-09-21T09:52:14.803 INFO:tasks.rados.rados.0.smithi076.stdout:3382: writing smithi07685715-38 from 410434 to 1089270 tid 1 2024-09-21T09:52:14.810 INFO:tasks.rados.rados.0.smithi076.stdout:3382: writing smithi07685715-38 from 1565805 to 2114235 tid 2 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3382: writing smithi07685715-38 from 2573111 to 3210434 tid 3 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3383: rollback oid 43 current snap is 424 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 423 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3378: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 976 (ObjNum 257 snap 0 seq_num 865233648) dirty dne 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3382: finishing write tid 1 to smithi07685715-38 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3378: done (3 left) 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3381: done (2 left) 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:3384: rollback oid 30 current snap is 424 2024-09-21T09:52:14.817 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 411 2024-09-21T09:52:14.818 INFO:tasks.rados.rados.0.smithi076.stdout:3385: snap_remove snap 420 2024-09-21T09:52:15.801 INFO:tasks.rados.rados.0.smithi076.stdout:3385: done (3 left) 2024-09-21T09:52:15.801 INFO:tasks.rados.rados.0.smithi076.stdout:3386: delete oid 5 current snap is 424 2024-09-21T09:52:15.801 INFO:tasks.rados.rados.0.smithi076.stdout:3382: finishing write tid 2 to smithi07685715-38 2024-09-21T09:52:15.801 INFO:tasks.rados.rados.0.smithi076.stdout:3383: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:52:15.801 INFO:tasks.rados.rados.0.smithi076.stdout:3384: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:52:15.801 INFO:tasks.rados.rados.0.smithi076.stdout:3383: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3384: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2609 (ObjNum 1217 snap 406 seq_num 1217) dirty exists 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3383: finishing rollback tid 2 to smithi07685715-43 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 764 (ObjNum 1170 snap 393 seq_num 1170) dirty exists 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: finishing write tid 3 to smithi07685715-38 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: finishing write tid 4 to smithi07685715-38 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: finishing write tid 5 to smithi07685715-38 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: oid 38 updating version 0 to 2127 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: oid 38 version 2127 is already newer than 2124 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: oid 38 version 2127 is already newer than 2119 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: oid 38 updating version 2127 to 2128 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 2128 (ObjNum 1262 snap 424 seq_num 1262) dirty exists 2024-09-21T09:52:15.802 INFO:tasks.rados.rados.0.smithi076.stdout:3382: left oid 38 (ObjNum 1262 snap 424 seq_num 1262) 2024-09-21T09:52:15.806 INFO:tasks.rados.rados.0.smithi076.stdout:3382: done (3 left) 2024-09-21T09:52:15.806 INFO:tasks.rados.rados.0.smithi076.stdout:3383: done (2 left) 2024-09-21T09:52:15.806 INFO:tasks.rados.rados.0.smithi076.stdout:3384: done (1 left) 2024-09-21T09:52:15.806 INFO:tasks.rados.rados.0.smithi076.stdout:3386: done (0 left) 2024-09-21T09:52:15.806 INFO:tasks.rados.rados.0.smithi076.stdout:3387: delete oid 27 current snap is 424 2024-09-21T09:52:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:3387: done (0 left) 2024-09-21T09:52:15.811 INFO:tasks.rados.rados.0.smithi076.stdout:3388: snap_create 2024-09-21T09:52:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:15 smithi191 ceph-mon[102206]: pgmap v850: 89 pgs: 89 active+clean; 234 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:52:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:15 smithi191 ceph-mon[102206]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:52:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:15 smithi076 ceph-mon[106482]: pgmap v850: 89 pgs: 89 active+clean; 234 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:52:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:15 smithi076 ceph-mon[106482]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:52:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:15 smithi076 ceph-mon[110963]: pgmap v850: 89 pgs: 89 active+clean; 234 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 16 op/s 2024-09-21T09:52:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:15 smithi076 ceph-mon[110963]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T09:52:16.806 INFO:tasks.rados.rados.0.smithi076.stdout:3388: done (0 left) 2024-09-21T09:52:16.806 INFO:tasks.rados.rados.0.smithi076.stdout:3389: write oid 4 current snap is 425 2024-09-21T09:52:16.806 INFO:tasks.rados.rados.0.smithi076.stdout:3389: seq_num 1265 ranges {460855=504583,1624898=412110,2539027=572508} 2024-09-21T09:52:16.811 INFO:tasks.rados.rados.0.smithi076.stdout:3389: writing smithi07685715-4 from 460855 to 965438 tid 1 2024-09-21T09:52:16.816 INFO:tasks.rados.rados.0.smithi076.stdout:3389: writing smithi07685715-4 from 1624898 to 2037008 tid 2 2024-09-21T09:52:16.822 INFO:tasks.rados.rados.0.smithi076.stdout:3389: writing smithi07685715-4 from 2539027 to 3111535 tid 3 2024-09-21T09:52:16.822 INFO:tasks.rados.rados.0.smithi076.stdout:3390: read oid 29 snap -1 2024-09-21T09:52:16.823 INFO:tasks.rados.rados.0.smithi076.stdout:3390: expect deleted 2024-09-21T09:52:16.823 INFO:tasks.rados.rados.0.smithi076.stdout:3391: write oid 19 current snap is 425 2024-09-21T09:52:16.823 INFO:tasks.rados.rados.0.smithi076.stdout:3391: seq_num 1266 ranges {477662=579832,1683380=555768,2966450=311212} 2024-09-21T09:52:16.828 INFO:tasks.rados.rados.0.smithi076.stdout:3391: writing smithi07685715-19 from 477662 to 1057494 tid 1 2024-09-21T09:52:16.835 INFO:tasks.rados.rados.0.smithi076.stdout:3391: writing smithi07685715-19 from 1683380 to 2239148 tid 2 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3391: writing smithi07685715-19 from 2966450 to 3277662 tid 3 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3392: read oid 49 snap -1 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3392: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3393: rollback oid 30 current snap is 425 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 411 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3389: finishing write tid 1 to smithi07685715-4 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3394: snap_create 2024-09-21T09:52:16.841 INFO:tasks.rados.rados.0.smithi076.stdout:3389: finishing write tid 2 to smithi07685715-4 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: finishing write tid 3 to smithi07685715-4 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: finishing write tid 4 to smithi07685715-4 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: finishing write tid 5 to smithi07685715-4 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: oid 4 updating version 0 to 1092 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: oid 4 updating version 1092 to 1094 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: oid 4 updating version 1094 to 1095 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: oid 4 version 1095 is already newer than 1093 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 1095 (ObjNum 1265 snap 425 seq_num 1265) dirty exists 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3389: left oid 4 (ObjNum 1265 snap 425 seq_num 1265) 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3391: finishing write tid 1 to smithi07685715-19 2024-09-21T09:52:16.842 INFO:tasks.rados.rados.0.smithi076.stdout:3391: finishing write tid 2 to smithi07685715-19 2024-09-21T09:52:16.843 INFO:tasks.rados.rados.0.smithi076.stdout:3391: finishing write tid 3 to smithi07685715-19 2024-09-21T09:52:16.843 INFO:tasks.rados.rados.0.smithi076.stdout:3391: finishing write tid 4 to smithi07685715-19 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:3391: finishing write tid 5 to smithi07685715-19 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:3391: oid 19 updating version 0 to 1018 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:3391: oid 19 version 1018 is already newer than 1016 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:3391: oid 19 version 1018 is already newer than 1017 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:3391: oid 19 version 1018 is already newer than 1015 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 1018 (ObjNum 1266 snap 425 seq_num 1266) dirty exists 2024-09-21T09:52:16.844 INFO:tasks.rados.rados.0.smithi076.stdout:3391: left oid 19 (ObjNum 1266 snap 425 seq_num 1266) 2024-09-21T09:52:16.845 INFO:tasks.rados.rados.0.smithi076.stdout:3392: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:52:16.963 INFO:tasks.rados.rados.0.smithi076.stdout:3393: finishing rollback tid 0 to smithi07685715-30 2024-09-21T09:52:16.963 INFO:tasks.rados.rados.0.smithi076.stdout:3393: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:52:16.963 INFO:tasks.rados.rados.0.smithi076.stdout:3393: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:52:16.963 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2621 (ObjNum 1217 snap 406 seq_num 1217) dirty exists 2024-09-21T09:52:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:16 smithi191 ceph-mon[102206]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:52:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:16 smithi076 ceph-mon[110963]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:52:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:16 smithi076 ceph-mon[106482]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3389: done (5 left) 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3390: done (4 left) 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3391: done (3 left) 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3392: done (2 left) 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3393: done (1 left) 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3394: done (0 left) 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3395: read oid 3 snap 412 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3395: expect deleted 2024-09-21T09:52:17.819 INFO:tasks.rados.rados.0.smithi076.stdout:3396: snap_create 2024-09-21T09:52:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:17 smithi191 ceph-mon[102206]: pgmap v853: 89 pgs: 89 active+clean; 216 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 294 KiB/s wr, 14 op/s 2024-09-21T09:52:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:17 smithi191 ceph-mon[102206]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:52:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:17 smithi076 ceph-mon[106482]: pgmap v853: 89 pgs: 89 active+clean; 216 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 294 KiB/s wr, 14 op/s 2024-09-21T09:52:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:17 smithi076 ceph-mon[106482]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:52:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:17 smithi076 ceph-mon[110963]: pgmap v853: 89 pgs: 89 active+clean; 216 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 294 KiB/s wr, 14 op/s 2024-09-21T09:52:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:17 smithi076 ceph-mon[110963]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T09:52:18.799 INFO:tasks.rados.rados.0.smithi076.stdout:3395: done (1 left) 2024-09-21T09:52:18.799 INFO:tasks.rados.rados.0.smithi076.stdout:3396: done (0 left) 2024-09-21T09:52:18.799 INFO:tasks.rados.rados.0.smithi076.stdout:3397: read oid 2 snap -1 2024-09-21T09:52:18.799 INFO:tasks.rados.rados.0.smithi076.stdout:3397: expect deleted 2024-09-21T09:52:18.799 INFO:tasks.rados.rados.0.smithi076.stdout:3398: delete oid 48 current snap is 427 2024-09-21T09:52:18.804 INFO:tasks.rados.rados.0.smithi076.stdout:3397: done (1 left) 2024-09-21T09:52:18.804 INFO:tasks.rados.rados.0.smithi076.stdout:3398: done (0 left) 2024-09-21T09:52:18.804 INFO:tasks.rados.rados.0.smithi076.stdout:3399: delete oid 45 current snap is 427 2024-09-21T09:52:18.806 INFO:tasks.rados.rados.0.smithi076.stdout:3399: done (0 left) 2024-09-21T09:52:18.807 INFO:tasks.rados.rados.0.smithi076.stdout:3400: rollback oid 39 current snap is 427 2024-09-21T09:52:18.807 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 387 2024-09-21T09:52:18.807 INFO:tasks.rados.rados.0.smithi076.stdout:3401: write oid 5 current snap is 427 2024-09-21T09:52:18.807 INFO:tasks.rados.rados.0.smithi076.stdout:3401: seq_num 1269 ranges {528083=405579,1342473=419448,2248719=733949} 2024-09-21T09:52:18.812 INFO:tasks.rados.rados.0.smithi076.stdout:3401: writing smithi07685715-5 from 528083 to 933662 tid 1 2024-09-21T09:52:18.817 INFO:tasks.rados.rados.0.smithi076.stdout:3401: writing smithi07685715-5 from 1342473 to 1761921 tid 2 2024-09-21T09:52:18.823 INFO:tasks.rados.rados.0.smithi076.stdout:3401: writing smithi07685715-5 from 2248719 to 2982668 tid 3 2024-09-21T09:52:18.823 INFO:tasks.rados.rados.0.smithi076.stdout:3402: snap_create 2024-09-21T09:52:18.824 INFO:tasks.rados.rados.0.smithi076.stdout:3400: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:52:18.824 INFO:tasks.rados.rados.0.smithi076.stdout:3400: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:52:18.824 INFO:tasks.rados.rados.0.smithi076.stdout:3400: finishing rollback tid 2 to smithi07685715-39 2024-09-21T09:52:18.824 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2646 (ObjNum 1097 snap 364 seq_num 1097) dirty exists 2024-09-21T09:52:18.824 INFO:tasks.rados.rados.0.smithi076.stdout:3401: finishing write tid 1 to smithi07685715-5 2024-09-21T09:52:18.824 INFO:tasks.rados.rados.0.smithi076.stdout:3401: finishing write tid 2 to smithi07685715-5 2024-09-21T09:52:18.828 INFO:tasks.rados.rados.0.smithi076.stdout:3401: finishing write tid 3 to smithi07685715-5 2024-09-21T09:52:18.828 INFO:tasks.rados.rados.0.smithi076.stdout:3401: finishing write tid 4 to smithi07685715-5 2024-09-21T09:52:18.828 INFO:tasks.rados.rados.0.smithi076.stdout:3401: finishing write tid 5 to smithi07685715-5 2024-09-21T09:52:18.828 INFO:tasks.rados.rados.0.smithi076.stdout:3401: oid 5 updating version 0 to 2650 2024-09-21T09:52:18.829 INFO:tasks.rados.rados.0.smithi076.stdout:3401: oid 5 version 2650 is already newer than 2649 2024-09-21T09:52:18.829 INFO:tasks.rados.rados.0.smithi076.stdout:3401: oid 5 version 2650 is already newer than 2647 2024-09-21T09:52:18.829 INFO:tasks.rados.rados.0.smithi076.stdout:3401: oid 5 version 2650 is already newer than 2648 2024-09-21T09:52:18.829 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2650 (ObjNum 1269 snap 427 seq_num 1269) dirty exists 2024-09-21T09:52:18.829 INFO:tasks.rados.rados.0.smithi076.stdout:3401: left oid 5 (ObjNum 1269 snap 427 seq_num 1269) 2024-09-21T09:52:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:18 smithi191 ceph-mon[102206]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:52:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:18 smithi191 ceph-mon[102206]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:52:19.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:18 smithi076 ceph-mon[110963]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:52:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:18 smithi076 ceph-mon[110963]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:52:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:18 smithi076 ceph-mon[106482]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T09:52:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:18 smithi076 ceph-mon[106482]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T09:52:19.826 INFO:tasks.rados.rados.0.smithi076.stdout:3400: done (2 left) 2024-09-21T09:52:19.826 INFO:tasks.rados.rados.0.smithi076.stdout:3401: done (1 left) 2024-09-21T09:52:19.826 INFO:tasks.rados.rados.0.smithi076.stdout:3402: done (0 left) 2024-09-21T09:52:19.826 INFO:tasks.rados.rados.0.smithi076.stdout:3403: read oid 46 snap -1 2024-09-21T09:52:19.826 INFO:tasks.rados.rados.0.smithi076.stdout:3403: expect deleted 2024-09-21T09:52:19.826 INFO:tasks.rados.rados.0.smithi076.stdout:3404: snap_create 2024-09-21T09:52:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:19 smithi191 ceph-mon[102206]: pgmap v856: 89 pgs: 89 active+clean; 216 MiB data, 783 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:52:20.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:19 smithi076 ceph-mon[106482]: pgmap v856: 89 pgs: 89 active+clean; 216 MiB data, 783 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:52:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:19 smithi076 ceph-mon[110963]: pgmap v856: 89 pgs: 89 active+clean; 216 MiB data, 783 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:52:20.832 INFO:tasks.rados.rados.0.smithi076.stdout:3403: done (1 left) 2024-09-21T09:52:20.832 INFO:tasks.rados.rados.0.smithi076.stdout:3404: done (0 left) 2024-09-21T09:52:20.832 INFO:tasks.rados.rados.0.smithi076.stdout:3405: rollback oid 32 current snap is 429 2024-09-21T09:52:20.832 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 426 2024-09-21T09:52:20.832 INFO:tasks.rados.rados.0.smithi076.stdout:3406: delete oid 6 current snap is 429 2024-09-21T09:52:20.836 INFO:tasks.rados.rados.0.smithi076.stdout:3405: finishing rollback tid 0 to smithi07685715-32 2024-09-21T09:52:20.836 INFO:tasks.rados.rados.0.smithi076.stdout:3405: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:52:20.837 INFO:tasks.rados.rados.0.smithi076.stdout:3405: finishing rollback tid 2 to smithi07685715-32 2024-09-21T09:52:20.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 1022 (ObjNum 1199 snap 401 seq_num 1199) dirty exists 2024-09-21T09:52:20.837 INFO:tasks.rados.rados.0.smithi076.stdout:3405: done (1 left) 2024-09-21T09:52:20.837 INFO:tasks.rados.rados.0.smithi076.stdout:3406: done (0 left) 2024-09-21T09:52:20.837 INFO:tasks.rados.rados.0.smithi076.stdout:3407: snap_create 2024-09-21T09:52:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:20 smithi191 ceph-mon[102206]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:52:21.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:20 smithi076 ceph-mon[106482]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:52:21.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:20 smithi076 ceph-mon[110963]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T09:52:21.836 INFO:tasks.rados.rados.0.smithi076.stdout:3407: done (0 left) 2024-09-21T09:52:21.836 INFO:tasks.rados.rados.0.smithi076.stdout:3408: read oid 2 snap -1 2024-09-21T09:52:21.836 INFO:tasks.rados.rados.0.smithi076.stdout:3408: expect deleted 2024-09-21T09:52:21.836 INFO:tasks.rados.rados.0.smithi076.stdout:3409: read oid 23 snap 411 2024-09-21T09:52:21.837 INFO:tasks.rados.rados.0.smithi076.stdout:3409: expect (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:52:21.837 INFO:tasks.rados.rados.0.smithi076.stdout:3410: delete oid 47 current snap is 430 2024-09-21T09:52:21.851 INFO:tasks.rados.rados.0.smithi076.stdout:3409: expect (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:52:21.949 INFO:tasks.rados.rados.0.smithi076.stdout:3408: done (2 left) 2024-09-21T09:52:21.949 INFO:tasks.rados.rados.0.smithi076.stdout:3409: done (1 left) 2024-09-21T09:52:21.950 INFO:tasks.rados.rados.0.smithi076.stdout:3410: done (0 left) 2024-09-21T09:52:21.950 INFO:tasks.rados.rados.0.smithi076.stdout:3411: read oid 32 snap 424 2024-09-21T09:52:21.950 INFO:tasks.rados.rados.0.smithi076.stdout:3411: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:52:21.950 INFO:tasks.rados.rados.0.smithi076.stdout:3412: snap_remove snap 411 2024-09-21T09:52:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:21 smithi191 ceph-mon[102206]: pgmap v859: 89 pgs: 89 active+clean; 216 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 809 KiB/s wr, 3 op/s 2024-09-21T09:52:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:21 smithi191 ceph-mon[102206]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:52:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:21 smithi076 ceph-mon[110963]: pgmap v859: 89 pgs: 89 active+clean; 216 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 809 KiB/s wr, 3 op/s 2024-09-21T09:52:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:21 smithi076 ceph-mon[110963]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:52:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:21 smithi076 ceph-mon[106482]: pgmap v859: 89 pgs: 89 active+clean; 216 MiB data, 786 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 809 KiB/s wr, 3 op/s 2024-09-21T09:52:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:21 smithi076 ceph-mon[106482]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T09:52:22.851 INFO:tasks.rados.rados.0.smithi076.stdout:3412: done (1 left) 2024-09-21T09:52:22.851 INFO:tasks.rados.rados.0.smithi076.stdout:3413: snap_remove snap 423 2024-09-21T09:52:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:22 smithi191 ceph-mon[102206]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:52:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:22 smithi076 ceph-mon[110963]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:52:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:22 smithi076 ceph-mon[106482]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T09:52:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:52:23] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-09-21T09:52:23.799 INFO:tasks.rados.rados.0.smithi076.stdout:3413: done (1 left) 2024-09-21T09:52:23.799 INFO:tasks.rados.rados.0.smithi076.stdout:3414: read oid 49 snap -1 2024-09-21T09:52:23.799 INFO:tasks.rados.rados.0.smithi076.stdout:3414: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:52:23.799 INFO:tasks.rados.rados.0.smithi076.stdout:3415: snap_create 2024-09-21T09:52:23.799 INFO:tasks.rados.rados.0.smithi076.stdout:3411: expect (ObjNum 1199 snap 401 seq_num 1199) 2024-09-21T09:52:23.874 INFO:tasks.rados.rados.0.smithi076.stdout:3414: expect (ObjNum 1106 snap 367 seq_num 1106) 2024-09-21T09:52:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:23 smithi191 ceph-mon[102206]: pgmap v862: 89 pgs: 89 active+clean; 225 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-21T09:52:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:23 smithi191 ceph-mon[102206]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:52:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:23 smithi191 ceph-mon[102206]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:52:24.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-mon[106482]: pgmap v862: 89 pgs: 89 active+clean; 225 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-21T09:52:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-mon[106482]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:52:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-mon[106482]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:52:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-mon[110963]: pgmap v862: 89 pgs: 89 active+clean; 225 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 1.5 MiB/s wr, 9 op/s 2024-09-21T09:52:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-mon[110963]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T09:52:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:23 smithi076 ceph-mon[110963]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T09:52:24.797 INFO:tasks.rados.rados.0.smithi076.stdout:3411: done (2 left) 2024-09-21T09:52:24.797 INFO:tasks.rados.rados.0.smithi076.stdout:3414: done (1 left) 2024-09-21T09:52:24.797 INFO:tasks.rados.rados.0.smithi076.stdout:3415: done (0 left) 2024-09-21T09:52:24.797 INFO:tasks.rados.rados.0.smithi076.stdout:3416: read oid 6 snap -1 2024-09-21T09:52:24.797 INFO:tasks.rados.rados.0.smithi076.stdout:3416: expect deleted 2024-09-21T09:52:24.797 INFO:tasks.rados.rados.0.smithi076.stdout:3417: delete oid 45 current snap is 431 2024-09-21T09:52:24.799 INFO:tasks.rados.rados.0.smithi076.stdout:3417: done (1 left) 2024-09-21T09:52:24.799 INFO:tasks.rados.rados.0.smithi076.stdout:3418: read oid 44 snap -1 2024-09-21T09:52:24.799 INFO:tasks.rados.rados.0.smithi076.stdout:3418: expect (ObjNum 1225 snap 406 seq_num 1225) 2024-09-21T09:52:24.799 INFO:tasks.rados.rados.0.smithi076.stdout:3419: snap_create 2024-09-21T09:52:24.805 INFO:tasks.rados.rados.0.smithi076.stdout:3418: expect (ObjNum 1225 snap 406 seq_num 1225) 2024-09-21T09:52:25.801 INFO:tasks.rados.rados.0.smithi076.stdout:3416: done (2 left) 2024-09-21T09:52:25.801 INFO:tasks.rados.rados.0.smithi076.stdout:3418: done (1 left) 2024-09-21T09:52:25.801 INFO:tasks.rados.rados.0.smithi076.stdout:3419: done (0 left) 2024-09-21T09:52:25.801 INFO:tasks.rados.rados.0.smithi076.stdout:3420: delete oid 8 current snap is 432 2024-09-21T09:52:25.806 INFO:tasks.rados.rados.0.smithi076.stdout:3420: done (0 left) 2024-09-21T09:52:25.806 INFO:tasks.rados.rados.0.smithi076.stdout:3421: delete oid 35 current snap is 432 2024-09-21T09:52:25.810 INFO:tasks.rados.rados.0.smithi076.stdout:3421: done (0 left) 2024-09-21T09:52:25.810 INFO:tasks.rados.rados.0.smithi076.stdout:3422: write oid 45 current snap is 432 2024-09-21T09:52:25.810 INFO:tasks.rados.rados.0.smithi076.stdout:3422: seq_num 1275 ranges {628925=573426,1693365=714102,3096904=332021} 2024-09-21T09:52:25.817 INFO:tasks.rados.rados.0.smithi076.stdout:3422: writing smithi07685715-45 from 628925 to 1202351 tid 1 2024-09-21T09:52:25.823 INFO:tasks.rados.rados.0.smithi076.stdout:3422: writing smithi07685715-45 from 1693365 to 2407467 tid 2 2024-09-21T09:52:25.829 INFO:tasks.rados.rados.0.smithi076.stdout:3422: writing smithi07685715-45 from 3096904 to 3428925 tid 3 2024-09-21T09:52:25.829 INFO:tasks.rados.rados.0.smithi076.stdout:3423: snap_create 2024-09-21T09:52:25.829 INFO:tasks.rados.rados.0.smithi076.stdout:3422: finishing write tid 1 to smithi07685715-45 2024-09-21T09:52:25.829 INFO:tasks.rados.rados.0.smithi076.stdout:3422: finishing write tid 2 to smithi07685715-45 2024-09-21T09:52:25.831 INFO:tasks.rados.rados.0.smithi076.stdout:3422: finishing write tid 3 to smithi07685715-45 2024-09-21T09:52:25.831 INFO:tasks.rados.rados.0.smithi076.stdout:3422: finishing write tid 4 to smithi07685715-45 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:3422: finishing write tid 5 to smithi07685715-45 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:3422: oid 45 updating version 0 to 754 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:3422: oid 45 version 754 is already newer than 752 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:3422: oid 45 version 754 is already newer than 751 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:3422: oid 45 version 754 is already newer than 753 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 754 (ObjNum 1275 snap 432 seq_num 1275) dirty exists 2024-09-21T09:52:25.832 INFO:tasks.rados.rados.0.smithi076.stdout:3422: left oid 45 (ObjNum 1275 snap 432 seq_num 1275) 2024-09-21T09:52:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:25 smithi191 ceph-mon[102206]: pgmap v865: 89 pgs: 89 active+clean; 225 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 745 KiB/s wr, 5 op/s 2024-09-21T09:52:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:25 smithi191 ceph-mon[102206]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:52:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:25 smithi076 ceph-mon[106482]: pgmap v865: 89 pgs: 89 active+clean; 225 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 745 KiB/s wr, 5 op/s 2024-09-21T09:52:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:25 smithi076 ceph-mon[106482]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:52:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:25 smithi076 ceph-mon[110963]: pgmap v865: 89 pgs: 89 active+clean; 225 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 973 KiB/s rd, 745 KiB/s wr, 5 op/s 2024-09-21T09:52:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:25 smithi076 ceph-mon[110963]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T09:52:26.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:26.809 INFO:tasks.rados.rados.0.smithi076.stdout:3422: done (1 left) 2024-09-21T09:52:26.809 INFO:tasks.rados.rados.0.smithi076.stdout:3423: done (0 left) 2024-09-21T09:52:26.809 INFO:tasks.rados.rados.0.smithi076.stdout:3424: write oid 37 current snap is 433 2024-09-21T09:52:26.809 INFO:tasks.rados.rados.0.smithi076.stdout:3424: seq_num 1276 ranges {645732=648675,1751847=574113,3124327=321405} 2024-09-21T09:52:26.816 INFO:tasks.rados.rados.0.smithi076.stdout:3424: writing smithi07685715-37 from 645732 to 1294407 tid 1 2024-09-21T09:52:26.822 INFO:tasks.rados.rados.0.smithi076.stdout:3424: writing smithi07685715-37 from 1751847 to 2325960 tid 2 2024-09-21T09:52:26.828 INFO:tasks.rados.rados.0.smithi076.stdout:3424: writing smithi07685715-37 from 3124327 to 3445732 tid 3 2024-09-21T09:52:26.828 INFO:tasks.rados.rados.0.smithi076.stdout:3425: write oid 25 current snap is 433 2024-09-21T09:52:26.828 INFO:tasks.rados.rados.0.smithi076.stdout:3425: seq_num 1277 ranges {662539=723924,2093976=717771,3435397=27142} 2024-09-21T09:52:26.835 INFO:tasks.rados.rados.0.smithi076.stdout:3425: writing smithi07685715-25 from 662539 to 1386463 tid 1 2024-09-21T09:52:26.844 INFO:tasks.rados.rados.0.smithi076.stdout:3425: writing smithi07685715-25 from 2093976 to 2811747 tid 2 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3425: writing smithi07685715-25 from 3435397 to 3462539 tid 3 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3426: rollback oid 18 current snap is 433 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 18 to 429 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: finishing write tid 1 to smithi07685715-37 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: finishing write tid 2 to smithi07685715-37 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: finishing write tid 3 to smithi07685715-37 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: finishing write tid 4 to smithi07685715-37 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: finishing write tid 5 to smithi07685715-37 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: oid 37 updating version 0 to 1027 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: oid 37 version 1027 is already newer than 1026 2024-09-21T09:52:26.849 INFO:tasks.rados.rados.0.smithi076.stdout:3424: oid 37 updating version 1027 to 1028 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3424: oid 37 updating version 1028 to 1029 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 1029 (ObjNum 1276 snap 433 seq_num 1276) dirty exists 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3424: left oid 37 (ObjNum 1276 snap 433 seq_num 1276) 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3425: finishing write tid 1 to smithi07685715-25 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3425: finishing write tid 2 to smithi07685715-25 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3424: done (2 left) 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3427: read oid 13 snap -1 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3427: expect (ObjNum 1203 snap 403 seq_num 1203) 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3428: write oid 3 current snap is 433 2024-09-21T09:52:26.850 INFO:tasks.rados.rados.0.smithi076.stdout:3428: seq_num 1278 ranges {679346=515526,1868811=577782,3179173=300173} 2024-09-21T09:52:26.855 INFO:tasks.rados.rados.0.smithi076.stdout:3428: writing smithi07685715-3 from 679346 to 1194872 tid 1 2024-09-21T09:52:26.862 INFO:tasks.rados.rados.0.smithi076.stdout:3428: writing smithi07685715-3 from 1868811 to 2446593 tid 2 2024-09-21T09:52:26.868 INFO:tasks.rados.rados.0.smithi076.stdout:3428: writing smithi07685715-3 from 3179173 to 3479346 tid 3 2024-09-21T09:52:26.868 INFO:tasks.rados.rados.0.smithi076.stdout:3429: write oid 5 current snap is 433 2024-09-21T09:52:26.868 INFO:tasks.rados.rados.0.smithi076.stdout:3425: finishing write tid 3 to smithi07685715-25 2024-09-21T09:52:26.868 INFO:tasks.rados.rados.0.smithi076.stdout:3425: finishing write tid 4 to smithi07685715-25 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3425: finishing write tid 5 to smithi07685715-25 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3425: oid 25 updating version 0 to 743 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3425: oid 25 updating version 743 to 744 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3425: oid 25 version 744 is already newer than 742 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3425: oid 25 version 744 is already newer than 741 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 744 (ObjNum 1277 snap 433 seq_num 1277) dirty exists 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3425: left oid 25 (ObjNum 1277 snap 433 seq_num 1277) 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3426: finishing rollback tid 0 to smithi07685715-18 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3426: finishing rollback tid 1 to smithi07685715-18 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3426: finishing rollback tid 2 to smithi07685715-18 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 864 (ObjNum 1205 snap 404 seq_num 1205) dirty exists 2024-09-21T09:52:26.869 INFO:tasks.rados.rados.0.smithi076.stdout:3427: expect (ObjNum 1203 snap 403 seq_num 1203) 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: finishing write tid 1 to smithi07685715-3 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: finishing write tid 2 to smithi07685715-3 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: finishing write tid 3 to smithi07685715-3 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: finishing write tid 4 to smithi07685715-3 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: finishing write tid 5 to smithi07685715-3 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: oid 3 updating version 0 to 264 2024-09-21T09:52:26.994 INFO:tasks.rados.rados.0.smithi076.stdout:3428: oid 3 updating version 264 to 265 2024-09-21T09:52:26.995 INFO:tasks.rados.rados.0.smithi076.stdout:3428: oid 3 version 265 is already newer than 262 2024-09-21T09:52:26.995 INFO:tasks.rados.rados.0.smithi076.stdout:3428: oid 3 version 265 is already newer than 263 2024-09-21T09:52:26.995 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 265 (ObjNum 1278 snap 433 seq_num 1278) dirty exists 2024-09-21T09:52:26.995 INFO:tasks.rados.rados.0.smithi076.stdout:3428: left oid 3 (ObjNum 1278 snap 433 seq_num 1278) 2024-09-21T09:52:26.995 INFO:tasks.rados.rados.0.smithi076.stdout:3429: seq_num 1279 ranges {696153=590775,1927293=721440,3206596=289557} 2024-09-21T09:52:27.000 INFO:tasks.rados.rados.0.smithi076.stdout:3429: writing smithi07685715-5 from 696153 to 1286928 tid 1 2024-09-21T09:52:27.007 INFO:tasks.rados.rados.0.smithi076.stdout:3429: writing smithi07685715-5 from 1927293 to 2648733 tid 2 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3429: writing smithi07685715-5 from 3206596 to 3496153 tid 3 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3425: done (4 left) 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3426: done (3 left) 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3427: done (2 left) 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3428: done (1 left) 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3430: rollback oid 41 current snap is 433 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 425 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3429: finishing write tid 1 to smithi07685715-5 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3429: finishing write tid 2 to smithi07685715-5 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3431: read oid 37 snap -1 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3431: expect (ObjNum 1276 snap 433 seq_num 1276) 2024-09-21T09:52:27.013 INFO:tasks.rados.rados.0.smithi076.stdout:3432: delete oid 36 current snap is 433 2024-09-21T09:52:27.015 INFO:tasks.rados.rados.0.smithi076.stdout:3432: done (3 left) 2024-09-21T09:52:27.015 INFO:tasks.rados.rados.0.smithi076.stdout:3433: write oid 2 current snap is 433 2024-09-21T09:52:27.015 INFO:tasks.rados.rados.0.smithi076.stdout:3433: seq_num 1281 ranges {729767=741273,1927904=725109,3145089=384678} 2024-09-21T09:52:27.023 INFO:tasks.rados.rados.0.smithi076.stdout:3433: writing smithi07685715-2 from 729767 to 1471040 tid 1 2024-09-21T09:52:27.029 INFO:tasks.rados.rados.0.smithi076.stdout:3433: writing smithi07685715-2 from 1927904 to 2653013 tid 2 2024-09-21T09:52:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:3433: writing smithi07685715-2 from 3145089 to 3529767 tid 3 2024-09-21T09:52:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:3434: read oid 45 snap 422 2024-09-21T09:52:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:3434: expect (ObjNum 1257 snap 422 seq_num 1257) 2024-09-21T09:52:27.035 INFO:tasks.rados.rados.0.smithi076.stdout:3435: write oid 34 current snap is 433 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3430: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1632 (ObjNum 1283431648 snap 32593 seq_num 304) dirty dne 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: finishing write tid 3 to smithi07685715-5 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: finishing write tid 4 to smithi07685715-5 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: finishing write tid 5 to smithi07685715-5 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: oid 5 updating version 0 to 2660 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: oid 5 updating version 2660 to 2662 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: oid 5 version 2662 is already newer than 2659 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: oid 5 version 2662 is already newer than 2661 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2662 (ObjNum 1279 snap 433 seq_num 1279) dirty exists 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3429: left oid 5 (ObjNum 1279 snap 433 seq_num 1279) 2024-09-21T09:52:27.036 INFO:tasks.rados.rados.0.smithi076.stdout:3431: expect (ObjNum 1276 snap 433 seq_num 1276) 2024-09-21T09:52:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:26 smithi191 ceph-mon[102206]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:52:27.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:26 smithi076 ceph-mon[106482]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:52:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:26 smithi076 ceph-mon[110963]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T09:52:27.192 INFO:tasks.rados.rados.0.smithi076.stdout:3433: finishing write tid 1 to smithi07685715-2 2024-09-21T09:52:27.192 INFO:tasks.rados.rados.0.smithi076.stdout:3433: finishing write tid 2 to smithi07685715-2 2024-09-21T09:52:27.192 INFO:tasks.rados.rados.0.smithi076.stdout:3433: finishing write tid 3 to smithi07685715-2 2024-09-21T09:52:27.192 INFO:tasks.rados.rados.0.smithi076.stdout:3435: seq_num 1282 ranges {746574=416522,1586386=468767,2772512=629544} 2024-09-21T09:52:27.197 INFO:tasks.rados.rados.0.smithi076.stdout:3435: writing smithi07685715-34 from 746574 to 1163096 tid 1 2024-09-21T09:52:27.202 INFO:tasks.rados.rados.0.smithi076.stdout:3435: writing smithi07685715-34 from 1586386 to 2055153 tid 2 2024-09-21T09:52:27.210 INFO:tasks.rados.rados.0.smithi076.stdout:3435: writing smithi07685715-34 from 2772512 to 3402056 tid 3 2024-09-21T09:52:27.210 INFO:tasks.rados.rados.0.smithi076.stdout:3429: done (5 left) 2024-09-21T09:52:27.210 INFO:tasks.rados.rados.0.smithi076.stdout:3430: done (4 left) 2024-09-21T09:52:27.210 INFO:tasks.rados.rados.0.smithi076.stdout:3431: done (3 left) 2024-09-21T09:52:27.210 INFO:tasks.rados.rados.0.smithi076.stdout:3436: snap_remove snap 426 2024-09-21T09:52:27.815 INFO:tasks.rados.rados.0.smithi076.stdout:3436: done (3 left) 2024-09-21T09:52:27.815 INFO:tasks.rados.rados.0.smithi076.stdout:3437: write oid 7 current snap is 433 2024-09-21T09:52:27.816 INFO:tasks.rados.rados.0.smithi076.stdout:3433: finishing write tid 4 to smithi07685715-2 2024-09-21T09:52:27.816 INFO:tasks.rados.rados.0.smithi076.stdout:3437: seq_num 1283 ranges {763381=491771,2044868=728778,3199935=363446} 2024-09-21T09:52:27.823 INFO:tasks.rados.rados.0.smithi076.stdout:3437: writing smithi07685715-7 from 763381 to 1255152 tid 1 2024-09-21T09:52:27.831 INFO:tasks.rados.rados.0.smithi076.stdout:3437: writing smithi07685715-7 from 2044868 to 2773646 tid 2 2024-09-21T09:52:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:3437: writing smithi07685715-7 from 3199935 to 3563381 tid 3 2024-09-21T09:52:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:3438: snap_create 2024-09-21T09:52:27.836 INFO:tasks.rados.rados.0.smithi076.stdout:3433: finishing write tid 5 to smithi07685715-2 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:3433: oid 2 updating version 0 to 694 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:3433: oid 2 version 694 is already newer than 691 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:3433: oid 2 version 694 is already newer than 693 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:3433: oid 2 version 694 is already newer than 692 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 694 (ObjNum 1281 snap 433 seq_num 1281) dirty exists 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:3433: left oid 2 (ObjNum 1281 snap 433 seq_num 1281) 2024-09-21T09:52:27.837 INFO:tasks.rados.rados.0.smithi076.stdout:3434: expect (ObjNum 1257 snap 422 seq_num 1257) 2024-09-21T09:52:27.997 INFO:tasks.rados.rados.0.smithi076.stdout:3435: finishing write tid 1 to smithi07685715-34 2024-09-21T09:52:27.997 INFO:tasks.rados.rados.0.smithi076.stdout:3435: finishing write tid 2 to smithi07685715-34 2024-09-21T09:52:27.997 INFO:tasks.rados.rados.0.smithi076.stdout:3435: finishing write tid 3 to smithi07685715-34 2024-09-21T09:52:27.997 INFO:tasks.rados.rados.0.smithi076.stdout:3435: finishing write tid 4 to smithi07685715-34 2024-09-21T09:52:27.997 INFO:tasks.rados.rados.0.smithi076.stdout:3435: finishing write tid 5 to smithi07685715-34 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3435: oid 34 updating version 0 to 772 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3435: oid 34 updating version 772 to 773 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3435: oid 34 updating version 773 to 774 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3435: oid 34 version 774 is already newer than 771 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 774 (ObjNum 1282 snap 433 seq_num 1282) dirty exists 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3435: left oid 34 (ObjNum 1282 snap 433 seq_num 1282) 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3437: finishing write tid 1 to smithi07685715-7 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3437: finishing write tid 2 to smithi07685715-7 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3437: finishing write tid 3 to smithi07685715-7 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3437: finishing write tid 4 to smithi07685715-7 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3437: finishing write tid 5 to smithi07685715-7 2024-09-21T09:52:27.998 INFO:tasks.rados.rados.0.smithi076.stdout:3437: oid 7 updating version 0 to 2139 2024-09-21T09:52:27.999 INFO:tasks.rados.rados.0.smithi076.stdout:3437: oid 7 updating version 2139 to 2140 2024-09-21T09:52:27.999 INFO:tasks.rados.rados.0.smithi076.stdout:3437: oid 7 updating version 2140 to 2141 2024-09-21T09:52:27.999 INFO:tasks.rados.rados.0.smithi076.stdout:3437: oid 7 version 2141 is already newer than 2138 2024-09-21T09:52:27.999 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2141 (ObjNum 1283 snap 433 seq_num 1283) dirty exists 2024-09-21T09:52:27.999 INFO:tasks.rados.rados.0.smithi076.stdout:3437: left oid 7 (ObjNum 1283 snap 433 seq_num 1283) 2024-09-21T09:52:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:27 smithi191 ceph-mon[102206]: pgmap v868: 89 pgs: 89 active+clean; 225 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:52:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:27 smithi191 ceph-mon[102206]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:52:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:27 smithi076 ceph-mon[106482]: pgmap v868: 89 pgs: 89 active+clean; 225 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:52:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:27 smithi076 ceph-mon[106482]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:52:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:27 smithi076 ceph-mon[110963]: pgmap v868: 89 pgs: 89 active+clean; 225 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 0 B/s wr, 9 op/s 2024-09-21T09:52:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:27 smithi076 ceph-mon[110963]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T09:52:28.827 INFO:tasks.rados.rados.0.smithi076.stdout:3433: done (4 left) 2024-09-21T09:52:28.827 INFO:tasks.rados.rados.0.smithi076.stdout:3434: done (3 left) 2024-09-21T09:52:28.827 INFO:tasks.rados.rados.0.smithi076.stdout:3435: done (2 left) 2024-09-21T09:52:28.827 INFO:tasks.rados.rados.0.smithi076.stdout:3437: done (1 left) 2024-09-21T09:52:28.827 INFO:tasks.rados.rados.0.smithi076.stdout:3438: done (0 left) 2024-09-21T09:52:28.828 INFO:tasks.rados.rados.0.smithi076.stdout:3439: write oid 16 current snap is 434 2024-09-21T09:52:28.828 INFO:tasks.rados.rados.0.smithi076.stdout:3439: seq_num 1284 ranges {780188=567020,2103350=472436,3227358=352830} 2024-09-21T09:52:28.834 INFO:tasks.rados.rados.0.smithi076.stdout:3439: writing smithi07685715-16 from 780188 to 1347208 tid 1 2024-09-21T09:52:28.841 INFO:tasks.rados.rados.0.smithi076.stdout:3439: writing smithi07685715-16 from 2103350 to 2575786 tid 2 2024-09-21T09:52:28.847 INFO:tasks.rados.rados.0.smithi076.stdout:3439: writing smithi07685715-16 from 3227358 to 3580188 tid 3 2024-09-21T09:52:28.847 INFO:tasks.rados.rados.0.smithi076.stdout:3440: write oid 24 current snap is 434 2024-09-21T09:52:28.847 INFO:tasks.rados.rados.0.smithi076.stdout:3440: seq_num 1285 ranges {796995=758622,2161832=732447} 2024-09-21T09:52:28.855 INFO:tasks.rados.rados.0.smithi076.stdout:3440: writing smithi07685715-24 from 796995 to 1555617 tid 1 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3440: writing smithi07685715-24 from 2161832 to 2894279 tid 2 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3441: rollback oid 35 current snap is 434 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 35 to 428 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3439: finishing write tid 1 to smithi07685715-16 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3439: finishing write tid 2 to smithi07685715-16 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3442: write oid 22 current snap is 434 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3439: finishing write tid 3 to smithi07685715-16 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3439: finishing write tid 4 to smithi07685715-16 2024-09-21T09:52:28.863 INFO:tasks.rados.rados.0.smithi076.stdout:3442: seq_num 1286 ranges {413802=433871,1420314=476105,2482204=790985} 2024-09-21T09:52:28.867 INFO:tasks.rados.rados.0.smithi076.stdout:3442: writing smithi07685715-22 from 413802 to 847673 tid 1 2024-09-21T09:52:28.873 INFO:tasks.rados.rados.0.smithi076.stdout:3442: writing smithi07685715-22 from 1420314 to 1896419 tid 2 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3442: writing smithi07685715-22 from 2482204 to 3273189 tid 3 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3443: snap_create 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3439: finishing write tid 5 to smithi07685715-16 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3439: oid 16 updating version 0 to 1701 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3439: oid 16 version 1701 is already newer than 1698 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3439: oid 16 version 1701 is already newer than 1699 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3439: oid 16 version 1701 is already newer than 1700 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1701 (ObjNum 1284 snap 434 seq_num 1284) dirty exists 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3439: left oid 16 (ObjNum 1284 snap 434 seq_num 1284) 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3440: finishing write tid 1 to smithi07685715-24 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3440: finishing write tid 2 to smithi07685715-24 2024-09-21T09:52:28.882 INFO:tasks.rados.rados.0.smithi076.stdout:3440: finishing write tid 3 to smithi07685715-24 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3440: finishing write tid 4 to smithi07685715-24 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3440: oid 24 updating version 0 to 344 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3440: oid 24 updating version 344 to 345 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3440: oid 24 updating version 345 to 346 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 346 (ObjNum 1285 snap 434 seq_num 1285) dirty exists 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3440: left oid 24 (ObjNum 1285 snap 434 seq_num 1285) 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3441: finishing rollback tid 1 to smithi07685715-35 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2600 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3442: finishing write tid 1 to smithi07685715-22 2024-09-21T09:52:28.883 INFO:tasks.rados.rados.0.smithi076.stdout:3442: finishing write tid 2 to smithi07685715-22 2024-09-21T09:52:28.885 INFO:tasks.rados.rados.0.smithi076.stdout:3442: finishing write tid 3 to smithi07685715-22 2024-09-21T09:52:28.885 INFO:tasks.rados.rados.0.smithi076.stdout:3442: finishing write tid 4 to smithi07685715-22 2024-09-21T09:52:28.885 INFO:tasks.rados.rados.0.smithi076.stdout:3442: finishing write tid 5 to smithi07685715-22 2024-09-21T09:52:28.886 INFO:tasks.rados.rados.0.smithi076.stdout:3442: oid 22 updating version 0 to 1705 2024-09-21T09:52:28.886 INFO:tasks.rados.rados.0.smithi076.stdout:3442: oid 22 version 1705 is already newer than 1703 2024-09-21T09:52:28.886 INFO:tasks.rados.rados.0.smithi076.stdout:3442: oid 22 updating version 1705 to 1706 2024-09-21T09:52:28.886 INFO:tasks.rados.rados.0.smithi076.stdout:3442: oid 22 version 1706 is already newer than 1704 2024-09-21T09:52:28.886 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1706 (ObjNum 1286 snap 434 seq_num 1286) dirty exists 2024-09-21T09:52:28.886 INFO:tasks.rados.rados.0.smithi076.stdout:3442: left oid 22 (ObjNum 1286 snap 434 seq_num 1286) 2024-09-21T09:52:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:28 smithi191 ceph-mon[102206]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:52:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:28 smithi076 ceph-mon[106482]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:52:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:28 smithi076 ceph-mon[110963]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T09:52:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3439: done (4 left) 2024-09-21T09:52:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3440: done (3 left) 2024-09-21T09:52:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3441: done (2 left) 2024-09-21T09:52:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3442: done (1 left) 2024-09-21T09:52:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3443: done (0 left) 2024-09-21T09:52:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3444: snap_remove snap 387 2024-09-21T09:52:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:29 smithi191 ceph-mon[102206]: pgmap v871: 89 pgs: 89 active+clean; 226 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 359 KiB/s wr, 10 op/s 2024-09-21T09:52:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:29 smithi191 ceph-mon[102206]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:52:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:29 smithi076 ceph-mon[106482]: pgmap v871: 89 pgs: 89 active+clean; 226 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 359 KiB/s wr, 10 op/s 2024-09-21T09:52:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:29 smithi076 ceph-mon[106482]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:52:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:29 smithi076 ceph-mon[110963]: pgmap v871: 89 pgs: 89 active+clean; 226 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 359 KiB/s wr, 10 op/s 2024-09-21T09:52:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:29 smithi076 ceph-mon[110963]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T09:52:30.843 INFO:tasks.rados.rados.0.smithi076.stdout:3444: done (0 left) 2024-09-21T09:52:30.843 INFO:tasks.rados.rados.0.smithi076.stdout:3445: read oid 31 snap 433 2024-09-21T09:52:30.843 INFO:tasks.rados.rados.0.smithi076.stdout:3445: expect deleted 2024-09-21T09:52:30.843 INFO:tasks.rados.rados.0.smithi076.stdout:3446: write oid 25 current snap is 435 2024-09-21T09:52:30.843 INFO:tasks.rados.rados.0.smithi076.stdout:3446: seq_num 1287 ranges {430609=509120,1478796=736116,2909627=602257} 2024-09-21T09:52:30.848 INFO:tasks.rados.rados.0.smithi076.stdout:3446: writing smithi07685715-25 from 430609 to 939729 tid 1 2024-09-21T09:52:30.856 INFO:tasks.rados.rados.0.smithi076.stdout:3446: writing smithi07685715-25 from 1478796 to 2214912 tid 2 2024-09-21T09:52:30.864 INFO:tasks.rados.rados.0.smithi076.stdout:3446: writing smithi07685715-25 from 2909627 to 3511884 tid 3 2024-09-21T09:52:30.865 INFO:tasks.rados.rados.0.smithi076.stdout:3447: snap_remove snap 434 2024-09-21T09:52:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:30 smithi191 ceph-mon[102206]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:52:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:30 smithi076 ceph-mon[106482]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:52:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:30 smithi076 ceph-mon[110963]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3447: done (2 left) 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3448: read oid 21 snap -1 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3448: expect (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3449: snap_create 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3446: finishing write tid 1 to smithi07685715-25 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3446: finishing write tid 2 to smithi07685715-25 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3446: finishing write tid 3 to smithi07685715-25 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3446: finishing write tid 4 to smithi07685715-25 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3446: finishing write tid 5 to smithi07685715-25 2024-09-21T09:52:31.846 INFO:tasks.rados.rados.0.smithi076.stdout:3446: oid 25 updating version 0 to 750 2024-09-21T09:52:31.847 INFO:tasks.rados.rados.0.smithi076.stdout:3446: oid 25 updating version 750 to 753 2024-09-21T09:52:31.847 INFO:tasks.rados.rados.0.smithi076.stdout:3446: oid 25 version 753 is already newer than 752 2024-09-21T09:52:31.847 INFO:tasks.rados.rados.0.smithi076.stdout:3446: oid 25 version 753 is already newer than 751 2024-09-21T09:52:31.847 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 753 (ObjNum 1287 snap 435 seq_num 1287) dirty exists 2024-09-21T09:52:31.847 INFO:tasks.rados.rados.0.smithi076.stdout:3446: left oid 25 (ObjNum 1287 snap 435 seq_num 1287) 2024-09-21T09:52:31.854 INFO:tasks.rados.rados.0.smithi076.stdout:3448: expect (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:52:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:31 smithi191 ceph-mon[102206]: pgmap v874: 89 pgs: 89 active+clean; 234 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:52:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:31 smithi191 ceph-mon[102206]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:52:32.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:31 smithi076 ceph-mon[110963]: pgmap v874: 89 pgs: 89 active+clean; 234 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:52:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:31 smithi076 ceph-mon[110963]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:52:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:31 smithi076 ceph-mon[106482]: pgmap v874: 89 pgs: 89 active+clean; 234 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T09:52:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:31 smithi076 ceph-mon[106482]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T09:52:32.850 INFO:tasks.rados.rados.0.smithi076.stdout:3445: done (3 left) 2024-09-21T09:52:32.850 INFO:tasks.rados.rados.0.smithi076.stdout:3446: done (2 left) 2024-09-21T09:52:32.850 INFO:tasks.rados.rados.0.smithi076.stdout:3448: done (1 left) 2024-09-21T09:52:32.850 INFO:tasks.rados.rados.0.smithi076.stdout:3449: done (0 left) 2024-09-21T09:52:32.850 INFO:tasks.rados.rados.0.smithi076.stdout:3450: snap_remove snap 422 2024-09-21T09:52:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:32 smithi191 ceph-mon[102206]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:52:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:32 smithi076 ceph-mon[106482]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:52:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:32 smithi076 ceph-mon[110963]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T09:52:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:52:33] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-09-21T09:52:33.797 INFO:tasks.rados.rados.0.smithi076.stdout:3450: done (0 left) 2024-09-21T09:52:33.797 INFO:tasks.rados.rados.0.smithi076.stdout:3451: rollback oid 11 current snap is 436 2024-09-21T09:52:33.797 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 11 to 408 2024-09-21T09:52:33.797 INFO:tasks.rados.rados.0.smithi076.stdout:3452: snap_remove snap 432 2024-09-21T09:52:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:33 smithi191 ceph-mon[102206]: pgmap v877: 89 pgs: 89 active+clean; 204 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.4 MiB/s wr, 20 op/s 2024-09-21T09:52:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:33 smithi191 ceph-mon[102206]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:52:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:33 smithi191 ceph-mon[102206]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:52:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-mon[106482]: pgmap v877: 89 pgs: 89 active+clean; 204 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.4 MiB/s wr, 20 op/s 2024-09-21T09:52:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-mon[106482]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:52:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-mon[106482]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:52:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-mon[110963]: pgmap v877: 89 pgs: 89 active+clean; 204 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.4 MiB/s wr, 20 op/s 2024-09-21T09:52:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-mon[110963]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T09:52:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:33 smithi076 ceph-mon[110963]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T09:52:34.800 INFO:tasks.rados.rados.0.smithi076.stdout:3452: done (1 left) 2024-09-21T09:52:34.800 INFO:tasks.rados.rados.0.smithi076.stdout:3453: rollback oid 17 current snap is 436 2024-09-21T09:52:34.800 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 417 2024-09-21T09:52:34.800 INFO:tasks.rados.rados.0.smithi076.stdout:3451: finishing rollback tid 1 to smithi07685715-11 2024-09-21T09:52:34.801 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:34.801 INFO:tasks.rados.rados.0.smithi076.stdout:3451: done (1 left) 2024-09-21T09:52:34.801 INFO:tasks.rados.rados.0.smithi076.stdout:3454: snap_create 2024-09-21T09:52:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:3453: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:52:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:3453: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:52:34.808 INFO:tasks.rados.rados.0.smithi076.stdout:3453: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:52:34.809 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2157 (ObjNum 1230 snap 410 seq_num 1230) dirty exists 2024-09-21T09:52:35.800 INFO:tasks.rados.rados.0.smithi076.stdout:3453: done (1 left) 2024-09-21T09:52:35.800 INFO:tasks.rados.rados.0.smithi076.stdout:3454: done (0 left) 2024-09-21T09:52:35.800 INFO:tasks.rados.rados.0.smithi076.stdout:3455: snap_remove snap 421 2024-09-21T09:52:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:35 smithi191 ceph-mon[102206]: pgmap v880: 89 pgs: 89 active+clean; 198 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 13 op/s 2024-09-21T09:52:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:35 smithi191 ceph-mon[102206]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:52:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:35 smithi076 ceph-mon[106482]: pgmap v880: 89 pgs: 89 active+clean; 198 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 13 op/s 2024-09-21T09:52:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:35 smithi076 ceph-mon[106482]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:52:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:35 smithi076 ceph-mon[110963]: pgmap v880: 89 pgs: 89 active+clean; 198 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.0 MiB/s wr, 13 op/s 2024-09-21T09:52:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:35 smithi076 ceph-mon[110963]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T09:52:36.804 INFO:tasks.rados.rados.0.smithi076.stdout:3455: done (0 left) 2024-09-21T09:52:36.805 INFO:tasks.rados.rados.0.smithi076.stdout:3456: snap_remove snap 429 2024-09-21T09:52:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:36 smithi191 ceph-mon[102206]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:52:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:36 smithi076 ceph-mon[106482]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:52:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:36 smithi076 ceph-mon[110963]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T09:52:37.807 INFO:tasks.rados.rados.0.smithi076.stdout:3456: done (0 left) 2024-09-21T09:52:37.807 INFO:tasks.rados.rados.0.smithi076.stdout:3457: rollback oid 36 current snap is 437 2024-09-21T09:52:37.807 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 406 2024-09-21T09:52:37.807 INFO:tasks.rados.rados.0.smithi076.stdout:3458: snap_create 2024-09-21T09:52:37.813 INFO:tasks.rados.rados.0.smithi076.stdout:3457: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:52:37.814 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1103 (ObjNum 129 snap 0 seq_num 865189336) dirty dne 2024-09-21T09:52:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:37 smithi191 ceph-mon[102206]: pgmap v883: 89 pgs: 89 active+clean; 183 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 816 KiB/s wr, 5 op/s 2024-09-21T09:52:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:37 smithi191 ceph-mon[102206]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:52:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:37 smithi191 ceph-mon[102206]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:52:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:37 smithi076 ceph-mon[106482]: pgmap v883: 89 pgs: 89 active+clean; 183 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 816 KiB/s wr, 5 op/s 2024-09-21T09:52:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:37 smithi076 ceph-mon[106482]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:52:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:37 smithi076 ceph-mon[106482]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:52:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:37 smithi076 ceph-mon[110963]: pgmap v883: 89 pgs: 89 active+clean; 183 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 816 KiB/s wr, 5 op/s 2024-09-21T09:52:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:37 smithi076 ceph-mon[110963]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T09:52:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:37 smithi076 ceph-mon[110963]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T09:52:38.801 INFO:tasks.rados.rados.0.smithi076.stdout:3457: done (1 left) 2024-09-21T09:52:38.801 INFO:tasks.rados.rados.0.smithi076.stdout:3458: done (0 left) 2024-09-21T09:52:38.801 INFO:tasks.rados.rados.0.smithi076.stdout:3459: rollback oid 33 current snap is 438 2024-09-21T09:52:38.801 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 417 2024-09-21T09:52:38.801 INFO:tasks.rados.rados.0.smithi076.stdout:3460: write oid 20 current snap is 438 2024-09-21T09:52:38.801 INFO:tasks.rados.rados.0.smithi076.stdout:3460: seq_num 1288 ranges {447416=584369,1537278=479774,2537050=529882,3524784=122632} 2024-09-21T09:52:38.807 INFO:tasks.rados.rados.0.smithi076.stdout:3460: writing smithi07685715-20 from 447416 to 1031785 tid 1 2024-09-21T09:52:38.812 INFO:tasks.rados.rados.0.smithi076.stdout:3460: writing smithi07685715-20 from 1537278 to 2017052 tid 2 2024-09-21T09:52:38.818 INFO:tasks.rados.rados.0.smithi076.stdout:3460: writing smithi07685715-20 from 2537050 to 3066932 tid 3 2024-09-21T09:52:38.822 INFO:tasks.rados.rados.0.smithi076.stdout:3460: writing smithi07685715-20 from 3524784 to 3647416 tid 4 2024-09-21T09:52:38.822 INFO:tasks.rados.rados.0.smithi076.stdout:3461: write oid 18 current snap is 438 2024-09-21T09:52:38.822 INFO:tasks.rados.rados.0.smithi076.stdout:3459: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:52:38.822 INFO:tasks.rados.rados.0.smithi076.stdout:3459: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:52:38.822 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 765 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2024-09-21T09:52:38.822 INFO:tasks.rados.rados.0.smithi076.stdout:3460: finishing write tid 1 to smithi07685715-20 2024-09-21T09:52:38.823 INFO:tasks.rados.rados.0.smithi076.stdout:3460: finishing write tid 2 to smithi07685715-20 2024-09-21T09:52:38.823 INFO:tasks.rados.rados.0.smithi076.stdout:3461: seq_num 1289 ranges {464223=659618,1879407=739785,3248120=416103} 2024-09-21T09:52:38.828 INFO:tasks.rados.rados.0.smithi076.stdout:3461: writing smithi07685715-18 from 464223 to 1123841 tid 1 2024-09-21T09:52:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3461: writing smithi07685715-18 from 1879407 to 2619192 tid 2 2024-09-21T09:52:38.842 INFO:tasks.rados.rados.0.smithi076.stdout:3461: writing smithi07685715-18 from 3248120 to 3664223 tid 3 2024-09-21T09:52:38.842 INFO:tasks.rados.rados.0.smithi076.stdout:3459: done (2 left) 2024-09-21T09:52:38.842 INFO:tasks.rados.rados.0.smithi076.stdout:3462: snap_create 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: finishing write tid 3 to smithi07685715-20 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: finishing write tid 4 to smithi07685715-20 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: finishing write tid 5 to smithi07685715-20 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: finishing write tid 6 to smithi07685715-20 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: oid 20 updating version 0 to 1055 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: oid 20 version 1055 is already newer than 1051 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: oid 20 version 1055 is already newer than 1052 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: oid 20 version 1055 is already newer than 1053 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: oid 20 version 1055 is already newer than 1054 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 1055 (ObjNum 1288 snap 438 seq_num 1288) dirty exists 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3460: left oid 20 (ObjNum 1288 snap 438 seq_num 1288) 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3461: finishing write tid 1 to smithi07685715-18 2024-09-21T09:52:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3461: finishing write tid 2 to smithi07685715-18 2024-09-21T09:52:38.845 INFO:tasks.rados.rados.0.smithi076.stdout:3461: finishing write tid 3 to smithi07685715-18 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: finishing write tid 4 to smithi07685715-18 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: finishing write tid 5 to smithi07685715-18 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: oid 18 updating version 0 to 885 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: oid 18 version 885 is already newer than 884 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: oid 18 version 885 is already newer than 883 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: oid 18 version 885 is already newer than 882 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 885 (ObjNum 1289 snap 438 seq_num 1289) dirty exists 2024-09-21T09:52:38.846 INFO:tasks.rados.rados.0.smithi076.stdout:3461: left oid 18 (ObjNum 1289 snap 438 seq_num 1289) 2024-09-21T09:52:39.824 INFO:tasks.rados.rados.0.smithi076.stdout:3460: done (2 left) 2024-09-21T09:52:39.824 INFO:tasks.rados.rados.0.smithi076.stdout:3461: done (1 left) 2024-09-21T09:52:39.825 INFO:tasks.rados.rados.0.smithi076.stdout:3462: done (0 left) 2024-09-21T09:52:39.825 INFO:tasks.rados.rados.0.smithi076.stdout:3463: write oid 49 current snap is 439 2024-09-21T09:52:39.825 INFO:tasks.rados.rados.0.smithi076.stdout:3463: seq_num 1290 ranges {481030=734867,1937889=483443,2875543=552426} 2024-09-21T09:52:39.831 INFO:tasks.rados.rados.0.smithi076.stdout:3463: writing smithi07685715-49 from 481030 to 1215897 tid 1 2024-09-21T09:52:39.838 INFO:tasks.rados.rados.0.smithi076.stdout:3463: writing smithi07685715-49 from 1937889 to 2421332 tid 2 2024-09-21T09:52:39.844 INFO:tasks.rados.rados.0.smithi076.stdout:3463: writing smithi07685715-49 from 2875543 to 3427969 tid 3 2024-09-21T09:52:39.844 INFO:tasks.rados.rados.0.smithi076.stdout:3464: write oid 11 current snap is 439 2024-09-21T09:52:39.844 INFO:tasks.rados.rados.0.smithi076.stdout:3463: finishing write tid 1 to smithi07685715-49 2024-09-21T09:52:39.844 INFO:tasks.rados.rados.0.smithi076.stdout:3463: finishing write tid 2 to smithi07685715-49 2024-09-21T09:52:39.844 INFO:tasks.rados.rados.0.smithi076.stdout:3464: seq_num 1291 ranges {497837=410116,1596371=743454,2902966=763698} 2024-09-21T09:52:39.849 INFO:tasks.rados.rados.0.smithi076.stdout:3464: writing smithi07685715-11 from 497837 to 907953 tid 1 2024-09-21T09:52:39.857 INFO:tasks.rados.rados.0.smithi076.stdout:3464: writing smithi07685715-11 from 1596371 to 2339825 tid 2 2024-09-21T09:52:39.864 INFO:tasks.rados.rados.0.smithi076.stdout:3464: writing smithi07685715-11 from 2902966 to 3666664 tid 3 2024-09-21T09:52:39.864 INFO:tasks.rados.rados.0.smithi076.stdout:3465: write oid 7 current snap is 439 2024-09-21T09:52:39.865 INFO:tasks.rados.rados.0.smithi076.stdout:3465: seq_num 1292 ranges {514644=485365,1654853=487112,2930389=574970} 2024-09-21T09:52:39.870 INFO:tasks.rados.rados.0.smithi076.stdout:3465: writing smithi07685715-7 from 514644 to 1000009 tid 1 2024-09-21T09:52:39.876 INFO:tasks.rados.rados.0.smithi076.stdout:3465: writing smithi07685715-7 from 1654853 to 2141965 tid 2 2024-09-21T09:52:39.884 INFO:tasks.rados.rados.0.smithi076.stdout:3465: writing smithi07685715-7 from 2930389 to 3505359 tid 3 2024-09-21T09:52:39.884 INFO:tasks.rados.rados.0.smithi076.stdout:3466: delete oid 28 current snap is 439 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: finishing write tid 3 to smithi07685715-49 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: finishing write tid 4 to smithi07685715-49 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: finishing write tid 5 to smithi07685715-49 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: oid 49 updating version 0 to 2673 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: oid 49 version 2673 is already newer than 2672 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: oid 49 version 2673 is already newer than 2671 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: oid 49 version 2673 is already newer than 2670 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2673 (ObjNum 1290 snap 439 seq_num 1290) dirty exists 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3463: left oid 49 (ObjNum 1290 snap 439 seq_num 1290) 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3464: finishing write tid 1 to smithi07685715-11 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3464: finishing write tid 2 to smithi07685715-11 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3464: finishing write tid 3 to smithi07685715-11 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3464: finishing write tid 4 to smithi07685715-11 2024-09-21T09:52:39.885 INFO:tasks.rados.rados.0.smithi076.stdout:3464: finishing write tid 5 to smithi07685715-11 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3464: oid 11 updating version 0 to 2685 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3464: oid 11 updating version 2685 to 2686 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3464: oid 11 version 2686 is already newer than 2684 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3464: oid 11 updating version 2686 to 2687 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2687 (ObjNum 1291 snap 439 seq_num 1291) dirty exists 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3464: left oid 11 (ObjNum 1291 snap 439 seq_num 1291) 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3465: finishing write tid 1 to smithi07685715-7 2024-09-21T09:52:39.886 INFO:tasks.rados.rados.0.smithi076.stdout:3465: finishing write tid 2 to smithi07685715-7 2024-09-21T09:52:39.887 INFO:tasks.rados.rados.0.smithi076.stdout:3463: done (3 left) 2024-09-21T09:52:39.887 INFO:tasks.rados.rados.0.smithi076.stdout:3464: done (2 left) 2024-09-21T09:52:39.887 INFO:tasks.rados.rados.0.smithi076.stdout:3466: done (1 left) 2024-09-21T09:52:39.887 INFO:tasks.rados.rados.0.smithi076.stdout:3467: snap_create 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: finishing write tid 3 to smithi07685715-7 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: finishing write tid 4 to smithi07685715-7 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: finishing write tid 5 to smithi07685715-7 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: oid 7 updating version 0 to 2174 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: oid 7 version 2174 is already newer than 2172 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: oid 7 version 2174 is already newer than 2173 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: oid 7 version 2174 is already newer than 2171 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2174 (ObjNum 1292 snap 439 seq_num 1292) dirty exists 2024-09-21T09:52:39.888 INFO:tasks.rados.rados.0.smithi076.stdout:3465: left oid 7 (ObjNum 1292 snap 439 seq_num 1292) 2024-09-21T09:52:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:39 smithi191 ceph-mon[102206]: pgmap v886: 89 pgs: 89 active+clean; 183 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 2 op/s 2024-09-21T09:52:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:39 smithi191 ceph-mon[102206]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:52:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:39 smithi076 ceph-mon[106482]: pgmap v886: 89 pgs: 89 active+clean; 183 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 2 op/s 2024-09-21T09:52:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:39 smithi076 ceph-mon[106482]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:52:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:39 smithi076 ceph-mon[110963]: pgmap v886: 89 pgs: 89 active+clean; 183 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 2 op/s 2024-09-21T09:52:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:39 smithi076 ceph-mon[110963]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T09:52:40.833 INFO:tasks.rados.rados.0.smithi076.stdout:3465: done (1 left) 2024-09-21T09:52:40.833 INFO:tasks.rados.rados.0.smithi076.stdout:3467: done (0 left) 2024-09-21T09:52:40.833 INFO:tasks.rados.rados.0.smithi076.stdout:3468: snap_remove snap 438 2024-09-21T09:52:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:40 smithi191 ceph-mon[102206]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:52:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:40 smithi076 ceph-mon[106482]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:52:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:40 smithi076 ceph-mon[110963]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T09:52:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:41.833 INFO:tasks.rados.rados.0.smithi076.stdout:3468: done (0 left) 2024-09-21T09:52:41.833 INFO:tasks.rados.rados.0.smithi076.stdout:3469: snap_remove snap 406 2024-09-21T09:52:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:41 smithi191 ceph-mon[102206]: pgmap v889: 89 pgs: 89 active+clean; 185 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 444 KiB/s wr, 1 op/s 2024-09-21T09:52:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:41 smithi191 ceph-mon[102206]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:52:42.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:41 smithi076 ceph-mon[106482]: pgmap v889: 89 pgs: 89 active+clean; 185 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 444 KiB/s wr, 1 op/s 2024-09-21T09:52:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:41 smithi076 ceph-mon[106482]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:52:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:41 smithi076 ceph-mon[110963]: pgmap v889: 89 pgs: 89 active+clean; 185 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 444 KiB/s wr, 1 op/s 2024-09-21T09:52:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:41 smithi076 ceph-mon[110963]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T09:52:42.832 INFO:tasks.rados.rados.0.smithi076.stdout:3469: done (0 left) 2024-09-21T09:52:42.833 INFO:tasks.rados.rados.0.smithi076.stdout:3470: delete oid 7 current snap is 440 2024-09-21T09:52:42.842 INFO:tasks.rados.rados.0.smithi076.stdout:3470: done (0 left) 2024-09-21T09:52:42.842 INFO:tasks.rados.rados.0.smithi076.stdout:3471: write oid 7 current snap is 440 2024-09-21T09:52:42.842 INFO:tasks.rados.rados.0.smithi076.stdout:3471: seq_num 1295 ranges {565065=427465,1430299=634439,2496305=525139,3472104=292961} 2024-09-21T09:52:42.848 INFO:tasks.rados.rados.0.smithi076.stdout:3471: writing smithi07685715-7 from 565065 to 992530 tid 1 2024-09-21T09:52:42.854 INFO:tasks.rados.rados.0.smithi076.stdout:3471: writing smithi07685715-7 from 1430299 to 2064738 tid 2 2024-09-21T09:52:42.860 INFO:tasks.rados.rados.0.smithi076.stdout:3471: writing smithi07685715-7 from 2496305 to 3021444 tid 3 2024-09-21T09:52:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3471: writing smithi07685715-7 from 3472104 to 3765065 tid 4 2024-09-21T09:52:42.864 INFO:tasks.rados.rados.0.smithi076.stdout:3472: snap_remove snap 425 2024-09-21T09:52:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:42 smithi191 ceph-mon[102206]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:52:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:42 smithi191 ceph-mon[102206]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:52:43.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:42 smithi076 ceph-mon[106482]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:52:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:42 smithi076 ceph-mon[106482]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:52:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:42 smithi076 ceph-mon[110963]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T09:52:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:42 smithi076 ceph-mon[110963]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T09:52:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:52:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:52:43] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:52:43.799 INFO:tasks.rados.rados.0.smithi076.stdout:3472: done (1 left) 2024-09-21T09:52:43.815 INFO:tasks.rados.rados.0.smithi076.stdout:3473: write oid 5 current snap is 440 2024-09-21T09:52:43.815 INFO:tasks.rados.rados.0.smithi076.stdout:3471: finishing write tid 1 to smithi07685715-7 2024-09-21T09:52:43.815 INFO:tasks.rados.rados.0.smithi076.stdout:3473: seq_num 1296 ranges {581872=502714,1488781=494450,2640081=736411} 2024-09-21T09:52:43.816 INFO:tasks.rados.rados.0.smithi076.stdout:3473: writing smithi07685715-5 from 581872 to 1084586 tid 1 2024-09-21T09:52:43.816 INFO:tasks.rados.rados.0.smithi076.stdout:3473: writing smithi07685715-5 from 1488781 to 1983231 tid 2 2024-09-21T09:52:43.821 INFO:tasks.rados.rados.0.smithi076.stdout:3473: writing smithi07685715-5 from 2640081 to 3376492 tid 3 2024-09-21T09:52:43.821 INFO:tasks.rados.rados.0.smithi076.stdout:3474: snap_remove snap 430 2024-09-21T09:52:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:43 smithi191 ceph-mon[102206]: pgmap v892: 89 pgs: 89 active+clean; 193 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 6 op/s 2024-09-21T09:52:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:43 smithi191 ceph-mon[102206]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:52:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:43 smithi076 ceph-mon[110963]: pgmap v892: 89 pgs: 89 active+clean; 193 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 6 op/s 2024-09-21T09:52:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:43 smithi076 ceph-mon[110963]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:52:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:43 smithi076 ceph-mon[106482]: pgmap v892: 89 pgs: 89 active+clean; 193 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.7 MiB/s wr, 6 op/s 2024-09-21T09:52:44.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:43 smithi076 ceph-mon[106482]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3474: done (2 left) 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3475: snap_create 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3471: finishing write tid 2 to smithi07685715-7 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3471: finishing write tid 3 to smithi07685715-7 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3471: finishing write tid 4 to smithi07685715-7 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3471: finishing write tid 5 to smithi07685715-7 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3471: finishing write tid 6 to smithi07685715-7 2024-09-21T09:52:44.804 INFO:tasks.rados.rados.0.smithi076.stdout:3471: oid 7 updating version 0 to 2185 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3471: oid 7 updating version 2185 to 2188 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3471: oid 7 updating version 2188 to 2189 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3471: oid 7 version 2189 is already newer than 2187 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3471: oid 7 version 2189 is already newer than 2186 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2189 (ObjNum 1295 snap 440 seq_num 1295) dirty exists 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3471: left oid 7 (ObjNum 1295 snap 440 seq_num 1295) 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: finishing write tid 1 to smithi07685715-5 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: finishing write tid 2 to smithi07685715-5 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: finishing write tid 3 to smithi07685715-5 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: finishing write tid 4 to smithi07685715-5 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: finishing write tid 5 to smithi07685715-5 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: oid 5 updating version 0 to 2699 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: oid 5 version 2699 is already newer than 2697 2024-09-21T09:52:44.805 INFO:tasks.rados.rados.0.smithi076.stdout:3473: oid 5 version 2699 is already newer than 2698 2024-09-21T09:52:44.806 INFO:tasks.rados.rados.0.smithi076.stdout:3473: oid 5 updating version 2699 to 2700 2024-09-21T09:52:44.806 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2700 (ObjNum 1296 snap 440 seq_num 1296) dirty exists 2024-09-21T09:52:44.806 INFO:tasks.rados.rados.0.smithi076.stdout:3473: left oid 5 (ObjNum 1296 snap 440 seq_num 1296) 2024-09-21T09:52:45.805 INFO:tasks.rados.rados.0.smithi076.stdout:3471: done (2 left) 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3473: done (1 left) 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3475: done (0 left) 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3476: read oid 25 snap -1 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3476: expect (ObjNum 1287 snap 435 seq_num 1287) 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3477: read oid 23 snap -1 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3477: expect (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:52:45.806 INFO:tasks.rados.rados.0.smithi076.stdout:3478: delete oid 45 current snap is 441 2024-09-21T09:52:45.809 INFO:tasks.rados.rados.0.smithi076.stdout:3477: expect (ObjNum 1214 snap 404 seq_num 1214) 2024-09-21T09:52:45.909 INFO:tasks.rados.rados.0.smithi076.stdout:3476: expect (ObjNum 1287 snap 435 seq_num 1287) 2024-09-21T09:52:46.084 INFO:tasks.rados.rados.0.smithi076.stdout:3476: done (2 left) 2024-09-21T09:52:47.280 INFO:tasks.rados.rados.0.smithi076.stdout:3477: done (1 left) 2024-09-21T09:52:47.280 INFO:tasks.rados.rados.0.smithi076.stdout:3478: done (0 left) 2024-09-21T09:52:47.281 INFO:tasks.rados.rados.0.smithi076.stdout:3479: write oid 34 current snap is 441 2024-09-21T09:52:47.281 INFO:tasks.rados.rados.0.smithi076.stdout:3479: seq_num 1298 ranges {615486=653212,1889392=498119,2978574=475308} 2024-09-21T09:52:47.281 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:45 smithi191 ceph-mon[102206]: pgmap v895: 89 pgs: 89 active+clean; 202 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:52:47.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:45 smithi191 ceph-mon[102206]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:52:47.282 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:46 smithi191 ceph-mon[102206]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:52:47.282 INFO:tasks.rados.rados.0.smithi076.stdout:3479: writing smithi07685715-34 from 615486 to 1268698 tid 1 2024-09-21T09:52:47.282 INFO:tasks.rados.rados.0.smithi076.stdout:3479: writing smithi07685715-34 from 1889392 to 2387511 tid 2 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3479: writing smithi07685715-34 from 2978574 to 3453882 tid 3 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3480: read oid 6 snap -1 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3480: expect deleted 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3481: write oid 36 current snap is 441 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3481: seq_num 1299 ranges {632293=728461,1947874=641777,3005997=686580} 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3481: writing smithi07685715-36 from 632293 to 1360754 tid 1 2024-09-21T09:52:47.283 INFO:tasks.rados.rados.0.smithi076.stdout:3481: writing smithi07685715-36 from 1947874 to 2589651 tid 2 2024-09-21T09:52:47.283 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:45 smithi076 ceph-mon[106482]: pgmap v895: 89 pgs: 89 active+clean; 202 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:52:47.283 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:45 smithi076 ceph-mon[106482]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:52:47.284 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:45 smithi076 ceph-mon[110963]: pgmap v895: 89 pgs: 89 active+clean; 202 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 8 op/s 2024-09-21T09:52:47.284 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:45 smithi076 ceph-mon[110963]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T09:52:47.288 INFO:tasks.rados.rados.0.smithi076.stdout:3481: writing smithi07685715-36 from 3005997 to 3692577 tid 3 2024-09-21T09:52:47.288 INFO:tasks.rados.rados.0.smithi076.stdout:3482: write oid 1 current snap is 441 2024-09-21T09:52:47.288 INFO:tasks.rados.rados.0.smithi076.stdout:3479: finishing write tid 1 to smithi07685715-34 2024-09-21T09:52:47.288 INFO:tasks.rados.rados.0.smithi076.stdout:3479: finishing write tid 2 to smithi07685715-34 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: finishing write tid 3 to smithi07685715-34 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: finishing write tid 4 to smithi07685715-34 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: finishing write tid 5 to smithi07685715-34 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: oid 34 updating version 0 to 785 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: oid 34 updating version 785 to 786 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: oid 34 updating version 786 to 787 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: oid 34 version 787 is already newer than 784 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 787 (ObjNum 1298 snap 441 seq_num 1298) dirty exists 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3479: left oid 34 (ObjNum 1298 snap 441 seq_num 1298) 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3481: finishing write tid 1 to smithi07685715-36 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3482: seq_num 1300 ranges {649100=403710,1606356=501788,2633420=497852,3636005=213095} 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3482: writing smithi07685715-1 from 649100 to 1052810 tid 1 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3482: writing smithi07685715-1 from 1606356 to 2108144 tid 2 2024-09-21T09:52:47.289 INFO:tasks.rados.rados.0.smithi076.stdout:3482: writing smithi07685715-1 from 2633420 to 3131272 tid 3 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3482: writing smithi07685715-1 from 3636005 to 3849100 tid 4 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3479: done (3 left) 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3480: done (2 left) 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3483: write oid 26 current snap is 441 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: finishing write tid 2 to smithi07685715-36 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: finishing write tid 3 to smithi07685715-36 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: finishing write tid 4 to smithi07685715-36 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: finishing write tid 5 to smithi07685715-36 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: oid 36 updating version 0 to 1130 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: oid 36 version 1130 is already newer than 1128 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: oid 36 version 1130 is already newer than 1129 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: oid 36 version 1130 is already newer than 1127 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1130 (ObjNum 1299 snap 441 seq_num 1299) dirty exists 2024-09-21T09:52:47.290 INFO:tasks.rados.rados.0.smithi076.stdout:3481: left oid 36 (ObjNum 1299 snap 441 seq_num 1299) 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3482: finishing write tid 1 to smithi07685715-1 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3482: finishing write tid 2 to smithi07685715-1 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3482: finishing write tid 3 to smithi07685715-1 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3483: seq_num 1301 ranges {665907=595312,1781191=645446,3177196=688711} 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3483: writing smithi07685715-26 from 665907 to 1261219 tid 1 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3483: writing smithi07685715-26 from 1781191 to 2426637 tid 2 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3483: writing smithi07685715-26 from 3177196 to 3865907 tid 3 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3481: done (2 left) 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3484: snap_remove snap 435 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3484: done (2 left) 2024-09-21T09:52:47.291 INFO:tasks.rados.rados.0.smithi076.stdout:3485: read oid 34 snap -1 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3485: expect (ObjNum 1298 snap 441 seq_num 1298) 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3486: read oid 33 snap -1 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3486: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3487: read oid 39 snap -1 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3487: expect (ObjNum 1097 snap 364 seq_num 1097) 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3488: rollback oid 17 current snap is 441 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 427 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3489: read oid 49 snap -1 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3489: expect (ObjNum 1290 snap 439 seq_num 1290) 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3490: write oid 12 current snap is 441 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3490: seq_num 1302 ranges {682714=670561,2123320=505457,3088266=636749} 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3490: writing smithi07685715-12 from 682714 to 1353275 tid 1 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3490: writing smithi07685715-12 from 2123320 to 2628777 tid 2 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3490: writing smithi07685715-12 from 3088266 to 3725015 tid 3 2024-09-21T09:52:47.292 INFO:tasks.rados.rados.0.smithi076.stdout:3491: delete oid 10 current snap is 441 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: finishing write tid 4 to smithi07685715-1 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: finishing write tid 5 to smithi07685715-1 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: finishing write tid 6 to smithi07685715-1 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: oid 1 updating version 0 to 2706 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: oid 1 version 2706 is already newer than 2705 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: oid 1 updating version 2706 to 2707 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: oid 1 version 2707 is already newer than 2704 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: oid 1 updating version 2707 to 2708 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2708 (ObjNum 1300 snap 441 seq_num 1300) dirty exists 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3482: left oid 1 (ObjNum 1300 snap 441 seq_num 1300) 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3483: finishing write tid 1 to smithi07685715-26 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3483: finishing write tid 2 to smithi07685715-26 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3483: finishing write tid 3 to smithi07685715-26 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3483: finishing write tid 4 to smithi07685715-26 2024-09-21T09:52:47.293 INFO:tasks.rados.rados.0.smithi076.stdout:3483: finishing write tid 5 to smithi07685715-26 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3483: oid 26 updating version 0 to 1210 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3483: oid 26 updating version 1210 to 1213 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3483: oid 26 version 1213 is already newer than 1211 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3483: oid 26 version 1213 is already newer than 1212 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1213 (ObjNum 1301 snap 441 seq_num 1301) dirty exists 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3483: left oid 26 (ObjNum 1301 snap 441 seq_num 1301) 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3485: expect (ObjNum 1298 snap 441 seq_num 1298) 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3488: finishing rollback tid 0 to smithi07685715-17 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3488: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3488: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2201 (ObjNum 1230 snap 410 seq_num 1230) dirty exists 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3482: done (8 left) 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3483: done (7 left) 2024-09-21T09:52:47.294 INFO:tasks.rados.rados.0.smithi076.stdout:3485: done (6 left) 2024-09-21T09:52:47.295 INFO:tasks.rados.rados.0.smithi076.stdout:3488: done (5 left) 2024-09-21T09:52:47.295 INFO:tasks.rados.rados.0.smithi076.stdout:3491: done (4 left) 2024-09-21T09:52:47.295 INFO:tasks.rados.rados.0.smithi076.stdout:3492: read oid 50 snap -1 2024-09-21T09:52:47.295 INFO:tasks.rados.rados.0.smithi076.stdout:3492: expect deleted 2024-09-21T09:52:47.295 INFO:tasks.rados.rados.0.smithi076.stdout:3493: write oid 5 current snap is 441 2024-09-21T09:52:47.295 INFO:tasks.rados.rados.0.smithi076.stdout:3486: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:52:47.295 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:46 smithi076 ceph-mon[106482]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:52:47.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:46 smithi076 ceph-mon[110963]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T09:52:47.296 INFO:tasks.rados.rados.0.smithi076.stdout:3487: expect (ObjNum 1097 snap 364 seq_num 1097) 2024-09-21T09:52:47.401 INFO:tasks.rados.rados.0.smithi076.stdout:3489: expect (ObjNum 1290 snap 439 seq_num 1290) 2024-09-21T09:52:47.605 INFO:tasks.rados.rados.0.smithi076.stdout:3490: finishing write tid 1 to smithi07685715-12 2024-09-21T09:52:47.605 INFO:tasks.rados.rados.0.smithi076.stdout:3490: finishing write tid 2 to smithi07685715-12 2024-09-21T09:52:47.605 INFO:tasks.rados.rados.0.smithi076.stdout:3490: finishing write tid 3 to smithi07685715-12 2024-09-21T09:52:47.605 INFO:tasks.rados.rados.0.smithi076.stdout:3490: finishing write tid 4 to smithi07685715-12 2024-09-21T09:52:47.605 INFO:tasks.rados.rados.0.smithi076.stdout:3490: finishing write tid 5 to smithi07685715-12 2024-09-21T09:52:47.605 INFO:tasks.rados.rados.0.smithi076.stdout:3490: oid 12 updating version 0 to 1215 2024-09-21T09:52:47.606 INFO:tasks.rados.rados.0.smithi076.stdout:3490: oid 12 updating version 1215 to 1220 2024-09-21T09:52:47.606 INFO:tasks.rados.rados.0.smithi076.stdout:3490: oid 12 version 1220 is already newer than 1216 2024-09-21T09:52:47.606 INFO:tasks.rados.rados.0.smithi076.stdout:3490: oid 12 version 1220 is already newer than 1219 2024-09-21T09:52:47.606 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1220 (ObjNum 1302 snap 441 seq_num 1302) dirty exists 2024-09-21T09:52:47.606 INFO:tasks.rados.rados.0.smithi076.stdout:3490: left oid 12 (ObjNum 1302 snap 441 seq_num 1302) 2024-09-21T09:52:47.606 INFO:tasks.rados.rados.0.smithi076.stdout:3493: seq_num 1304 ranges {716328=421059,1840284=509126,3143112=659293} 2024-09-21T09:52:47.610 INFO:tasks.rados.rados.0.smithi076.stdout:3493: writing smithi07685715-5 from 716328 to 1137387 tid 1 2024-09-21T09:52:47.617 INFO:tasks.rados.rados.0.smithi076.stdout:3493: writing smithi07685715-5 from 1840284 to 2349410 tid 2 2024-09-21T09:52:47.625 INFO:tasks.rados.rados.0.smithi076.stdout:3493: writing smithi07685715-5 from 3143112 to 3802405 tid 3 2024-09-21T09:52:47.625 INFO:tasks.rados.rados.0.smithi076.stdout:3486: done (5 left) 2024-09-21T09:52:47.625 INFO:tasks.rados.rados.0.smithi076.stdout:3487: done (4 left) 2024-09-21T09:52:47.625 INFO:tasks.rados.rados.0.smithi076.stdout:3489: done (3 left) 2024-09-21T09:52:47.626 INFO:tasks.rados.rados.0.smithi076.stdout:3490: done (2 left) 2024-09-21T09:52:47.626 INFO:tasks.rados.rados.0.smithi076.stdout:3492: done (1 left) 2024-09-21T09:52:47.626 INFO:tasks.rados.rados.0.smithi076.stdout:3494: write oid 35 current snap is 441 2024-09-21T09:52:47.626 INFO:tasks.rados.rados.0.smithi076.stdout:3493: finishing write tid 1 to smithi07685715-5 2024-09-21T09:52:47.626 INFO:tasks.rados.rados.0.smithi076.stdout:3493: finishing write tid 2 to smithi07685715-5 2024-09-21T09:52:47.626 INFO:tasks.rados.rados.0.smithi076.stdout:3494: seq_num 1305 ranges {733135=496308,1898766=652784,3170535=470565} 2024-09-21T09:52:47.632 INFO:tasks.rados.rados.0.smithi076.stdout:3494: writing smithi07685715-35 from 733135 to 1229443 tid 1 2024-09-21T09:52:47.638 INFO:tasks.rados.rados.0.smithi076.stdout:3494: writing smithi07685715-35 from 1898766 to 2551550 tid 2 2024-09-21T09:52:47.645 INFO:tasks.rados.rados.0.smithi076.stdout:3494: writing smithi07685715-35 from 3170535 to 3641100 tid 3 2024-09-21T09:52:47.645 INFO:tasks.rados.rados.0.smithi076.stdout:3495: write oid 25 current snap is 441 2024-09-21T09:52:47.645 INFO:tasks.rados.rados.0.smithi076.stdout:3495: seq_num 1306 ranges {749942=571557,1840895=796442,3365252=584690} 2024-09-21T09:52:47.652 INFO:tasks.rados.rados.0.smithi076.stdout:3495: writing smithi07685715-25 from 749942 to 1321499 tid 1 2024-09-21T09:52:47.659 INFO:tasks.rados.rados.0.smithi076.stdout:3495: writing smithi07685715-25 from 1840895 to 2637337 tid 2 2024-09-21T09:52:47.666 INFO:tasks.rados.rados.0.smithi076.stdout:3495: writing smithi07685715-25 from 3365252 to 3949942 tid 3 2024-09-21T09:52:47.666 INFO:tasks.rados.rados.0.smithi076.stdout:3496: read oid 15 snap -1 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3496: expect deleted 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3497: snap_create 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: finishing write tid 3 to smithi07685715-5 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: finishing write tid 4 to smithi07685715-5 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: finishing write tid 5 to smithi07685715-5 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: oid 5 updating version 0 to 2714 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: oid 5 updating version 2714 to 2716 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: oid 5 updating version 2716 to 2717 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: oid 5 version 2717 is already newer than 2715 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2717 (ObjNum 1304 snap 441 seq_num 1304) dirty exists 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3493: left oid 5 (ObjNum 1304 snap 441 seq_num 1304) 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3494: finishing write tid 1 to smithi07685715-35 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3494: finishing write tid 2 to smithi07685715-35 2024-09-21T09:52:47.667 INFO:tasks.rados.rados.0.smithi076.stdout:3494: finishing write tid 3 to smithi07685715-35 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: finishing write tid 4 to smithi07685715-35 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: finishing write tid 5 to smithi07685715-35 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: oid 35 updating version 0 to 2718 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: oid 35 updating version 2718 to 2721 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: oid 35 version 2721 is already newer than 2720 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: oid 35 version 2721 is already newer than 2719 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2721 (ObjNum 1305 snap 441 seq_num 1305) dirty exists 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3494: left oid 35 (ObjNum 1305 snap 441 seq_num 1305) 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3495: finishing write tid 1 to smithi07685715-25 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3495: finishing write tid 2 to smithi07685715-25 2024-09-21T09:52:47.668 INFO:tasks.rados.rados.0.smithi076.stdout:3495: finishing write tid 3 to smithi07685715-25 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:3495: finishing write tid 4 to smithi07685715-25 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:3495: finishing write tid 5 to smithi07685715-25 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:3495: oid 25 updating version 0 to 775 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:3495: oid 25 updating version 775 to 776 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:3495: oid 25 updating version 776 to 777 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:3495: oid 25 updating version 777 to 778 2024-09-21T09:52:47.669 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 778 (ObjNum 1306 snap 441 seq_num 1306) dirty exists 2024-09-21T09:52:47.670 INFO:tasks.rados.rados.0.smithi076.stdout:3495: left oid 25 (ObjNum 1306 snap 441 seq_num 1306) 2024-09-21T09:52:47.813 INFO:tasks.rados.rados.0.smithi076.stdout:3493: done (4 left) 2024-09-21T09:52:47.813 INFO:tasks.rados.rados.0.smithi076.stdout:3494: done (3 left) 2024-09-21T09:52:47.814 INFO:tasks.rados.rados.0.smithi076.stdout:3495: done (2 left) 2024-09-21T09:52:47.814 INFO:tasks.rados.rados.0.smithi076.stdout:3496: done (1 left) 2024-09-21T09:52:47.814 INFO:tasks.rados.rados.0.smithi076.stdout:3497: done (0 left) 2024-09-21T09:52:47.814 INFO:tasks.rados.rados.0.smithi076.stdout:3498: write oid 42 current snap is 442 2024-09-21T09:52:47.814 INFO:tasks.rados.rados.0.smithi076.stdout:3498: seq_num 1307 ranges {766749=646806,1899377=656453,3109028=609462} 2024-09-21T09:52:47.821 INFO:tasks.rados.rados.0.smithi076.stdout:3498: writing smithi07685715-42 from 766749 to 1413555 tid 1 2024-09-21T09:52:47.828 INFO:tasks.rados.rados.0.smithi076.stdout:3498: writing smithi07685715-42 from 1899377 to 2555830 tid 2 2024-09-21T09:52:47.836 INFO:tasks.rados.rados.0.smithi076.stdout:3498: writing smithi07685715-42 from 3109028 to 3718490 tid 3 2024-09-21T09:52:47.836 INFO:tasks.rados.rados.0.smithi076.stdout:3499: read oid 26 snap -1 2024-09-21T09:52:47.836 INFO:tasks.rados.rados.0.smithi076.stdout:3499: expect (ObjNum 1301 snap 441 seq_num 1301) 2024-09-21T09:52:47.836 INFO:tasks.rados.rados.0.smithi076.stdout:3500: write oid 2 current snap is 442 2024-09-21T09:52:47.836 INFO:tasks.rados.rados.0.smithi076.stdout:3500: seq_num 1308 ranges {783556=438408,1674212=400111,2736451=420734,3882270=101286} 2024-09-21T09:52:47.842 INFO:tasks.rados.rados.0.smithi076.stdout:3500: writing smithi07685715-2 from 783556 to 1221964 tid 1 2024-09-21T09:52:47.847 INFO:tasks.rados.rados.0.smithi076.stdout:3500: writing smithi07685715-2 from 1674212 to 2074323 tid 2 2024-09-21T09:52:47.853 INFO:tasks.rados.rados.0.smithi076.stdout:3500: writing smithi07685715-2 from 2736451 to 3157185 tid 3 2024-09-21T09:52:47.858 INFO:tasks.rados.rados.0.smithi076.stdout:3500: writing smithi07685715-2 from 3882270 to 3983556 tid 4 2024-09-21T09:52:47.858 INFO:tasks.rados.rados.0.smithi076.stdout:3501: write oid 36 current snap is 442 2024-09-21T09:52:47.858 INFO:tasks.rados.rados.0.smithi076.stdout:3501: seq_num 1309 ranges {400363=513657,1332694=660122} 2024-09-21T09:52:47.863 INFO:tasks.rados.rados.0.smithi076.stdout:3501: writing smithi07685715-36 from 400363 to 914020 tid 1 2024-09-21T09:52:47.869 INFO:tasks.rados.rados.0.smithi076.stdout:3501: writing smithi07685715-36 from 1332694 to 1992816 tid 2 2024-09-21T09:52:47.869 INFO:tasks.rados.rados.0.smithi076.stdout:3498: finishing write tid 1 to smithi07685715-42 2024-09-21T09:52:47.869 INFO:tasks.rados.rados.0.smithi076.stdout:3498: finishing write tid 2 to smithi07685715-42 2024-09-21T09:52:47.869 INFO:tasks.rados.rados.0.smithi076.stdout:3498: finishing write tid 3 to smithi07685715-42 2024-09-21T09:52:47.869 INFO:tasks.rados.rados.0.smithi076.stdout:3498: finishing write tid 4 to smithi07685715-42 2024-09-21T09:52:47.869 INFO:tasks.rados.rados.0.smithi076.stdout:3502: read oid 14 snap -1 2024-09-21T09:52:47.870 INFO:tasks.rados.rados.0.smithi076.stdout:3502: expect deleted 2024-09-21T09:52:47.870 INFO:tasks.rados.rados.0.smithi076.stdout:3503: rollback oid 19 current snap is 442 2024-09-21T09:52:47.870 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 19 to 436 2024-09-21T09:52:47.870 INFO:tasks.rados.rados.0.smithi076.stdout:3499: expect (ObjNum 1301 snap 441 seq_num 1301) 2024-09-21T09:52:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:47 smithi191 ceph-mon[102206]: pgmap v898: 89 pgs: 89 active+clean; 207 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-21T09:52:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:47 smithi191 ceph-mon[102206]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:52:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:47 smithi076 ceph-mon[106482]: pgmap v898: 89 pgs: 89 active+clean; 207 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-21T09:52:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:47 smithi076 ceph-mon[106482]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:52:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:47 smithi076 ceph-mon[110963]: pgmap v898: 89 pgs: 89 active+clean; 207 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.1 MiB/s wr, 10 op/s 2024-09-21T09:52:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:47 smithi076 ceph-mon[110963]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3498: finishing write tid 5 to smithi07685715-42 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3498: oid 42 updating version 0 to 900 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3498: oid 42 version 900 is already newer than 899 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3498: oid 42 version 900 is already newer than 898 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3498: oid 42 version 900 is already newer than 897 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 900 (ObjNum 1307 snap 442 seq_num 1307) dirty exists 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3498: left oid 42 (ObjNum 1307 snap 442 seq_num 1307) 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3500: finishing write tid 1 to smithi07685715-2 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3500: finishing write tid 2 to smithi07685715-2 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3500: finishing write tid 3 to smithi07685715-2 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3500: finishing write tid 4 to smithi07685715-2 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3500: finishing write tid 5 to smithi07685715-2 2024-09-21T09:52:48.134 INFO:tasks.rados.rados.0.smithi076.stdout:3500: finishing write tid 6 to smithi07685715-2 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: oid 2 updating version 0 to 699 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: oid 2 version 699 is already newer than 698 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: oid 2 updating version 699 to 702 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: oid 2 version 702 is already newer than 701 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: oid 2 version 702 is already newer than 700 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 702 (ObjNum 1308 snap 442 seq_num 1308) dirty exists 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: left oid 2 (ObjNum 1308 snap 442 seq_num 1308) 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3501: finishing write tid 1 to smithi07685715-36 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3498: done (5 left) 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3499: done (4 left) 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3500: done (3 left) 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3504: read oid 43 snap -1 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3504: expect (ObjNum 1170 snap 393 seq_num 1170) 2024-09-21T09:52:48.135 INFO:tasks.rados.rados.0.smithi076.stdout:3505: write oid 41 current snap is 442 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:3503: finishing rollback tid 0 to smithi07685715-19 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:3503: finishing rollback tid 1 to smithi07685715-19 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:3503: finishing rollback tid 2 to smithi07685715-19 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 1054 (ObjNum 1266 snap 425 seq_num 1266) dirty exists 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:3501: finishing write tid 2 to smithi07685715-36 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:3501: finishing write tid 3 to smithi07685715-36 2024-09-21T09:52:48.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: seq_num 1310 ranges {417170=588906,1674823=342347} 2024-09-21T09:52:48.140 INFO:tasks.rados.rados.0.smithi076.stdout:3505: writing smithi07685715-41 from 417170 to 1006076 tid 1 2024-09-21T09:52:48.144 INFO:tasks.rados.rados.0.smithi076.stdout:3505: writing smithi07685715-41 from 1674823 to 2017170 tid 2 2024-09-21T09:52:48.145 INFO:tasks.rados.rados.0.smithi076.stdout:3502: done (4 left) 2024-09-21T09:52:48.145 INFO:tasks.rados.rados.0.smithi076.stdout:3503: done (3 left) 2024-09-21T09:52:48.145 INFO:tasks.rados.rados.0.smithi076.stdout:3506: read oid 13 snap -1 2024-09-21T09:52:48.145 INFO:tasks.rados.rados.0.smithi076.stdout:3506: expect (ObjNum 1203 snap 403 seq_num 1203) 2024-09-21T09:52:48.145 INFO:tasks.rados.rados.0.smithi076.stdout:3507: snap_remove snap 439 2024-09-21T09:52:48.829 INFO:tasks.rados.rados.0.smithi076.stdout:3507: done (4 left) 2024-09-21T09:52:48.829 INFO:tasks.rados.rados.0.smithi076.stdout:3508: write oid 48 current snap is 442 2024-09-21T09:52:48.830 INFO:tasks.rados.rados.0.smithi076.stdout:3508: seq_num 1311 ranges {433977=664155,1733305=300672} 2024-09-21T09:52:48.836 INFO:tasks.rados.rados.0.smithi076.stdout:3508: writing smithi07685715-48 from 433977 to 1098132 tid 1 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3508: writing smithi07685715-48 from 1733305 to 2033977 tid 2 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3509: write oid 1 current snap is 442 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3501: finishing write tid 4 to smithi07685715-36 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3501: oid 36 updating version 0 to 1133 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3501: oid 36 version 1133 is already newer than 1132 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3501: oid 36 updating version 1133 to 1134 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1134 (ObjNum 1309 snap 442 seq_num 1309) dirty exists 2024-09-21T09:52:48.842 INFO:tasks.rados.rados.0.smithi076.stdout:3501: left oid 36 (ObjNum 1309 snap 442 seq_num 1309) 2024-09-21T09:52:48.843 INFO:tasks.rados.rados.0.smithi076.stdout:3504: expect (ObjNum 1170 snap 393 seq_num 1170) 2024-09-21T09:52:49.009 INFO:tasks.rados.rados.0.smithi076.stdout:3505: finishing write tid 1 to smithi07685715-41 2024-09-21T09:52:49.009 INFO:tasks.rados.rados.0.smithi076.stdout:3506: expect (ObjNum 1203 snap 403 seq_num 1203) 2024-09-21T09:52:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:48 smithi191 ceph-mon[102206]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:52:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:48 smithi076 ceph-mon[106482]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:52:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:48 smithi076 ceph-mon[110963]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T09:52:49.135 INFO:tasks.rados.rados.0.smithi076.stdout:3505: finishing write tid 2 to smithi07685715-41 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: finishing write tid 3 to smithi07685715-41 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: finishing write tid 4 to smithi07685715-41 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: oid 41 updating version 0 to 1750 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: oid 41 updating version 1750 to 1751 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: oid 41 version 1751 is already newer than 1749 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1751 (ObjNum 1310 snap 442 seq_num 1310) dirty exists 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3505: left oid 41 (ObjNum 1310 snap 442 seq_num 1310) 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3508: finishing write tid 1 to smithi07685715-48 2024-09-21T09:52:49.136 INFO:tasks.rados.rados.0.smithi076.stdout:3509: seq_num 1312 ranges {450784=739404,1791787=258997} 2024-09-21T09:52:49.142 INFO:tasks.rados.rados.0.smithi076.stdout:3509: writing smithi07685715-1 from 450784 to 1190188 tid 1 2024-09-21T09:52:49.147 INFO:tasks.rados.rados.0.smithi076.stdout:3509: writing smithi07685715-1 from 1791787 to 2050784 tid 2 2024-09-21T09:52:49.147 INFO:tasks.rados.rados.0.smithi076.stdout:3501: done (5 left) 2024-09-21T09:52:49.147 INFO:tasks.rados.rados.0.smithi076.stdout:3504: done (4 left) 2024-09-21T09:52:49.147 INFO:tasks.rados.rados.0.smithi076.stdout:3505: done (3 left) 2024-09-21T09:52:49.147 INFO:tasks.rados.rados.0.smithi076.stdout:3506: done (2 left) 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3510: write oid 19 current snap is 442 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: finishing write tid 2 to smithi07685715-48 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: finishing write tid 3 to smithi07685715-48 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: finishing write tid 4 to smithi07685715-48 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: oid 48 updating version 0 to 2702 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: oid 48 updating version 2702 to 2703 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: oid 48 version 2703 is already newer than 2701 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2703 (ObjNum 1311 snap 442 seq_num 1311) dirty exists 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3508: left oid 48 (ObjNum 1311 snap 442 seq_num 1311) 2024-09-21T09:52:49.148 INFO:tasks.rados.rados.0.smithi076.stdout:3510: seq_num 1313 ranges {467591=414653,1450269=617322} 2024-09-21T09:52:49.151 INFO:tasks.rados.rados.0.smithi076.stdout:3510: writing smithi07685715-19 from 467591 to 882244 tid 1 2024-09-21T09:52:49.158 INFO:tasks.rados.rados.0.smithi076.stdout:3510: writing smithi07685715-19 from 1450269 to 2067591 tid 2 2024-09-21T09:52:49.158 INFO:tasks.rados.rados.0.smithi076.stdout:3508: done (2 left) 2024-09-21T09:52:49.158 INFO:tasks.rados.rados.0.smithi076.stdout:3511: write oid 37 current snap is 442 2024-09-21T09:52:49.158 INFO:tasks.rados.rados.0.smithi076.stdout:3511: seq_num 1314 ranges {484398=489902,1392398=411118} 2024-09-21T09:52:49.163 INFO:tasks.rados.rados.0.smithi076.stdout:3511: writing smithi07685715-37 from 484398 to 974300 tid 1 2024-09-21T09:52:49.169 INFO:tasks.rados.rados.0.smithi076.stdout:3511: writing smithi07685715-37 from 1392398 to 1803516 tid 2 2024-09-21T09:52:49.169 INFO:tasks.rados.rados.0.smithi076.stdout:3512: rollback oid 24 current snap is 442 2024-09-21T09:52:49.169 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 24 to 408 2024-09-21T09:52:49.169 INFO:tasks.rados.rados.0.smithi076.stdout:3509: finishing write tid 1 to smithi07685715-1 2024-09-21T09:52:49.169 INFO:tasks.rados.rados.0.smithi076.stdout:3509: finishing write tid 2 to smithi07685715-1 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3509: finishing write tid 3 to smithi07685715-1 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3509: finishing write tid 4 to smithi07685715-1 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3509: oid 1 updating version 0 to 2727 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3509: oid 1 updating version 2727 to 2728 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3509: oid 1 updating version 2728 to 2729 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2729 (ObjNum 1312 snap 442 seq_num 1312) dirty exists 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3509: left oid 1 (ObjNum 1312 snap 442 seq_num 1312) 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3510: finishing write tid 1 to smithi07685715-19 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3510: finishing write tid 2 to smithi07685715-19 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3510: finishing write tid 3 to smithi07685715-19 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3510: finishing write tid 4 to smithi07685715-19 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3510: oid 19 updating version 0 to 1059 2024-09-21T09:52:49.170 INFO:tasks.rados.rados.0.smithi076.stdout:3510: oid 19 version 1059 is already newer than 1058 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3510: oid 19 version 1059 is already newer than 1057 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 1059 (ObjNum 1313 snap 442 seq_num 1313) dirty exists 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3510: left oid 19 (ObjNum 1313 snap 442 seq_num 1313) 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3511: finishing write tid 1 to smithi07685715-37 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3509: done (3 left) 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3510: done (2 left) 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3513: write oid 22 current snap is 442 2024-09-21T09:52:49.171 INFO:tasks.rados.rados.0.smithi076.stdout:3513: seq_num 1315 ranges {501205=565151,1850880=250325} 2024-09-21T09:52:49.174 INFO:tasks.rados.rados.0.smithi076.stdout:3513: writing smithi07685715-22 from 501205 to 1066356 tid 1 2024-09-21T09:52:49.180 INFO:tasks.rados.rados.0.smithi076.stdout:3513: writing smithi07685715-22 from 1850880 to 2101205 tid 2 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3514: rollback oid 27 current snap is 442 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 431 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3512: finishing rollback tid 0 to smithi07685715-24 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3512: finishing rollback tid 1 to smithi07685715-24 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3512: finishing rollback tid 2 to smithi07685715-24 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 364 (ObjNum 1218 snap 406 seq_num 1218) dirty exists 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3511: finishing write tid 2 to smithi07685715-37 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3511: finishing write tid 3 to smithi07685715-37 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3511: finishing write tid 4 to smithi07685715-37 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3511: oid 37 updating version 0 to 1073 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3511: oid 37 version 1073 is already newer than 1071 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:3511: oid 37 version 1073 is already newer than 1072 2024-09-21T09:52:49.181 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 1073 (ObjNum 1314 snap 442 seq_num 1314) dirty exists 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3511: left oid 37 (ObjNum 1314 snap 442 seq_num 1314) 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3513: finishing write tid 1 to smithi07685715-22 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3511: done (3 left) 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3512: done (2 left) 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3515: read oid 11 snap -1 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3515: expect (ObjNum 1291 snap 439 seq_num 1291) 2024-09-21T09:52:49.182 INFO:tasks.rados.rados.0.smithi076.stdout:3516: snap_create 2024-09-21T09:52:49.184 INFO:tasks.rados.rados.0.smithi076.stdout:3513: finishing write tid 2 to smithi07685715-22 2024-09-21T09:52:49.185 INFO:tasks.rados.rados.0.smithi076.stdout:3513: finishing write tid 3 to smithi07685715-22 2024-09-21T09:52:49.185 INFO:tasks.rados.rados.0.smithi076.stdout:3513: finishing write tid 4 to smithi07685715-22 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:3513: oid 22 updating version 0 to 1753 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:3513: oid 22 updating version 1753 to 1754 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:3513: oid 22 updating version 1754 to 1755 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1755 (ObjNum 1315 snap 442 seq_num 1315) dirty exists 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:3513: left oid 22 (ObjNum 1315 snap 442 seq_num 1315) 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:3514: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:52:49.186 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1687 (ObjNum 33 snap 0 seq_num 865207792) dirty dne 2024-09-21T09:52:49.194 INFO:tasks.rados.rados.0.smithi076.stdout:3515: expect (ObjNum 1291 snap 439 seq_num 1291) 2024-09-21T09:52:49.827 INFO:tasks.rados.rados.0.smithi076.stdout:3513: done (3 left) 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:3514: done (2 left) 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:3515: done (1 left) 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:3516: done (0 left) 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:3517: rollback oid 36 current snap is 443 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 36 to 440 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:3518: rollback oid 29 current snap is 443 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 431 2024-09-21T09:52:49.828 INFO:tasks.rados.rados.0.smithi076.stdout:3519: snap_create 2024-09-21T09:52:49.831 INFO:tasks.rados.rados.0.smithi076.stdout:3518: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:52:49.831 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 0 (ObjNum 129 snap 0 seq_num 864377888) dirty dne 2024-09-21T09:52:49.834 INFO:tasks.rados.rados.0.smithi076.stdout:3517: finishing rollback tid 0 to smithi07685715-36 2024-09-21T09:52:49.837 INFO:tasks.rados.rados.0.smithi076.stdout:3517: finishing rollback tid 1 to smithi07685715-36 2024-09-21T09:52:49.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1137 (ObjNum 305 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:49 smithi191 ceph-mon[102206]: pgmap v901: 89 pgs: 89 active+clean; 210 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-21T09:52:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:49 smithi191 ceph-mon[102206]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:52:50.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:49 smithi076 ceph-mon[106482]: pgmap v901: 89 pgs: 89 active+clean; 210 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-21T09:52:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:49 smithi076 ceph-mon[106482]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:52:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:49 smithi076 ceph-mon[110963]: pgmap v901: 89 pgs: 89 active+clean; 210 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.1 MiB/s wr, 16 op/s 2024-09-21T09:52:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:49 smithi076 ceph-mon[110963]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T09:52:50.828 INFO:tasks.rados.rados.0.smithi076.stdout:3517: done (2 left) 2024-09-21T09:52:50.829 INFO:tasks.rados.rados.0.smithi076.stdout:3518: done (1 left) 2024-09-21T09:52:50.829 INFO:tasks.rados.rados.0.smithi076.stdout:3519: done (0 left) 2024-09-21T09:52:50.829 INFO:tasks.rados.rados.0.smithi076.stdout:3520: snap_create 2024-09-21T09:52:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:50 smithi191 ceph-mon[102206]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:52:51.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:50 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:52:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:50 smithi191 ceph-mon[102206]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:52:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:50 smithi076 ceph-mon[106482]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:52:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:50 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:52:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:50 smithi076 ceph-mon[106482]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:52:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:50 smithi076 ceph-mon[110963]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T09:52:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:50 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:52:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:50 smithi076 ceph-mon[110963]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T09:52:51.829 INFO:tasks.rados.rados.0.smithi076.stdout:3520: done (0 left) 2024-09-21T09:52:51.830 INFO:tasks.rados.rados.0.smithi076.stdout:3521: delete oid 2 current snap is 445 2024-09-21T09:52:51.834 INFO:tasks.rados.rados.0.smithi076.stdout:3521: done (0 left) 2024-09-21T09:52:51.834 INFO:tasks.rados.rados.0.smithi076.stdout:3522: rollback oid 20 current snap is 445 2024-09-21T09:52:51.834 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 20 to 424 2024-09-21T09:52:51.834 INFO:tasks.rados.rados.0.smithi076.stdout:3523: write oid 39 current snap is 445 2024-09-21T09:52:51.834 INFO:tasks.rados.rados.0.smithi076.stdout:3523: seq_num 1317 ranges {534819=432002,1684197=450622} 2024-09-21T09:52:51.839 INFO:tasks.rados.rados.0.smithi076.stdout:3523: writing smithi07685715-39 from 534819 to 966821 tid 1 2024-09-21T09:52:51.846 INFO:tasks.rados.rados.0.smithi076.stdout:3523: writing smithi07685715-39 from 1684197 to 2134819 tid 2 2024-09-21T09:52:51.846 INFO:tasks.rados.rados.0.smithi076.stdout:3524: snap_remove snap 442 2024-09-21T09:52:52.104 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:51 smithi191 ceph-mon[102206]: pgmap v904: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 220 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-09-21T09:52:52.104 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:51 smithi191 ceph-mon[102206]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:52:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:51 smithi076 ceph-mon[106482]: pgmap v904: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 220 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-09-21T09:52:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:51 smithi076 ceph-mon[106482]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:52:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:51 smithi076 ceph-mon[110963]: pgmap v904: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 220 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-09-21T09:52:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:51 smithi076 ceph-mon[110963]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T09:52:52.848 INFO:tasks.rados.rados.0.smithi076.stdout:3524: done (2 left) 2024-09-21T09:52:52.848 INFO:tasks.rados.rados.0.smithi076.stdout:3525: write oid 7 current snap is 445 2024-09-21T09:52:52.848 INFO:tasks.rados.rados.0.smithi076.stdout:3522: finishing rollback tid 0 to smithi07685715-20 2024-09-21T09:52:52.848 INFO:tasks.rados.rados.0.smithi076.stdout:3525: seq_num 1318 ranges {551626=507251,1626326=418456} 2024-09-21T09:52:52.853 INFO:tasks.rados.rados.0.smithi076.stdout:3525: writing smithi07685715-7 from 551626 to 1058877 tid 1 2024-09-21T09:52:52.860 INFO:tasks.rados.rados.0.smithi076.stdout:3525: writing smithi07685715-7 from 1626326 to 2044782 tid 2 2024-09-21T09:52:52.860 INFO:tasks.rados.rados.0.smithi076.stdout:3526: rollback oid 26 current snap is 445 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 433 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:3522: finishing rollback tid 1 to smithi07685715-20 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:3522: finishing rollback tid 2 to smithi07685715-20 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 1077 (ObjNum 1127 snap 371 seq_num 1127) dirty exists 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:3523: finishing write tid 1 to smithi07685715-39 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:3522: done (3 left) 2024-09-21T09:52:52.861 INFO:tasks.rados.rados.0.smithi076.stdout:3527: snap_create 2024-09-21T09:52:52.862 INFO:tasks.rados.rados.0.smithi076.stdout:3523: finishing write tid 2 to smithi07685715-39 2024-09-21T09:52:52.862 INFO:tasks.rados.rados.0.smithi076.stdout:3523: finishing write tid 3 to smithi07685715-39 2024-09-21T09:52:52.862 INFO:tasks.rados.rados.0.smithi076.stdout:3523: finishing write tid 4 to smithi07685715-39 2024-09-21T09:52:52.862 INFO:tasks.rados.rados.0.smithi076.stdout:3523: oid 39 updating version 0 to 2732 2024-09-21T09:52:52.862 INFO:tasks.rados.rados.0.smithi076.stdout:3523: oid 39 version 2732 is already newer than 2731 2024-09-21T09:52:52.862 INFO:tasks.rados.rados.0.smithi076.stdout:3523: oid 39 updating version 2732 to 2733 2024-09-21T09:52:52.863 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2733 (ObjNum 1317 snap 445 seq_num 1317) dirty exists 2024-09-21T09:52:52.863 INFO:tasks.rados.rados.0.smithi076.stdout:3523: left oid 39 (ObjNum 1317 snap 445 seq_num 1317) 2024-09-21T09:52:52.863 INFO:tasks.rados.rados.0.smithi076.stdout:3525: finishing write tid 1 to smithi07685715-7 2024-09-21T09:52:52.864 INFO:tasks.rados.rados.0.smithi076.stdout:3525: finishing write tid 2 to smithi07685715-7 2024-09-21T09:52:52.865 INFO:tasks.rados.rados.0.smithi076.stdout:3525: finishing write tid 3 to smithi07685715-7 2024-09-21T09:52:52.865 INFO:tasks.rados.rados.0.smithi076.stdout:3525: finishing write tid 4 to smithi07685715-7 2024-09-21T09:52:52.865 INFO:tasks.rados.rados.0.smithi076.stdout:3525: oid 7 updating version 0 to 2217 2024-09-21T09:52:52.865 INFO:tasks.rados.rados.0.smithi076.stdout:3525: oid 7 version 2217 is already newer than 2215 2024-09-21T09:52:52.865 INFO:tasks.rados.rados.0.smithi076.stdout:3525: oid 7 version 2217 is already newer than 2216 2024-09-21T09:52:52.865 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2217 (ObjNum 1318 snap 445 seq_num 1318) dirty exists 2024-09-21T09:52:52.866 INFO:tasks.rados.rados.0.smithi076.stdout:3525: left oid 7 (ObjNum 1318 snap 445 seq_num 1318) 2024-09-21T09:52:52.866 INFO:tasks.rados.rados.0.smithi076.stdout:3526: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:52:52.867 INFO:tasks.rados.rados.0.smithi076.stdout:3526: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:52:52.867 INFO:tasks.rados.rados.0.smithi076.stdout:3526: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:52:52.867 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1228 (ObjNum 1084 snap 363 seq_num 1084) dirty exists 2024-09-21T09:52:53.564 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:52:53] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.51.0" 2024-09-21T09:52:53.801 INFO:tasks.rados.rados.0.smithi076.stdout:3523: done (3 left) 2024-09-21T09:52:53.801 INFO:tasks.rados.rados.0.smithi076.stdout:3525: done (2 left) 2024-09-21T09:52:53.801 INFO:tasks.rados.rados.0.smithi076.stdout:3526: done (1 left) 2024-09-21T09:52:53.801 INFO:tasks.rados.rados.0.smithi076.stdout:3527: done (0 left) 2024-09-21T09:52:53.802 INFO:tasks.rados.rados.0.smithi076.stdout:3528: delete oid 23 current snap is 446 2024-09-21T09:52:53.807 INFO:tasks.rados.rados.0.smithi076.stdout:3528: done (0 left) 2024-09-21T09:52:53.807 INFO:tasks.rados.rados.0.smithi076.stdout:3529: read oid 34 snap -1 2024-09-21T09:52:53.807 INFO:tasks.rados.rados.0.smithi076.stdout:3529: expect (ObjNum 1298 snap 441 seq_num 1298) 2024-09-21T09:52:53.807 INFO:tasks.rados.rados.0.smithi076.stdout:3530: snap_remove snap 433 2024-09-21T09:52:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:53 smithi191 ceph-mon[102206]: pgmap v907: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 233 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.1 MiB/s wr, 33 op/s 2024-09-21T09:52:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:53 smithi191 ceph-mon[102206]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:52:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:53 smithi191 ceph-mon[102206]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:52:54.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-mon[106482]: pgmap v907: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 233 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.1 MiB/s wr, 33 op/s 2024-09-21T09:52:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-mon[106482]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:52:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-mon[106482]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:52:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-mon[110963]: pgmap v907: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 233 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 4.1 MiB/s wr, 33 op/s 2024-09-21T09:52:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-mon[110963]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T09:52:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:53 smithi076 ceph-mon[110963]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T09:52:54.808 INFO:tasks.rados.rados.0.smithi076.stdout:3530: done (1 left) 2024-09-21T09:52:54.809 INFO:tasks.rados.rados.0.smithi076.stdout:3531: write oid 22 current snap is 446 2024-09-21T09:52:54.809 INFO:tasks.rados.rados.0.smithi076.stdout:3531: seq_num 1320 ranges {585240=657749,1743290=422125} 2024-09-21T09:52:54.816 INFO:tasks.rados.rados.0.smithi076.stdout:3531: writing smithi07685715-22 from 585240 to 1242989 tid 1 2024-09-21T09:52:54.822 INFO:tasks.rados.rados.0.smithi076.stdout:3531: writing smithi07685715-22 from 1743290 to 2165415 tid 2 2024-09-21T09:52:54.822 INFO:tasks.rados.rados.0.smithi076.stdout:3532: delete oid 7 current snap is 446 2024-09-21T09:52:54.822 INFO:tasks.rados.rados.0.smithi076.stdout:3529: expect (ObjNum 1298 snap 441 seq_num 1298) 2024-09-21T09:52:54.999 INFO:tasks.rados.rados.0.smithi076.stdout:3531: finishing write tid 1 to smithi07685715-22 2024-09-21T09:52:54.999 INFO:tasks.rados.rados.0.smithi076.stdout:3531: finishing write tid 2 to smithi07685715-22 2024-09-21T09:52:54.999 INFO:tasks.rados.rados.0.smithi076.stdout:3531: finishing write tid 3 to smithi07685715-22 2024-09-21T09:52:54.999 INFO:tasks.rados.rados.0.smithi076.stdout:3531: finishing write tid 4 to smithi07685715-22 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3531: oid 22 updating version 0 to 1764 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3531: oid 22 version 1764 is already newer than 1762 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3531: oid 22 version 1764 is already newer than 1763 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1764 (ObjNum 1320 snap 446 seq_num 1320) dirty exists 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3531: left oid 22 (ObjNum 1320 snap 446 seq_num 1320) 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3529: done (2 left) 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3531: done (1 left) 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3532: done (0 left) 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3533: read oid 28 snap -1 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3533: expect deleted 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3534: write oid 48 current snap is 446 2024-09-21T09:52:55.000 INFO:tasks.rados.rados.0.smithi076.stdout:3534: seq_num 1322 ranges {618854=408247,1743901=425794} 2024-09-21T09:52:55.004 INFO:tasks.rados.rados.0.smithi076.stdout:3534: writing smithi07685715-48 from 618854 to 1027101 tid 1 2024-09-21T09:52:55.011 INFO:tasks.rados.rados.0.smithi076.stdout:3534: writing smithi07685715-48 from 1743901 to 2169695 tid 2 2024-09-21T09:52:55.011 INFO:tasks.rados.rados.0.smithi076.stdout:3535: delete oid 39 current snap is 446 2024-09-21T09:52:55.011 INFO:tasks.rados.rados.0.smithi076.stdout:3534: finishing write tid 1 to smithi07685715-48 2024-09-21T09:52:55.013 INFO:tasks.rados.rados.0.smithi076.stdout:3533: done (2 left) 2024-09-21T09:52:55.014 INFO:tasks.rados.rados.0.smithi076.stdout:3535: done (1 left) 2024-09-21T09:52:55.014 INFO:tasks.rados.rados.0.smithi076.stdout:3536: write oid 41 current snap is 446 2024-09-21T09:52:55.014 INFO:tasks.rados.rados.0.smithi076.stdout:3536: seq_num 1324 ranges {652468=675098,1977218=275250} 2024-09-21T09:52:55.021 INFO:tasks.rados.rados.0.smithi076.stdout:3536: writing smithi07685715-41 from 652468 to 1327566 tid 1 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3536: writing smithi07685715-41 from 1977218 to 2252468 tid 2 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3537: read oid 26 snap 412 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3537: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3538: rollback oid 5 current snap is 446 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 428 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3534: finishing write tid 2 to smithi07685715-48 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3534: finishing write tid 3 to smithi07685715-48 2024-09-21T09:52:55.027 INFO:tasks.rados.rados.0.smithi076.stdout:3534: finishing write tid 4 to smithi07685715-48 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3534: oid 48 updating version 0 to 2711 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3534: oid 48 updating version 2711 to 2713 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3534: oid 48 version 2713 is already newer than 2712 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2713 (ObjNum 1322 snap 446 seq_num 1322) dirty exists 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3534: left oid 48 (ObjNum 1322 snap 446 seq_num 1322) 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3536: finishing write tid 1 to smithi07685715-41 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3534: done (3 left) 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3539: read oid 47 snap 424 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3539: expect deleted 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3540: rollback oid 12 current snap is 446 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 12 to 441 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3541: read oid 11 snap -1 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3541: expect (ObjNum 1291 snap 439 seq_num 1291) 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:3542: rollback oid 29 current snap is 446 2024-09-21T09:52:55.028 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 29 to 417 2024-09-21T09:52:55.029 INFO:tasks.rados.rados.0.smithi076.stdout:3543: write oid 1 current snap is 446 2024-09-21T09:52:55.029 INFO:tasks.rados.rados.0.smithi076.stdout:3543: seq_num 1325 ranges {669275=750347,2035700=233575} 2024-09-21T09:52:55.034 INFO:tasks.rados.rados.0.smithi076.stdout:3543: writing smithi07685715-1 from 669275 to 1419622 tid 1 2024-09-21T09:52:55.039 INFO:tasks.rados.rados.0.smithi076.stdout:3543: writing smithi07685715-1 from 2035700 to 2269275 tid 2 2024-09-21T09:52:55.039 INFO:tasks.rados.rados.0.smithi076.stdout:3544: snap_remove snap 427 2024-09-21T09:52:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:54 smithi191 ceph-mon[102206]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:52:55.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:54 smithi076 ceph-mon[106482]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:52:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:54 smithi076 ceph-mon[110963]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:3544: done (8 left) 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:3545: read oid 42 snap -1 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:3545: expect (ObjNum 1307 snap 442 seq_num 1307) 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:3546: snap_create 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:3542: finishing rollback tid 1 to smithi07685715-29 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:55.907 INFO:tasks.rados.rados.0.smithi076.stdout:3537: expect (ObjNum 1084 snap 363 seq_num 1084) 2024-09-21T09:52:56.124 INFO:tasks.rados.rados.0.smithi076.stdout:3536: finishing write tid 2 to smithi07685715-41 2024-09-21T09:52:56.124 INFO:tasks.rados.rados.0.smithi076.stdout:3536: finishing write tid 3 to smithi07685715-41 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3536: finishing write tid 4 to smithi07685715-41 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3536: oid 41 updating version 0 to 1767 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3536: oid 41 version 1767 is already newer than 1766 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3536: oid 41 updating version 1767 to 1768 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1768 (ObjNum 1324 snap 446 seq_num 1324) dirty exists 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3536: left oid 41 (ObjNum 1324 snap 446 seq_num 1324) 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3540: finishing rollback tid 0 to smithi07685715-12 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3540: finishing rollback tid 1 to smithi07685715-12 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3540: finishing rollback tid 2 to smithi07685715-12 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1236 (ObjNum 1302 snap 441 seq_num 1302) dirty exists 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3538: finishing rollback tid 0 to smithi07685715-5 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3538: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:3538: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:52:56.125 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2747 (ObjNum 1269 snap 427 seq_num 1269) dirty exists 2024-09-21T09:52:56.126 INFO:tasks.rados.rados.0.smithi076.stdout:3541: expect (ObjNum 1291 snap 439 seq_num 1291) 2024-09-21T09:52:56.240 INFO:tasks.rados.rados.0.smithi076.stdout:3543: finishing write tid 1 to smithi07685715-1 2024-09-21T09:52:56.240 INFO:tasks.rados.rados.0.smithi076.stdout:3543: finishing write tid 2 to smithi07685715-1 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3543: finishing write tid 3 to smithi07685715-1 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3543: finishing write tid 4 to smithi07685715-1 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3543: oid 1 updating version 0 to 2750 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3543: oid 1 updating version 2750 to 2751 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3543: oid 1 version 2751 is already newer than 2749 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2751 (ObjNum 1325 snap 446 seq_num 1325) dirty exists 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3543: left oid 1 (ObjNum 1325 snap 446 seq_num 1325) 2024-09-21T09:52:56.241 INFO:tasks.rados.rados.0.smithi076.stdout:3545: expect (ObjNum 1307 snap 442 seq_num 1307) 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: pgmap v910: 89 pgs: 89 active+clean; 233 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: pgmap v910: 89 pgs: 89 active+clean; 233 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:56.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: pgmap v910: 89 pgs: 89 active+clean; 233 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 2.4 MiB/s wr, 21 op/s 2024-09-21T09:52:56.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:52:56.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:52:56.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:52:56.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:52:56.910 INFO:tasks.rados.rados.0.smithi076.stdout:3536: done (9 left) 2024-09-21T09:52:56.910 INFO:tasks.rados.rados.0.smithi076.stdout:3537: done (8 left) 2024-09-21T09:52:56.910 INFO:tasks.rados.rados.0.smithi076.stdout:3538: done (7 left) 2024-09-21T09:52:56.910 INFO:tasks.rados.rados.0.smithi076.stdout:3539: done (6 left) 2024-09-21T09:52:56.910 INFO:tasks.rados.rados.0.smithi076.stdout:3540: done (5 left) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3541: done (4 left) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3542: done (3 left) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3543: done (2 left) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3545: done (1 left) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3546: done (0 left) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3547: read oid 49 snap -1 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3547: expect (ObjNum 1290 snap 439 seq_num 1290) 2024-09-21T09:52:56.911 INFO:tasks.rados.rados.0.smithi076.stdout:3548: snap_create 2024-09-21T09:52:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:3547: expect (ObjNum 1290 snap 439 seq_num 1290) 2024-09-21T09:52:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:56 smithi191 ceph-mon[102206]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:52:57.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:56 smithi076 ceph-mon[106482]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:52:57.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:56 smithi076 ceph-mon[110963]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T09:52:57.913 INFO:tasks.rados.rados.0.smithi076.stdout:3547: done (1 left) 2024-09-21T09:52:57.913 INFO:tasks.rados.rados.0.smithi076.stdout:3548: done (0 left) 2024-09-21T09:52:57.913 INFO:tasks.rados.rados.0.smithi076.stdout:3549: write oid 33 current snap is 448 2024-09-21T09:52:57.913 INFO:tasks.rados.rados.0.smithi076.stdout:3549: seq_num 1326 ranges {686082=425596,1577829=433132} 2024-09-21T09:52:57.919 INFO:tasks.rados.rados.0.smithi076.stdout:3549: writing smithi07685715-33 from 686082 to 1111678 tid 1 2024-09-21T09:52:57.924 INFO:tasks.rados.rados.0.smithi076.stdout:3549: writing smithi07685715-33 from 1577829 to 2010961 tid 2 2024-09-21T09:52:57.924 INFO:tasks.rados.rados.0.smithi076.stdout:3550: write oid 11 current snap is 448 2024-09-21T09:52:57.924 INFO:tasks.rados.rados.0.smithi076.stdout:3550: seq_num 1327 ranges {702889=500845,1636311=576790} 2024-09-21T09:52:57.930 INFO:tasks.rados.rados.0.smithi076.stdout:3550: writing smithi07685715-11 from 702889 to 1203734 tid 1 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3550: writing smithi07685715-11 from 1636311 to 2213101 tid 2 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3551: rollback oid 15 current snap is 448 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 445 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: finishing write tid 1 to smithi07685715-33 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: finishing write tid 2 to smithi07685715-33 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: finishing write tid 3 to smithi07685715-33 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: finishing write tid 4 to smithi07685715-33 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: oid 33 updating version 0 to 786 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: oid 33 updating version 786 to 788 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:3549: oid 33 version 788 is already newer than 787 2024-09-21T09:52:57.937 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 788 (ObjNum 1326 snap 448 seq_num 1326) dirty exists 2024-09-21T09:52:57.938 INFO:tasks.rados.rados.0.smithi076.stdout:3549: left oid 33 (ObjNum 1326 snap 448 seq_num 1326) 2024-09-21T09:52:57.938 INFO:tasks.rados.rados.0.smithi076.stdout:3550: finishing write tid 1 to smithi07685715-11 2024-09-21T09:52:57.938 INFO:tasks.rados.rados.0.smithi076.stdout:3549: done (2 left) 2024-09-21T09:52:57.938 INFO:tasks.rados.rados.0.smithi076.stdout:3552: write oid 13 current snap is 448 2024-09-21T09:52:57.938 INFO:tasks.rados.rados.0.smithi076.stdout:3552: seq_num 1328 ranges {719696=576094,2094793=224903} 2024-09-21T09:52:57.943 INFO:tasks.rados.rados.0.smithi076.stdout:3552: writing smithi07685715-13 from 719696 to 1295790 tid 1 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3552: writing smithi07685715-13 from 2094793 to 2319696 tid 2 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3553: rollback oid 6 current snap is 448 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 6 to 428 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3551: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 0 (ObjNum 129 snap 0 seq_num 864458080) dirty dne 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3551: done (3 left) 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3554: delete oid 32 current snap is 448 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3550: finishing write tid 2 to smithi07685715-11 2024-09-21T09:52:57.950 INFO:tasks.rados.rados.0.smithi076.stdout:3550: finishing write tid 3 to smithi07685715-11 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:3550: finishing write tid 4 to smithi07685715-11 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:3550: oid 11 updating version 0 to 2759 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:3550: oid 11 updating version 2759 to 2760 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:3550: oid 11 updating version 2760 to 2761 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2761 (ObjNum 1327 snap 448 seq_num 1327) dirty exists 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:3550: left oid 11 (ObjNum 1327 snap 448 seq_num 1327) 2024-09-21T09:52:57.951 INFO:tasks.rados.rados.0.smithi076.stdout:3552: finishing write tid 1 to smithi07685715-13 2024-09-21T09:52:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:3553: finishing rollback tid 1 to smithi07685715-6 2024-09-21T09:52:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:3552: finishing write tid 2 to smithi07685715-13 2024-09-21T09:52:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:3550: done (3 left) 2024-09-21T09:52:57.953 INFO:tasks.rados.rados.0.smithi076.stdout:3553: done (2 left) 2024-09-21T09:52:57.954 INFO:tasks.rados.rados.0.smithi076.stdout:3554: done (1 left) 2024-09-21T09:52:57.954 INFO:tasks.rados.rados.0.smithi076.stdout:3555: write oid 19 current snap is 448 2024-09-21T09:52:57.954 INFO:tasks.rados.rados.0.smithi076.stdout:3555: seq_num 1330 ranges {753310=726592,2095404=257906} 2024-09-21T09:52:57.959 INFO:tasks.rados.rados.0.smithi076.stdout:3555: writing smithi07685715-19 from 753310 to 1479902 tid 1 2024-09-21T09:52:57.964 INFO:tasks.rados.rados.0.smithi076.stdout:3555: writing smithi07685715-19 from 2095404 to 2353310 tid 2 2024-09-21T09:52:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:3556: write oid 36 current snap is 448 2024-09-21T09:52:57.965 INFO:tasks.rados.rados.0.smithi076.stdout:3556: seq_num 1331 ranges {770117=518194,1870239=499878} 2024-09-21T09:52:57.971 INFO:tasks.rados.rados.0.smithi076.stdout:3556: writing smithi07685715-36 from 770117 to 1288311 tid 1 2024-09-21T09:52:57.977 INFO:tasks.rados.rados.0.smithi076.stdout:3556: writing smithi07685715-36 from 1870239 to 2370117 tid 2 2024-09-21T09:52:57.977 INFO:tasks.rados.rados.0.smithi076.stdout:3557: read oid 27 snap -1 2024-09-21T09:52:57.977 INFO:tasks.rados.rados.0.smithi076.stdout:3557: expect deleted 2024-09-21T09:52:57.977 INFO:tasks.rados.rados.0.smithi076.stdout:3558: snap_remove snap 437 2024-09-21T09:52:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:57 smithi191 ceph-mon[102206]: pgmap v913: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 241 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:52:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:57 smithi191 ceph-mon[102206]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:52:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:57 smithi076 ceph-mon[106482]: pgmap v913: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 241 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:52:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:57 smithi076 ceph-mon[106482]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:52:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:57 smithi076 ceph-mon[110963]: pgmap v913: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 241 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T09:52:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:57 smithi076 ceph-mon[110963]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T09:52:58.803 INFO:tasks.rados.rados.0.smithi076.stdout:3558: done (4 left) 2024-09-21T09:52:58.803 INFO:tasks.rados.rados.0.smithi076.stdout:3559: snap_remove snap 440 2024-09-21T09:52:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:58 smithi191 ceph-mon[102206]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:52:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:58 smithi191 ceph-mon[102206]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:52:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:58 smithi076 ceph-mon[106482]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:52:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:58 smithi076 ceph-mon[106482]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:52:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:58 smithi076 ceph-mon[110963]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T09:52:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:58 smithi076 ceph-mon[110963]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T09:52:59.805 INFO:tasks.rados.rados.0.smithi076.stdout:3559: done (4 left) 2024-09-21T09:52:59.806 INFO:tasks.rados.rados.0.smithi076.stdout:3560: write oid 5 current snap is 448 2024-09-21T09:52:59.806 INFO:tasks.rados.rados.0.smithi076.stdout:3552: finishing write tid 3 to smithi07685715-13 2024-09-21T09:52:59.806 INFO:tasks.rados.rados.0.smithi076.stdout:3552: finishing write tid 4 to smithi07685715-13 2024-09-21T09:52:59.806 INFO:tasks.rados.rados.0.smithi076.stdout:3552: oid 13 updating version 0 to 712 2024-09-21T09:52:59.806 INFO:tasks.rados.rados.0.smithi076.stdout:3552: oid 13 updating version 712 to 713 2024-09-21T09:52:59.806 INFO:tasks.rados.rados.0.smithi076.stdout:3552: oid 13 updating version 713 to 714 2024-09-21T09:52:59.807 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 714 (ObjNum 1328 snap 448 seq_num 1328) dirty exists 2024-09-21T09:52:59.807 INFO:tasks.rados.rados.0.smithi076.stdout:3552: left oid 13 (ObjNum 1328 snap 448 seq_num 1328) 2024-09-21T09:52:59.807 INFO:tasks.rados.rados.0.smithi076.stdout:3555: finishing write tid 1 to smithi07685715-19 2024-09-21T09:52:59.807 INFO:tasks.rados.rados.0.smithi076.stdout:3555: finishing write tid 2 to smithi07685715-19 2024-09-21T09:52:59.807 INFO:tasks.rados.rados.0.smithi076.stdout:3560: seq_num 1332 ranges {786924=593443,1928721=458203} 2024-09-21T09:52:59.813 INFO:tasks.rados.rados.0.smithi076.stdout:3560: writing smithi07685715-5 from 786924 to 1380367 tid 1 2024-09-21T09:52:59.821 INFO:tasks.rados.rados.0.smithi076.stdout:3560: writing smithi07685715-5 from 1928721 to 2386924 tid 2 2024-09-21T09:52:59.821 INFO:tasks.rados.rados.0.smithi076.stdout:3552: done (4 left) 2024-09-21T09:52:59.822 INFO:tasks.rados.rados.0.smithi076.stdout:3561: write oid 37 current snap is 448 2024-09-21T09:52:59.822 INFO:tasks.rados.rados.0.smithi076.stdout:3555: finishing write tid 3 to smithi07685715-19 2024-09-21T09:52:59.822 INFO:tasks.rados.rados.0.smithi076.stdout:3555: finishing write tid 4 to smithi07685715-19 2024-09-21T09:52:59.822 INFO:tasks.rados.rados.0.smithi076.stdout:3555: oid 19 updating version 0 to 1069 2024-09-21T09:52:59.822 INFO:tasks.rados.rados.0.smithi076.stdout:3555: oid 19 updating version 1069 to 1071 2024-09-21T09:52:59.823 INFO:tasks.rados.rados.0.smithi076.stdout:3555: oid 19 version 1071 is already newer than 1070 2024-09-21T09:52:59.823 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 1071 (ObjNum 1330 snap 448 seq_num 1330) dirty exists 2024-09-21T09:52:59.823 INFO:tasks.rados.rados.0.smithi076.stdout:3555: left oid 19 (ObjNum 1330 snap 448 seq_num 1330) 2024-09-21T09:52:59.823 INFO:tasks.rados.rados.0.smithi076.stdout:3556: finishing write tid 1 to smithi07685715-36 2024-09-21T09:52:59.823 INFO:tasks.rados.rados.0.smithi076.stdout:3556: finishing write tid 2 to smithi07685715-36 2024-09-21T09:52:59.824 INFO:tasks.rados.rados.0.smithi076.stdout:3556: finishing write tid 3 to smithi07685715-36 2024-09-21T09:52:59.824 INFO:tasks.rados.rados.0.smithi076.stdout:3556: finishing write tid 4 to smithi07685715-36 2024-09-21T09:52:59.824 INFO:tasks.rados.rados.0.smithi076.stdout:3556: oid 36 updating version 0 to 1143 2024-09-21T09:52:59.824 INFO:tasks.rados.rados.0.smithi076.stdout:3556: oid 36 version 1143 is already newer than 1142 2024-09-21T09:52:59.824 INFO:tasks.rados.rados.0.smithi076.stdout:3556: oid 36 updating version 1143 to 1144 2024-09-21T09:52:59.825 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1144 (ObjNum 1331 snap 448 seq_num 1331) dirty exists 2024-09-21T09:52:59.825 INFO:tasks.rados.rados.0.smithi076.stdout:3556: left oid 36 (ObjNum 1331 snap 448 seq_num 1331) 2024-09-21T09:52:59.825 INFO:tasks.rados.rados.0.smithi076.stdout:3561: seq_num 1333 ranges {403731=668692,1587203=587797} 2024-09-21T09:52:59.826 INFO:tasks.rados.rados.0.smithi076.stdout:3561: writing smithi07685715-37 from 403731 to 1072423 tid 1 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3561: writing smithi07685715-37 from 1587203 to 2175000 tid 2 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3555: done (4 left) 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3556: done (3 left) 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3557: done (2 left) 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3562: delete oid 10 current snap is 448 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3560: finishing write tid 1 to smithi07685715-5 2024-09-21T09:52:59.833 INFO:tasks.rados.rados.0.smithi076.stdout:3560: finishing write tid 2 to smithi07685715-5 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3560: finishing write tid 3 to smithi07685715-5 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3560: finishing write tid 4 to smithi07685715-5 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3560: oid 5 updating version 0 to 2779 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3560: oid 5 version 2779 is already newer than 2778 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3560: oid 5 version 2779 is already newer than 2777 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2779 (ObjNum 1332 snap 448 seq_num 1332) dirty exists 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3560: left oid 5 (ObjNum 1332 snap 448 seq_num 1332) 2024-09-21T09:52:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3561: finishing write tid 1 to smithi07685715-37 2024-09-21T09:52:59.836 INFO:tasks.rados.rados.0.smithi076.stdout:3560: done (2 left) 2024-09-21T09:52:59.836 INFO:tasks.rados.rados.0.smithi076.stdout:3562: done (1 left) 2024-09-21T09:52:59.836 INFO:tasks.rados.rados.0.smithi076.stdout:3563: write oid 45 current snap is 448 2024-09-21T09:52:59.837 INFO:tasks.rados.rados.0.smithi076.stdout:3563: seq_num 1335 ranges {437345=419190,1587814=591466} 2024-09-21T09:52:59.841 INFO:tasks.rados.rados.0.smithi076.stdout:3563: writing smithi07685715-45 from 437345 to 856535 tid 1 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3563: writing smithi07685715-45 from 1587814 to 2179280 tid 2 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3564: delete oid 5 current snap is 448 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3561: finishing write tid 2 to smithi07685715-37 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3561: finishing write tid 3 to smithi07685715-37 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3561: finishing write tid 4 to smithi07685715-37 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3561: oid 37 updating version 0 to 1099 2024-09-21T09:52:59.852 INFO:tasks.rados.rados.0.smithi076.stdout:3561: oid 37 updating version 1099 to 1100 2024-09-21T09:52:59.853 INFO:tasks.rados.rados.0.smithi076.stdout:3561: oid 37 version 1100 is already newer than 1098 2024-09-21T09:52:59.853 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 1100 (ObjNum 1333 snap 448 seq_num 1333) dirty exists 2024-09-21T09:52:59.853 INFO:tasks.rados.rados.0.smithi076.stdout:3561: left oid 37 (ObjNum 1333 snap 448 seq_num 1333) 2024-09-21T09:52:59.853 INFO:tasks.rados.rados.0.smithi076.stdout:3563: finishing write tid 1 to smithi07685715-45 2024-09-21T09:52:59.853 INFO:tasks.rados.rados.0.smithi076.stdout:3563: finishing write tid 2 to smithi07685715-45 2024-09-21T09:52:59.853 INFO:tasks.rados.rados.0.smithi076.stdout:3561: done (2 left) 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3564: done (1 left) 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3565: snap_create 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3563: finishing write tid 3 to smithi07685715-45 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3563: finishing write tid 4 to smithi07685715-45 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3563: oid 45 updating version 0 to 780 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3563: oid 45 updating version 780 to 781 2024-09-21T09:52:59.854 INFO:tasks.rados.rados.0.smithi076.stdout:3563: oid 45 version 781 is already newer than 779 2024-09-21T09:52:59.855 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 781 (ObjNum 1335 snap 448 seq_num 1335) dirty exists 2024-09-21T09:52:59.855 INFO:tasks.rados.rados.0.smithi076.stdout:3563: left oid 45 (ObjNum 1335 snap 448 seq_num 1335) 2024-09-21T09:53:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:59 smithi191 ceph-mon[102206]: pgmap v916: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 241 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-21T09:53:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:52:59 smithi191 ceph-mon[102206]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:53:00.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:59 smithi076 ceph-mon[106482]: pgmap v916: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 241 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-21T09:53:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:52:59 smithi076 ceph-mon[106482]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:53:00.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:59 smithi076 ceph-mon[110963]: pgmap v916: 89 pgs: 2 active+clean+snaptrim, 3 active+clean+snaptrim_wait, 84 active+clean; 241 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-09-21T09:53:00.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:52:59 smithi076 ceph-mon[110963]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T09:53:00.807 INFO:tasks.rados.rados.0.smithi076.stdout:3563: done (1 left) 2024-09-21T09:53:00.807 INFO:tasks.rados.rados.0.smithi076.stdout:3565: done (0 left) 2024-09-21T09:53:00.807 INFO:tasks.rados.rados.0.smithi076.stdout:3566: snap_create 2024-09-21T09:53:01.809 INFO:tasks.rados.rados.0.smithi076.stdout:3566: done (0 left) 2024-09-21T09:53:01.810 INFO:tasks.rados.rados.0.smithi076.stdout:3567: write oid 17 current snap is 450 2024-09-21T09:53:01.810 INFO:tasks.rados.rados.0.smithi076.stdout:3567: seq_num 1337 ranges {470959=569688,1704778=595135} 2024-09-21T09:53:01.815 INFO:tasks.rados.rados.0.smithi076.stdout:3567: writing smithi07685715-17 from 470959 to 1040647 tid 1 2024-09-21T09:53:01.823 INFO:tasks.rados.rados.0.smithi076.stdout:3567: writing smithi07685715-17 from 1704778 to 2299913 tid 2 2024-09-21T09:53:01.824 INFO:tasks.rados.rados.0.smithi076.stdout:3568: rollback oid 8 current snap is 450 2024-09-21T09:53:01.824 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 8 to 408 2024-09-21T09:53:01.824 INFO:tasks.rados.rados.0.smithi076.stdout:3569: write oid 34 current snap is 450 2024-09-21T09:53:01.824 INFO:tasks.rados.rados.0.smithi076.stdout:3569: seq_num 1338 ranges {487766=644937,1646907=738793} 2024-09-21T09:53:01.830 INFO:tasks.rados.rados.0.smithi076.stdout:3569: writing smithi07685715-34 from 487766 to 1132703 tid 1 2024-09-21T09:53:01.837 INFO:tasks.rados.rados.0.smithi076.stdout:3569: writing smithi07685715-34 from 1646907 to 2385700 tid 2 2024-09-21T09:53:01.837 INFO:tasks.rados.rados.0.smithi076.stdout:3570: delete oid 11 current snap is 450 2024-09-21T09:53:01.837 INFO:tasks.rados.rados.0.smithi076.stdout:3567: finishing write tid 1 to smithi07685715-17 2024-09-21T09:53:01.837 INFO:tasks.rados.rados.0.smithi076.stdout:3568: finishing rollback tid 1 to smithi07685715-8 2024-09-21T09:53:01.837 INFO:tasks.rados.rados.0.smithi076.stdout:3568: finishing rollback tid 2 to smithi07685715-8 2024-09-21T09:53:01.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2727 (ObjNum 1223 snap 406 seq_num 1223) dirty exists 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: finishing write tid 2 to smithi07685715-17 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: finishing write tid 3 to smithi07685715-17 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: finishing write tid 4 to smithi07685715-17 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: oid 17 updating version 0 to 2246 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: oid 17 updating version 2246 to 2248 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: oid 17 version 2248 is already newer than 2247 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2248 (ObjNum 1337 snap 450 seq_num 1337) dirty exists 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3567: left oid 17 (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:01.838 INFO:tasks.rados.rados.0.smithi076.stdout:3569: finishing write tid 1 to smithi07685715-34 2024-09-21T09:53:01.841 INFO:tasks.rados.rados.0.smithi076.stdout:3567: done (3 left) 2024-09-21T09:53:01.841 INFO:tasks.rados.rados.0.smithi076.stdout:3568: done (2 left) 2024-09-21T09:53:01.841 INFO:tasks.rados.rados.0.smithi076.stdout:3570: done (1 left) 2024-09-21T09:53:01.841 INFO:tasks.rados.rados.0.smithi076.stdout:3571: read oid 27 snap -1 2024-09-21T09:53:01.841 INFO:tasks.rados.rados.0.smithi076.stdout:3571: expect deleted 2024-09-21T09:53:01.841 INFO:tasks.rados.rados.0.smithi076.stdout:3572: snap_remove snap 444 2024-09-21T09:53:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:01 smithi191 ceph-mon[102206]: pgmap v919: 89 pgs: 89 active+clean; 243 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 436 KiB/s wr, 9 op/s 2024-09-21T09:53:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:01 smithi191 ceph-mon[102206]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:53:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:01 smithi076 ceph-mon[106482]: pgmap v919: 89 pgs: 89 active+clean; 243 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 436 KiB/s wr, 9 op/s 2024-09-21T09:53:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:01 smithi076 ceph-mon[106482]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:53:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:01 smithi076 ceph-mon[110963]: pgmap v919: 89 pgs: 89 active+clean; 243 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 436 KiB/s wr, 9 op/s 2024-09-21T09:53:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:01 smithi076 ceph-mon[110963]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T09:53:02.824 INFO:tasks.rados.rados.0.smithi076.stdout:3572: done (2 left) 2024-09-21T09:53:02.824 INFO:tasks.rados.rados.0.smithi076.stdout:3573: delete oid 30 current snap is 450 2024-09-21T09:53:02.824 INFO:tasks.rados.rados.0.smithi076.stdout:3569: finishing write tid 2 to smithi07685715-34 2024-09-21T09:53:02.824 INFO:tasks.rados.rados.0.smithi076.stdout:3569: finishing write tid 3 to smithi07685715-34 2024-09-21T09:53:02.824 INFO:tasks.rados.rados.0.smithi076.stdout:3569: finishing write tid 4 to smithi07685715-34 2024-09-21T09:53:02.825 INFO:tasks.rados.rados.0.smithi076.stdout:3569: oid 34 updating version 0 to 800 2024-09-21T09:53:02.825 INFO:tasks.rados.rados.0.smithi076.stdout:3569: oid 34 version 800 is already newer than 799 2024-09-21T09:53:02.825 INFO:tasks.rados.rados.0.smithi076.stdout:3569: oid 34 updating version 800 to 801 2024-09-21T09:53:02.825 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 801 (ObjNum 1338 snap 450 seq_num 1338) dirty exists 2024-09-21T09:53:02.825 INFO:tasks.rados.rados.0.smithi076.stdout:3569: left oid 34 (ObjNum 1338 snap 450 seq_num 1338) 2024-09-21T09:53:02.831 INFO:tasks.rados.rados.0.smithi076.stdout:3569: done (2 left) 2024-09-21T09:53:02.831 INFO:tasks.rados.rados.0.smithi076.stdout:3571: done (1 left) 2024-09-21T09:53:02.831 INFO:tasks.rados.rados.0.smithi076.stdout:3573: done (0 left) 2024-09-21T09:53:02.831 INFO:tasks.rados.rados.0.smithi076.stdout:3574: read oid 21 snap -1 2024-09-21T09:53:02.831 INFO:tasks.rados.rados.0.smithi076.stdout:3574: expect (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:53:02.831 INFO:tasks.rados.rados.0.smithi076.stdout:3575: delete oid 36 current snap is 450 2024-09-21T09:53:02.836 INFO:tasks.rados.rados.0.smithi076.stdout:3575: done (1 left) 2024-09-21T09:53:02.836 INFO:tasks.rados.rados.0.smithi076.stdout:3576: read oid 31 snap -1 2024-09-21T09:53:02.836 INFO:tasks.rados.rados.0.smithi076.stdout:3576: expect deleted 2024-09-21T09:53:02.836 INFO:tasks.rados.rados.0.smithi076.stdout:3577: write oid 44 current snap is 450 2024-09-21T09:53:02.836 INFO:tasks.rados.rados.0.smithi076.stdout:3577: seq_num 1342 ranges {554994=662286,1997188=557806} 2024-09-21T09:53:02.843 INFO:tasks.rados.rados.0.smithi076.stdout:3577: writing smithi07685715-44 from 554994 to 1217280 tid 1 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3577: writing smithi07685715-44 from 1997188 to 2554994 tid 2 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3578: read oid 24 snap -1 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3578: expect (ObjNum 1218 snap 406 seq_num 1218) 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3579: read oid 17 snap -1 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3579: expect (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3580: read oid 32 snap -1 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3580: expect deleted 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3581: read oid 9 snap -1 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3581: expect (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3582: rollback oid 47 current snap is 450 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 445 2024-09-21T09:53:02.852 INFO:tasks.rados.rados.0.smithi076.stdout:3576: done (7 left) 2024-09-21T09:53:02.853 INFO:tasks.rados.rados.0.smithi076.stdout:3583: read oid 31 snap -1 2024-09-21T09:53:02.853 INFO:tasks.rados.rados.0.smithi076.stdout:3583: expect deleted 2024-09-21T09:53:02.853 INFO:tasks.rados.rados.0.smithi076.stdout:3584: snap_remove snap 412 2024-09-21T09:53:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:02 smithi191 ceph-mon[102206]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:53:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:02 smithi076 ceph-mon[106482]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:53:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:02 smithi076 ceph-mon[110963]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T09:53:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:53:03] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:53:03.805 INFO:tasks.rados.rados.0.smithi076.stdout:3584: done (8 left) 2024-09-21T09:53:03.805 INFO:tasks.rados.rados.0.smithi076.stdout:3585: write oid 14 current snap is 450 2024-09-21T09:53:03.805 INFO:tasks.rados.rados.0.smithi076.stdout:3585: seq_num 1343 ranges {571801=737535,1939317=489789} 2024-09-21T09:53:03.813 INFO:tasks.rados.rados.0.smithi076.stdout:3585: writing smithi07685715-14 from 571801 to 1309336 tid 1 2024-09-21T09:53:03.821 INFO:tasks.rados.rados.0.smithi076.stdout:3585: writing smithi07685715-14 from 1939317 to 2429106 tid 2 2024-09-21T09:53:03.821 INFO:tasks.rados.rados.0.smithi076.stdout:3586: rollback oid 15 current snap is 450 2024-09-21T09:53:03.821 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 15 to 449 2024-09-21T09:53:03.822 INFO:tasks.rados.rados.0.smithi076.stdout:3574: expect (ObjNum 1231 snap 410 seq_num 1231) 2024-09-21T09:53:04.054 INFO:tasks.rados.rados.0.smithi076.stdout:3577: finishing write tid 1 to smithi07685715-44 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3577: finishing write tid 2 to smithi07685715-44 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3582: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2559 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3577: finishing write tid 3 to smithi07685715-44 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3574: done (9 left) 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3580: done (8 left) 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3582: done (7 left) 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3583: done (6 left) 2024-09-21T09:53:04.055 INFO:tasks.rados.rados.0.smithi076.stdout:3587: snap_remove snap 436 2024-09-21T09:53:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:03 smithi191 ceph-mon[102206]: pgmap v922: 89 pgs: 89 active+clean; 248 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-21T09:53:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:03 smithi191 ceph-mon[102206]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:53:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:03 smithi191 ceph-mon[102206]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:53:04.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-mon[106482]: pgmap v922: 89 pgs: 89 active+clean; 248 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-21T09:53:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-mon[106482]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:53:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-mon[106482]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:53:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-mon[110963]: pgmap v922: 89 pgs: 89 active+clean; 248 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-09-21T09:53:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-mon[110963]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T09:53:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:03 smithi076 ceph-mon[110963]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T09:53:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3587: done (6 left) 2024-09-21T09:53:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3588: write oid 5 current snap is 450 2024-09-21T09:53:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3577: finishing write tid 4 to smithi07685715-44 2024-09-21T09:53:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3577: oid 44 updating version 0 to 771 2024-09-21T09:53:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3577: oid 44 version 771 is already newer than 769 2024-09-21T09:53:04.836 INFO:tasks.rados.rados.0.smithi076.stdout:3577: oid 44 version 771 is already newer than 770 2024-09-21T09:53:04.836 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 771 (ObjNum 1342 snap 450 seq_num 1342) dirty exists 2024-09-21T09:53:04.836 INFO:tasks.rados.rados.0.smithi076.stdout:3577: left oid 44 (ObjNum 1342 snap 450 seq_num 1342) 2024-09-21T09:53:04.836 INFO:tasks.rados.rados.0.smithi076.stdout:3581: expect (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:53:04.916 INFO:tasks.rados.rados.0.smithi076.stdout:3579: expect (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:05.027 INFO:tasks.rados.rados.0.smithi076.stdout:3578: expect (ObjNum 1218 snap 406 seq_num 1218) 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: finishing write tid 1 to smithi07685715-14 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3586: finishing rollback tid 1 to smithi07685715-15 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: finishing write tid 2 to smithi07685715-14 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: finishing write tid 3 to smithi07685715-14 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: finishing write tid 4 to smithi07685715-14 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: oid 14 updating version 0 to 2257 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: oid 14 updating version 2257 to 2259 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:3585: oid 14 version 2259 is already newer than 2258 2024-09-21T09:53:05.104 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 2259 (ObjNum 1343 snap 450 seq_num 1343) dirty exists 2024-09-21T09:53:05.105 INFO:tasks.rados.rados.0.smithi076.stdout:3585: left oid 14 (ObjNum 1343 snap 450 seq_num 1343) 2024-09-21T09:53:05.105 INFO:tasks.rados.rados.0.smithi076.stdout:3588: seq_num 1344 ranges {588608=412784,1597799=749800} 2024-09-21T09:53:05.109 INFO:tasks.rados.rados.0.smithi076.stdout:3588: writing smithi07685715-5 from 588608 to 1001392 tid 1 2024-09-21T09:53:05.116 INFO:tasks.rados.rados.0.smithi076.stdout:3588: writing smithi07685715-5 from 1597799 to 2347599 tid 2 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3577: done (6 left) 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3578: done (5 left) 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3579: done (4 left) 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3581: done (3 left) 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3585: done (2 left) 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3586: done (1 left) 2024-09-21T09:53:05.117 INFO:tasks.rados.rados.0.smithi076.stdout:3589: snap_remove snap 417 2024-09-21T09:53:05.845 INFO:tasks.rados.rados.0.smithi076.stdout:3589: done (1 left) 2024-09-21T09:53:05.845 INFO:tasks.rados.rados.0.smithi076.stdout:3590: write oid 33 current snap is 450 2024-09-21T09:53:05.845 INFO:tasks.rados.rados.0.smithi076.stdout:3588: finishing write tid 1 to smithi07685715-5 2024-09-21T09:53:05.845 INFO:tasks.rados.rados.0.smithi076.stdout:3590: seq_num 1345 ranges {605415=488033,1656281=493458} 2024-09-21T09:53:05.851 INFO:tasks.rados.rados.0.smithi076.stdout:3590: writing smithi07685715-33 from 605415 to 1093448 tid 1 2024-09-21T09:53:05.858 INFO:tasks.rados.rados.0.smithi076.stdout:3590: writing smithi07685715-33 from 1656281 to 2149739 tid 2 2024-09-21T09:53:05.858 INFO:tasks.rados.rados.0.smithi076.stdout:3591: read oid 37 snap -1 2024-09-21T09:53:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3591: expect (ObjNum 1333 snap 448 seq_num 1333) 2024-09-21T09:53:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3592: write oid 42 current snap is 450 2024-09-21T09:53:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3592: seq_num 1346 ranges {622222=679635,1831116=753469} 2024-09-21T09:53:05.867 INFO:tasks.rados.rados.0.smithi076.stdout:3592: writing smithi07685715-42 from 622222 to 1301857 tid 1 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3592: writing smithi07685715-42 from 1831116 to 2584585 tid 2 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3593: write oid 49 current snap is 450 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: finishing write tid 2 to smithi07685715-5 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: finishing write tid 3 to smithi07685715-5 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: finishing write tid 4 to smithi07685715-5 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: oid 5 updating version 0 to 2804 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: oid 5 version 2804 is already newer than 2803 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: oid 5 updating version 2804 to 2805 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2805 (ObjNum 1344 snap 450 seq_num 1344) dirty exists 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3588: left oid 5 (ObjNum 1344 snap 450 seq_num 1344) 2024-09-21T09:53:05.875 INFO:tasks.rados.rados.0.smithi076.stdout:3590: finishing write tid 1 to smithi07685715-33 2024-09-21T09:53:05.876 INFO:tasks.rados.rados.0.smithi076.stdout:3593: seq_num 1347 ranges {639029=754884,2173245=465784} 2024-09-21T09:53:05.882 INFO:tasks.rados.rados.0.smithi076.stdout:3593: writing smithi07685715-49 from 639029 to 1393913 tid 1 2024-09-21T09:53:05.889 INFO:tasks.rados.rados.0.smithi076.stdout:3593: writing smithi07685715-49 from 2173245 to 2639029 tid 2 2024-09-21T09:53:05.889 INFO:tasks.rados.rados.0.smithi076.stdout:3588: done (4 left) 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3594: delete oid 8 current snap is 450 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: finishing write tid 2 to smithi07685715-33 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: finishing write tid 3 to smithi07685715-33 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: finishing write tid 4 to smithi07685715-33 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: oid 33 updating version 0 to 808 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: oid 33 updating version 808 to 809 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: oid 33 updating version 809 to 810 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 810 (ObjNum 1345 snap 450 seq_num 1345) dirty exists 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3590: left oid 33 (ObjNum 1345 snap 450 seq_num 1345) 2024-09-21T09:53:05.890 INFO:tasks.rados.rados.0.smithi076.stdout:3591: expect (ObjNum 1333 snap 448 seq_num 1333) 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: finishing write tid 1 to smithi07685715-42 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: finishing write tid 2 to smithi07685715-42 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: finishing write tid 3 to smithi07685715-42 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: finishing write tid 4 to smithi07685715-42 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: oid 42 updating version 0 to 931 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: oid 42 version 931 is already newer than 926 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: oid 42 updating version 931 to 932 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 932 (ObjNum 1346 snap 450 seq_num 1346) dirty exists 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3592: left oid 42 (ObjNum 1346 snap 450 seq_num 1346) 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3593: finishing write tid 1 to smithi07685715-49 2024-09-21T09:53:06.034 INFO:tasks.rados.rados.0.smithi076.stdout:3590: done (4 left) 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3591: done (3 left) 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3592: done (2 left) 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3594: done (1 left) 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3595: snap_create 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: finishing write tid 2 to smithi07685715-49 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: finishing write tid 3 to smithi07685715-49 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: finishing write tid 4 to smithi07685715-49 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: oid 49 updating version 0 to 2768 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: oid 49 version 2768 is already newer than 2766 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: oid 49 version 2768 is already newer than 2767 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2768 (ObjNum 1347 snap 450 seq_num 1347) dirty exists 2024-09-21T09:53:06.035 INFO:tasks.rados.rados.0.smithi076.stdout:3593: left oid 49 (ObjNum 1347 snap 450 seq_num 1347) 2024-09-21T09:53:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:05 smithi191 ceph-mon[102206]: pgmap v925: 89 pgs: 89 active+clean; 251 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T09:53:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:05 smithi191 ceph-mon[102206]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:53:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:05 smithi076 ceph-mon[106482]: pgmap v925: 89 pgs: 89 active+clean; 251 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T09:53:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:05 smithi076 ceph-mon[106482]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:53:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:05 smithi076 ceph-mon[110963]: pgmap v925: 89 pgs: 89 active+clean; 251 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T09:53:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:05 smithi076 ceph-mon[110963]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T09:53:06.857 INFO:tasks.rados.rados.0.smithi076.stdout:3593: done (1 left) 2024-09-21T09:53:06.857 INFO:tasks.rados.rados.0.smithi076.stdout:3595: done (0 left) 2024-09-21T09:53:06.857 INFO:tasks.rados.rados.0.smithi076.stdout:3596: read oid 9 snap -1 2024-09-21T09:53:06.857 INFO:tasks.rados.rados.0.smithi076.stdout:3596: expect (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:53:06.857 INFO:tasks.rados.rados.0.smithi076.stdout:3597: snap_create 2024-09-21T09:53:06.861 INFO:tasks.rados.rados.0.smithi076.stdout:3596: expect (ObjNum 1211 snap 404 seq_num 1211) 2024-09-21T09:53:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:06 smithi191 ceph-mon[102206]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:53:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:06 smithi076 ceph-mon[106482]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:53:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:06 smithi076 ceph-mon[110963]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T09:53:07.857 INFO:tasks.rados.rados.0.smithi076.stdout:3596: done (1 left) 2024-09-21T09:53:07.857 INFO:tasks.rados.rados.0.smithi076.stdout:3597: done (0 left) 2024-09-21T09:53:07.857 INFO:tasks.rados.rados.0.smithi076.stdout:3598: snap_create 2024-09-21T09:53:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:07 smithi191 ceph-mon[102206]: pgmap v928: 89 pgs: 89 active+clean; 247 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-09-21T09:53:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:07 smithi191 ceph-mon[102206]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:53:08.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:07 smithi076 ceph-mon[106482]: pgmap v928: 89 pgs: 89 active+clean; 247 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-09-21T09:53:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:07 smithi076 ceph-mon[106482]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:53:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:07 smithi076 ceph-mon[110963]: pgmap v928: 89 pgs: 89 active+clean; 247 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-09-21T09:53:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:07 smithi076 ceph-mon[110963]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T09:53:08.806 INFO:tasks.rados.rados.0.smithi076.stdout:3598: done (0 left) 2024-09-21T09:53:08.806 INFO:tasks.rados.rados.0.smithi076.stdout:3599: snap_remove snap 428 2024-09-21T09:53:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:08 smithi191 ceph-mon[102206]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:53:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:08 smithi191 ceph-mon[102206]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:53:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:08 smithi076 ceph-mon[106482]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:53:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:08 smithi076 ceph-mon[106482]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:53:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:08 smithi076 ceph-mon[110963]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T09:53:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:08 smithi076 ceph-mon[110963]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T09:53:09.808 INFO:tasks.rados.rados.0.smithi076.stdout:3599: done (0 left) 2024-09-21T09:53:09.808 INFO:tasks.rados.rados.0.smithi076.stdout:3600: write oid 1 current snap is 453 2024-09-21T09:53:09.808 INFO:tasks.rados.rados.0.smithi076.stdout:3600: seq_num 1349 ranges {672643=505382,1890209=500796} 2024-09-21T09:53:09.815 INFO:tasks.rados.rados.0.smithi076.stdout:3600: writing smithi07685715-1 from 672643 to 1178025 tid 1 2024-09-21T09:53:09.823 INFO:tasks.rados.rados.0.smithi076.stdout:3600: writing smithi07685715-1 from 1890209 to 2391005 tid 2 2024-09-21T09:53:09.823 INFO:tasks.rados.rados.0.smithi076.stdout:3601: read oid 2 snap -1 2024-09-21T09:53:09.823 INFO:tasks.rados.rados.0.smithi076.stdout:3601: expect deleted 2024-09-21T09:53:09.823 INFO:tasks.rados.rados.0.smithi076.stdout:3602: write oid 13 current snap is 453 2024-09-21T09:53:09.823 INFO:tasks.rados.rados.0.smithi076.stdout:3600: finishing write tid 1 to smithi07685715-1 2024-09-21T09:53:09.823 INFO:tasks.rados.rados.0.smithi076.stdout:3602: seq_num 1350 ranges {689450=580631,1948691=740759} 2024-09-21T09:53:09.830 INFO:tasks.rados.rados.0.smithi076.stdout:3602: writing smithi07685715-13 from 689450 to 1270081 tid 1 2024-09-21T09:53:09.838 INFO:tasks.rados.rados.0.smithi076.stdout:3602: writing smithi07685715-13 from 1948691 to 2689450 tid 2 2024-09-21T09:53:09.838 INFO:tasks.rados.rados.0.smithi076.stdout:3603: delete oid 34 current snap is 453 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: finishing write tid 2 to smithi07685715-1 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: finishing write tid 3 to smithi07685715-1 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: finishing write tid 4 to smithi07685715-1 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: oid 1 updating version 0 to 2815 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: oid 1 updating version 2815 to 2818 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: oid 1 updating version 2818 to 2819 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2819 (ObjNum 1349 snap 453 seq_num 1349) dirty exists 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3600: left oid 1 (ObjNum 1349 snap 453 seq_num 1349) 2024-09-21T09:53:09.839 INFO:tasks.rados.rados.0.smithi076.stdout:3602: finishing write tid 1 to smithi07685715-13 2024-09-21T09:53:09.840 INFO:tasks.rados.rados.0.smithi076.stdout:3600: done (3 left) 2024-09-21T09:53:09.840 INFO:tasks.rados.rados.0.smithi076.stdout:3601: done (2 left) 2024-09-21T09:53:09.840 INFO:tasks.rados.rados.0.smithi076.stdout:3603: done (1 left) 2024-09-21T09:53:09.840 INFO:tasks.rados.rados.0.smithi076.stdout:3604: snap_remove snap 452 2024-09-21T09:53:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:09 smithi076 ceph-mon[106482]: pgmap v931: 89 pgs: 89 active+clean; 245 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-09-21T09:53:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:09 smithi076 ceph-mon[106482]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:53:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:09 smithi076 ceph-mon[110963]: pgmap v931: 89 pgs: 89 active+clean; 245 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-09-21T09:53:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:09 smithi076 ceph-mon[110963]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:53:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:09 smithi191 ceph-mon[102206]: pgmap v931: 89 pgs: 89 active+clean; 245 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-09-21T09:53:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:09 smithi191 ceph-mon[102206]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T09:53:10.809 INFO:tasks.rados.rados.0.smithi076.stdout:3604: done (1 left) 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3605: rollback oid 41 current snap is 453 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 446 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: finishing write tid 2 to smithi07685715-13 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: finishing write tid 3 to smithi07685715-13 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: finishing write tid 4 to smithi07685715-13 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: oid 13 updating version 0 to 740 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: oid 13 version 740 is already newer than 738 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: oid 13 version 740 is already newer than 739 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 740 (ObjNum 1350 snap 453 seq_num 1350) dirty exists 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: left oid 13 (ObjNum 1350 snap 453 seq_num 1350) 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3602: done (1 left) 2024-09-21T09:53:10.810 INFO:tasks.rados.rados.0.smithi076.stdout:3606: write oid 16 current snap is 453 2024-09-21T09:53:10.811 INFO:tasks.rados.rados.0.smithi076.stdout:3606: seq_num 1352 ranges {723064=731129,1949302=764476} 2024-09-21T09:53:10.818 INFO:tasks.rados.rados.0.smithi076.stdout:3606: writing smithi07685715-16 from 723064 to 1454193 tid 1 2024-09-21T09:53:10.825 INFO:tasks.rados.rados.0.smithi076.stdout:3606: writing smithi07685715-16 from 1949302 to 2713778 tid 2 2024-09-21T09:53:10.825 INFO:tasks.rados.rados.0.smithi076.stdout:3607: delete oid 22 current snap is 453 2024-09-21T09:53:10.825 INFO:tasks.rados.rados.0.smithi076.stdout:3605: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:53:10.825 INFO:tasks.rados.rados.0.smithi076.stdout:3605: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:53:10.826 INFO:tasks.rados.rados.0.smithi076.stdout:3605: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:53:10.826 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1806 (ObjNum 1324 snap 446 seq_num 1324) dirty exists 2024-09-21T09:53:10.826 INFO:tasks.rados.rados.0.smithi076.stdout:3606: finishing write tid 1 to smithi07685715-16 2024-09-21T09:53:10.831 INFO:tasks.rados.rados.0.smithi076.stdout:3606: finishing write tid 2 to smithi07685715-16 2024-09-21T09:53:10.831 INFO:tasks.rados.rados.0.smithi076.stdout:3606: finishing write tid 3 to smithi07685715-16 2024-09-21T09:53:10.831 INFO:tasks.rados.rados.0.smithi076.stdout:3606: finishing write tid 4 to smithi07685715-16 2024-09-21T09:53:10.831 INFO:tasks.rados.rados.0.smithi076.stdout:3606: oid 16 updating version 0 to 1810 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3606: oid 16 version 1810 is already newer than 1808 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3606: oid 16 version 1810 is already newer than 1809 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1810 (ObjNum 1352 snap 453 seq_num 1352) dirty exists 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3606: left oid 16 (ObjNum 1352 snap 453 seq_num 1352) 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3605: done (2 left) 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3606: done (1 left) 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3607: done (0 left) 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3608: read oid 48 snap 449 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3608: expect (ObjNum 1322 snap 446 seq_num 1322) 2024-09-21T09:53:10.832 INFO:tasks.rados.rados.0.smithi076.stdout:3609: snap_remove snap 445 2024-09-21T09:53:11.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:10 smithi191 ceph-mon[102206]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:53:11.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:10 smithi076 ceph-mon[106482]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:53:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:10 smithi076 ceph-mon[110963]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T09:53:11.814 INFO:tasks.rados.rados.0.smithi076.stdout:3609: done (1 left) 2024-09-21T09:53:11.815 INFO:tasks.rados.rados.0.smithi076.stdout:3610: read oid 17 snap -1 2024-09-21T09:53:11.815 INFO:tasks.rados.rados.0.smithi076.stdout:3610: expect (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:11.815 INFO:tasks.rados.rados.0.smithi076.stdout:3611: read oid 2 snap -1 2024-09-21T09:53:11.815 INFO:tasks.rados.rados.0.smithi076.stdout:3611: expect deleted 2024-09-21T09:53:11.815 INFO:tasks.rados.rados.0.smithi076.stdout:3612: write oid 40 current snap is 453 2024-09-21T09:53:11.815 INFO:tasks.rados.rados.0.smithi076.stdout:3612: seq_num 1354 ranges {756678=597980,1782619=768145} 2024-09-21T09:53:11.823 INFO:tasks.rados.rados.0.smithi076.stdout:3612: writing smithi07685715-40 from 756678 to 1354658 tid 1 2024-09-21T09:53:11.830 INFO:tasks.rados.rados.0.smithi076.stdout:3612: writing smithi07685715-40 from 1782619 to 2550764 tid 2 2024-09-21T09:53:11.830 INFO:tasks.rados.rados.0.smithi076.stdout:3613: write oid 44 current snap is 453 2024-09-21T09:53:11.830 INFO:tasks.rados.rados.0.smithi076.stdout:3613: seq_num 1355 ranges {773485=673229,2124748=511803} 2024-09-21T09:53:11.838 INFO:tasks.rados.rados.0.smithi076.stdout:3613: writing smithi07685715-44 from 773485 to 1446714 tid 1 2024-09-21T09:53:11.845 INFO:tasks.rados.rados.0.smithi076.stdout:3613: writing smithi07685715-44 from 2124748 to 2636551 tid 2 2024-09-21T09:53:11.845 INFO:tasks.rados.rados.0.smithi076.stdout:3614: delete oid 3 current snap is 453 2024-09-21T09:53:11.845 INFO:tasks.rados.rados.0.smithi076.stdout:3608: expect (ObjNum 1322 snap 446 seq_num 1322) 2024-09-21T09:53:11.950 INFO:tasks.rados.rados.0.smithi076.stdout:3610: expect (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:12.062 INFO:tasks.rados.rados.0.smithi076.stdout:3612: finishing write tid 1 to smithi07685715-40 2024-09-21T09:53:12.062 INFO:tasks.rados.rados.0.smithi076.stdout:3612: finishing write tid 2 to smithi07685715-40 2024-09-21T09:53:12.062 INFO:tasks.rados.rados.0.smithi076.stdout:3612: finishing write tid 3 to smithi07685715-40 2024-09-21T09:53:12.062 INFO:tasks.rados.rados.0.smithi076.stdout:3612: finishing write tid 4 to smithi07685715-40 2024-09-21T09:53:12.062 INFO:tasks.rados.rados.0.smithi076.stdout:3612: oid 40 updating version 0 to 1821 2024-09-21T09:53:12.062 INFO:tasks.rados.rados.0.smithi076.stdout:3612: oid 40 updating version 1821 to 1822 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3612: oid 40 version 1822 is already newer than 1814 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1822 (ObjNum 1354 snap 453 seq_num 1354) dirty exists 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3612: left oid 40 (ObjNum 1354 snap 453 seq_num 1354) 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3613: finishing write tid 1 to smithi07685715-44 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3608: done (5 left) 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3610: done (4 left) 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3611: done (3 left) 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3612: done (2 left) 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3614: done (1 left) 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3615: read oid 10 snap -1 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3615: expect deleted 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3616: rollback oid 42 current snap is 453 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 42 to 448 2024-09-21T09:53:12.063 INFO:tasks.rados.rados.0.smithi076.stdout:3613: finishing write tid 2 to smithi07685715-44 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: finishing write tid 3 to smithi07685715-44 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: finishing write tid 4 to smithi07685715-44 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: oid 44 updating version 0 to 784 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: oid 44 updating version 784 to 785 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: oid 44 version 785 is already newer than 783 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 785 (ObjNum 1355 snap 453 seq_num 1355) dirty exists 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: left oid 44 (ObjNum 1355 snap 453 seq_num 1355) 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3613: done (2 left) 2024-09-21T09:53:12.064 INFO:tasks.rados.rados.0.smithi076.stdout:3617: snap_create 2024-09-21T09:53:12.065 INFO:tasks.rados.rados.0.smithi076.stdout:3616: finishing rollback tid 0 to smithi07685715-42 2024-09-21T09:53:12.065 INFO:tasks.rados.rados.0.smithi076.stdout:3616: finishing rollback tid 1 to smithi07685715-42 2024-09-21T09:53:12.065 INFO:tasks.rados.rados.0.smithi076.stdout:3616: finishing rollback tid 2 to smithi07685715-42 2024-09-21T09:53:12.065 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 942 (ObjNum 1307 snap 442 seq_num 1307) dirty exists 2024-09-21T09:53:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:11 smithi191 ceph-mon[102206]: pgmap v934: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 243 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 709 KiB/s wr, 5 op/s 2024-09-21T09:53:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:11 smithi191 ceph-mon[102206]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:53:12.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:11 smithi076 ceph-mon[106482]: pgmap v934: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 243 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 709 KiB/s wr, 5 op/s 2024-09-21T09:53:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:11 smithi076 ceph-mon[106482]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:53:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:11 smithi076 ceph-mon[110963]: pgmap v934: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 243 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 709 KiB/s wr, 5 op/s 2024-09-21T09:53:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:11 smithi076 ceph-mon[110963]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T09:53:12.887 INFO:tasks.rados.rados.0.smithi076.stdout:3615: done (2 left) 2024-09-21T09:53:12.887 INFO:tasks.rados.rados.0.smithi076.stdout:3616: done (1 left) 2024-09-21T09:53:12.887 INFO:tasks.rados.rados.0.smithi076.stdout:3617: done (0 left) 2024-09-21T09:53:12.887 INFO:tasks.rados.rados.0.smithi076.stdout:3618: read oid 2 snap -1 2024-09-21T09:53:12.887 INFO:tasks.rados.rados.0.smithi076.stdout:3618: expect deleted 2024-09-21T09:53:12.887 INFO:tasks.rados.rados.0.smithi076.stdout:3619: snap_remove snap 448 2024-09-21T09:53:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:53:13] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.51.0" 2024-09-21T09:53:13.808 INFO:tasks.rados.rados.0.smithi076.stdout:3619: done (1 left) 2024-09-21T09:53:13.808 INFO:tasks.rados.rados.0.smithi076.stdout:3620: write oid 18 current snap is 454 2024-09-21T09:53:13.808 INFO:tasks.rados.rados.0.smithi076.stdout:3620: seq_num 1357 ranges {407099=423727,1441712=515472,2582060=225039} 2024-09-21T09:53:13.813 INFO:tasks.rados.rados.0.smithi076.stdout:3620: writing smithi07685715-18 from 407099 to 830826 tid 1 2024-09-21T09:53:13.820 INFO:tasks.rados.rados.0.smithi076.stdout:3620: writing smithi07685715-18 from 1441712 to 1957184 tid 2 2024-09-21T09:53:13.825 INFO:tasks.rados.rados.0.smithi076.stdout:3620: writing smithi07685715-18 from 2582060 to 2807099 tid 3 2024-09-21T09:53:13.825 INFO:tasks.rados.rados.0.smithi076.stdout:3621: delete oid 32 current snap is 454 2024-09-21T09:53:13.825 INFO:tasks.rados.rados.0.smithi076.stdout:3620: finishing write tid 1 to smithi07685715-18 2024-09-21T09:53:13.825 INFO:tasks.rados.rados.0.smithi076.stdout:3620: finishing write tid 2 to smithi07685715-18 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3618: done (2 left) 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3621: done (1 left) 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3622: read oid 1 snap -1 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3622: expect (ObjNum 1349 snap 453 seq_num 1349) 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3620: finishing write tid 3 to smithi07685715-18 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3623: write oid 23 current snap is 454 2024-09-21T09:53:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3623: seq_num 1359 ranges {440713=574225,1442323=519141,2636906=203807} 2024-09-21T09:53:13.832 INFO:tasks.rados.rados.0.smithi076.stdout:3623: writing smithi07685715-23 from 440713 to 1014938 tid 1 2024-09-21T09:53:13.838 INFO:tasks.rados.rados.0.smithi076.stdout:3623: writing smithi07685715-23 from 1442323 to 1961464 tid 2 2024-09-21T09:53:13.843 INFO:tasks.rados.rados.0.smithi076.stdout:3623: writing smithi07685715-23 from 2636906 to 2840713 tid 3 2024-09-21T09:53:13.843 INFO:tasks.rados.rados.0.smithi076.stdout:3624: delete oid 27 current snap is 454 2024-09-21T09:53:13.843 INFO:tasks.rados.rados.0.smithi076.stdout:3620: finishing write tid 4 to smithi07685715-18 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3620: finishing write tid 5 to smithi07685715-18 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3620: oid 18 updating version 0 to 949 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3620: oid 18 version 949 is already newer than 944 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3620: oid 18 version 949 is already newer than 948 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3620: oid 18 version 949 is already newer than 945 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 949 (ObjNum 1357 snap 454 seq_num 1357) dirty exists 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3620: left oid 18 (ObjNum 1357 snap 454 seq_num 1357) 2024-09-21T09:53:13.844 INFO:tasks.rados.rados.0.smithi076.stdout:3622: expect (ObjNum 1349 snap 453 seq_num 1349) 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3623: finishing write tid 1 to smithi07685715-23 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3623: finishing write tid 2 to smithi07685715-23 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3620: done (3 left) 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3622: done (2 left) 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3624: done (1 left) 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3625: delete oid 3 current snap is 454 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3623: finishing write tid 3 to smithi07685715-23 2024-09-21T09:53:14.024 INFO:tasks.rados.rados.0.smithi076.stdout:3623: finishing write tid 4 to smithi07685715-23 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: finishing write tid 5 to smithi07685715-23 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: oid 23 updating version 0 to 2792 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: oid 23 updating version 2792 to 2793 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: oid 23 version 2793 is already newer than 2791 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: oid 23 version 2793 is already newer than 2790 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2793 (ObjNum 1359 snap 454 seq_num 1359) dirty exists 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: left oid 23 (ObjNum 1359 snap 454 seq_num 1359) 2024-09-21T09:53:14.025 INFO:tasks.rados.rados.0.smithi076.stdout:3623: done (1 left) 2024-09-21T09:53:14.026 INFO:tasks.rados.rados.0.smithi076.stdout:3625: done (0 left) 2024-09-21T09:53:14.026 INFO:tasks.rados.rados.0.smithi076.stdout:3626: write oid 32 current snap is 454 2024-09-21T09:53:14.026 INFO:tasks.rados.rados.0.smithi076.stdout:3626: seq_num 1362 ranges {491134=516325,1734122=666468} 2024-09-21T09:53:14.031 INFO:tasks.rados.rados.0.smithi076.stdout:3626: writing smithi07685715-32 from 491134 to 1007459 tid 1 2024-09-21T09:53:14.038 INFO:tasks.rados.rados.0.smithi076.stdout:3626: writing smithi07685715-32 from 1734122 to 2400590 tid 2 2024-09-21T09:53:14.038 INFO:tasks.rados.rados.0.smithi076.stdout:3627: write oid 41 current snap is 454 2024-09-21T09:53:14.039 INFO:tasks.rados.rados.0.smithi076.stdout:3626: finishing write tid 1 to smithi07685715-32 2024-09-21T09:53:14.039 INFO:tasks.rados.rados.0.smithi076.stdout:3627: seq_num 1363 ranges {507941=591574,1676251=526479,2746598=161343} 2024-09-21T09:53:14.044 INFO:tasks.rados.rados.0.smithi076.stdout:3627: writing smithi07685715-41 from 507941 to 1099515 tid 1 2024-09-21T09:53:14.050 INFO:tasks.rados.rados.0.smithi076.stdout:3627: writing smithi07685715-41 from 1676251 to 2202730 tid 2 2024-09-21T09:53:14.054 INFO:tasks.rados.rados.0.smithi076.stdout:3627: writing smithi07685715-41 from 2746598 to 2907941 tid 3 2024-09-21T09:53:14.054 INFO:tasks.rados.rados.0.smithi076.stdout:3628: snap_remove snap 431 2024-09-21T09:53:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:13 smithi191 ceph-mon[102206]: pgmap v937: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 246 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:53:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:13 smithi191 ceph-mon[102206]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:53:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:13 smithi191 ceph-mon[102206]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:53:14.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-mon[106482]: pgmap v937: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 246 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:53:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-mon[106482]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:53:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-mon[106482]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:53:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-mon[110963]: pgmap v937: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 246 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-21T09:53:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-mon[110963]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T09:53:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:13 smithi076 ceph-mon[110963]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T09:53:14.891 INFO:tasks.rados.rados.0.smithi076.stdout:3628: done (2 left) 2024-09-21T09:53:14.891 INFO:tasks.rados.rados.0.smithi076.stdout:3629: snap_create 2024-09-21T09:53:14.891 INFO:tasks.rados.rados.0.smithi076.stdout:3626: finishing write tid 2 to smithi07685715-32 2024-09-21T09:53:14.891 INFO:tasks.rados.rados.0.smithi076.stdout:3626: finishing write tid 3 to smithi07685715-32 2024-09-21T09:53:14.891 INFO:tasks.rados.rados.0.smithi076.stdout:3626: finishing write tid 4 to smithi07685715-32 2024-09-21T09:53:14.891 INFO:tasks.rados.rados.0.smithi076.stdout:3626: oid 32 updating version 0 to 1107 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3626: oid 32 version 1107 is already newer than 1105 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3626: oid 32 version 1107 is already newer than 1106 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 1107 (ObjNum 1362 snap 454 seq_num 1362) dirty exists 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3626: left oid 32 (ObjNum 1362 snap 454 seq_num 1362) 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: finishing write tid 1 to smithi07685715-41 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: finishing write tid 2 to smithi07685715-41 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: finishing write tid 3 to smithi07685715-41 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: finishing write tid 4 to smithi07685715-41 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: finishing write tid 5 to smithi07685715-41 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: oid 41 updating version 0 to 1833 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: oid 41 updating version 1833 to 1834 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: oid 41 updating version 1834 to 1836 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:3627: oid 41 version 1836 is already newer than 1835 2024-09-21T09:53:14.892 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1836 (ObjNum 1363 snap 454 seq_num 1363) dirty exists 2024-09-21T09:53:14.893 INFO:tasks.rados.rados.0.smithi076.stdout:3627: left oid 41 (ObjNum 1363 snap 454 seq_num 1363) 2024-09-21T09:53:15.893 INFO:tasks.rados.rados.0.smithi076.stdout:3626: done (2 left) 2024-09-21T09:53:15.894 INFO:tasks.rados.rados.0.smithi076.stdout:3627: done (1 left) 2024-09-21T09:53:15.894 INFO:tasks.rados.rados.0.smithi076.stdout:3629: done (0 left) 2024-09-21T09:53:15.894 INFO:tasks.rados.rados.0.smithi076.stdout:3630: snap_create 2024-09-21T09:53:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:15 smithi191 ceph-mon[102206]: pgmap v940: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 245 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 611 KiB/s wr, 8 op/s 2024-09-21T09:53:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:15 smithi191 ceph-mon[102206]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:53:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:15 smithi076 ceph-mon[106482]: pgmap v940: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 245 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 611 KiB/s wr, 8 op/s 2024-09-21T09:53:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:15 smithi076 ceph-mon[106482]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:53:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:15 smithi076 ceph-mon[110963]: pgmap v940: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 245 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 611 KiB/s wr, 8 op/s 2024-09-21T09:53:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:15 smithi076 ceph-mon[110963]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T09:53:16.894 INFO:tasks.rados.rados.0.smithi076.stdout:3630: done (0 left) 2024-09-21T09:53:16.894 INFO:tasks.rados.rados.0.smithi076.stdout:3631: write oid 9 current snap is 456 2024-09-21T09:53:16.895 INFO:tasks.rados.rados.0.smithi076.stdout:3631: seq_num 1364 ranges {524748=666823,1734733=670137} 2024-09-21T09:53:16.901 INFO:tasks.rados.rados.0.smithi076.stdout:3631: writing smithi07685715-9 from 524748 to 1191571 tid 1 2024-09-21T09:53:16.908 INFO:tasks.rados.rados.0.smithi076.stdout:3631: writing smithi07685715-9 from 1734733 to 2404870 tid 2 2024-09-21T09:53:16.909 INFO:tasks.rados.rados.0.smithi076.stdout:3632: write oid 7 current snap is 456 2024-09-21T09:53:16.909 INFO:tasks.rados.rados.0.smithi076.stdout:3632: seq_num 1365 ranges {541555=742072,1793215=530148,2801444=140111} 2024-09-21T09:53:16.915 INFO:tasks.rados.rados.0.smithi076.stdout:3632: writing smithi07685715-7 from 541555 to 1283627 tid 1 2024-09-21T09:53:16.921 INFO:tasks.rados.rados.0.smithi076.stdout:3632: writing smithi07685715-7 from 1793215 to 2323363 tid 2 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3632: writing smithi07685715-7 from 2801444 to 2941555 tid 3 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3633: delete oid 18 current snap is 456 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3631: finishing write tid 1 to smithi07685715-9 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3631: finishing write tid 2 to smithi07685715-9 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3631: finishing write tid 3 to smithi07685715-9 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3631: finishing write tid 4 to smithi07685715-9 2024-09-21T09:53:16.925 INFO:tasks.rados.rados.0.smithi076.stdout:3631: oid 9 updating version 0 to 1277 2024-09-21T09:53:16.926 INFO:tasks.rados.rados.0.smithi076.stdout:3631: oid 9 updating version 1277 to 1278 2024-09-21T09:53:16.926 INFO:tasks.rados.rados.0.smithi076.stdout:3631: oid 9 version 1278 is already newer than 1276 2024-09-21T09:53:16.926 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 1278 (ObjNum 1364 snap 456 seq_num 1364) dirty exists 2024-09-21T09:53:16.926 INFO:tasks.rados.rados.0.smithi076.stdout:3631: left oid 9 (ObjNum 1364 snap 456 seq_num 1364) 2024-09-21T09:53:16.926 INFO:tasks.rados.rados.0.smithi076.stdout:3632: finishing write tid 1 to smithi07685715-7 2024-09-21T09:53:16.926 INFO:tasks.rados.rados.0.smithi076.stdout:3632: finishing write tid 2 to smithi07685715-7 2024-09-21T09:53:16.927 INFO:tasks.rados.rados.0.smithi076.stdout:3631: done (2 left) 2024-09-21T09:53:16.927 INFO:tasks.rados.rados.0.smithi076.stdout:3633: done (1 left) 2024-09-21T09:53:16.927 INFO:tasks.rados.rados.0.smithi076.stdout:3634: delete oid 31 current snap is 456 2024-09-21T09:53:16.928 INFO:tasks.rados.rados.0.smithi076.stdout:3632: finishing write tid 3 to smithi07685715-7 2024-09-21T09:53:16.928 INFO:tasks.rados.rados.0.smithi076.stdout:3632: finishing write tid 4 to smithi07685715-7 2024-09-21T09:53:16.928 INFO:tasks.rados.rados.0.smithi076.stdout:3632: finishing write tid 5 to smithi07685715-7 2024-09-21T09:53:16.928 INFO:tasks.rados.rados.0.smithi076.stdout:3632: oid 7 updating version 0 to 2292 2024-09-21T09:53:16.928 INFO:tasks.rados.rados.0.smithi076.stdout:3632: oid 7 updating version 2292 to 2293 2024-09-21T09:53:16.928 INFO:tasks.rados.rados.0.smithi076.stdout:3632: oid 7 version 2293 is already newer than 2290 2024-09-21T09:53:16.929 INFO:tasks.rados.rados.0.smithi076.stdout:3632: oid 7 version 2293 is already newer than 2291 2024-09-21T09:53:16.929 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2293 (ObjNum 1365 snap 456 seq_num 1365) dirty exists 2024-09-21T09:53:16.929 INFO:tasks.rados.rados.0.smithi076.stdout:3632: left oid 7 (ObjNum 1365 snap 456 seq_num 1365) 2024-09-21T09:53:16.929 INFO:tasks.rados.rados.0.smithi076.stdout:3632: done (1 left) 2024-09-21T09:53:16.929 INFO:tasks.rados.rados.0.smithi076.stdout:3634: done (0 left) 2024-09-21T09:53:16.929 INFO:tasks.rados.rados.0.smithi076.stdout:3635: snap_create 2024-09-21T09:53:17.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:16 smithi191 ceph-mon[102206]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:53:17.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:16 smithi191 ceph-mon[102206]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:53:17.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:16 smithi076 ceph-mon[106482]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:53:17.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:16 smithi076 ceph-mon[106482]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:53:17.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:16 smithi076 ceph-mon[110963]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T09:53:17.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:16 smithi076 ceph-mon[110963]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T09:53:17.912 INFO:tasks.rados.rados.0.smithi076.stdout:3635: done (0 left) 2024-09-21T09:53:17.913 INFO:tasks.rados.rados.0.smithi076.stdout:3636: snap_remove snap 454 2024-09-21T09:53:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:17 smithi191 ceph-mon[102206]: pgmap v943: 89 pgs: 89 active+clean; 256 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:53:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:17 smithi076 ceph-mon[106482]: pgmap v943: 89 pgs: 89 active+clean; 256 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:53:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:17 smithi076 ceph-mon[110963]: pgmap v943: 89 pgs: 89 active+clean; 256 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T09:53:18.812 INFO:tasks.rados.rados.0.smithi076.stdout:3636: done (0 left) 2024-09-21T09:53:18.813 INFO:tasks.rados.rados.0.smithi076.stdout:3637: rollback oid 39 current snap is 457 2024-09-21T09:53:18.813 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 456 2024-09-21T09:53:18.813 INFO:tasks.rados.rados.0.smithi076.stdout:3638: delete oid 14 current snap is 457 2024-09-21T09:53:18.816 INFO:tasks.rados.rados.0.smithi076.stdout:3638: done (1 left) 2024-09-21T09:53:18.816 INFO:tasks.rados.rados.0.smithi076.stdout:3639: snap_create 2024-09-21T09:53:18.827 INFO:tasks.rados.rados.0.smithi076.stdout:3637: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:53:18.827 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2743 (ObjNum 129 snap 0 seq_num 864580176) dirty dne 2024-09-21T09:53:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:18 smithi191 ceph-mon[102206]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:53:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:18 smithi191 ceph-mon[102206]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:53:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:18 smithi076 ceph-mon[106482]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:53:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:18 smithi076 ceph-mon[106482]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:53:19.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:18 smithi076 ceph-mon[110963]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T09:53:19.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:18 smithi076 ceph-mon[110963]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T09:53:19.812 INFO:tasks.rados.rados.0.smithi076.stdout:3637: done (1 left) 2024-09-21T09:53:19.812 INFO:tasks.rados.rados.0.smithi076.stdout:3639: done (0 left) 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3640: read oid 43 snap 450 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3640: expect (ObjNum 1170 snap 393 seq_num 1170) 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3641: read oid 9 snap -1 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3641: expect (ObjNum 1364 snap 456 seq_num 1364) 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3642: read oid 10 snap 451 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3642: expect deleted 2024-09-21T09:53:19.813 INFO:tasks.rados.rados.0.smithi076.stdout:3643: snap_create 2024-09-21T09:53:19.818 INFO:tasks.rados.rados.0.smithi076.stdout:3641: expect (ObjNum 1364 snap 456 seq_num 1364) 2024-09-21T09:53:19.950 INFO:tasks.rados.rados.0.smithi076.stdout:3640: expect (ObjNum 1170 snap 393 seq_num 1170) 2024-09-21T09:53:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:19 smithi191 ceph-mon[102206]: pgmap v946: 89 pgs: 89 active+clean; 256 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-09-21T09:53:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:19 smithi191 ceph-mon[102206]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:53:20.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:19 smithi076 ceph-mon[106482]: pgmap v946: 89 pgs: 89 active+clean; 256 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-09-21T09:53:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:19 smithi076 ceph-mon[106482]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:53:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:19 smithi076 ceph-mon[110963]: pgmap v946: 89 pgs: 89 active+clean; 256 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 12 op/s 2024-09-21T09:53:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:19 smithi076 ceph-mon[110963]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T09:53:20.813 INFO:tasks.rados.rados.0.smithi076.stdout:3640: done (3 left) 2024-09-21T09:53:20.813 INFO:tasks.rados.rados.0.smithi076.stdout:3641: done (2 left) 2024-09-21T09:53:20.813 INFO:tasks.rados.rados.0.smithi076.stdout:3642: done (1 left) 2024-09-21T09:53:20.813 INFO:tasks.rados.rados.0.smithi076.stdout:3643: done (0 left) 2024-09-21T09:53:20.814 INFO:tasks.rados.rados.0.smithi076.stdout:3644: read oid 34 snap -1 2024-09-21T09:53:20.814 INFO:tasks.rados.rados.0.smithi076.stdout:3644: expect deleted 2024-09-21T09:53:20.814 INFO:tasks.rados.rados.0.smithi076.stdout:3645: delete oid 28 current snap is 459 2024-09-21T09:53:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3644: done (1 left) 2024-09-21T09:53:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3645: done (0 left) 2024-09-21T09:53:20.816 INFO:tasks.rados.rados.0.smithi076.stdout:3646: write oid 5 current snap is 459 2024-09-21T09:53:20.817 INFO:tasks.rados.rados.0.smithi076.stdout:3646: seq_num 1370 ranges {625590=434670,1685625=681144,2822206=203384} 2024-09-21T09:53:20.822 INFO:tasks.rados.rados.0.smithi076.stdout:3646: writing smithi07685715-5 from 625590 to 1060260 tid 1 2024-09-21T09:53:20.829 INFO:tasks.rados.rados.0.smithi076.stdout:3646: writing smithi07685715-5 from 1685625 to 2366769 tid 2 2024-09-21T09:53:20.833 INFO:tasks.rados.rados.0.smithi076.stdout:3646: writing smithi07685715-5 from 2822206 to 3025590 tid 3 2024-09-21T09:53:20.833 INFO:tasks.rados.rados.0.smithi076.stdout:3647: read oid 15 snap -1 2024-09-21T09:53:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:3647: expect deleted 2024-09-21T09:53:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:3648: snap_remove snap 449 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: finishing write tid 1 to smithi07685715-5 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: finishing write tid 2 to smithi07685715-5 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: finishing write tid 3 to smithi07685715-5 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: finishing write tid 4 to smithi07685715-5 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: finishing write tid 5 to smithi07685715-5 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: oid 5 updating version 0 to 2843 2024-09-21T09:53:21.828 INFO:tasks.rados.rados.0.smithi076.stdout:3646: oid 5 updating version 2843 to 2845 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3646: oid 5 version 2845 is already newer than 2842 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3646: oid 5 version 2845 is already newer than 2844 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2845 (ObjNum 1370 snap 459 seq_num 1370) dirty exists 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3646: left oid 5 (ObjNum 1370 snap 459 seq_num 1370) 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3646: done (2 left) 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3647: done (1 left) 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3648: done (0 left) 2024-09-21T09:53:21.829 INFO:tasks.rados.rados.0.smithi076.stdout:3649: snap_create 2024-09-21T09:53:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:21 smithi191 ceph-mon[102206]: pgmap v949: 89 pgs: 89 active+clean; 261 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 1020 KiB/s rd, 672 KiB/s wr, 5 op/s 2024-09-21T09:53:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:21 smithi191 ceph-mon[102206]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:53:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:21 smithi076 ceph-mon[106482]: pgmap v949: 89 pgs: 89 active+clean; 261 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 1020 KiB/s rd, 672 KiB/s wr, 5 op/s 2024-09-21T09:53:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:21 smithi076 ceph-mon[106482]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:53:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:21 smithi076 ceph-mon[110963]: pgmap v949: 89 pgs: 89 active+clean; 261 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 1020 KiB/s rd, 672 KiB/s wr, 5 op/s 2024-09-21T09:53:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:21 smithi076 ceph-mon[110963]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T09:53:22.829 INFO:tasks.rados.rados.0.smithi076.stdout:3649: done (0 left) 2024-09-21T09:53:22.829 INFO:tasks.rados.rados.0.smithi076.stdout:3650: delete oid 6 current snap is 460 2024-09-21T09:53:22.833 INFO:tasks.rados.rados.0.smithi076.stdout:3650: done (0 left) 2024-09-21T09:53:22.833 INFO:tasks.rados.rados.0.smithi076.stdout:3651: delete oid 32 current snap is 460 2024-09-21T09:53:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3651: done (0 left) 2024-09-21T09:53:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3652: read oid 33 snap -1 2024-09-21T09:53:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3652: expect (ObjNum 1345 snap 450 seq_num 1345) 2024-09-21T09:53:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3653: write oid 16 current snap is 460 2024-09-21T09:53:22.836 INFO:tasks.rados.rados.0.smithi076.stdout:3653: seq_num 1373 ranges {676011=660417,1744718=428471,2788122=287889} 2024-09-21T09:53:22.843 INFO:tasks.rados.rados.0.smithi076.stdout:3653: writing smithi07685715-16 from 676011 to 1336428 tid 1 2024-09-21T09:53:22.848 INFO:tasks.rados.rados.0.smithi076.stdout:3653: writing smithi07685715-16 from 1744718 to 2173189 tid 2 2024-09-21T09:53:22.853 INFO:tasks.rados.rados.0.smithi076.stdout:3653: writing smithi07685715-16 from 2788122 to 3076011 tid 3 2024-09-21T09:53:22.853 INFO:tasks.rados.rados.0.smithi076.stdout:3654: snap_remove snap 458 2024-09-21T09:53:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:22 smithi191 ceph-mon[102206]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:53:23.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:22 smithi076 ceph-mon[106482]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:53:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:22 smithi076 ceph-mon[110963]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T09:53:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:53:23] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.51.0" 2024-09-21T09:53:23.812 INFO:tasks.rados.rados.0.smithi076.stdout:3654: done (2 left) 2024-09-21T09:53:23.812 INFO:tasks.rados.rados.0.smithi076.stdout:3655: snap_remove snap 447 2024-09-21T09:53:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:23 smithi191 ceph-mon[102206]: pgmap v952: 89 pgs: 89 active+clean; 262 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 994 KiB/s wr, 8 op/s 2024-09-21T09:53:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:23 smithi191 ceph-mon[102206]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:53:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:23 smithi191 ceph-mon[102206]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:53:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-mon[106482]: pgmap v952: 89 pgs: 89 active+clean; 262 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 994 KiB/s wr, 8 op/s 2024-09-21T09:53:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-mon[106482]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:53:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-mon[106482]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:53:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-mon[110963]: pgmap v952: 89 pgs: 89 active+clean; 262 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 994 KiB/s wr, 8 op/s 2024-09-21T09:53:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-mon[110963]: osdmap e988: 8 total, 8 up, 8 in 2024-09-21T09:53:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:23 smithi076 ceph-mon[110963]: osdmap e989: 8 total, 8 up, 8 in 2024-09-21T09:53:24.812 INFO:tasks.rados.rados.0.smithi076.stdout:3655: done (2 left) 2024-09-21T09:53:24.812 INFO:tasks.rados.rados.0.smithi076.stdout:3656: delete oid 48 current snap is 460 2024-09-21T09:53:24.812 INFO:tasks.rados.rados.0.smithi076.stdout:3652: expect (ObjNum 1345 snap 450 seq_num 1345) 2024-09-21T09:53:25.002 INFO:tasks.rados.rados.0.smithi076.stdout:3653: finishing write tid 1 to smithi07685715-16 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: finishing write tid 2 to smithi07685715-16 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3652: done (2 left) 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3656: done (1 left) 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3657: snap_create 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: finishing write tid 3 to smithi07685715-16 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: finishing write tid 4 to smithi07685715-16 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: finishing write tid 5 to smithi07685715-16 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: oid 16 updating version 0 to 1852 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: oid 16 updating version 1852 to 1854 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: oid 16 updating version 1854 to 1855 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:3653: oid 16 version 1855 is already newer than 1853 2024-09-21T09:53:25.003 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1855 (ObjNum 1373 snap 460 seq_num 1373) dirty exists 2024-09-21T09:53:25.004 INFO:tasks.rados.rados.0.smithi076.stdout:3653: left oid 16 (ObjNum 1373 snap 460 seq_num 1373) 2024-09-21T09:53:25.834 INFO:tasks.rados.rados.0.smithi076.stdout:3653: done (1 left) 2024-09-21T09:53:25.834 INFO:tasks.rados.rados.0.smithi076.stdout:3657: done (0 left) 2024-09-21T09:53:25.834 INFO:tasks.rados.rados.0.smithi076.stdout:3658: read oid 47 snap -1 2024-09-21T09:53:25.834 INFO:tasks.rados.rados.0.smithi076.stdout:3658: expect deleted 2024-09-21T09:53:25.835 INFO:tasks.rados.rados.0.smithi076.stdout:3659: write oid 4 current snap is 461 2024-09-21T09:53:25.835 INFO:tasks.rados.rados.0.smithi076.stdout:3659: seq_num 1375 ranges {709625=410915,1745329=432140,2726615=383010} 2024-09-21T09:53:25.840 INFO:tasks.rados.rados.0.smithi076.stdout:3659: writing smithi07685715-4 from 709625 to 1120540 tid 1 2024-09-21T09:53:25.845 INFO:tasks.rados.rados.0.smithi076.stdout:3659: writing smithi07685715-4 from 1745329 to 2177469 tid 2 2024-09-21T09:53:25.851 INFO:tasks.rados.rados.0.smithi076.stdout:3659: writing smithi07685715-4 from 2726615 to 3109625 tid 3 2024-09-21T09:53:25.851 INFO:tasks.rados.rados.0.smithi076.stdout:3660: write oid 13 current snap is 461 2024-09-21T09:53:25.851 INFO:tasks.rados.rados.0.smithi076.stdout:3659: finishing write tid 1 to smithi07685715-4 2024-09-21T09:53:25.851 INFO:tasks.rados.rados.0.smithi076.stdout:3659: finishing write tid 2 to smithi07685715-4 2024-09-21T09:53:25.851 INFO:tasks.rados.rados.0.smithi076.stdout:3660: seq_num 1376 ranges {726432=486164,1803811=692151} 2024-09-21T09:53:25.858 INFO:tasks.rados.rados.0.smithi076.stdout:3660: writing smithi07685715-13 from 726432 to 1212596 tid 1 2024-09-21T09:53:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:3660: writing smithi07685715-13 from 1803811 to 2495962 tid 2 2024-09-21T09:53:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:3658: done (2 left) 2024-09-21T09:53:25.864 INFO:tasks.rados.rados.0.smithi076.stdout:3661: write oid 29 current snap is 461 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: finishing write tid 3 to smithi07685715-4 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: finishing write tid 4 to smithi07685715-4 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: finishing write tid 5 to smithi07685715-4 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: oid 4 updating version 0 to 1166 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: oid 4 version 1166 is already newer than 1164 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: oid 4 updating version 1166 to 1167 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: oid 4 version 1167 is already newer than 1165 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 1167 (ObjNum 1375 snap 461 seq_num 1375) dirty exists 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3659: left oid 4 (ObjNum 1375 snap 461 seq_num 1375) 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3660: finishing write tid 1 to smithi07685715-13 2024-09-21T09:53:25.865 INFO:tasks.rados.rados.0.smithi076.stdout:3661: seq_num 1377 ranges {743239=677766,1978646=435809,2897814=245425} 2024-09-21T09:53:25.872 INFO:tasks.rados.rados.0.smithi076.stdout:3661: writing smithi07685715-29 from 743239 to 1421005 tid 1 2024-09-21T09:53:25.877 INFO:tasks.rados.rados.0.smithi076.stdout:3661: writing smithi07685715-29 from 1978646 to 2414455 tid 2 2024-09-21T09:53:25.882 INFO:tasks.rados.rados.0.smithi076.stdout:3661: writing smithi07685715-29 from 2897814 to 3143239 tid 3 2024-09-21T09:53:25.882 INFO:tasks.rados.rados.0.smithi076.stdout:3659: done (2 left) 2024-09-21T09:53:25.882 INFO:tasks.rados.rados.0.smithi076.stdout:3662: write oid 19 current snap is 461 2024-09-21T09:53:25.882 INFO:tasks.rados.rados.0.smithi076.stdout:3660: finishing write tid 2 to smithi07685715-13 2024-09-21T09:53:25.882 INFO:tasks.rados.rados.0.smithi076.stdout:3662: seq_num 1378 ranges {760046=753015,2037128=695820} 2024-09-21T09:53:25.889 INFO:tasks.rados.rados.0.smithi076.stdout:3662: writing smithi07685715-19 from 760046 to 1513061 tid 1 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3662: writing smithi07685715-19 from 2037128 to 2732948 tid 2 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3663: snap_create 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3660: finishing write tid 3 to smithi07685715-13 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3660: finishing write tid 4 to smithi07685715-13 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3660: oid 13 updating version 0 to 756 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3660: oid 13 version 756 is already newer than 755 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3660: oid 13 version 756 is already newer than 754 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 756 (ObjNum 1376 snap 461 seq_num 1376) dirty exists 2024-09-21T09:53:25.897 INFO:tasks.rados.rados.0.smithi076.stdout:3660: left oid 13 (ObjNum 1376 snap 461 seq_num 1376) 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: finishing write tid 1 to smithi07685715-29 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: finishing write tid 2 to smithi07685715-29 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: finishing write tid 3 to smithi07685715-29 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: finishing write tid 4 to smithi07685715-29 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: finishing write tid 5 to smithi07685715-29 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: oid 29 updating version 0 to 1131 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: oid 29 updating version 1131 to 1132 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: oid 29 version 1132 is already newer than 1130 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: oid 29 version 1132 is already newer than 1129 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 1132 (ObjNum 1377 snap 461 seq_num 1377) dirty exists 2024-09-21T09:53:25.898 INFO:tasks.rados.rados.0.smithi076.stdout:3661: left oid 29 (ObjNum 1377 snap 461 seq_num 1377) 2024-09-21T09:53:25.899 INFO:tasks.rados.rados.0.smithi076.stdout:3662: finishing write tid 1 to smithi07685715-19 2024-09-21T09:53:25.899 INFO:tasks.rados.rados.0.smithi076.stdout:3662: finishing write tid 2 to smithi07685715-19 2024-09-21T09:53:25.899 INFO:tasks.rados.rados.0.smithi076.stdout:3662: finishing write tid 3 to smithi07685715-19 2024-09-21T09:53:25.900 INFO:tasks.rados.rados.0.smithi076.stdout:3662: finishing write tid 4 to smithi07685715-19 2024-09-21T09:53:25.900 INFO:tasks.rados.rados.0.smithi076.stdout:3662: oid 19 updating version 0 to 1122 2024-09-21T09:53:25.900 INFO:tasks.rados.rados.0.smithi076.stdout:3662: oid 19 updating version 1122 to 1124 2024-09-21T09:53:25.900 INFO:tasks.rados.rados.0.smithi076.stdout:3662: oid 19 version 1124 is already newer than 1123 2024-09-21T09:53:25.900 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 1124 (ObjNum 1378 snap 461 seq_num 1378) dirty exists 2024-09-21T09:53:25.900 INFO:tasks.rados.rados.0.smithi076.stdout:3662: left oid 19 (ObjNum 1378 snap 461 seq_num 1378) 2024-09-21T09:53:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:25 smithi191 ceph-mon[102206]: pgmap v955: 89 pgs: 89 active+clean; 262 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 322 KiB/s wr, 5 op/s 2024-09-21T09:53:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:25 smithi191 ceph-mon[102206]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:53:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:25 smithi076 ceph-mon[106482]: pgmap v955: 89 pgs: 89 active+clean; 262 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 322 KiB/s wr, 5 op/s 2024-09-21T09:53:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:25 smithi076 ceph-mon[106482]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:53:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:25 smithi076 ceph-mon[110963]: pgmap v955: 89 pgs: 89 active+clean; 262 MiB data, 891 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 322 KiB/s wr, 5 op/s 2024-09-21T09:53:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:25 smithi076 ceph-mon[110963]: osdmap e990: 8 total, 8 up, 8 in 2024-09-21T09:53:26.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:26.846 INFO:tasks.rados.rados.0.smithi076.stdout:3660: done (3 left) 2024-09-21T09:53:26.846 INFO:tasks.rados.rados.0.smithi076.stdout:3661: done (2 left) 2024-09-21T09:53:26.846 INFO:tasks.rados.rados.0.smithi076.stdout:3662: done (1 left) 2024-09-21T09:53:26.846 INFO:tasks.rados.rados.0.smithi076.stdout:3663: done (0 left) 2024-09-21T09:53:26.846 INFO:tasks.rados.rados.0.smithi076.stdout:3664: delete oid 14 current snap is 462 2024-09-21T09:53:26.852 INFO:tasks.rados.rados.0.smithi076.stdout:3664: done (0 left) 2024-09-21T09:53:26.852 INFO:tasks.rados.rados.0.smithi076.stdout:3665: rollback oid 7 current snap is 462 2024-09-21T09:53:26.852 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 424 2024-09-21T09:53:26.852 INFO:tasks.rados.rados.0.smithi076.stdout:3666: write oid 20 current snap is 462 2024-09-21T09:53:26.852 INFO:tasks.rados.rados.0.smithi076.stdout:3666: seq_num 1380 ranges {793660=503513,2037739=583136} 2024-09-21T09:53:26.859 INFO:tasks.rados.rados.0.smithi076.stdout:3666: writing smithi07685715-20 from 793660 to 1297173 tid 1 2024-09-21T09:53:26.866 INFO:tasks.rados.rados.0.smithi076.stdout:3666: writing smithi07685715-20 from 2037739 to 2620875 tid 2 2024-09-21T09:53:26.866 INFO:tasks.rados.rados.0.smithi076.stdout:3667: snap_remove snap 455 2024-09-21T09:53:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:26 smithi191 ceph-mon[102206]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:53:27.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:26 smithi076 ceph-mon[106482]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:53:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:26 smithi076 ceph-mon[110963]: osdmap e991: 8 total, 8 up, 8 in 2024-09-21T09:53:27.855 INFO:tasks.rados.rados.0.smithi076.stdout:3667: done (2 left) 2024-09-21T09:53:27.856 INFO:tasks.rados.rados.0.smithi076.stdout:3668: write oid 18 current snap is 462 2024-09-21T09:53:27.856 INFO:tasks.rados.rados.0.smithi076.stdout:3665: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:53:27.856 INFO:tasks.rados.rados.0.smithi076.stdout:3665: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:53:27.856 INFO:tasks.rados.rados.0.smithi076.stdout:3668: seq_num 1381 ranges {410467=578762,1696221=443147,2891153=319314} 2024-09-21T09:53:27.861 INFO:tasks.rados.rados.0.smithi076.stdout:3668: writing smithi07685715-18 from 410467 to 989229 tid 1 2024-09-21T09:53:27.868 INFO:tasks.rados.rados.0.smithi076.stdout:3668: writing smithi07685715-18 from 1696221 to 2139368 tid 2 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3668: writing smithi07685715-18 from 2891153 to 3210467 tid 3 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3669: delete oid 36 current snap is 462 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3665: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2304 (ObjNum 1186 snap 399 seq_num 1186) dirty exists 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: finishing write tid 1 to smithi07685715-20 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: finishing write tid 2 to smithi07685715-20 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: finishing write tid 3 to smithi07685715-20 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: finishing write tid 4 to smithi07685715-20 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: oid 20 updating version 0 to 1136 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: oid 20 version 1136 is already newer than 1134 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:3666: oid 20 version 1136 is already newer than 1135 2024-09-21T09:53:27.875 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 1136 (ObjNum 1380 snap 462 seq_num 1380) dirty exists 2024-09-21T09:53:27.876 INFO:tasks.rados.rados.0.smithi076.stdout:3666: left oid 20 (ObjNum 1380 snap 462 seq_num 1380) 2024-09-21T09:53:27.876 INFO:tasks.rados.rados.0.smithi076.stdout:3668: finishing write tid 1 to smithi07685715-18 2024-09-21T09:53:27.876 INFO:tasks.rados.rados.0.smithi076.stdout:3668: finishing write tid 2 to smithi07685715-18 2024-09-21T09:53:27.877 INFO:tasks.rados.rados.0.smithi076.stdout:3668: finishing write tid 3 to smithi07685715-18 2024-09-21T09:53:27.877 INFO:tasks.rados.rados.0.smithi076.stdout:3665: done (3 left) 2024-09-21T09:53:27.877 INFO:tasks.rados.rados.0.smithi076.stdout:3666: done (2 left) 2024-09-21T09:53:27.877 INFO:tasks.rados.rados.0.smithi076.stdout:3669: done (1 left) 2024-09-21T09:53:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3670: write oid 22 current snap is 462 2024-09-21T09:53:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3670: seq_num 1383 ranges {444081=729260,1813185=446816,2945999=298082} 2024-09-21T09:53:27.884 INFO:tasks.rados.rados.0.smithi076.stdout:3670: writing smithi07685715-22 from 444081 to 1173341 tid 1 2024-09-21T09:53:27.890 INFO:tasks.rados.rados.0.smithi076.stdout:3670: writing smithi07685715-22 from 1813185 to 2260001 tid 2 2024-09-21T09:53:27.896 INFO:tasks.rados.rados.0.smithi076.stdout:3670: writing smithi07685715-22 from 2945999 to 3244081 tid 3 2024-09-21T09:53:27.896 INFO:tasks.rados.rados.0.smithi076.stdout:3671: read oid 33 snap 408 2024-09-21T09:53:27.896 INFO:tasks.rados.rados.0.smithi076.stdout:3671: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3672: rollback oid 44 current snap is 462 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 460 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: finishing write tid 4 to smithi07685715-18 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: finishing write tid 5 to smithi07685715-18 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: oid 18 updating version 0 to 971 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: oid 18 version 971 is already newer than 966 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: oid 18 version 971 is already newer than 969 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: oid 18 version 971 is already newer than 970 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 971 (ObjNum 1381 snap 462 seq_num 1381) dirty exists 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3668: left oid 18 (ObjNum 1381 snap 462 seq_num 1381) 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3670: finishing write tid 1 to smithi07685715-22 2024-09-21T09:53:27.897 INFO:tasks.rados.rados.0.smithi076.stdout:3670: finishing write tid 2 to smithi07685715-22 2024-09-21T09:53:27.898 INFO:tasks.rados.rados.0.smithi076.stdout:3668: done (3 left) 2024-09-21T09:53:27.898 INFO:tasks.rados.rados.0.smithi076.stdout:3673: write oid 47 current snap is 462 2024-09-21T09:53:27.898 INFO:tasks.rados.rados.0.smithi076.stdout:3673: seq_num 1384 ranges {460888=520862,1471667=590474,2573422=455407} 2024-09-21T09:53:27.901 INFO:tasks.rados.rados.0.smithi076.stdout:3673: writing smithi07685715-47 from 460888 to 981750 tid 1 2024-09-21T09:53:27.907 INFO:tasks.rados.rados.0.smithi076.stdout:3673: writing smithi07685715-47 from 1471667 to 2062141 tid 2 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3673: writing smithi07685715-47 from 2573422 to 3028829 tid 3 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3674: read oid 41 snap -1 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3674: expect (ObjNum 1363 snap 454 seq_num 1363) 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3675: write oid 9 current snap is 462 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3672: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3672: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:53:27.914 INFO:tasks.rados.rados.0.smithi076.stdout:3670: finishing write tid 3 to smithi07685715-22 2024-09-21T09:53:27.915 INFO:tasks.rados.rados.0.smithi076.stdout:3670: finishing write tid 4 to smithi07685715-22 2024-09-21T09:53:27.915 INFO:tasks.rados.rados.0.smithi076.stdout:3675: seq_num 1385 ranges {477695=596111,1530149=450485,2600845=666679} 2024-09-21T09:53:27.919 INFO:tasks.rados.rados.0.smithi076.stdout:3675: writing smithi07685715-9 from 477695 to 1073806 tid 1 2024-09-21T09:53:27.924 INFO:tasks.rados.rados.0.smithi076.stdout:3675: writing smithi07685715-9 from 1530149 to 1980634 tid 2 2024-09-21T09:53:27.931 INFO:tasks.rados.rados.0.smithi076.stdout:3675: writing smithi07685715-9 from 2600845 to 3267524 tid 3 2024-09-21T09:53:27.931 INFO:tasks.rados.rados.0.smithi076.stdout:3676: read oid 40 snap -1 2024-09-21T09:53:27.931 INFO:tasks.rados.rados.0.smithi076.stdout:3676: expect (ObjNum 1354 snap 453 seq_num 1354) 2024-09-21T09:53:27.931 INFO:tasks.rados.rados.0.smithi076.stdout:3677: snap_create 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3672: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 798 (ObjNum 1355 snap 453 seq_num 1355) dirty exists 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3670: finishing write tid 5 to smithi07685715-22 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3670: oid 22 updating version 0 to 1869 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3670: oid 22 updating version 1869 to 1871 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3670: oid 22 version 1871 is already newer than 1868 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3670: oid 22 version 1871 is already newer than 1870 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1871 (ObjNum 1383 snap 462 seq_num 1383) dirty exists 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3670: left oid 22 (ObjNum 1383 snap 462 seq_num 1383) 2024-09-21T09:53:27.932 INFO:tasks.rados.rados.0.smithi076.stdout:3671: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:53:28.103 INFO:tasks.rados.rados.0.smithi076.stdout:3673: finishing write tid 1 to smithi07685715-47 2024-09-21T09:53:28.103 INFO:tasks.rados.rados.0.smithi076.stdout:3673: finishing write tid 2 to smithi07685715-47 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: finishing write tid 3 to smithi07685715-47 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: finishing write tid 4 to smithi07685715-47 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: finishing write tid 5 to smithi07685715-47 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: oid 47 updating version 0 to 2817 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: oid 47 version 2817 is already newer than 2815 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: oid 47 updating version 2817 to 2818 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: oid 47 version 2818 is already newer than 2816 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2818 (ObjNum 1384 snap 462 seq_num 1384) dirty exists 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3673: left oid 47 (ObjNum 1384 snap 462 seq_num 1384) 2024-09-21T09:53:28.104 INFO:tasks.rados.rados.0.smithi076.stdout:3675: finishing write tid 1 to smithi07685715-9 2024-09-21T09:53:28.105 INFO:tasks.rados.rados.0.smithi076.stdout:3674: expect (ObjNum 1363 snap 454 seq_num 1363) 2024-09-21T09:53:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:27 smithi191 ceph-mon[102206]: pgmap v958: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 264 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 968 KiB/s wr, 7 op/s 2024-09-21T09:53:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:27 smithi191 ceph-mon[102206]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:53:28.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:27 smithi076 ceph-mon[106482]: pgmap v958: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 264 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 968 KiB/s wr, 7 op/s 2024-09-21T09:53:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:27 smithi076 ceph-mon[106482]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:53:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:27 smithi076 ceph-mon[110963]: pgmap v958: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 264 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 968 KiB/s wr, 7 op/s 2024-09-21T09:53:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:27 smithi076 ceph-mon[110963]: osdmap e992: 8 total, 8 up, 8 in 2024-09-21T09:53:28.253 INFO:tasks.rados.rados.0.smithi076.stdout:3675: finishing write tid 2 to smithi07685715-9 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: finishing write tid 3 to smithi07685715-9 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: finishing write tid 4 to smithi07685715-9 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: finishing write tid 5 to smithi07685715-9 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: oid 9 updating version 0 to 1290 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: oid 9 version 1290 is already newer than 1288 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: oid 9 updating version 1290 to 1291 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: oid 9 version 1291 is already newer than 1289 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 1291 (ObjNum 1385 snap 462 seq_num 1385) dirty exists 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3675: left oid 9 (ObjNum 1385 snap 462 seq_num 1385) 2024-09-21T09:53:28.254 INFO:tasks.rados.rados.0.smithi076.stdout:3676: expect (ObjNum 1354 snap 453 seq_num 1354) 2024-09-21T09:53:28.812 INFO:tasks.rados.rados.0.smithi076.stdout:3670: done (7 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3671: done (6 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3672: done (5 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3673: done (4 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3674: done (3 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3675: done (2 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3676: done (1 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3677: done (0 left) 2024-09-21T09:53:28.813 INFO:tasks.rados.rados.0.smithi076.stdout:3678: write oid 16 current snap is 463 2024-09-21T09:53:28.814 INFO:tasks.rados.rados.0.smithi076.stdout:3678: seq_num 1386 ranges {494502=671360,1588631=594143,2628268=477951} 2024-09-21T09:53:28.818 INFO:tasks.rados.rados.0.smithi076.stdout:3678: writing smithi07685715-16 from 494502 to 1165862 tid 1 2024-09-21T09:53:28.824 INFO:tasks.rados.rados.0.smithi076.stdout:3678: writing smithi07685715-16 from 1588631 to 2182774 tid 2 2024-09-21T09:53:28.829 INFO:tasks.rados.rados.0.smithi076.stdout:3678: writing smithi07685715-16 from 2628268 to 3106219 tid 3 2024-09-21T09:53:28.829 INFO:tasks.rados.rados.0.smithi076.stdout:3679: snap_remove snap 451 2024-09-21T09:53:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:28 smithi191 ceph-mon[102206]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:53:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:28 smithi191 ceph-mon[102206]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:53:29.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:28 smithi076 ceph-mon[106482]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:53:29.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:28 smithi076 ceph-mon[106482]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:53:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:28 smithi076 ceph-mon[110963]: osdmap e993: 8 total, 8 up, 8 in 2024-09-21T09:53:29.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:28 smithi076 ceph-mon[110963]: osdmap e994: 8 total, 8 up, 8 in 2024-09-21T09:53:29.819 INFO:tasks.rados.rados.0.smithi076.stdout:3679: done (1 left) 2024-09-21T09:53:29.819 INFO:tasks.rados.rados.0.smithi076.stdout:3680: delete oid 28 current snap is 463 2024-09-21T09:53:29.819 INFO:tasks.rados.rados.0.smithi076.stdout:3678: finishing write tid 1 to smithi07685715-16 2024-09-21T09:53:29.819 INFO:tasks.rados.rados.0.smithi076.stdout:3678: finishing write tid 2 to smithi07685715-16 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: finishing write tid 3 to smithi07685715-16 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: finishing write tid 4 to smithi07685715-16 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: finishing write tid 5 to smithi07685715-16 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: oid 16 updating version 0 to 1875 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: oid 16 updating version 1875 to 1876 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: oid 16 version 1876 is already newer than 1874 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: oid 16 version 1876 is already newer than 1873 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1876 (ObjNum 1386 snap 463 seq_num 1386) dirty exists 2024-09-21T09:53:29.820 INFO:tasks.rados.rados.0.smithi076.stdout:3678: left oid 16 (ObjNum 1386 snap 463 seq_num 1386) 2024-09-21T09:53:29.826 INFO:tasks.rados.rados.0.smithi076.stdout:3678: done (1 left) 2024-09-21T09:53:29.826 INFO:tasks.rados.rados.0.smithi076.stdout:3680: done (0 left) 2024-09-21T09:53:29.826 INFO:tasks.rados.rados.0.smithi076.stdout:3681: snap_create 2024-09-21T09:53:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:29 smithi191 ceph-mon[102206]: pgmap v961: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 264 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 968 KiB/s wr, 5 op/s 2024-09-21T09:53:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:29 smithi191 ceph-mon[102206]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:53:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:29 smithi076 ceph-mon[106482]: pgmap v961: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 264 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 968 KiB/s wr, 5 op/s 2024-09-21T09:53:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:29 smithi076 ceph-mon[106482]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:53:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:29 smithi076 ceph-mon[110963]: pgmap v961: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 264 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 968 KiB/s wr, 5 op/s 2024-09-21T09:53:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:29 smithi076 ceph-mon[110963]: osdmap e995: 8 total, 8 up, 8 in 2024-09-21T09:53:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:3681: done (0 left) 2024-09-21T09:53:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:3682: rollback oid 30 current snap is 464 2024-09-21T09:53:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 460 2024-09-21T09:53:30.815 INFO:tasks.rados.rados.0.smithi076.stdout:3683: snap_remove snap 441 2024-09-21T09:53:31.832 INFO:tasks.rados.rados.0.smithi076.stdout:3683: done (1 left) 2024-09-21T09:53:31.832 INFO:tasks.rados.rados.0.smithi076.stdout:3684: read oid 9 snap 461 2024-09-21T09:53:31.832 INFO:tasks.rados.rados.0.smithi076.stdout:3684: expect (ObjNum 1364 snap 456 seq_num 1364) 2024-09-21T09:53:31.832 INFO:tasks.rados.rados.0.smithi076.stdout:3685: rollback oid 3 current snap is 464 2024-09-21T09:53:31.832 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 463 2024-09-21T09:53:31.833 INFO:tasks.rados.rados.0.smithi076.stdout:3682: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:53:31.833 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2733 (ObjNum 864031872 snap 21884 seq_num 304) dirty dne 2024-09-21T09:53:31.833 INFO:tasks.rados.rados.0.smithi076.stdout:3682: done (2 left) 2024-09-21T09:53:31.833 INFO:tasks.rados.rados.0.smithi076.stdout:3686: rollback oid 33 current snap is 464 2024-09-21T09:53:31.833 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 408 2024-09-21T09:53:31.833 INFO:tasks.rados.rados.0.smithi076.stdout:3687: snap_remove snap 456 2024-09-21T09:53:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:31 smithi191 ceph-mon[102206]: pgmap v964: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 275 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-21T09:53:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:31 smithi191 ceph-mon[102206]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:53:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:31 smithi076 ceph-mon[106482]: pgmap v964: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 275 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-21T09:53:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:31 smithi076 ceph-mon[106482]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:53:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:31 smithi076 ceph-mon[110963]: pgmap v964: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 275 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.4 MiB/s wr, 5 op/s 2024-09-21T09:53:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:31 smithi076 ceph-mon[110963]: osdmap e996: 8 total, 8 up, 8 in 2024-09-21T09:53:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:3687: done (3 left) 2024-09-21T09:53:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:3688: rollback oid 48 current snap is 464 2024-09-21T09:53:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 461 2024-09-21T09:53:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:3685: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:53:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 285 (ObjNum 113 snap 0 seq_num 1683069495) dirty dne 2024-09-21T09:53:32.835 INFO:tasks.rados.rados.0.smithi076.stdout:3686: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:53:32.836 INFO:tasks.rados.rados.0.smithi076.stdout:3686: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:53:32.836 INFO:tasks.rados.rados.0.smithi076.stdout:3686: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:53:32.836 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 828 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2024-09-21T09:53:32.836 INFO:tasks.rados.rados.0.smithi076.stdout:3684: expect (ObjNum 1364 snap 456 seq_num 1364) 2024-09-21T09:53:32.969 INFO:tasks.rados.rados.0.smithi076.stdout:3688: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:53:32.969 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2812 (ObjNum 129 snap 0 seq_num 865156448) dirty dne 2024-09-21T09:53:32.969 INFO:tasks.rados.rados.0.smithi076.stdout:3684: done (3 left) 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3685: done (2 left) 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3686: done (1 left) 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3688: done (0 left) 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3689: read oid 15 snap 461 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3689: expect deleted 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3690: write oid 13 current snap is 464 2024-09-21T09:53:32.970 INFO:tasks.rados.rados.0.smithi076.stdout:3690: seq_num 1388 ranges {528116=421858,1589242=597812,2966761=361355} 2024-09-21T09:53:32.974 INFO:tasks.rados.rados.0.smithi076.stdout:3690: writing smithi07685715-13 from 528116 to 949974 tid 1 2024-09-21T09:53:32.981 INFO:tasks.rados.rados.0.smithi076.stdout:3690: writing smithi07685715-13 from 1589242 to 2187054 tid 2 2024-09-21T09:53:32.987 INFO:tasks.rados.rados.0.smithi076.stdout:3690: writing smithi07685715-13 from 2966761 to 3328116 tid 3 2024-09-21T09:53:32.987 INFO:tasks.rados.rados.0.smithi076.stdout:3691: write oid 41 current snap is 464 2024-09-21T09:53:32.987 INFO:tasks.rados.rados.0.smithi076.stdout:3691: seq_num 1389 ranges {544923=497107,1647724=457823,2710537=428120} 2024-09-21T09:53:32.993 INFO:tasks.rados.rados.0.smithi076.stdout:3691: writing smithi07685715-41 from 544923 to 1042030 tid 1 2024-09-21T09:53:32.998 INFO:tasks.rados.rados.0.smithi076.stdout:3691: writing smithi07685715-41 from 1647724 to 2105547 tid 2 2024-09-21T09:53:33.004 INFO:tasks.rados.rados.0.smithi076.stdout:3691: writing smithi07685715-41 from 2710537 to 3138657 tid 3 2024-09-21T09:53:33.005 INFO:tasks.rados.rados.0.smithi076.stdout:3692: write oid 49 current snap is 464 2024-09-21T09:53:33.005 INFO:tasks.rados.rados.0.smithi076.stdout:3692: seq_num 1390 ranges {561730=572356,1706206=601481,3021607=340123} 2024-09-21T09:53:33.010 INFO:tasks.rados.rados.0.smithi076.stdout:3692: writing smithi07685715-49 from 561730 to 1134086 tid 1 2024-09-21T09:53:33.017 INFO:tasks.rados.rados.0.smithi076.stdout:3692: writing smithi07685715-49 from 1706206 to 2307687 tid 2 2024-09-21T09:53:33.024 INFO:tasks.rados.rados.0.smithi076.stdout:3692: writing smithi07685715-49 from 3021607 to 3361730 tid 3 2024-09-21T09:53:33.024 INFO:tasks.rados.rados.0.smithi076.stdout:3693: read oid 2 snap 453 2024-09-21T09:53:33.024 INFO:tasks.rados.rados.0.smithi076.stdout:3693: expect deleted 2024-09-21T09:53:33.024 INFO:tasks.rados.rados.0.smithi076.stdout:3694: snap_remove snap 462 2024-09-21T09:53:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:32 smithi191 ceph-mon[102206]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:53:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:32 smithi076 ceph-mon[106482]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:53:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:32 smithi076 ceph-mon[110963]: osdmap e997: 8 total, 8 up, 8 in 2024-09-21T09:53:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:53:33] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3694: done (5 left) 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3695: delete oid 26 current snap is 464 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3690: finishing write tid 1 to smithi07685715-13 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3690: finishing write tid 2 to smithi07685715-13 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3690: finishing write tid 3 to smithi07685715-13 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3690: finishing write tid 4 to smithi07685715-13 2024-09-21T09:53:33.814 INFO:tasks.rados.rados.0.smithi076.stdout:3690: finishing write tid 5 to smithi07685715-13 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3690: oid 13 updating version 0 to 769 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3690: oid 13 updating version 769 to 770 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3690: oid 13 updating version 770 to 771 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3690: oid 13 version 771 is already newer than 768 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 771 (ObjNum 1388 snap 464 seq_num 1388) dirty exists 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3690: left oid 13 (ObjNum 1388 snap 464 seq_num 1388) 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3691: finishing write tid 1 to smithi07685715-41 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3691: finishing write tid 2 to smithi07685715-41 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3691: finishing write tid 3 to smithi07685715-41 2024-09-21T09:53:33.815 INFO:tasks.rados.rados.0.smithi076.stdout:3691: finishing write tid 4 to smithi07685715-41 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3691: finishing write tid 5 to smithi07685715-41 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3691: oid 41 updating version 0 to 1894 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3691: oid 41 updating version 1894 to 1895 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3691: oid 41 updating version 1895 to 1897 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3691: oid 41 version 1897 is already newer than 1896 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 1897 (ObjNum 1389 snap 464 seq_num 1389) dirty exists 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3691: left oid 41 (ObjNum 1389 snap 464 seq_num 1389) 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3692: finishing write tid 1 to smithi07685715-49 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3692: finishing write tid 2 to smithi07685715-49 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3692: finishing write tid 3 to smithi07685715-49 2024-09-21T09:53:33.816 INFO:tasks.rados.rados.0.smithi076.stdout:3692: finishing write tid 4 to smithi07685715-49 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:3692: finishing write tid 5 to smithi07685715-49 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:3692: oid 49 updating version 0 to 2833 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:3692: oid 49 updating version 2833 to 2834 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:3692: oid 49 updating version 2834 to 2835 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:3692: oid 49 updating version 2835 to 2836 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 49 v 2836 (ObjNum 1390 snap 464 seq_num 1390) dirty exists 2024-09-21T09:53:33.817 INFO:tasks.rados.rados.0.smithi076.stdout:3692: left oid 49 (ObjNum 1390 snap 464 seq_num 1390) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3689: done (5 left) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3690: done (4 left) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3691: done (3 left) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3692: done (2 left) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3693: done (1 left) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3695: done (0 left) 2024-09-21T09:53:33.818 INFO:tasks.rados.rados.0.smithi076.stdout:3696: delete oid 41 current snap is 464 2024-09-21T09:53:33.824 INFO:tasks.rados.rados.0.smithi076.stdout:3696: done (0 left) 2024-09-21T09:53:33.824 INFO:tasks.rados.rados.0.smithi076.stdout:3697: read oid 39 snap -1 2024-09-21T09:53:33.824 INFO:tasks.rados.rados.0.smithi076.stdout:3697: expect deleted 2024-09-21T09:53:33.824 INFO:tasks.rados.rados.0.smithi076.stdout:3698: write oid 45 current snap is 464 2024-09-21T09:53:33.825 INFO:tasks.rados.rados.0.smithi076.stdout:3698: seq_num 1393 ranges {612151=514456,1881652=748808,3103876=308275} 2024-09-21T09:53:33.830 INFO:tasks.rados.rados.0.smithi076.stdout:3698: writing smithi07685715-45 from 612151 to 1126607 tid 1 2024-09-21T09:53:33.838 INFO:tasks.rados.rados.0.smithi076.stdout:3698: writing smithi07685715-45 from 1881652 to 2630460 tid 2 2024-09-21T09:53:33.843 INFO:tasks.rados.rados.0.smithi076.stdout:3698: writing smithi07685715-45 from 3103876 to 3412151 tid 3 2024-09-21T09:53:33.843 INFO:tasks.rados.rados.0.smithi076.stdout:3699: delete oid 43 current snap is 464 2024-09-21T09:53:33.843 INFO:tasks.rados.rados.0.smithi076.stdout:3698: finishing write tid 1 to smithi07685715-45 2024-09-21T09:53:33.843 INFO:tasks.rados.rados.0.smithi076.stdout:3698: finishing write tid 2 to smithi07685715-45 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: finishing write tid 3 to smithi07685715-45 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3697: done (2 left) 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3699: done (1 left) 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3700: delete oid 25 current snap is 464 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: finishing write tid 4 to smithi07685715-45 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: finishing write tid 5 to smithi07685715-45 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: oid 45 updating version 0 to 793 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: oid 45 version 793 is already newer than 791 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: oid 45 version 793 is already newer than 792 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:3698: oid 45 version 793 is already newer than 790 2024-09-21T09:53:33.846 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 793 (ObjNum 1393 snap 464 seq_num 1393) dirty exists 2024-09-21T09:53:33.847 INFO:tasks.rados.rados.0.smithi076.stdout:3698: left oid 45 (ObjNum 1393 snap 464 seq_num 1393) 2024-09-21T09:53:33.847 INFO:tasks.rados.rados.0.smithi076.stdout:3698: done (1 left) 2024-09-21T09:53:33.847 INFO:tasks.rados.rados.0.smithi076.stdout:3700: done (0 left) 2024-09-21T09:53:33.847 INFO:tasks.rados.rados.0.smithi076.stdout:3701: delete oid 40 current snap is 464 2024-09-21T09:53:33.849 INFO:tasks.rados.rados.0.smithi076.stdout:3701: done (0 left) 2024-09-21T09:53:33.849 INFO:tasks.rados.rados.0.smithi076.stdout:3702: snap_remove snap 457 2024-09-21T09:53:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:33 smithi191 ceph-mon[102206]: pgmap v967: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 284 MiB data, 946 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.8 MiB/s wr, 16 op/s 2024-09-21T09:53:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:33 smithi191 ceph-mon[102206]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:53:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:33 smithi191 ceph-mon[102206]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:53:34.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-mon[106482]: pgmap v967: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 284 MiB data, 946 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.8 MiB/s wr, 16 op/s 2024-09-21T09:53:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-mon[106482]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:53:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-mon[106482]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:53:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-mon[110963]: pgmap v967: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 284 MiB data, 946 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.8 MiB/s wr, 16 op/s 2024-09-21T09:53:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-mon[110963]: osdmap e998: 8 total, 8 up, 8 in 2024-09-21T09:53:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:33 smithi076 ceph-mon[110963]: osdmap e999: 8 total, 8 up, 8 in 2024-09-21T09:53:34.842 INFO:tasks.rados.rados.0.smithi076.stdout:3702: done (0 left) 2024-09-21T09:53:34.843 INFO:tasks.rados.rados.0.smithi076.stdout:3703: snap_remove snap 424 2024-09-21T09:53:35.843 INFO:tasks.rados.rados.0.smithi076.stdout:3703: done (0 left) 2024-09-21T09:53:35.843 INFO:tasks.rados.rados.0.smithi076.stdout:3704: write oid 31 current snap is 464 2024-09-21T09:53:35.843 INFO:tasks.rados.rados.0.smithi076.stdout:3704: seq_num 1397 ranges {679379=415452,1599227=756146,3097215=382164} 2024-09-21T09:53:35.850 INFO:tasks.rados.rados.0.smithi076.stdout:3704: writing smithi07685715-31 from 679379 to 1094831 tid 1 2024-09-21T09:53:35.858 INFO:tasks.rados.rados.0.smithi076.stdout:3704: writing smithi07685715-31 from 1599227 to 2355373 tid 2 2024-09-21T09:53:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3704: writing smithi07685715-31 from 3097215 to 3479379 tid 3 2024-09-21T09:53:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3705: snap_create 2024-09-21T09:53:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3704: finishing write tid 1 to smithi07685715-31 2024-09-21T09:53:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3704: finishing write tid 2 to smithi07685715-31 2024-09-21T09:53:35.867 INFO:tasks.rados.rados.0.smithi076.stdout:3704: finishing write tid 3 to smithi07685715-31 2024-09-21T09:53:35.867 INFO:tasks.rados.rados.0.smithi076.stdout:3704: finishing write tid 4 to smithi07685715-31 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:3704: finishing write tid 5 to smithi07685715-31 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:3704: oid 31 updating version 0 to 1150 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:3704: oid 31 version 1150 is already newer than 1149 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:3704: oid 31 version 1150 is already newer than 1148 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:3704: oid 31 version 1150 is already newer than 1143 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 1150 (ObjNum 1397 snap 464 seq_num 1397) dirty exists 2024-09-21T09:53:35.868 INFO:tasks.rados.rados.0.smithi076.stdout:3704: left oid 31 (ObjNum 1397 snap 464 seq_num 1397) 2024-09-21T09:53:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:35 smithi191 ceph-mon[102206]: pgmap v970: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 286 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:53:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:35 smithi191 ceph-mon[102206]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:53:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:35 smithi076 ceph-mon[106482]: pgmap v970: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 286 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:53:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:35 smithi076 ceph-mon[106482]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:53:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:35 smithi076 ceph-mon[110963]: pgmap v970: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 286 MiB data, 945 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-09-21T09:53:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:35 smithi076 ceph-mon[110963]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-21T09:53:36.858 INFO:tasks.rados.rados.0.smithi076.stdout:3704: done (1 left) 2024-09-21T09:53:36.859 INFO:tasks.rados.rados.0.smithi076.stdout:3705: done (0 left) 2024-09-21T09:53:36.859 INFO:tasks.rados.rados.0.smithi076.stdout:3706: rollback oid 14 current snap is 465 2024-09-21T09:53:36.859 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 463 2024-09-21T09:53:36.859 INFO:tasks.rados.rados.0.smithi076.stdout:3707: write oid 3 current snap is 465 2024-09-21T09:53:36.859 INFO:tasks.rados.rados.0.smithi076.stdout:3707: seq_num 1398 ranges {696186=490701,1657709=616157,2724638=562274} 2024-09-21T09:53:36.865 INFO:tasks.rados.rados.0.smithi076.stdout:3707: writing smithi07685715-3 from 696186 to 1186887 tid 1 2024-09-21T09:53:36.870 INFO:tasks.rados.rados.0.smithi076.stdout:3707: writing smithi07685715-3 from 1657709 to 2273866 tid 2 2024-09-21T09:53:36.876 INFO:tasks.rados.rados.0.smithi076.stdout:3707: writing smithi07685715-3 from 2724638 to 3286912 tid 3 2024-09-21T09:53:36.876 INFO:tasks.rados.rados.0.smithi076.stdout:3708: delete oid 10 current snap is 465 2024-09-21T09:53:36.876 INFO:tasks.rados.rados.0.smithi076.stdout:3706: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:53:36.876 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 2295 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-09-21T09:53:36.876 INFO:tasks.rados.rados.0.smithi076.stdout:3707: finishing write tid 1 to smithi07685715-3 2024-09-21T09:53:36.876 INFO:tasks.rados.rados.0.smithi076.stdout:3707: finishing write tid 2 to smithi07685715-3 2024-09-21T09:53:36.879 INFO:tasks.rados.rados.0.smithi076.stdout:3707: finishing write tid 3 to smithi07685715-3 2024-09-21T09:53:36.879 INFO:tasks.rados.rados.0.smithi076.stdout:3707: finishing write tid 4 to smithi07685715-3 2024-09-21T09:53:36.879 INFO:tasks.rados.rados.0.smithi076.stdout:3707: finishing write tid 5 to smithi07685715-3 2024-09-21T09:53:36.879 INFO:tasks.rados.rados.0.smithi076.stdout:3707: oid 3 updating version 0 to 305 2024-09-21T09:53:36.879 INFO:tasks.rados.rados.0.smithi076.stdout:3707: oid 3 version 305 is already newer than 304 2024-09-21T09:53:36.880 INFO:tasks.rados.rados.0.smithi076.stdout:3707: oid 3 version 305 is already newer than 302 2024-09-21T09:53:36.880 INFO:tasks.rados.rados.0.smithi076.stdout:3707: oid 3 version 305 is already newer than 303 2024-09-21T09:53:36.880 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 305 (ObjNum 1398 snap 465 seq_num 1398) dirty exists 2024-09-21T09:53:36.880 INFO:tasks.rados.rados.0.smithi076.stdout:3707: left oid 3 (ObjNum 1398 snap 465 seq_num 1398) 2024-09-21T09:53:36.881 INFO:tasks.rados.rados.0.smithi076.stdout:3706: done (2 left) 2024-09-21T09:53:36.881 INFO:tasks.rados.rados.0.smithi076.stdout:3707: done (1 left) 2024-09-21T09:53:36.881 INFO:tasks.rados.rados.0.smithi076.stdout:3708: done (0 left) 2024-09-21T09:53:36.881 INFO:tasks.rados.rados.0.smithi076.stdout:3709: snap_create 2024-09-21T09:53:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:36 smithi191 ceph-mon[102206]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:53:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:36 smithi076 ceph-mon[106482]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:53:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:36 smithi076 ceph-mon[110963]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3709: done (0 left) 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3710: read oid 18 snap -1 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3710: expect (ObjNum 1381 snap 462 seq_num 1381) 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3711: rollback oid 32 current snap is 466 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 32 to 464 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3712: write oid 8 current snap is 466 2024-09-21T09:53:37.866 INFO:tasks.rados.rados.0.smithi076.stdout:3712: seq_num 1400 ranges {729800=757552,2174673=619826} 2024-09-21T09:53:37.874 INFO:tasks.rados.rados.0.smithi076.stdout:3712: writing smithi07685715-8 from 729800 to 1487352 tid 1 2024-09-21T09:53:37.882 INFO:tasks.rados.rados.0.smithi076.stdout:3712: writing smithi07685715-8 from 2174673 to 2794499 tid 2 2024-09-21T09:53:37.882 INFO:tasks.rados.rados.0.smithi076.stdout:3711: finishing rollback tid 1 to smithi07685715-32 2024-09-21T09:53:37.882 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 32 v 1114 (ObjNum 129 snap 0 seq_num 865155616) dirty dne 2024-09-21T09:53:37.882 INFO:tasks.rados.rados.0.smithi076.stdout:3710: expect (ObjNum 1381 snap 462 seq_num 1381) 2024-09-21T09:53:37.986 INFO:tasks.rados.rados.0.smithi076.stdout:3712: finishing write tid 1 to smithi07685715-8 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: finishing write tid 2 to smithi07685715-8 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: finishing write tid 3 to smithi07685715-8 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: finishing write tid 4 to smithi07685715-8 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: oid 8 updating version 0 to 2854 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: oid 8 version 2854 is already newer than 2853 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: oid 8 updating version 2854 to 2855 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 8 v 2855 (ObjNum 1400 snap 466 seq_num 1400) dirty exists 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: left oid 8 (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3713: read oid 16 snap -1 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3713: expect (ObjNum 1386 snap 463 seq_num 1386) 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3710: done (3 left) 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3711: done (2 left) 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3712: done (1 left) 2024-09-21T09:53:37.987 INFO:tasks.rados.rados.0.smithi076.stdout:3714: delete oid 30 current snap is 466 2024-09-21T09:53:37.989 INFO:tasks.rados.rados.0.smithi076.stdout:3714: done (1 left) 2024-09-21T09:53:37.989 INFO:tasks.rados.rados.0.smithi076.stdout:3715: rollback oid 14 current snap is 466 2024-09-21T09:53:37.990 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 14 to 463 2024-09-21T09:53:37.990 INFO:tasks.rados.rados.0.smithi076.stdout:3716: rollback oid 10 current snap is 466 2024-09-21T09:53:37.990 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 10 to 460 2024-09-21T09:53:37.990 INFO:tasks.rados.rados.0.smithi076.stdout:3717: write oid 28 current snap is 466 2024-09-21T09:53:37.990 INFO:tasks.rados.rados.0.smithi076.stdout:3717: seq_num 1402 ranges {763414=508050,1891637=623495,3234330=329084} 2024-09-21T09:53:37.997 INFO:tasks.rados.rados.0.smithi076.stdout:3717: writing smithi07685715-28 from 763414 to 1271464 tid 1 2024-09-21T09:53:38.004 INFO:tasks.rados.rados.0.smithi076.stdout:3717: writing smithi07685715-28 from 1891637 to 2515132 tid 2 2024-09-21T09:53:38.010 INFO:tasks.rados.rados.0.smithi076.stdout:3717: writing smithi07685715-28 from 3234330 to 3563414 tid 3 2024-09-21T09:53:38.011 INFO:tasks.rados.rados.0.smithi076.stdout:3718: read oid 5 snap -1 2024-09-21T09:53:38.011 INFO:tasks.rados.rados.0.smithi076.stdout:3718: expect (ObjNum 1370 snap 459 seq_num 1370) 2024-09-21T09:53:38.011 INFO:tasks.rados.rados.0.smithi076.stdout:3719: write oid 2 current snap is 466 2024-09-21T09:53:38.011 INFO:tasks.rados.rados.0.smithi076.stdout:3719: seq_num 1403 ranges {780221=583299,1950119=767153,3261753=318468} 2024-09-21T09:53:38.018 INFO:tasks.rados.rados.0.smithi076.stdout:3719: writing smithi07685715-2 from 780221 to 1363520 tid 1 2024-09-21T09:53:38.025 INFO:tasks.rados.rados.0.smithi076.stdout:3719: writing smithi07685715-2 from 1950119 to 2717272 tid 2 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:3719: writing smithi07685715-2 from 3261753 to 3580221 tid 3 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:3720: delete oid 13 current snap is 466 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:3715: finishing rollback tid 1 to smithi07685715-14 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 14 v 2295 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:3716: finishing rollback tid 1 to smithi07685715-10 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 10 v 2207 (ObjNum 33 snap 0 seq_num 865134736) dirty dne 2024-09-21T09:53:38.031 INFO:tasks.rados.rados.0.smithi076.stdout:3713: expect (ObjNum 1386 snap 463 seq_num 1386) 2024-09-21T09:53:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:37 smithi191 ceph-mon[102206]: pgmap v973: 89 pgs: 89 active+clean; 267 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:53:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:37 smithi191 ceph-mon[102206]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:53:38.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:37 smithi076 ceph-mon[110963]: pgmap v973: 89 pgs: 89 active+clean; 267 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:53:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:37 smithi076 ceph-mon[110963]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:53:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:37 smithi076 ceph-mon[106482]: pgmap v973: 89 pgs: 89 active+clean; 267 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T09:53:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:37 smithi076 ceph-mon[106482]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: finishing write tid 1 to smithi07685715-28 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: finishing write tid 2 to smithi07685715-28 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: finishing write tid 3 to smithi07685715-28 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: finishing write tid 4 to smithi07685715-28 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: finishing write tid 5 to smithi07685715-28 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: oid 28 updating version 0 to 2860 2024-09-21T09:53:38.237 INFO:tasks.rados.rados.0.smithi076.stdout:3717: oid 28 version 2860 is already newer than 2858 2024-09-21T09:53:38.238 INFO:tasks.rados.rados.0.smithi076.stdout:3717: oid 28 version 2860 is already newer than 2859 2024-09-21T09:53:38.238 INFO:tasks.rados.rados.0.smithi076.stdout:3717: oid 28 version 2860 is already newer than 2857 2024-09-21T09:53:38.238 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 2860 (ObjNum 1402 snap 466 seq_num 1402) dirty exists 2024-09-21T09:53:38.238 INFO:tasks.rados.rados.0.smithi076.stdout:3717: left oid 28 (ObjNum 1402 snap 466 seq_num 1402) 2024-09-21T09:53:38.238 INFO:tasks.rados.rados.0.smithi076.stdout:3718: expect (ObjNum 1370 snap 459 seq_num 1370) 2024-09-21T09:53:38.372 INFO:tasks.rados.rados.0.smithi076.stdout:3719: finishing write tid 1 to smithi07685715-2 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: finishing write tid 2 to smithi07685715-2 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: finishing write tid 3 to smithi07685715-2 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: finishing write tid 4 to smithi07685715-2 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: finishing write tid 5 to smithi07685715-2 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: oid 2 updating version 0 to 779 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: oid 2 updating version 779 to 780 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: oid 2 updating version 780 to 781 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: oid 2 version 781 is already newer than 778 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 781 (ObjNum 1403 snap 466 seq_num 1403) dirty exists 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3719: left oid 2 (ObjNum 1403 snap 466 seq_num 1403) 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3713: done (6 left) 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3715: done (5 left) 2024-09-21T09:53:38.373 INFO:tasks.rados.rados.0.smithi076.stdout:3716: done (4 left) 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:3717: done (3 left) 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:3718: done (2 left) 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:3719: done (1 left) 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:3720: done (0 left) 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:3721: rollback oid 7 current snap is 466 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 7 to 461 2024-09-21T09:53:38.374 INFO:tasks.rados.rados.0.smithi076.stdout:3722: snap_remove snap 443 2024-09-21T09:53:38.817 INFO:tasks.rados.rados.0.smithi076.stdout:3722: done (1 left) 2024-09-21T09:53:38.817 INFO:tasks.rados.rados.0.smithi076.stdout:3723: read oid 4 snap -1 2024-09-21T09:53:38.817 INFO:tasks.rados.rados.0.smithi076.stdout:3723: expect (ObjNum 1375 snap 461 seq_num 1375) 2024-09-21T09:53:38.817 INFO:tasks.rados.rados.0.smithi076.stdout:3724: snap_remove snap 450 2024-09-21T09:53:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:38 smithi076 ceph-mon[106482]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:53:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:38 smithi076 ceph-mon[106482]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:53:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:38 smithi076 ceph-mon[110963]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:53:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:38 smithi076 ceph-mon[110963]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:53:39.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:38 smithi191 ceph-mon[102206]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-21T09:53:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:38 smithi191 ceph-mon[102206]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:3724: done (2 left) 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:3725: delete oid 33 current snap is 466 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:3721: finishing rollback tid 0 to smithi07685715-7 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:3721: finishing rollback tid 1 to smithi07685715-7 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:3721: finishing rollback tid 2 to smithi07685715-7 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 7 v 2332 (ObjNum 1365 snap 456 seq_num 1365) dirty exists 2024-09-21T09:53:39.820 INFO:tasks.rados.rados.0.smithi076.stdout:3723: expect (ObjNum 1375 snap 461 seq_num 1375) 2024-09-21T09:53:39.993 INFO:tasks.rados.rados.0.smithi076.stdout:3721: done (2 left) 2024-09-21T09:53:39.993 INFO:tasks.rados.rados.0.smithi076.stdout:3723: done (1 left) 2024-09-21T09:53:39.994 INFO:tasks.rados.rados.0.smithi076.stdout:3725: done (0 left) 2024-09-21T09:53:39.994 INFO:tasks.rados.rados.0.smithi076.stdout:3726: snap_create 2024-09-21T09:53:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:39 smithi076 ceph-mon[110963]: pgmap v976: 89 pgs: 89 active+clean; 267 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-09-21T09:53:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:39 smithi076 ceph-mon[110963]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:53:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:39 smithi076 ceph-mon[106482]: pgmap v976: 89 pgs: 89 active+clean; 267 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-09-21T09:53:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:39 smithi076 ceph-mon[106482]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:53:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:39 smithi191 ceph-mon[102206]: pgmap v976: 89 pgs: 89 active+clean; 267 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-09-21T09:53:40.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:39 smithi191 ceph-mon[102206]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-21T09:53:40.874 INFO:tasks.rados.rados.0.smithi076.stdout:3726: done (0 left) 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:3727: rollback oid 3 current snap is 467 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 3 to 459 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:3728: rollback oid 16 current snap is 467 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 461 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:3729: rollback oid 33 current snap is 467 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 408 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:3730: write oid 27 current snap is 467 2024-09-21T09:53:40.875 INFO:tasks.rados.rados.0.smithi076.stdout:3730: seq_num 1406 ranges {430642=409046,1609212=514480,2827669=768803} 2024-09-21T09:53:40.879 INFO:tasks.rados.rados.0.smithi076.stdout:3730: writing smithi07685715-27 from 430642 to 839688 tid 1 2024-09-21T09:53:40.887 INFO:tasks.rados.rados.0.smithi076.stdout:3730: writing smithi07685715-27 from 1609212 to 2123692 tid 2 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:3730: writing smithi07685715-27 from 2827669 to 3596472 tid 3 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:3731: snap_create 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:3727: finishing rollback tid 0 to smithi07685715-3 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:3727: finishing rollback tid 1 to smithi07685715-3 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 312 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:3729: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:53:40.896 INFO:tasks.rados.rados.0.smithi076.stdout:3729: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 852 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:3728: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:3728: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:3728: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 1936 (ObjNum 1373 snap 460 seq_num 1373) dirty exists 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:3730: finishing write tid 1 to smithi07685715-27 2024-09-21T09:53:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:3730: finishing write tid 2 to smithi07685715-27 2024-09-21T09:53:40.900 INFO:tasks.rados.rados.0.smithi076.stdout:3730: finishing write tid 3 to smithi07685715-27 2024-09-21T09:53:40.900 INFO:tasks.rados.rados.0.smithi076.stdout:3730: finishing write tid 4 to smithi07685715-27 2024-09-21T09:53:40.900 INFO:tasks.rados.rados.0.smithi076.stdout:3730: finishing write tid 5 to smithi07685715-27 2024-09-21T09:53:40.901 INFO:tasks.rados.rados.0.smithi076.stdout:3730: oid 27 updating version 0 to 1937 2024-09-21T09:53:40.901 INFO:tasks.rados.rados.0.smithi076.stdout:3730: oid 27 updating version 1937 to 1939 2024-09-21T09:53:40.901 INFO:tasks.rados.rados.0.smithi076.stdout:3730: oid 27 version 1939 is already newer than 1938 2024-09-21T09:53:40.901 INFO:tasks.rados.rados.0.smithi076.stdout:3730: oid 27 updating version 1939 to 1940 2024-09-21T09:53:40.901 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1940 (ObjNum 1406 snap 467 seq_num 1406) dirty exists 2024-09-21T09:53:40.901 INFO:tasks.rados.rados.0.smithi076.stdout:3730: left oid 27 (ObjNum 1406 snap 467 seq_num 1406) 2024-09-21T09:53:41.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:41.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:41.893 INFO:tasks.rados.rados.0.smithi076.stdout:3727: done (4 left) 2024-09-21T09:53:41.893 INFO:tasks.rados.rados.0.smithi076.stdout:3728: done (3 left) 2024-09-21T09:53:41.893 INFO:tasks.rados.rados.0.smithi076.stdout:3729: done (2 left) 2024-09-21T09:53:41.894 INFO:tasks.rados.rados.0.smithi076.stdout:3730: done (1 left) 2024-09-21T09:53:41.894 INFO:tasks.rados.rados.0.smithi076.stdout:3731: done (0 left) 2024-09-21T09:53:41.894 INFO:tasks.rados.rados.0.smithi076.stdout:3732: snap_create 2024-09-21T09:53:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:41 smithi191 ceph-mon[102206]: pgmap v979: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 257 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 788 KiB/s wr, 6 op/s 2024-09-21T09:53:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:41 smithi191 ceph-mon[102206]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:53:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:41 smithi076 ceph-mon[106482]: pgmap v979: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 257 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 788 KiB/s wr, 6 op/s 2024-09-21T09:53:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:41 smithi076 ceph-mon[106482]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:53:42.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:41 smithi076 ceph-mon[110963]: pgmap v979: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 257 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 788 KiB/s wr, 6 op/s 2024-09-21T09:53:42.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:41 smithi076 ceph-mon[110963]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-21T09:53:42.899 INFO:tasks.rados.rados.0.smithi076.stdout:3732: done (0 left) 2024-09-21T09:53:42.899 INFO:tasks.rados.rados.0.smithi076.stdout:3733: write oid 6 current snap is 469 2024-09-21T09:53:42.899 INFO:tasks.rados.rados.0.smithi076.stdout:3733: seq_num 1407 ranges {447449=600648,1784047=774491,2971445=676004} 2024-09-21T09:53:42.905 INFO:tasks.rados.rados.0.smithi076.stdout:3733: writing smithi07685715-6 from 447449 to 1048097 tid 1 2024-09-21T09:53:42.913 INFO:tasks.rados.rados.0.smithi076.stdout:3733: writing smithi07685715-6 from 1784047 to 2558538 tid 2 2024-09-21T09:53:42.920 INFO:tasks.rados.rados.0.smithi076.stdout:3733: writing smithi07685715-6 from 2971445 to 3647449 tid 3 2024-09-21T09:53:42.920 INFO:tasks.rados.rados.0.smithi076.stdout:3734: read oid 23 snap -1 2024-09-21T09:53:42.920 INFO:tasks.rados.rados.0.smithi076.stdout:3734: expect (ObjNum 1359 snap 454 seq_num 1359) 2024-09-21T09:53:42.920 INFO:tasks.rados.rados.0.smithi076.stdout:3735: snap_remove snap 468 2024-09-21T09:53:43.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:42 smithi076 ceph-mon[106482]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:53:43.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:42 smithi076 ceph-mon[110963]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:53:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:42 smithi191 ceph-mon[102206]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-21T09:53:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:53:43] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-09-21T09:53:43.817 INFO:tasks.rados.rados.0.smithi076.stdout:3735: done (2 left) 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3736: rollback oid 48 current snap is 469 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 48 to 464 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: finishing write tid 1 to smithi07685715-6 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: finishing write tid 2 to smithi07685715-6 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: finishing write tid 3 to smithi07685715-6 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: finishing write tid 4 to smithi07685715-6 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: finishing write tid 5 to smithi07685715-6 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: oid 6 updating version 0 to 816 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: oid 6 version 816 is already newer than 814 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: oid 6 version 816 is already newer than 815 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: oid 6 version 816 is already newer than 813 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 816 (ObjNum 1407 snap 469 seq_num 1407) dirty exists 2024-09-21T09:53:43.818 INFO:tasks.rados.rados.0.smithi076.stdout:3733: left oid 6 (ObjNum 1407 snap 469 seq_num 1407) 2024-09-21T09:53:43.819 INFO:tasks.rados.rados.0.smithi076.stdout:3734: expect (ObjNum 1359 snap 454 seq_num 1359) 2024-09-21T09:53:43.910 INFO:tasks.rados.rados.0.smithi076.stdout:3736: finishing rollback tid 1 to smithi07685715-48 2024-09-21T09:53:43.911 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 48 v 2812 (ObjNum 1265 snap 0 seq_num 865132464) dirty dne 2024-09-21T09:53:43.911 INFO:tasks.rados.rados.0.smithi076.stdout:3733: done (2 left) 2024-09-21T09:53:43.911 INFO:tasks.rados.rados.0.smithi076.stdout:3734: done (1 left) 2024-09-21T09:53:43.911 INFO:tasks.rados.rados.0.smithi076.stdout:3736: done (0 left) 2024-09-21T09:53:43.911 INFO:tasks.rados.rados.0.smithi076.stdout:3737: snap_remove snap 463 2024-09-21T09:53:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-mon[106482]: pgmap v982: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 260 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:53:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-mon[106482]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:53:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-mon[106482]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:53:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-mon[110963]: pgmap v982: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 260 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:53:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-mon[110963]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:53:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:43 smithi076 ceph-mon[110963]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:53:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:43 smithi191 ceph-mon[102206]: pgmap v982: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 260 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 2.2 MiB/s wr, 18 op/s 2024-09-21T09:53:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:43 smithi191 ceph-mon[102206]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-21T09:53:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:43 smithi191 ceph-mon[102206]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-21T09:53:44.907 INFO:tasks.rados.rados.0.smithi076.stdout:3737: done (0 left) 2024-09-21T09:53:44.907 INFO:tasks.rados.rados.0.smithi076.stdout:3738: rollback oid 39 current snap is 469 2024-09-21T09:53:44.907 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 466 2024-09-21T09:53:44.907 INFO:tasks.rados.rados.0.smithi076.stdout:3739: snap_create 2024-09-21T09:53:44.915 INFO:tasks.rados.rados.0.smithi076.stdout:3738: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:53:44.915 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2743 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:53:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:3738: done (1 left) 2024-09-21T09:53:45.910 INFO:tasks.rados.rados.0.smithi076.stdout:3739: done (0 left) 2024-09-21T09:53:45.911 INFO:tasks.rados.rados.0.smithi076.stdout:3740: snap_remove snap 453 2024-09-21T09:53:46.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:45 smithi191 ceph-mon[102206]: pgmap v985: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 264 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2024-09-21T09:53:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:45 smithi191 ceph-mon[102206]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:53:46.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:45 smithi076 ceph-mon[106482]: pgmap v985: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 264 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2024-09-21T09:53:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:45 smithi076 ceph-mon[106482]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:53:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:45 smithi076 ceph-mon[110963]: pgmap v985: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 264 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2024-09-21T09:53:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:45 smithi076 ceph-mon[110963]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-21T09:53:46.920 INFO:tasks.rados.rados.0.smithi076.stdout:3740: done (0 left) 2024-09-21T09:53:46.920 INFO:tasks.rados.rados.0.smithi076.stdout:3741: delete oid 41 current snap is 470 2024-09-21T09:53:46.925 INFO:tasks.rados.rados.0.smithi076.stdout:3741: done (0 left) 2024-09-21T09:53:46.925 INFO:tasks.rados.rados.0.smithi076.stdout:3742: write oid 36 current snap is 470 2024-09-21T09:53:46.925 INFO:tasks.rados.rados.0.smithi076.stdout:3742: seq_num 1409 ranges {481063=751146,1784658=778160,3309938=371125} 2024-09-21T09:53:46.932 INFO:tasks.rados.rados.0.smithi076.stdout:3742: writing smithi07685715-36 from 481063 to 1232209 tid 1 2024-09-21T09:53:46.939 INFO:tasks.rados.rados.0.smithi076.stdout:3742: writing smithi07685715-36 from 1784658 to 2562818 tid 2 2024-09-21T09:53:46.945 INFO:tasks.rados.rados.0.smithi076.stdout:3742: writing smithi07685715-36 from 3309938 to 3681063 tid 3 2024-09-21T09:53:46.945 INFO:tasks.rados.rados.0.smithi076.stdout:3743: write oid 9 current snap is 470 2024-09-21T09:53:46.945 INFO:tasks.rados.rados.0.smithi076.stdout:3743: seq_num 1410 ranges {497870=426395,1443140=521818,2537361=530244,3583943=113927} 2024-09-21T09:53:46.950 INFO:tasks.rados.rados.0.smithi076.stdout:3743: writing smithi07685715-9 from 497870 to 924265 tid 1 2024-09-21T09:53:46.956 INFO:tasks.rados.rados.0.smithi076.stdout:3743: writing smithi07685715-9 from 1443140 to 1964958 tid 2 2024-09-21T09:53:46.962 INFO:tasks.rados.rados.0.smithi076.stdout:3743: writing smithi07685715-9 from 2537361 to 3067605 tid 3 2024-09-21T09:53:46.966 INFO:tasks.rados.rados.0.smithi076.stdout:3743: writing smithi07685715-9 from 3583943 to 3697870 tid 4 2024-09-21T09:53:46.966 INFO:tasks.rados.rados.0.smithi076.stdout:3744: write oid 3 current snap is 470 2024-09-21T09:53:46.966 INFO:tasks.rados.rados.0.smithi076.stdout:3742: finishing write tid 1 to smithi07685715-36 2024-09-21T09:53:46.966 INFO:tasks.rados.rados.0.smithi076.stdout:3744: seq_num 1411 ranges {514677=501644,1501622=781829,2964784=741516} 2024-09-21T09:53:46.971 INFO:tasks.rados.rados.0.smithi076.stdout:3744: writing smithi07685715-3 from 514677 to 1016321 tid 1 2024-09-21T09:53:46.978 INFO:tasks.rados.rados.0.smithi076.stdout:3744: writing smithi07685715-3 from 1501622 to 2283451 tid 2 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3744: writing smithi07685715-3 from 2964784 to 3706300 tid 3 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3745: write oid 27 current snap is 470 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: finishing write tid 2 to smithi07685715-36 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: finishing write tid 3 to smithi07685715-36 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: finishing write tid 4 to smithi07685715-36 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: finishing write tid 5 to smithi07685715-36 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: oid 36 updating version 0 to 1194 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: oid 36 version 1194 is already newer than 1192 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: oid 36 version 1194 is already newer than 1191 2024-09-21T09:53:46.986 INFO:tasks.rados.rados.0.smithi076.stdout:3742: oid 36 version 1194 is already newer than 1193 2024-09-21T09:53:46.987 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1194 (ObjNum 1409 snap 470 seq_num 1409) dirty exists 2024-09-21T09:53:46.987 INFO:tasks.rados.rados.0.smithi076.stdout:3742: left oid 36 (ObjNum 1409 snap 470 seq_num 1409) 2024-09-21T09:53:46.987 INFO:tasks.rados.rados.0.smithi076.stdout:3743: finishing write tid 1 to smithi07685715-9 2024-09-21T09:53:46.987 INFO:tasks.rados.rados.0.smithi076.stdout:3743: finishing write tid 2 to smithi07685715-9 2024-09-21T09:53:46.987 INFO:tasks.rados.rados.0.smithi076.stdout:3743: finishing write tid 3 to smithi07685715-9 2024-09-21T09:53:46.987 INFO:tasks.rados.rados.0.smithi076.stdout:3745: seq_num 1412 ranges {531484=576893,1843751=525487,2875854=669141} 2024-09-21T09:53:46.991 INFO:tasks.rados.rados.0.smithi076.stdout:3745: writing smithi07685715-27 from 531484 to 1108377 tid 1 2024-09-21T09:53:46.998 INFO:tasks.rados.rados.0.smithi076.stdout:3745: writing smithi07685715-27 from 1843751 to 2369238 tid 2 2024-09-21T09:53:47.006 INFO:tasks.rados.rados.0.smithi076.stdout:3745: writing smithi07685715-27 from 2875854 to 3544995 tid 3 2024-09-21T09:53:47.006 INFO:tasks.rados.rados.0.smithi076.stdout:3742: done (3 left) 2024-09-21T09:53:47.006 INFO:tasks.rados.rados.0.smithi076.stdout:3746: delete oid 18 current snap is 470 2024-09-21T09:53:47.006 INFO:tasks.rados.rados.0.smithi076.stdout:3743: finishing write tid 4 to smithi07685715-9 2024-09-21T09:53:47.007 INFO:tasks.rados.rados.0.smithi076.stdout:3743: finishing write tid 5 to smithi07685715-9 2024-09-21T09:53:47.007 INFO:tasks.rados.rados.0.smithi076.stdout:3743: finishing write tid 6 to smithi07685715-9 2024-09-21T09:53:47.007 INFO:tasks.rados.rados.0.smithi076.stdout:3743: oid 9 updating version 0 to 1335 2024-09-21T09:53:47.007 INFO:tasks.rados.rados.0.smithi076.stdout:3743: oid 9 version 1335 is already newer than 1333 2024-09-21T09:53:47.007 INFO:tasks.rados.rados.0.smithi076.stdout:3743: oid 9 version 1335 is already newer than 1331 2024-09-21T09:53:47.007 INFO:tasks.rados.rados.0.smithi076.stdout:3743: oid 9 version 1335 is already newer than 1334 2024-09-21T09:53:47.008 INFO:tasks.rados.rados.0.smithi076.stdout:3743: oid 9 version 1335 is already newer than 1332 2024-09-21T09:53:47.008 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 1335 (ObjNum 1410 snap 470 seq_num 1410) dirty exists 2024-09-21T09:53:47.008 INFO:tasks.rados.rados.0.smithi076.stdout:3743: left oid 9 (ObjNum 1410 snap 470 seq_num 1410) 2024-09-21T09:53:47.008 INFO:tasks.rados.rados.0.smithi076.stdout:3744: finishing write tid 1 to smithi07685715-3 2024-09-21T09:53:47.008 INFO:tasks.rados.rados.0.smithi076.stdout:3744: finishing write tid 2 to smithi07685715-3 2024-09-21T09:53:47.008 INFO:tasks.rados.rados.0.smithi076.stdout:3744: finishing write tid 3 to smithi07685715-3 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:3744: finishing write tid 4 to smithi07685715-3 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:3744: finishing write tid 5 to smithi07685715-3 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:3744: oid 3 updating version 0 to 316 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:3744: oid 3 version 316 is already newer than 314 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:3744: oid 3 version 316 is already newer than 315 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:3744: oid 3 version 316 is already newer than 313 2024-09-21T09:53:47.009 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 316 (ObjNum 1411 snap 470 seq_num 1411) dirty exists 2024-09-21T09:53:47.010 INFO:tasks.rados.rados.0.smithi076.stdout:3744: left oid 3 (ObjNum 1411 snap 470 seq_num 1411) 2024-09-21T09:53:47.010 INFO:tasks.rados.rados.0.smithi076.stdout:3745: finishing write tid 1 to smithi07685715-27 2024-09-21T09:53:47.010 INFO:tasks.rados.rados.0.smithi076.stdout:3745: finishing write tid 2 to smithi07685715-27 2024-09-21T09:53:47.010 INFO:tasks.rados.rados.0.smithi076.stdout:3743: done (3 left) 2024-09-21T09:53:47.011 INFO:tasks.rados.rados.0.smithi076.stdout:3744: done (2 left) 2024-09-21T09:53:47.011 INFO:tasks.rados.rados.0.smithi076.stdout:3746: done (1 left) 2024-09-21T09:53:47.011 INFO:tasks.rados.rados.0.smithi076.stdout:3747: snap_remove snap 460 2024-09-21T09:53:47.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:46 smithi191 ceph-mon[102206]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:53:47.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:46 smithi076 ceph-mon[106482]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:53:47.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:46 smithi076 ceph-mon[110963]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-21T09:53:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:3747: done (1 left) 2024-09-21T09:53:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:3748: read oid 16 snap -1 2024-09-21T09:53:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:3748: expect (ObjNum 1373 snap 460 seq_num 1373) 2024-09-21T09:53:47.921 INFO:tasks.rados.rados.0.smithi076.stdout:3749: read oid 25 snap -1 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3749: expect deleted 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3750: delete oid 42 current snap is 470 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: finishing write tid 3 to smithi07685715-27 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: finishing write tid 4 to smithi07685715-27 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: finishing write tid 5 to smithi07685715-27 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: oid 27 updating version 0 to 1957 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: oid 27 version 1957 is already newer than 1956 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: oid 27 updating version 1957 to 1958 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: oid 27 version 1958 is already newer than 1955 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1958 (ObjNum 1412 snap 470 seq_num 1412) dirty exists 2024-09-21T09:53:47.922 INFO:tasks.rados.rados.0.smithi076.stdout:3745: left oid 27 (ObjNum 1412 snap 470 seq_num 1412) 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3745: done (3 left) 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3749: done (2 left) 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3750: done (1 left) 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3751: read oid 42 snap -1 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3751: expect deleted 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3752: write oid 26 current snap is 470 2024-09-21T09:53:47.933 INFO:tasks.rados.rados.0.smithi076.stdout:3752: seq_num 1415 ranges {581905=518993,1735550=672814,2958123=502957} 2024-09-21T09:53:47.939 INFO:tasks.rados.rados.0.smithi076.stdout:3752: writing smithi07685715-26 from 581905 to 1100898 tid 1 2024-09-21T09:53:47.946 INFO:tasks.rados.rados.0.smithi076.stdout:3752: writing smithi07685715-26 from 1735550 to 2408364 tid 2 2024-09-21T09:53:47.952 INFO:tasks.rados.rados.0.smithi076.stdout:3752: writing smithi07685715-26 from 2958123 to 3461080 tid 3 2024-09-21T09:53:47.952 INFO:tasks.rados.rados.0.smithi076.stdout:3753: delete oid 46 current snap is 470 2024-09-21T09:53:47.953 INFO:tasks.rados.rados.0.smithi076.stdout:3748: expect (ObjNum 1373 snap 460 seq_num 1373) 2024-09-21T09:53:48.109 INFO:tasks.rados.rados.0.smithi076.stdout:3752: finishing write tid 1 to smithi07685715-26 2024-09-21T09:53:48.109 INFO:tasks.rados.rados.0.smithi076.stdout:3752: finishing write tid 2 to smithi07685715-26 2024-09-21T09:53:48.109 INFO:tasks.rados.rados.0.smithi076.stdout:3748: done (3 left) 2024-09-21T09:53:48.109 INFO:tasks.rados.rados.0.smithi076.stdout:3751: done (2 left) 2024-09-21T09:53:48.109 INFO:tasks.rados.rados.0.smithi076.stdout:3753: done (1 left) 2024-09-21T09:53:48.109 INFO:tasks.rados.rados.0.smithi076.stdout:3754: read oid 44 snap -1 2024-09-21T09:53:48.110 INFO:tasks.rados.rados.0.smithi076.stdout:3754: expect (ObjNum 1355 snap 453 seq_num 1355) 2024-09-21T09:53:48.110 INFO:tasks.rados.rados.0.smithi076.stdout:3755: read oid 33 snap 408 2024-09-21T09:53:48.110 INFO:tasks.rados.rados.0.smithi076.stdout:3755: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:53:48.110 INFO:tasks.rados.rados.0.smithi076.stdout:3756: snap_remove snap 464 2024-09-21T09:53:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:47 smithi191 ceph-mon[102206]: pgmap v988: 89 pgs: 89 active+clean; 266 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 501 KiB/s wr, 4 op/s 2024-09-21T09:53:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:47 smithi191 ceph-mon[102206]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:53:48.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:47 smithi076 ceph-mon[106482]: pgmap v988: 89 pgs: 89 active+clean; 266 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 501 KiB/s wr, 4 op/s 2024-09-21T09:53:48.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:47 smithi076 ceph-mon[106482]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:53:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:47 smithi076 ceph-mon[110963]: pgmap v988: 89 pgs: 89 active+clean; 266 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 501 KiB/s wr, 4 op/s 2024-09-21T09:53:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:47 smithi076 ceph-mon[110963]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-21T09:53:48.819 INFO:tasks.rados.rados.0.smithi076.stdout:3756: done (3 left) 2024-09-21T09:53:48.820 INFO:tasks.rados.rados.0.smithi076.stdout:3757: read oid 17 snap -1 2024-09-21T09:53:48.820 INFO:tasks.rados.rados.0.smithi076.stdout:3757: expect (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:48.820 INFO:tasks.rados.rados.0.smithi076.stdout:3758: write oid 29 current snap is 470 2024-09-21T09:53:48.820 INFO:tasks.rados.rados.0.smithi076.stdout:3758: seq_num 1417 ranges {615519=669491,1736161=676483,2896616=641854} 2024-09-21T09:53:48.826 INFO:tasks.rados.rados.0.smithi076.stdout:3758: writing smithi07685715-29 from 615519 to 1285010 tid 1 2024-09-21T09:53:48.833 INFO:tasks.rados.rados.0.smithi076.stdout:3758: writing smithi07685715-29 from 1736161 to 2412644 tid 2 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3758: writing smithi07685715-29 from 2896616 to 3538470 tid 3 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3759: snap_remove snap 467 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: finishing write tid 3 to smithi07685715-26 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: finishing write tid 4 to smithi07685715-26 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: finishing write tid 5 to smithi07685715-26 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: oid 26 updating version 0 to 1343 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: oid 26 version 1343 is already newer than 1340 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: oid 26 version 1343 is already newer than 1341 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: oid 26 version 1343 is already newer than 1342 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1343 (ObjNum 1415 snap 470 seq_num 1415) dirty exists 2024-09-21T09:53:48.840 INFO:tasks.rados.rados.0.smithi076.stdout:3752: left oid 26 (ObjNum 1415 snap 470 seq_num 1415) 2024-09-21T09:53:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:48 smithi191 ceph-mon[102206]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:53:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:48 smithi191 ceph-mon[102206]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:53:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:48 smithi076 ceph-mon[106482]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:53:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:48 smithi076 ceph-mon[106482]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:53:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:48 smithi076 ceph-mon[110963]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-21T09:53:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:48 smithi076 ceph-mon[110963]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-21T09:53:49.819 INFO:tasks.rados.rados.0.smithi076.stdout:3752: done (5 left) 2024-09-21T09:53:49.819 INFO:tasks.rados.rados.0.smithi076.stdout:3759: done (4 left) 2024-09-21T09:53:49.819 INFO:tasks.rados.rados.0.smithi076.stdout:3760: snap_create 2024-09-21T09:53:49.819 INFO:tasks.rados.rados.0.smithi076.stdout:3755: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:53:49.997 INFO:tasks.rados.rados.0.smithi076.stdout:3754: expect (ObjNum 1355 snap 453 seq_num 1355) 2024-09-21T09:53:50.204 INFO:tasks.rados.rados.0.smithi076.stdout:3757: expect (ObjNum 1337 snap 450 seq_num 1337) 2024-09-21T09:53:50.317 INFO:tasks.rados.rados.0.smithi076.stdout:3758: finishing write tid 1 to smithi07685715-29 2024-09-21T09:53:50.317 INFO:tasks.rados.rados.0.smithi076.stdout:3758: finishing write tid 2 to smithi07685715-29 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: finishing write tid 3 to smithi07685715-29 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: finishing write tid 4 to smithi07685715-29 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: finishing write tid 5 to smithi07685715-29 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: oid 29 updating version 0 to 1164 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: oid 29 updating version 1164 to 1166 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: oid 29 updating version 1166 to 1167 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: oid 29 version 1167 is already newer than 1165 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 1167 (ObjNum 1417 snap 470 seq_num 1417) dirty exists 2024-09-21T09:53:50.318 INFO:tasks.rados.rados.0.smithi076.stdout:3758: left oid 29 (ObjNum 1417 snap 470 seq_num 1417) 2024-09-21T09:53:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:49 smithi191 ceph-mon[102206]: pgmap v991: 89 pgs: 89 active+clean; 266 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 501 KiB/s wr, 3 op/s 2024-09-21T09:53:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:49 smithi191 ceph-mon[102206]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:53:50.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:49 smithi076 ceph-mon[106482]: pgmap v991: 89 pgs: 89 active+clean; 266 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 501 KiB/s wr, 3 op/s 2024-09-21T09:53:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:49 smithi076 ceph-mon[106482]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:53:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:49 smithi076 ceph-mon[110963]: pgmap v991: 89 pgs: 89 active+clean; 266 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 501 KiB/s wr, 3 op/s 2024-09-21T09:53:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:49 smithi076 ceph-mon[110963]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-21T09:53:50.822 INFO:tasks.rados.rados.0.smithi076.stdout:3754: done (4 left) 2024-09-21T09:53:50.822 INFO:tasks.rados.rados.0.smithi076.stdout:3755: done (3 left) 2024-09-21T09:53:50.822 INFO:tasks.rados.rados.0.smithi076.stdout:3757: done (2 left) 2024-09-21T09:53:50.822 INFO:tasks.rados.rados.0.smithi076.stdout:3758: done (1 left) 2024-09-21T09:53:50.823 INFO:tasks.rados.rados.0.smithi076.stdout:3760: done (0 left) 2024-09-21T09:53:50.823 INFO:tasks.rados.rados.0.smithi076.stdout:3761: read oid 2 snap -1 2024-09-21T09:53:50.823 INFO:tasks.rados.rados.0.smithi076.stdout:3761: expect (ObjNum 1403 snap 466 seq_num 1403) 2024-09-21T09:53:50.823 INFO:tasks.rados.rados.0.smithi076.stdout:3762: snap_create 2024-09-21T09:53:50.830 INFO:tasks.rados.rados.0.smithi076.stdout:3761: expect (ObjNum 1403 snap 466 seq_num 1403) 2024-09-21T09:53:51.825 INFO:tasks.rados.rados.0.smithi076.stdout:3761: done (1 left) 2024-09-21T09:53:51.825 INFO:tasks.rados.rados.0.smithi076.stdout:3762: done (0 left) 2024-09-21T09:53:51.825 INFO:tasks.rados.rados.0.smithi076.stdout:3763: snap_create 2024-09-21T09:53:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:51 smithi191 ceph-mon[102206]: pgmap v994: 89 pgs: 89 active+clean; 269 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 803 KiB/s wr, 6 op/s 2024-09-21T09:53:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:51 smithi191 ceph-mon[102206]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:53:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:51 smithi076 ceph-mon[110963]: pgmap v994: 89 pgs: 89 active+clean; 269 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 803 KiB/s wr, 6 op/s 2024-09-21T09:53:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:51 smithi076 ceph-mon[110963]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:53:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:51 smithi076 ceph-mon[106482]: pgmap v994: 89 pgs: 89 active+clean; 269 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 803 KiB/s wr, 6 op/s 2024-09-21T09:53:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:51 smithi076 ceph-mon[106482]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-21T09:53:52.825 INFO:tasks.rados.rados.0.smithi076.stdout:3763: done (0 left) 2024-09-21T09:53:52.825 INFO:tasks.rados.rados.0.smithi076.stdout:3764: snap_remove snap 446 2024-09-21T09:53:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:52 smithi191 ceph-mon[102206]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:53:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:52 smithi191 ceph-mon[102206]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:53:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:52 smithi076 ceph-mon[106482]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:53:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:52 smithi076 ceph-mon[106482]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:53:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:52 smithi076 ceph-mon[110963]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-21T09:53:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:52 smithi076 ceph-mon[110963]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-21T09:53:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:53:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:53:53] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-09-21T09:53:53.821 INFO:tasks.rados.rados.0.smithi076.stdout:3764: done (0 left) 2024-09-21T09:53:53.821 INFO:tasks.rados.rados.0.smithi076.stdout:3765: snap_remove snap 461 2024-09-21T09:53:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:53 smithi191 ceph-mon[102206]: pgmap v997: 89 pgs: 89 active+clean; 279 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:53:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:53 smithi191 ceph-mon[102206]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:53:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:53 smithi076 ceph-mon[110963]: pgmap v997: 89 pgs: 89 active+clean; 279 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:53:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:53 smithi076 ceph-mon[110963]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:53:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:53 smithi076 ceph-mon[106482]: pgmap v997: 89 pgs: 89 active+clean; 279 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.6 MiB/s wr, 15 op/s 2024-09-21T09:53:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:53 smithi076 ceph-mon[106482]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-21T09:53:54.822 INFO:tasks.rados.rados.0.smithi076.stdout:3765: done (0 left) 2024-09-21T09:53:54.822 INFO:tasks.rados.rados.0.smithi076.stdout:3766: write oid 39 current snap is 473 2024-09-21T09:53:54.822 INFO:tasks.rados.rados.0.smithi076.stdout:3766: seq_num 1418 ranges {632326=744740,1794643=536494,3040392=453126} 2024-09-21T09:53:54.830 INFO:tasks.rados.rados.0.smithi076.stdout:3766: writing smithi07685715-39 from 632326 to 1377066 tid 1 2024-09-21T09:53:54.836 INFO:tasks.rados.rados.0.smithi076.stdout:3766: writing smithi07685715-39 from 1794643 to 2331137 tid 2 2024-09-21T09:53:54.842 INFO:tasks.rados.rados.0.smithi076.stdout:3766: writing smithi07685715-39 from 3040392 to 3493518 tid 3 2024-09-21T09:53:54.842 INFO:tasks.rados.rados.0.smithi076.stdout:3767: write oid 30 current snap is 473 2024-09-21T09:53:54.843 INFO:tasks.rados.rados.0.smithi076.stdout:3767: seq_num 1419 ranges {649133=419989,1853125=680152,2951462=664398} 2024-09-21T09:53:54.848 INFO:tasks.rados.rados.0.smithi076.stdout:3767: writing smithi07685715-30 from 649133 to 1069122 tid 1 2024-09-21T09:53:54.856 INFO:tasks.rados.rados.0.smithi076.stdout:3767: writing smithi07685715-30 from 1853125 to 2533277 tid 2 2024-09-21T09:53:54.862 INFO:tasks.rados.rados.0.smithi076.stdout:3767: writing smithi07685715-30 from 2951462 to 3615860 tid 3 2024-09-21T09:53:54.863 INFO:tasks.rados.rados.0.smithi076.stdout:3768: write oid 1 current snap is 473 2024-09-21T09:53:54.863 INFO:tasks.rados.rados.0.smithi076.stdout:3768: seq_num 1420 ranges {665940=495238,1795254=540163,2978885=475670} 2024-09-21T09:53:54.869 INFO:tasks.rados.rados.0.smithi076.stdout:3768: writing smithi07685715-1 from 665940 to 1161178 tid 1 2024-09-21T09:53:54.875 INFO:tasks.rados.rados.0.smithi076.stdout:3768: writing smithi07685715-1 from 1795254 to 2335417 tid 2 2024-09-21T09:53:54.881 INFO:tasks.rados.rados.0.smithi076.stdout:3768: writing smithi07685715-1 from 2978885 to 3454555 tid 3 2024-09-21T09:53:54.881 INFO:tasks.rados.rados.0.smithi076.stdout:3769: snap_remove snap 470 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3769: done (3 left) 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3770: snap_create 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3766: finishing write tid 1 to smithi07685715-39 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3766: finishing write tid 2 to smithi07685715-39 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3766: finishing write tid 3 to smithi07685715-39 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3766: finishing write tid 4 to smithi07685715-39 2024-09-21T09:53:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3766: finishing write tid 5 to smithi07685715-39 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3766: oid 39 updating version 0 to 2902 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3766: oid 39 updating version 2902 to 2905 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3766: oid 39 version 2905 is already newer than 2903 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3766: oid 39 version 2905 is already newer than 2904 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2905 (ObjNum 1418 snap 473 seq_num 1418) dirty exists 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3766: left oid 39 (ObjNum 1418 snap 473 seq_num 1418) 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: finishing write tid 1 to smithi07685715-30 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: finishing write tid 2 to smithi07685715-30 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: finishing write tid 3 to smithi07685715-30 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: finishing write tid 4 to smithi07685715-30 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: finishing write tid 5 to smithi07685715-30 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: oid 30 updating version 0 to 2891 2024-09-21T09:53:55.844 INFO:tasks.rados.rados.0.smithi076.stdout:3767: oid 30 updating version 2891 to 2892 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3767: oid 30 version 2892 is already newer than 2890 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3767: oid 30 updating version 2892 to 2893 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2893 (ObjNum 1419 snap 473 seq_num 1419) dirty exists 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3767: left oid 30 (ObjNum 1419 snap 473 seq_num 1419) 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: finishing write tid 1 to smithi07685715-1 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: finishing write tid 2 to smithi07685715-1 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: finishing write tid 3 to smithi07685715-1 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: finishing write tid 4 to smithi07685715-1 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: finishing write tid 5 to smithi07685715-1 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: oid 1 updating version 0 to 2909 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: oid 1 updating version 2909 to 2910 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: oid 1 version 2910 is already newer than 2908 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: oid 1 version 2910 is already newer than 2907 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2910 (ObjNum 1420 snap 473 seq_num 1420) dirty exists 2024-09-21T09:53:55.845 INFO:tasks.rados.rados.0.smithi076.stdout:3768: left oid 1 (ObjNum 1420 snap 473 seq_num 1420) 2024-09-21T09:53:56.087 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:55 smithi191 ceph-mon[102206]: pgmap v1000: 89 pgs: 89 active+clean; 277 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:53:56.087 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:55 smithi191 ceph-mon[102206]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:53:56.087 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:53:56.087 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[106482]: pgmap v1000: 89 pgs: 89 active+clean; 277 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[106482]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[110963]: pgmap v1000: 89 pgs: 89 active+clean; 277 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 1.8 MiB/s wr, 11 op/s 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[110963]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:53:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:53:56.849 INFO:tasks.rados.rados.0.smithi076.stdout:3766: done (3 left) 2024-09-21T09:53:56.849 INFO:tasks.rados.rados.0.smithi076.stdout:3767: done (2 left) 2024-09-21T09:53:56.849 INFO:tasks.rados.rados.0.smithi076.stdout:3768: done (1 left) 2024-09-21T09:53:56.849 INFO:tasks.rados.rados.0.smithi076.stdout:3770: done (0 left) 2024-09-21T09:53:56.849 INFO:tasks.rados.rados.0.smithi076.stdout:3771: write oid 24 current snap is 474 2024-09-21T09:53:56.849 INFO:tasks.rados.rados.0.smithi076.stdout:3771: seq_num 1421 ranges {682747=570487,1853736=683821,3006308=403295} 2024-09-21T09:53:56.855 INFO:tasks.rados.rados.0.smithi076.stdout:3771: writing smithi07685715-24 from 682747 to 1253234 tid 1 2024-09-21T09:53:56.862 INFO:tasks.rados.rados.0.smithi076.stdout:3771: writing smithi07685715-24 from 1853736 to 2537557 tid 2 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:3771: writing smithi07685715-24 from 3006308 to 3409603 tid 3 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:3772: rollback oid 4 current snap is 474 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 4 to 408 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:3771: finishing write tid 1 to smithi07685715-24 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:3771: finishing write tid 2 to smithi07685715-24 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:3773: write oid 12 current snap is 474 2024-09-21T09:53:56.868 INFO:tasks.rados.rados.0.smithi076.stdout:3773: seq_num 1422 ranges {699554=762089,2028571=543832,3150084=614567} 2024-09-21T09:53:56.876 INFO:tasks.rados.rados.0.smithi076.stdout:3773: writing smithi07685715-12 from 699554 to 1461643 tid 1 2024-09-21T09:53:56.882 INFO:tasks.rados.rados.0.smithi076.stdout:3773: writing smithi07685715-12 from 2028571 to 2572403 tid 2 2024-09-21T09:53:56.889 INFO:tasks.rados.rados.0.smithi076.stdout:3773: writing smithi07685715-12 from 3150084 to 3764651 tid 3 2024-09-21T09:53:56.889 INFO:tasks.rados.rados.0.smithi076.stdout:3774: write oid 29 current snap is 474 2024-09-21T09:53:56.889 INFO:tasks.rados.rados.0.smithi076.stdout:3774: seq_num 1423 ranges {716361=437338,1687053=687490,2777507=425839,3877756=38605} 2024-09-21T09:53:56.895 INFO:tasks.rados.rados.0.smithi076.stdout:3774: writing smithi07685715-29 from 716361 to 1153699 tid 1 2024-09-21T09:53:56.902 INFO:tasks.rados.rados.0.smithi076.stdout:3774: writing smithi07685715-29 from 1687053 to 2374543 tid 2 2024-09-21T09:53:56.907 INFO:tasks.rados.rados.0.smithi076.stdout:3774: writing smithi07685715-29 from 2777507 to 3203346 tid 3 2024-09-21T09:53:56.912 INFO:tasks.rados.rados.0.smithi076.stdout:3774: writing smithi07685715-29 from 3877756 to 3916361 tid 4 2024-09-21T09:53:56.912 INFO:tasks.rados.rados.0.smithi076.stdout:3775: write oid 26 current snap is 474 2024-09-21T09:53:56.912 INFO:tasks.rados.rados.0.smithi076.stdout:3771: finishing write tid 3 to smithi07685715-24 2024-09-21T09:53:56.912 INFO:tasks.rados.rados.0.smithi076.stdout:3775: seq_num 1424 ranges {733168=512587,2029182=547501,3088577=637111} 2024-09-21T09:53:56.919 INFO:tasks.rados.rados.0.smithi076.stdout:3775: writing smithi07685715-26 from 733168 to 1245755 tid 1 2024-09-21T09:53:56.927 INFO:tasks.rados.rados.0.smithi076.stdout:3775: writing smithi07685715-26 from 2029182 to 2576683 tid 2 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3775: writing smithi07685715-26 from 3088577 to 3725688 tid 3 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3776: read oid 6 snap -1 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3776: expect (ObjNum 1407 snap 469 seq_num 1407) 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3777: write oid 34 current snap is 474 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: finishing write tid 4 to smithi07685715-24 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: finishing write tid 5 to smithi07685715-24 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: oid 24 updating version 0 to 390 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: oid 24 version 390 is already newer than 389 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: oid 24 updating version 390 to 391 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: oid 24 version 391 is already newer than 388 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 391 (ObjNum 1421 snap 474 seq_num 1421) dirty exists 2024-09-21T09:53:56.934 INFO:tasks.rados.rados.0.smithi076.stdout:3771: left oid 24 (ObjNum 1421 snap 474 seq_num 1421) 2024-09-21T09:53:56.935 INFO:tasks.rados.rados.0.smithi076.stdout:3772: finishing rollback tid 0 to smithi07685715-4 2024-09-21T09:53:56.935 INFO:tasks.rados.rados.0.smithi076.stdout:3772: finishing rollback tid 1 to smithi07685715-4 2024-09-21T09:53:56.935 INFO:tasks.rados.rados.0.smithi076.stdout:3777: seq_num 1425 ranges {749975=587836,2087664=691159,3516000=433975} 2024-09-21T09:53:56.940 INFO:tasks.rados.rados.0.smithi076.stdout:3777: writing smithi07685715-34 from 749975 to 1337811 tid 1 2024-09-21T09:53:56.948 INFO:tasks.rados.rados.0.smithi076.stdout:3777: writing smithi07685715-34 from 2087664 to 2778823 tid 2 2024-09-21T09:53:56.955 INFO:tasks.rados.rados.0.smithi076.stdout:3777: writing smithi07685715-34 from 3516000 to 3949975 tid 3 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3771: done (6 left) 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3778: snap_create 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3772: finishing rollback tid 2 to smithi07685715-4 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 4 v 1202 (ObjNum 1190 snap 399 seq_num 1190) dirty exists 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: finishing write tid 1 to smithi07685715-12 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: finishing write tid 2 to smithi07685715-12 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: finishing write tid 3 to smithi07685715-12 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: finishing write tid 4 to smithi07685715-12 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: finishing write tid 5 to smithi07685715-12 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: oid 12 updating version 0 to 1358 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: oid 12 version 1358 is already newer than 1357 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: oid 12 updating version 1358 to 1359 2024-09-21T09:53:56.956 INFO:tasks.rados.rados.0.smithi076.stdout:3773: oid 12 updating version 1359 to 1360 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 12 v 1360 (ObjNum 1422 snap 474 seq_num 1422) dirty exists 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3773: left oid 12 (ObjNum 1422 snap 474 seq_num 1422) 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: finishing write tid 1 to smithi07685715-29 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: finishing write tid 2 to smithi07685715-29 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: finishing write tid 3 to smithi07685715-29 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: finishing write tid 4 to smithi07685715-29 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: finishing write tid 5 to smithi07685715-29 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: finishing write tid 6 to smithi07685715-29 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: oid 29 updating version 0 to 1179 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: oid 29 updating version 1179 to 1183 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: oid 29 version 1183 is already newer than 1180 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: oid 29 version 1183 is already newer than 1181 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: oid 29 version 1183 is already newer than 1182 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 1183 (ObjNum 1423 snap 474 seq_num 1423) dirty exists 2024-09-21T09:53:56.957 INFO:tasks.rados.rados.0.smithi076.stdout:3774: left oid 29 (ObjNum 1423 snap 474 seq_num 1423) 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: finishing write tid 1 to smithi07685715-26 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: finishing write tid 2 to smithi07685715-26 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: finishing write tid 3 to smithi07685715-26 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: finishing write tid 4 to smithi07685715-26 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: finishing write tid 5 to smithi07685715-26 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: oid 26 updating version 0 to 1362 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: oid 26 updating version 1362 to 1365 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: oid 26 version 1365 is already newer than 1364 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: oid 26 version 1365 is already newer than 1363 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1365 (ObjNum 1424 snap 474 seq_num 1424) dirty exists 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3775: left oid 26 (ObjNum 1424 snap 474 seq_num 1424) 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3777: finishing write tid 1 to smithi07685715-34 2024-09-21T09:53:56.958 INFO:tasks.rados.rados.0.smithi076.stdout:3776: expect (ObjNum 1407 snap 469 seq_num 1407) 2024-09-21T09:53:57.070 INFO:tasks.rados.rados.0.smithi076.stdout:3777: finishing write tid 2 to smithi07685715-34 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: finishing write tid 3 to smithi07685715-34 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: finishing write tid 4 to smithi07685715-34 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: finishing write tid 5 to smithi07685715-34 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: oid 34 updating version 0 to 854 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: oid 34 version 854 is already newer than 853 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: oid 34 updating version 854 to 855 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: oid 34 version 855 is already newer than 852 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 855 (ObjNum 1425 snap 474 seq_num 1425) dirty exists 2024-09-21T09:53:57.071 INFO:tasks.rados.rados.0.smithi076.stdout:3777: left oid 34 (ObjNum 1425 snap 474 seq_num 1425) 2024-09-21T09:53:57.106 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:56 smithi076 ceph-mon[106482]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:53:57.107 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:56 smithi076 ceph-mon[110963]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:53:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:56 smithi191 ceph-mon[102206]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3772: done (6 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3773: done (5 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3774: done (4 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3775: done (3 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3776: done (2 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3777: done (1 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3778: done (0 left) 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3779: read oid 33 snap -1 2024-09-21T09:53:57.851 INFO:tasks.rados.rados.0.smithi076.stdout:3779: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:53:57.852 INFO:tasks.rados.rados.0.smithi076.stdout:3780: snap_remove snap 473 2024-09-21T09:53:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:57 smithi191 ceph-mon[102206]: pgmap v1003: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 258 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:53:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:57 smithi191 ceph-mon[102206]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:53:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:57 smithi076 ceph-mon[106482]: pgmap v1003: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 258 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:53:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:57 smithi076 ceph-mon[106482]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:53:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:57 smithi076 ceph-mon[110963]: pgmap v1003: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 258 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:53:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:57 smithi076 ceph-mon[110963]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-21T09:53:58.822 INFO:tasks.rados.rados.0.smithi076.stdout:3780: done (1 left) 2024-09-21T09:53:58.822 INFO:tasks.rados.rados.0.smithi076.stdout:3781: snap_remove snap 459 2024-09-21T09:53:59.109 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:58 smithi191 ceph-mon[102206]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:53:59.109 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:53:59.109 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:53:59.109 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:58 smithi191 ceph-mon[102206]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[106482]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[106482]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[110963]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:53:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:58 smithi076 ceph-mon[110963]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-21T09:53:59.825 INFO:tasks.rados.rados.0.smithi076.stdout:3781: done (1 left) 2024-09-21T09:53:59.826 INFO:tasks.rados.rados.0.smithi076.stdout:3782: read oid 4 snap -1 2024-09-21T09:53:59.826 INFO:tasks.rados.rados.0.smithi076.stdout:3782: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:53:59.826 INFO:tasks.rados.rados.0.smithi076.stdout:3783: write oid 1 current snap is 475 2024-09-21T09:53:59.826 INFO:tasks.rados.rados.0.smithi076.stdout:3783: seq_num 1426 ranges {766782=663085,2146146=551170,3143423=776008} 2024-09-21T09:53:59.834 INFO:tasks.rados.rados.0.smithi076.stdout:3783: writing smithi07685715-1 from 766782 to 1429867 tid 1 2024-09-21T09:53:59.842 INFO:tasks.rados.rados.0.smithi076.stdout:3783: writing smithi07685715-1 from 2146146 to 2697316 tid 2 2024-09-21T09:53:59.848 INFO:tasks.rados.rados.0.smithi076.stdout:3783: writing smithi07685715-1 from 3143423 to 3919431 tid 3 2024-09-21T09:53:59.848 INFO:tasks.rados.rados.0.smithi076.stdout:3784: rollback oid 44 current snap is 475 2024-09-21T09:53:59.848 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 44 to 466 2024-09-21T09:53:59.848 INFO:tasks.rados.rados.0.smithi076.stdout:3779: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:54:00.027 INFO:tasks.rados.rados.0.smithi076.stdout:3782: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:54:00.089 INFO:tasks.rados.rados.0.smithi076.stdout:3783: finishing write tid 1 to smithi07685715-1 2024-09-21T09:54:00.089 INFO:tasks.rados.rados.0.smithi076.stdout:3783: finishing write tid 2 to smithi07685715-1 2024-09-21T09:54:00.089 INFO:tasks.rados.rados.0.smithi076.stdout:3784: finishing rollback tid 0 to smithi07685715-44 2024-09-21T09:54:00.089 INFO:tasks.rados.rados.0.smithi076.stdout:3784: finishing rollback tid 1 to smithi07685715-44 2024-09-21T09:54:00.089 INFO:tasks.rados.rados.0.smithi076.stdout:3784: finishing rollback tid 2 to smithi07685715-44 2024-09-21T09:54:00.089 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 830 (ObjNum 1355 snap 453 seq_num 1355) dirty exists 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: finishing write tid 3 to smithi07685715-1 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: finishing write tid 4 to smithi07685715-1 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: finishing write tid 5 to smithi07685715-1 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: oid 1 updating version 0 to 2919 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: oid 1 version 2919 is already newer than 2916 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: oid 1 version 2919 is already newer than 2917 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: oid 1 version 2919 is already newer than 2918 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 1 v 2919 (ObjNum 1426 snap 475 seq_num 1426) dirty exists 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: left oid 1 (ObjNum 1426 snap 475 seq_num 1426) 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3779: done (3 left) 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3782: done (2 left) 2024-09-21T09:54:00.090 INFO:tasks.rados.rados.0.smithi076.stdout:3783: done (1 left) 2024-09-21T09:54:00.091 INFO:tasks.rados.rados.0.smithi076.stdout:3784: done (0 left) 2024-09-21T09:54:00.091 INFO:tasks.rados.rados.0.smithi076.stdout:3785: write oid 11 current snap is 475 2024-09-21T09:54:00.091 INFO:tasks.rados.rados.0.smithi076.stdout:3785: seq_num 1427 ranges {783589=738334,2204628=694828,3570846=412743} 2024-09-21T09:54:00.096 INFO:tasks.rados.rados.0.smithi076.stdout:3785: writing smithi07685715-11 from 783589 to 1521923 tid 1 2024-09-21T09:54:00.104 INFO:tasks.rados.rados.0.smithi076.stdout:3785: writing smithi07685715-11 from 2204628 to 2899456 tid 2 2024-09-21T09:54:00.110 INFO:tasks.rados.rados.0.smithi076.stdout:3785: writing smithi07685715-11 from 3570846 to 3983589 tid 3 2024-09-21T09:54:00.110 INFO:tasks.rados.rados.0.smithi076.stdout:3786: write oid 15 current snap is 475 2024-09-21T09:54:00.110 INFO:tasks.rados.rados.0.smithi076.stdout:3786: seq_num 1428 ranges {400396=413583,1463110=438486} 2024-09-21T09:54:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:59 smithi191 ceph-mon[102206]: pgmap v1006: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 252 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:54:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:53:59 smithi191 ceph-mon[102206]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:54:00.114 INFO:tasks.rados.rados.0.smithi076.stdout:3786: writing smithi07685715-15 from 400396 to 813979 tid 1 2024-09-21T09:54:00.120 INFO:tasks.rados.rados.0.smithi076.stdout:3786: writing smithi07685715-15 from 1463110 to 1901596 tid 2 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3787: snap_create 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: finishing write tid 1 to smithi07685715-11 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: finishing write tid 2 to smithi07685715-11 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: finishing write tid 3 to smithi07685715-11 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: finishing write tid 4 to smithi07685715-11 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: finishing write tid 5 to smithi07685715-11 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: oid 11 updating version 0 to 2920 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: oid 11 updating version 2920 to 2923 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: oid 11 version 2923 is already newer than 2922 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: oid 11 version 2923 is already newer than 2921 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2923 (ObjNum 1427 snap 475 seq_num 1427) dirty exists 2024-09-21T09:54:00.121 INFO:tasks.rados.rados.0.smithi076.stdout:3785: left oid 11 (ObjNum 1427 snap 475 seq_num 1427) 2024-09-21T09:54:00.122 INFO:tasks.rados.rados.0.smithi076.stdout:3786: finishing write tid 1 to smithi07685715-15 2024-09-21T09:54:00.122 INFO:tasks.rados.rados.0.smithi076.stdout:3786: finishing write tid 2 to smithi07685715-15 2024-09-21T09:54:00.122 INFO:tasks.rados.rados.0.smithi076.stdout:3786: finishing write tid 3 to smithi07685715-15 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[110963]: pgmap v1006: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 252 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[110963]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[106482]: pgmap v1006: 89 pgs: 1 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 86 active+clean; 252 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:53:59 smithi076 ceph-mon[106482]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-21T09:54:00.124 INFO:tasks.rados.rados.0.smithi076.stdout:3786: finishing write tid 4 to smithi07685715-15 2024-09-21T09:54:00.124 INFO:tasks.rados.rados.0.smithi076.stdout:3786: oid 15 updating version 0 to 811 2024-09-21T09:54:00.124 INFO:tasks.rados.rados.0.smithi076.stdout:3786: oid 15 updating version 811 to 812 2024-09-21T09:54:00.124 INFO:tasks.rados.rados.0.smithi076.stdout:3786: oid 15 version 812 is already newer than 810 2024-09-21T09:54:00.124 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 812 (ObjNum 1428 snap 475 seq_num 1428) dirty exists 2024-09-21T09:54:00.124 INFO:tasks.rados.rados.0.smithi076.stdout:3786: left oid 15 (ObjNum 1428 snap 475 seq_num 1428) 2024-09-21T09:54:00.921 INFO:tasks.rados.rados.0.smithi076.stdout:3785: done (2 left) 2024-09-21T09:54:00.921 INFO:tasks.rados.rados.0.smithi076.stdout:3786: done (1 left) 2024-09-21T09:54:00.921 INFO:tasks.rados.rados.0.smithi076.stdout:3787: done (0 left) 2024-09-21T09:54:00.921 INFO:tasks.rados.rados.0.smithi076.stdout:3788: read oid 30 snap 469 2024-09-21T09:54:00.921 INFO:tasks.rados.rados.0.smithi076.stdout:3788: expect deleted 2024-09-21T09:54:00.921 INFO:tasks.rados.rados.0.smithi076.stdout:3789: write oid 29 current snap is 476 2024-09-21T09:54:00.922 INFO:tasks.rados.rados.0.smithi076.stdout:3789: seq_num 1429 ranges {417203=488832,1405239=611964} 2024-09-21T09:54:00.926 INFO:tasks.rados.rados.0.smithi076.stdout:3789: writing smithi07685715-29 from 417203 to 906035 tid 1 2024-09-21T09:54:00.933 INFO:tasks.rados.rados.0.smithi076.stdout:3789: writing smithi07685715-29 from 1405239 to 2017203 tid 2 2024-09-21T09:54:00.933 INFO:tasks.rados.rados.0.smithi076.stdout:3790: snap_remove snap 474 2024-09-21T09:54:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:00 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:54:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:00 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:54:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:00 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:00 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:54:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:00 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:54:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:00 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:00 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:54:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:00 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:54:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:00 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:54:01.932 INFO:tasks.rados.rados.0.smithi076.stdout:3790: done (2 left) 2024-09-21T09:54:01.932 INFO:tasks.rados.rados.0.smithi076.stdout:3791: write oid 11 current snap is 476 2024-09-21T09:54:01.932 INFO:tasks.rados.rados.0.smithi076.stdout:3791: seq_num 1430 ranges {434010=680434,1580074=442155} 2024-09-21T09:54:01.938 INFO:tasks.rados.rados.0.smithi076.stdout:3791: writing smithi07685715-11 from 434010 to 1114444 tid 1 2024-09-21T09:54:01.944 INFO:tasks.rados.rados.0.smithi076.stdout:3791: writing smithi07685715-11 from 1580074 to 2022229 tid 2 2024-09-21T09:54:01.944 INFO:tasks.rados.rados.0.smithi076.stdout:3792: write oid 35 current snap is 476 2024-09-21T09:54:01.944 INFO:tasks.rados.rados.0.smithi076.stdout:3792: seq_num 1431 ranges {450817=755683,1638556=412261} 2024-09-21T09:54:01.950 INFO:tasks.rados.rados.0.smithi076.stdout:3792: writing smithi07685715-35 from 450817 to 1206500 tid 1 2024-09-21T09:54:01.955 INFO:tasks.rados.rados.0.smithi076.stdout:3792: writing smithi07685715-35 from 1638556 to 2050817 tid 2 2024-09-21T09:54:01.955 INFO:tasks.rados.rados.0.smithi076.stdout:3793: write oid 25 current snap is 476 2024-09-21T09:54:01.955 INFO:tasks.rados.rados.0.smithi076.stdout:3793: seq_num 1432 ranges {467624=430932,1697038=370586} 2024-09-21T09:54:01.960 INFO:tasks.rados.rados.0.smithi076.stdout:3793: writing smithi07685715-25 from 467624 to 898556 tid 1 2024-09-21T09:54:01.967 INFO:tasks.rados.rados.0.smithi076.stdout:3793: writing smithi07685715-25 from 1697038 to 2067624 tid 2 2024-09-21T09:54:01.967 INFO:tasks.rados.rados.0.smithi076.stdout:3794: rollback oid 47 current snap is 476 2024-09-21T09:54:01.967 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 47 to 472 2024-09-21T09:54:01.967 INFO:tasks.rados.rados.0.smithi076.stdout:3789: finishing write tid 1 to smithi07685715-29 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: finishing write tid 2 to smithi07685715-29 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: finishing write tid 3 to smithi07685715-29 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: finishing write tid 4 to smithi07685715-29 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: oid 29 updating version 0 to 1189 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: oid 29 updating version 1189 to 1190 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: oid 29 updating version 1190 to 1191 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 1191 (ObjNum 1429 snap 476 seq_num 1429) dirty exists 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3789: left oid 29 (ObjNum 1429 snap 476 seq_num 1429) 2024-09-21T09:54:01.968 INFO:tasks.rados.rados.0.smithi076.stdout:3791: finishing write tid 1 to smithi07685715-11 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: finishing write tid 2 to smithi07685715-11 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: finishing write tid 3 to smithi07685715-11 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3788: done (5 left) 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3789: done (4 left) 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3795: delete oid 29 current snap is 476 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: finishing write tid 4 to smithi07685715-11 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: oid 11 updating version 0 to 2929 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: oid 11 version 2929 is already newer than 2927 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: oid 11 version 2929 is already newer than 2928 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 11 v 2929 (ObjNum 1430 snap 476 seq_num 1430) dirty exists 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3791: left oid 11 (ObjNum 1430 snap 476 seq_num 1430) 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3792: finishing write tid 1 to smithi07685715-35 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3792: finishing write tid 2 to smithi07685715-35 2024-09-21T09:54:01.969 INFO:tasks.rados.rados.0.smithi076.stdout:3792: finishing write tid 3 to smithi07685715-35 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:3792: finishing write tid 4 to smithi07685715-35 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:3792: oid 35 updating version 0 to 2932 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:3792: oid 35 version 2932 is already newer than 2931 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:3792: oid 35 updating version 2932 to 2933 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 35 v 2933 (ObjNum 1431 snap 476 seq_num 1431) dirty exists 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:3792: left oid 35 (ObjNum 1431 snap 476 seq_num 1431) 2024-09-21T09:54:01.970 INFO:tasks.rados.rados.0.smithi076.stdout:3793: finishing write tid 1 to smithi07685715-25 2024-09-21T09:54:01.971 INFO:tasks.rados.rados.0.smithi076.stdout:3793: finishing write tid 2 to smithi07685715-25 2024-09-21T09:54:01.971 INFO:tasks.rados.rados.0.smithi076.stdout:3791: done (4 left) 2024-09-21T09:54:01.971 INFO:tasks.rados.rados.0.smithi076.stdout:3792: done (3 left) 2024-09-21T09:54:01.971 INFO:tasks.rados.rados.0.smithi076.stdout:3795: done (2 left) 2024-09-21T09:54:01.971 INFO:tasks.rados.rados.0.smithi076.stdout:3796: write oid 22 current snap is 476 2024-09-21T09:54:01.971 INFO:tasks.rados.rados.0.smithi076.stdout:3796: seq_num 1434 ranges {501238=581430,1697649=403589} 2024-09-21T09:54:01.975 INFO:tasks.rados.rados.0.smithi076.stdout:3796: writing smithi07685715-22 from 501238 to 1082668 tid 1 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3796: writing smithi07685715-22 from 1697649 to 2101238 tid 2 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3797: rollback oid 50 current snap is 476 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 50 to 475 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3793: finishing write tid 3 to smithi07685715-25 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3793: finishing write tid 4 to smithi07685715-25 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3793: oid 25 updating version 0 to 879 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3793: oid 25 updating version 879 to 880 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3793: oid 25 updating version 880 to 881 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 881 (ObjNum 1432 snap 476 seq_num 1432) dirty exists 2024-09-21T09:54:01.982 INFO:tasks.rados.rados.0.smithi076.stdout:3793: left oid 25 (ObjNum 1432 snap 476 seq_num 1432) 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3794: finishing rollback tid 0 to smithi07685715-47 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3794: finishing rollback tid 1 to smithi07685715-47 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3794: finishing rollback tid 2 to smithi07685715-47 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2901 (ObjNum 1384 snap 462 seq_num 1384) dirty exists 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3796: finishing write tid 1 to smithi07685715-22 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3793: done (3 left) 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3794: done (2 left) 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3798: read oid 11 snap 472 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3798: expect deleted 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3799: read oid 8 snap -1 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3799: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3800: rollback oid 31 current snap is 476 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 466 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3801: read oid 9 snap 475 2024-09-21T09:54:01.983 INFO:tasks.rados.rados.0.smithi076.stdout:3801: expect (ObjNum 1410 snap 470 seq_num 1410) 2024-09-21T09:54:01.984 INFO:tasks.rados.rados.0.smithi076.stdout:3802: snap_create 2024-09-21T09:54:01.985 INFO:tasks.rados.rados.0.smithi076.stdout:3796: finishing write tid 2 to smithi07685715-22 2024-09-21T09:54:01.986 INFO:tasks.rados.rados.0.smithi076.stdout:3796: finishing write tid 3 to smithi07685715-22 2024-09-21T09:54:01.986 INFO:tasks.rados.rados.0.smithi076.stdout:3801: expect (ObjNum 1410 snap 470 seq_num 1410) 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3800: finishing rollback tid 0 to smithi07685715-31 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3800: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3796: finishing write tid 4 to smithi07685715-22 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3796: oid 22 updating version 0 to 1992 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3796: oid 22 version 1992 is already newer than 1991 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3796: oid 22 version 1992 is already newer than 1990 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 1992 (ObjNum 1434 snap 476 seq_num 1434) dirty exists 2024-09-21T09:54:02.234 INFO:tasks.rados.rados.0.smithi076.stdout:3796: left oid 22 (ObjNum 1434 snap 476 seq_num 1434) 2024-09-21T09:54:02.235 INFO:tasks.rados.rados.0.smithi076.stdout:3800: finishing rollback tid 2 to smithi07685715-31 2024-09-21T09:54:02.235 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 1173 (ObjNum 1397 snap 464 seq_num 1397) dirty exists 2024-09-21T09:54:02.235 INFO:tasks.rados.rados.0.smithi076.stdout:3797: finishing rollback tid 1 to smithi07685715-50 2024-09-21T09:54:02.235 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 1067 (ObjNum 33 snap 0 seq_num 864018544) dirty dne 2024-09-21T09:54:02.235 INFO:tasks.rados.rados.0.smithi076.stdout:3799: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:01 smithi191 ceph-mon[102206]: pgmap v1009: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 254 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-21T09:54:02.457 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:01 smithi191 ceph-mon[102206]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:54:02.458 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:01 smithi076 ceph-mon[106482]: pgmap v1009: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 254 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-21T09:54:02.458 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:01 smithi076 ceph-mon[106482]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:54:02.458 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:01 smithi076 ceph-mon[110963]: pgmap v1009: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 254 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-21T09:54:02.458 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:01 smithi076 ceph-mon[110963]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-21T09:54:02.939 INFO:tasks.rados.rados.0.smithi076.stdout:3796: done (6 left) 2024-09-21T09:54:02.939 INFO:tasks.rados.rados.0.smithi076.stdout:3797: done (5 left) 2024-09-21T09:54:02.939 INFO:tasks.rados.rados.0.smithi076.stdout:3798: done (4 left) 2024-09-21T09:54:02.939 INFO:tasks.rados.rados.0.smithi076.stdout:3799: done (3 left) 2024-09-21T09:54:02.939 INFO:tasks.rados.rados.0.smithi076.stdout:3800: done (2 left) 2024-09-21T09:54:02.939 INFO:tasks.rados.rados.0.smithi076.stdout:3801: done (1 left) 2024-09-21T09:54:02.940 INFO:tasks.rados.rados.0.smithi076.stdout:3802: done (0 left) 2024-09-21T09:54:02.940 INFO:tasks.rados.rados.0.smithi076.stdout:3803: snap_create 2024-09-21T09:54:03.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:02 smithi076 ceph-mon[106482]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:54:03.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:02 smithi076 ceph-mon[110963]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:54:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:02 smithi191 ceph-mon[102206]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-21T09:54:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:54:03] "GET /metrics HTTP/1.1" 200 36255 "" "Prometheus/2.51.0" 2024-09-21T09:54:03.825 INFO:tasks.rados.rados.0.smithi076.stdout:3803: done (0 left) 2024-09-21T09:54:03.825 INFO:tasks.rados.rados.0.smithi076.stdout:3804: delete oid 32 current snap is 478 2024-09-21T09:54:03.830 INFO:tasks.rados.rados.0.smithi076.stdout:3804: done (0 left) 2024-09-21T09:54:03.830 INFO:tasks.rados.rados.0.smithi076.stdout:3805: write oid 40 current snap is 478 2024-09-21T09:54:03.830 INFO:tasks.rados.rados.0.smithi076.stdout:3805: seq_num 1436 ranges {534852=731928,1814613=320239} 2024-09-21T09:54:03.837 INFO:tasks.rados.rados.0.smithi076.stdout:3805: writing smithi07685715-40 from 534852 to 1266780 tid 1 2024-09-21T09:54:03.842 INFO:tasks.rados.rados.0.smithi076.stdout:3805: writing smithi07685715-40 from 1814613 to 2134852 tid 2 2024-09-21T09:54:03.842 INFO:tasks.rados.rados.0.smithi076.stdout:3806: snap_create 2024-09-21T09:54:03.846 INFO:tasks.rados.rados.0.smithi076.stdout:3805: finishing write tid 1 to smithi07685715-40 2024-09-21T09:54:03.846 INFO:tasks.rados.rados.0.smithi076.stdout:3805: finishing write tid 2 to smithi07685715-40 2024-09-21T09:54:03.846 INFO:tasks.rados.rados.0.smithi076.stdout:3805: finishing write tid 3 to smithi07685715-40 2024-09-21T09:54:03.847 INFO:tasks.rados.rados.0.smithi076.stdout:3805: finishing write tid 4 to smithi07685715-40 2024-09-21T09:54:03.847 INFO:tasks.rados.rados.0.smithi076.stdout:3805: oid 40 updating version 0 to 1994 2024-09-21T09:54:03.847 INFO:tasks.rados.rados.0.smithi076.stdout:3805: oid 40 version 1994 is already newer than 1993 2024-09-21T09:54:03.847 INFO:tasks.rados.rados.0.smithi076.stdout:3805: oid 40 updating version 1994 to 1995 2024-09-21T09:54:03.847 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 1995 (ObjNum 1436 snap 478 seq_num 1436) dirty exists 2024-09-21T09:54:03.847 INFO:tasks.rados.rados.0.smithi076.stdout:3805: left oid 40 (ObjNum 1436 snap 478 seq_num 1436) 2024-09-21T09:54:04.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:03 smithi191 ceph-mon[102206]: pgmap v1012: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 255 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-21T09:54:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:03 smithi191 ceph-mon[102206]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:54:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:03 smithi191 ceph-mon[102206]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:54:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-mon[110963]: pgmap v1012: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 255 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-21T09:54:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-mon[110963]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:54:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-mon[110963]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:54:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-mon[106482]: pgmap v1012: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 255 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-09-21T09:54:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-mon[106482]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-21T09:54:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:03 smithi076 ceph-mon[106482]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-21T09:54:04.824 INFO:tasks.rados.rados.0.smithi076.stdout:3805: done (1 left) 2024-09-21T09:54:04.824 INFO:tasks.rados.rados.0.smithi076.stdout:3806: done (0 left) 2024-09-21T09:54:04.824 INFO:tasks.rados.rados.0.smithi076.stdout:3807: delete oid 17 current snap is 479 2024-09-21T09:54:04.832 INFO:tasks.rados.rados.0.smithi076.stdout:3807: done (0 left) 2024-09-21T09:54:04.832 INFO:tasks.rados.rados.0.smithi076.stdout:3808: delete oid 17 current snap is 479 2024-09-21T09:54:04.834 INFO:tasks.rados.rados.0.smithi076.stdout:3808: done (0 left) 2024-09-21T09:54:04.834 INFO:tasks.rados.rados.0.smithi076.stdout:3809: read oid 32 snap -1 2024-09-21T09:54:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3809: expect deleted 2024-09-21T09:54:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3810: rollback oid 27 current snap is 479 2024-09-21T09:54:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 27 to 469 2024-09-21T09:54:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3811: write oid 28 current snap is 479 2024-09-21T09:54:04.835 INFO:tasks.rados.rados.0.smithi076.stdout:3811: seq_num 1439 ranges {585273=674028,1990059=195214} 2024-09-21T09:54:04.841 INFO:tasks.rados.rados.0.smithi076.stdout:3811: writing smithi07685715-28 from 585273 to 1259301 tid 1 2024-09-21T09:54:04.846 INFO:tasks.rados.rados.0.smithi076.stdout:3811: writing smithi07685715-28 from 1990059 to 2185273 tid 2 2024-09-21T09:54:04.847 INFO:tasks.rados.rados.0.smithi076.stdout:3812: read oid 38 snap -1 2024-09-21T09:54:04.847 INFO:tasks.rados.rados.0.smithi076.stdout:3812: expect (ObjNum 1262 snap 424 seq_num 1262) 2024-09-21T09:54:04.847 INFO:tasks.rados.rados.0.smithi076.stdout:3813: snap_remove snap 477 2024-09-21T09:54:05.832 INFO:tasks.rados.rados.0.smithi076.stdout:3813: done (4 left) 2024-09-21T09:54:05.833 INFO:tasks.rados.rados.0.smithi076.stdout:3814: read oid 34 snap -1 2024-09-21T09:54:05.833 INFO:tasks.rados.rados.0.smithi076.stdout:3814: expect (ObjNum 1425 snap 474 seq_num 1425) 2024-09-21T09:54:05.833 INFO:tasks.rados.rados.0.smithi076.stdout:3815: read oid 44 snap 469 2024-09-21T09:54:05.833 INFO:tasks.rados.rados.0.smithi076.stdout:3815: expect (ObjNum 1355 snap 453 seq_num 1355) 2024-09-21T09:54:05.833 INFO:tasks.rados.rados.0.smithi076.stdout:3816: write oid 17 current snap is 479 2024-09-21T09:54:05.833 INFO:tasks.rados.rados.0.smithi076.stdout:3816: seq_num 1440 ranges {602080=749277,2048541=153539} 2024-09-21T09:54:05.840 INFO:tasks.rados.rados.0.smithi076.stdout:3816: writing smithi07685715-17 from 602080 to 1351357 tid 1 2024-09-21T09:54:05.845 INFO:tasks.rados.rados.0.smithi076.stdout:3816: writing smithi07685715-17 from 2048541 to 2202080 tid 2 2024-09-21T09:54:05.846 INFO:tasks.rados.rados.0.smithi076.stdout:3817: write oid 41 current snap is 479 2024-09-21T09:54:05.846 INFO:tasks.rados.rados.0.smithi076.stdout:3817: seq_num 1441 ranges {618887=424526,1590670=604158} 2024-09-21T09:54:05.851 INFO:tasks.rados.rados.0.smithi076.stdout:3817: writing smithi07685715-41 from 618887 to 1043413 tid 1 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3817: writing smithi07685715-41 from 1590670 to 2194828 tid 2 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3818: delete oid 14 current snap is 479 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3810: finishing rollback tid 0 to smithi07685715-27 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3810: finishing rollback tid 1 to smithi07685715-27 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3810: finishing rollback tid 2 to smithi07685715-27 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 1999 (ObjNum 1406 snap 467 seq_num 1406) dirty exists 2024-09-21T09:54:05.859 INFO:tasks.rados.rados.0.smithi076.stdout:3811: finishing write tid 1 to smithi07685715-28 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: finishing write tid 2 to smithi07685715-28 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: finishing write tid 3 to smithi07685715-28 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: finishing write tid 4 to smithi07685715-28 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: oid 28 updating version 0 to 2905 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: oid 28 version 2905 is already newer than 2903 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: oid 28 version 2905 is already newer than 2904 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 28 v 2905 (ObjNum 1439 snap 479 seq_num 1439) dirty exists 2024-09-21T09:54:05.860 INFO:tasks.rados.rados.0.smithi076.stdout:3811: left oid 28 (ObjNum 1439 snap 479 seq_num 1439) 2024-09-21T09:54:05.861 INFO:tasks.rados.rados.0.smithi076.stdout:3812: expect (ObjNum 1262 snap 424 seq_num 1262) 2024-09-21T09:54:06.007 INFO:tasks.rados.rados.0.smithi076.stdout:3815: expect (ObjNum 1355 snap 453 seq_num 1355) 2024-09-21T09:54:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:05 smithi191 ceph-mon[102206]: pgmap v1015: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 249 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:54:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:05 smithi191 ceph-mon[102206]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:54:06.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:05 smithi076 ceph-mon[110963]: pgmap v1015: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 249 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:54:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:05 smithi076 ceph-mon[110963]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:54:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:05 smithi076 ceph-mon[106482]: pgmap v1015: 89 pgs: 2 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 85 active+clean; 249 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:54:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:05 smithi076 ceph-mon[106482]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-21T09:54:06.215 INFO:tasks.rados.rados.0.smithi076.stdout:3814: expect (ObjNum 1425 snap 474 seq_num 1425) 2024-09-21T09:54:06.343 INFO:tasks.rados.rados.0.smithi076.stdout:3816: finishing write tid 1 to smithi07685715-17 2024-09-21T09:54:06.343 INFO:tasks.rados.rados.0.smithi076.stdout:3816: finishing write tid 2 to smithi07685715-17 2024-09-21T09:54:06.343 INFO:tasks.rados.rados.0.smithi076.stdout:3816: finishing write tid 3 to smithi07685715-17 2024-09-21T09:54:06.343 INFO:tasks.rados.rados.0.smithi076.stdout:3816: finishing write tid 4 to smithi07685715-17 2024-09-21T09:54:06.343 INFO:tasks.rados.rados.0.smithi076.stdout:3816: oid 17 updating version 0 to 2358 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3816: oid 17 updating version 2358 to 2359 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3816: oid 17 updating version 2359 to 2360 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2360 (ObjNum 1440 snap 479 seq_num 1440) dirty exists 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3816: left oid 17 (ObjNum 1440 snap 479 seq_num 1440) 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3817: finishing write tid 1 to smithi07685715-41 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3809: done (8 left) 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3810: done (7 left) 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3811: done (6 left) 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3812: done (5 left) 2024-09-21T09:54:06.344 INFO:tasks.rados.rados.0.smithi076.stdout:3814: done (4 left) 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3815: done (3 left) 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3816: done (2 left) 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3818: done (1 left) 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3819: delete oid 17 current snap is 479 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3817: finishing write tid 2 to smithi07685715-41 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3817: finishing write tid 3 to smithi07685715-41 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3817: finishing write tid 4 to smithi07685715-41 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3817: oid 41 updating version 0 to 2004 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3817: oid 41 version 2004 is already newer than 2002 2024-09-21T09:54:06.345 INFO:tasks.rados.rados.0.smithi076.stdout:3817: oid 41 version 2004 is already newer than 2003 2024-09-21T09:54:06.346 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 2004 (ObjNum 1441 snap 479 seq_num 1441) dirty exists 2024-09-21T09:54:06.346 INFO:tasks.rados.rados.0.smithi076.stdout:3817: left oid 41 (ObjNum 1441 snap 479 seq_num 1441) 2024-09-21T09:54:06.346 INFO:tasks.rados.rados.0.smithi076.stdout:3817: done (1 left) 2024-09-21T09:54:06.346 INFO:tasks.rados.rados.0.smithi076.stdout:3819: done (0 left) 2024-09-21T09:54:06.346 INFO:tasks.rados.rados.0.smithi076.stdout:3820: snap_create 2024-09-21T09:54:06.844 INFO:tasks.rados.rados.0.smithi076.stdout:3820: done (0 left) 2024-09-21T09:54:06.844 INFO:tasks.rados.rados.0.smithi076.stdout:3821: snap_remove snap 472 2024-09-21T09:54:07.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:06 smithi191 ceph-mon[102206]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:54:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:06 smithi076 ceph-mon[106482]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:54:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:06 smithi076 ceph-mon[110963]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-21T09:54:07.840 INFO:tasks.rados.rados.0.smithi076.stdout:3821: done (0 left) 2024-09-21T09:54:07.840 INFO:tasks.rados.rados.0.smithi076.stdout:3822: rollback oid 34 current snap is 480 2024-09-21T09:54:07.840 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 34 to 471 2024-09-21T09:54:07.840 INFO:tasks.rados.rados.0.smithi076.stdout:3823: delete oid 9 current snap is 480 2024-09-21T09:54:07.849 INFO:tasks.rados.rados.0.smithi076.stdout:3822: finishing rollback tid 0 to smithi07685715-34 2024-09-21T09:54:07.849 INFO:tasks.rados.rados.0.smithi076.stdout:3822: finishing rollback tid 1 to smithi07685715-34 2024-09-21T09:54:07.849 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 860 (ObjNum 864362448 snap 21884 seq_num 304) dirty dne 2024-09-21T09:54:07.851 INFO:tasks.rados.rados.0.smithi076.stdout:3822: done (1 left) 2024-09-21T09:54:07.851 INFO:tasks.rados.rados.0.smithi076.stdout:3823: done (0 left) 2024-09-21T09:54:07.851 INFO:tasks.rados.rados.0.smithi076.stdout:3824: write oid 15 current snap is 480 2024-09-21T09:54:07.851 INFO:tasks.rados.rados.0.smithi076.stdout:3824: seq_num 1445 ranges {686115=441875,1824598=461517} 2024-09-21T09:54:07.857 INFO:tasks.rados.rados.0.smithi076.stdout:3824: writing smithi07685715-15 from 686115 to 1127990 tid 1 2024-09-21T09:54:07.864 INFO:tasks.rados.rados.0.smithi076.stdout:3824: writing smithi07685715-15 from 1824598 to 2286115 tid 2 2024-09-21T09:54:07.864 INFO:tasks.rados.rados.0.smithi076.stdout:3825: delete oid 24 current snap is 480 2024-09-21T09:54:07.864 INFO:tasks.rados.rados.0.smithi076.stdout:3824: finishing write tid 1 to smithi07685715-15 2024-09-21T09:54:07.866 INFO:tasks.rados.rados.0.smithi076.stdout:3825: done (1 left) 2024-09-21T09:54:07.866 INFO:tasks.rados.rados.0.smithi076.stdout:3826: delete oid 3 current snap is 480 2024-09-21T09:54:07.866 INFO:tasks.rados.rados.0.smithi076.stdout:3824: finishing write tid 2 to smithi07685715-15 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:3824: finishing write tid 3 to smithi07685715-15 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:3824: finishing write tid 4 to smithi07685715-15 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:3824: oid 15 updating version 0 to 816 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:3824: oid 15 version 816 is already newer than 815 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:3824: oid 15 version 816 is already newer than 814 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 816 (ObjNum 1445 snap 480 seq_num 1445) dirty exists 2024-09-21T09:54:07.867 INFO:tasks.rados.rados.0.smithi076.stdout:3824: left oid 15 (ObjNum 1445 snap 480 seq_num 1445) 2024-09-21T09:54:07.868 INFO:tasks.rados.rados.0.smithi076.stdout:3824: done (1 left) 2024-09-21T09:54:07.868 INFO:tasks.rados.rados.0.smithi076.stdout:3826: done (0 left) 2024-09-21T09:54:07.868 INFO:tasks.rados.rados.0.smithi076.stdout:3827: read oid 3 snap -1 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3827: expect deleted 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3828: read oid 8 snap 475 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3828: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3829: read oid 33 snap -1 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3829: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3830: write oid 19 current snap is 480 2024-09-21T09:54:07.869 INFO:tasks.rados.rados.0.smithi076.stdout:3830: seq_num 1448 ranges {736536=667622,2000044=336492} 2024-09-21T09:54:07.875 INFO:tasks.rados.rados.0.smithi076.stdout:3830: writing smithi07685715-19 from 736536 to 1404158 tid 1 2024-09-21T09:54:07.881 INFO:tasks.rados.rados.0.smithi076.stdout:3830: writing smithi07685715-19 from 2000044 to 2336536 tid 2 2024-09-21T09:54:07.881 INFO:tasks.rados.rados.0.smithi076.stdout:3831: snap_create 2024-09-21T09:54:07.881 INFO:tasks.rados.rados.0.smithi076.stdout:3829: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:54:08.054 INFO:tasks.rados.rados.0.smithi076.stdout:3830: finishing write tid 1 to smithi07685715-19 2024-09-21T09:54:08.055 INFO:tasks.rados.rados.0.smithi076.stdout:3828: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:07 smithi191 ceph-mon[102206]: pgmap v1018: 89 pgs: 89 active+clean; 254 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:54:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:07 smithi191 ceph-mon[102206]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:54:08.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:07 smithi076 ceph-mon[106482]: pgmap v1018: 89 pgs: 89 active+clean; 254 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:54:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:07 smithi076 ceph-mon[106482]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:54:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:07 smithi076 ceph-mon[110963]: pgmap v1018: 89 pgs: 89 active+clean; 254 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:54:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:07 smithi076 ceph-mon[110963]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: finishing write tid 2 to smithi07685715-19 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: finishing write tid 3 to smithi07685715-19 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: finishing write tid 4 to smithi07685715-19 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: oid 19 updating version 0 to 1179 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: oid 19 version 1179 is already newer than 1178 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: oid 19 updating version 1179 to 1180 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 19 v 1180 (ObjNum 1448 snap 480 seq_num 1448) dirty exists 2024-09-21T09:54:08.170 INFO:tasks.rados.rados.0.smithi076.stdout:3830: left oid 19 (ObjNum 1448 snap 480 seq_num 1448) 2024-09-21T09:54:08.824 INFO:tasks.rados.rados.0.smithi076.stdout:3827: done (4 left) 2024-09-21T09:54:08.824 INFO:tasks.rados.rados.0.smithi076.stdout:3828: done (3 left) 2024-09-21T09:54:08.824 INFO:tasks.rados.rados.0.smithi076.stdout:3829: done (2 left) 2024-09-21T09:54:08.824 INFO:tasks.rados.rados.0.smithi076.stdout:3830: done (1 left) 2024-09-21T09:54:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:3831: done (0 left) 2024-09-21T09:54:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:3832: write oid 25 current snap is 481 2024-09-21T09:54:08.825 INFO:tasks.rados.rados.0.smithi076.stdout:3832: seq_num 1449 ranges {753343=742871,1942173=411170} 2024-09-21T09:54:08.832 INFO:tasks.rados.rados.0.smithi076.stdout:3832: writing smithi07685715-25 from 753343 to 1496214 tid 1 2024-09-21T09:54:08.837 INFO:tasks.rados.rados.0.smithi076.stdout:3832: writing smithi07685715-25 from 1942173 to 2353343 tid 2 2024-09-21T09:54:08.837 INFO:tasks.rados.rados.0.smithi076.stdout:3833: read oid 43 snap -1 2024-09-21T09:54:08.837 INFO:tasks.rados.rados.0.smithi076.stdout:3833: expect deleted 2024-09-21T09:54:08.837 INFO:tasks.rados.rados.0.smithi076.stdout:3834: snap_remove snap 480 2024-09-21T09:54:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:08 smithi191 ceph-mon[102206]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:54:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:08 smithi191 ceph-mon[102206]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:54:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:08 smithi076 ceph-mon[106482]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:54:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:08 smithi076 ceph-mon[106482]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:54:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:08 smithi076 ceph-mon[110963]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-21T09:54:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:08 smithi076 ceph-mon[110963]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-21T09:54:09.828 INFO:tasks.rados.rados.0.smithi076.stdout:3834: done (2 left) 2024-09-21T09:54:09.828 INFO:tasks.rados.rados.0.smithi076.stdout:3835: read oid 15 snap -1 2024-09-21T09:54:09.828 INFO:tasks.rados.rados.0.smithi076.stdout:3835: expect (ObjNum 1445 snap 480 seq_num 1445) 2024-09-21T09:54:09.828 INFO:tasks.rados.rados.0.smithi076.stdout:3836: read oid 11 snap -1 2024-09-21T09:54:09.828 INFO:tasks.rados.rados.0.smithi076.stdout:3836: expect (ObjNum 1430 snap 476 seq_num 1430) 2024-09-21T09:54:09.828 INFO:tasks.rados.rados.0.smithi076.stdout:3837: delete oid 1 current snap is 481 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: finishing write tid 1 to smithi07685715-25 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: finishing write tid 2 to smithi07685715-25 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: finishing write tid 3 to smithi07685715-25 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: finishing write tid 4 to smithi07685715-25 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: oid 25 updating version 0 to 884 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: oid 25 version 884 is already newer than 883 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: oid 25 updating version 884 to 885 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 885 (ObjNum 1449 snap 481 seq_num 1449) dirty exists 2024-09-21T09:54:09.829 INFO:tasks.rados.rados.0.smithi076.stdout:3832: left oid 25 (ObjNum 1449 snap 481 seq_num 1449) 2024-09-21T09:54:09.838 INFO:tasks.rados.rados.0.smithi076.stdout:3835: expect (ObjNum 1445 snap 480 seq_num 1445) 2024-09-21T09:54:09.936 INFO:tasks.rados.rados.0.smithi076.stdout:3836: expect (ObjNum 1430 snap 476 seq_num 1430) 2024-09-21T09:54:10.017 INFO:tasks.rados.rados.0.smithi076.stdout:3832: done (4 left) 2024-09-21T09:54:10.018 INFO:tasks.rados.rados.0.smithi076.stdout:3833: done (3 left) 2024-09-21T09:54:10.018 INFO:tasks.rados.rados.0.smithi076.stdout:3835: done (2 left) 2024-09-21T09:54:10.018 INFO:tasks.rados.rados.0.smithi076.stdout:3836: done (1 left) 2024-09-21T09:54:10.018 INFO:tasks.rados.rados.0.smithi076.stdout:3837: done (0 left) 2024-09-21T09:54:10.018 INFO:tasks.rados.rados.0.smithi076.stdout:3838: write oid 38 current snap is 481 2024-09-21T09:54:10.018 INFO:tasks.rados.rados.0.smithi076.stdout:3838: seq_num 1451 ranges {786957=493369,2059137=327820} 2024-09-21T09:54:10.024 INFO:tasks.rados.rados.0.smithi076.stdout:3838: writing smithi07685715-38 from 786957 to 1280326 tid 1 2024-09-21T09:54:10.030 INFO:tasks.rados.rados.0.smithi076.stdout:3838: writing smithi07685715-38 from 2059137 to 2386957 tid 2 2024-09-21T09:54:10.030 INFO:tasks.rados.rados.0.smithi076.stdout:3839: snap_remove snap 465 2024-09-21T09:54:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:09 smithi191 ceph-mon[102206]: pgmap v1021: 89 pgs: 89 active+clean; 254 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:54:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:09 smithi191 ceph-mon[102206]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:54:10.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:09 smithi076 ceph-mon[106482]: pgmap v1021: 89 pgs: 89 active+clean; 254 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:54:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:09 smithi076 ceph-mon[106482]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:54:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:09 smithi076 ceph-mon[110963]: pgmap v1021: 89 pgs: 89 active+clean; 254 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:54:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:09 smithi076 ceph-mon[110963]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-21T09:54:10.867 INFO:tasks.rados.rados.0.smithi076.stdout:3838: finishing write tid 1 to smithi07685715-38 2024-09-21T09:54:10.867 INFO:tasks.rados.rados.0.smithi076.stdout:3838: finishing write tid 2 to smithi07685715-38 2024-09-21T09:54:10.867 INFO:tasks.rados.rados.0.smithi076.stdout:3838: finishing write tid 3 to smithi07685715-38 2024-09-21T09:54:10.867 INFO:tasks.rados.rados.0.smithi076.stdout:3838: finishing write tid 4 to smithi07685715-38 2024-09-21T09:54:10.867 INFO:tasks.rados.rados.0.smithi076.stdout:3838: oid 38 updating version 0 to 2368 2024-09-21T09:54:10.867 INFO:tasks.rados.rados.0.smithi076.stdout:3838: oid 38 version 2368 is already newer than 2367 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3838: oid 38 version 2368 is already newer than 2366 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 2368 (ObjNum 1451 snap 481 seq_num 1451) dirty exists 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3838: left oid 38 (ObjNum 1451 snap 481 seq_num 1451) 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3838: done (1 left) 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3839: done (0 left) 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3840: read oid 31 snap 471 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3840: expect (ObjNum 1397 snap 464 seq_num 1397) 2024-09-21T09:54:10.868 INFO:tasks.rados.rados.0.smithi076.stdout:3841: snap_create 2024-09-21T09:54:10.876 INFO:tasks.rados.rados.0.smithi076.stdout:3840: expect (ObjNum 1397 snap 464 seq_num 1397) 2024-09-21T09:54:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:11.876 INFO:tasks.rados.rados.0.smithi076.stdout:3840: done (1 left) 2024-09-21T09:54:11.876 INFO:tasks.rados.rados.0.smithi076.stdout:3841: done (0 left) 2024-09-21T09:54:11.876 INFO:tasks.rados.rados.0.smithi076.stdout:3842: read oid 18 snap -1 2024-09-21T09:54:11.876 INFO:tasks.rados.rados.0.smithi076.stdout:3842: expect deleted 2024-09-21T09:54:11.876 INFO:tasks.rados.rados.0.smithi076.stdout:3843: snap_create 2024-09-21T09:54:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:11 smithi191 ceph-mon[102206]: pgmap v1024: 89 pgs: 89 active+clean; 255 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 686 KiB/s wr, 14 op/s 2024-09-21T09:54:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:11 smithi191 ceph-mon[102206]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:54:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:11 smithi076 ceph-mon[110963]: pgmap v1024: 89 pgs: 89 active+clean; 255 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 686 KiB/s wr, 14 op/s 2024-09-21T09:54:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:11 smithi076 ceph-mon[110963]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:54:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:11 smithi076 ceph-mon[106482]: pgmap v1024: 89 pgs: 89 active+clean; 255 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 686 KiB/s wr, 14 op/s 2024-09-21T09:54:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:11 smithi076 ceph-mon[106482]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-21T09:54:12.885 INFO:tasks.rados.rados.0.smithi076.stdout:3842: done (1 left) 2024-09-21T09:54:12.885 INFO:tasks.rados.rados.0.smithi076.stdout:3843: done (0 left) 2024-09-21T09:54:12.885 INFO:tasks.rados.rados.0.smithi076.stdout:3844: write oid 20 current snap is 483 2024-09-21T09:54:12.886 INFO:tasks.rados.rados.0.smithi076.stdout:3844: seq_num 1452 ranges {403764=568618,1717619=686145} 2024-09-21T09:54:12.890 INFO:tasks.rados.rados.0.smithi076.stdout:3844: writing smithi07685715-20 from 403764 to 972382 tid 1 2024-09-21T09:54:12.898 INFO:tasks.rados.rados.0.smithi076.stdout:3844: writing smithi07685715-20 from 1717619 to 2403764 tid 2 2024-09-21T09:54:12.899 INFO:tasks.rados.rados.0.smithi076.stdout:3845: snap_create 2024-09-21T09:54:12.899 INFO:tasks.rados.rados.0.smithi076.stdout:3844: finishing write tid 1 to smithi07685715-20 2024-09-21T09:54:12.902 INFO:tasks.rados.rados.0.smithi076.stdout:3844: finishing write tid 2 to smithi07685715-20 2024-09-21T09:54:12.902 INFO:tasks.rados.rados.0.smithi076.stdout:3844: finishing write tid 3 to smithi07685715-20 2024-09-21T09:54:12.902 INFO:tasks.rados.rados.0.smithi076.stdout:3844: finishing write tid 4 to smithi07685715-20 2024-09-21T09:54:12.903 INFO:tasks.rados.rados.0.smithi076.stdout:3844: oid 20 updating version 0 to 1202 2024-09-21T09:54:12.903 INFO:tasks.rados.rados.0.smithi076.stdout:3844: oid 20 version 1202 is already newer than 1200 2024-09-21T09:54:12.903 INFO:tasks.rados.rados.0.smithi076.stdout:3844: oid 20 version 1202 is already newer than 1201 2024-09-21T09:54:12.903 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 1202 (ObjNum 1452 snap 483 seq_num 1452) dirty exists 2024-09-21T09:54:12.903 INFO:tasks.rados.rados.0.smithi076.stdout:3844: left oid 20 (ObjNum 1452 snap 483 seq_num 1452) 2024-09-21T09:54:13.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:12 smithi076 ceph-mon[106482]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:54:13.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:12 smithi076 ceph-mon[110963]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:54:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:12 smithi191 ceph-mon[102206]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-21T09:54:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:54:13] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:54:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3844: done (1 left) 2024-09-21T09:54:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3845: done (0 left) 2024-09-21T09:54:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3846: write oid 43 current snap is 484 2024-09-21T09:54:13.827 INFO:tasks.rados.rados.0.smithi076.stdout:3846: seq_num 1453 ranges {420571=760220,1776101=626172} 2024-09-21T09:54:13.833 INFO:tasks.rados.rados.0.smithi076.stdout:3846: writing smithi07685715-43 from 420571 to 1180791 tid 1 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:3846: writing smithi07685715-43 from 1776101 to 2402273 tid 2 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:3847: rollback oid 16 current snap is 484 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 16 to 469 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:3848: rollback oid 31 current snap is 484 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 31 to 471 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:3849: read oid 4 snap -1 2024-09-21T09:54:13.841 INFO:tasks.rados.rados.0.smithi076.stdout:3849: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:54:13.842 INFO:tasks.rados.rados.0.smithi076.stdout:3850: write oid 29 current snap is 484 2024-09-21T09:54:13.842 INFO:tasks.rados.rados.0.smithi076.stdout:3850: seq_num 1454 ranges {437378=435469,1434583=769830} 2024-09-21T09:54:13.845 INFO:tasks.rados.rados.0.smithi076.stdout:3850: writing smithi07685715-29 from 437378 to 872847 tid 1 2024-09-21T09:54:13.853 INFO:tasks.rados.rados.0.smithi076.stdout:3850: writing smithi07685715-29 from 1434583 to 2204413 tid 2 2024-09-21T09:54:13.854 INFO:tasks.rados.rados.0.smithi076.stdout:3851: write oid 44 current snap is 484 2024-09-21T09:54:13.854 INFO:tasks.rados.rados.0.smithi076.stdout:3851: seq_num 1455 ranges {454185=510718,1493065=629841} 2024-09-21T09:54:13.858 INFO:tasks.rados.rados.0.smithi076.stdout:3851: writing smithi07685715-44 from 454185 to 964903 tid 1 2024-09-21T09:54:13.865 INFO:tasks.rados.rados.0.smithi076.stdout:3851: writing smithi07685715-44 from 1493065 to 2122906 tid 2 2024-09-21T09:54:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:3852: read oid 33 snap -1 2024-09-21T09:54:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:3852: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:54:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:3853: snap_create 2024-09-21T09:54:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:3848: finishing rollback tid 0 to smithi07685715-31 2024-09-21T09:54:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:3848: finishing rollback tid 1 to smithi07685715-31 2024-09-21T09:54:13.866 INFO:tasks.rados.rados.0.smithi076.stdout:3849: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:54:13.931 INFO:tasks.rados.rados.0.smithi076.stdout:3848: finishing rollback tid 2 to smithi07685715-31 2024-09-21T09:54:13.931 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 31 v 1188 (ObjNum 1397 snap 464 seq_num 1397) dirty exists 2024-09-21T09:54:13.931 INFO:tasks.rados.rados.0.smithi076.stdout:3850: finishing write tid 1 to smithi07685715-29 2024-09-21T09:54:13.931 INFO:tasks.rados.rados.0.smithi076.stdout:3847: finishing rollback tid 0 to smithi07685715-16 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3847: finishing rollback tid 1 to smithi07685715-16 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3847: finishing rollback tid 2 to smithi07685715-16 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 2016 (ObjNum 1373 snap 460 seq_num 1373) dirty exists 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3846: finishing write tid 1 to smithi07685715-43 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3846: finishing write tid 2 to smithi07685715-43 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3846: finishing write tid 3 to smithi07685715-43 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3846: finishing write tid 4 to smithi07685715-43 2024-09-21T09:54:13.932 INFO:tasks.rados.rados.0.smithi076.stdout:3846: oid 43 updating version 0 to 861 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3846: oid 43 updating version 861 to 862 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3846: oid 43 updating version 862 to 863 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 863 (ObjNum 1453 snap 484 seq_num 1453) dirty exists 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3846: left oid 43 (ObjNum 1453 snap 484 seq_num 1453) 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3850: finishing write tid 2 to smithi07685715-29 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3850: finishing write tid 3 to smithi07685715-29 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3850: finishing write tid 4 to smithi07685715-29 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3850: oid 29 updating version 0 to 1203 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3850: oid 29 updating version 1203 to 1205 2024-09-21T09:54:13.933 INFO:tasks.rados.rados.0.smithi076.stdout:3850: oid 29 version 1205 is already newer than 1204 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 1205 (ObjNum 1454 snap 484 seq_num 1454) dirty exists 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3850: left oid 29 (ObjNum 1454 snap 484 seq_num 1454) 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: finishing write tid 1 to smithi07685715-44 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: finishing write tid 2 to smithi07685715-44 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: finishing write tid 3 to smithi07685715-44 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: finishing write tid 4 to smithi07685715-44 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: oid 44 updating version 0 to 840 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: oid 44 version 840 is already newer than 838 2024-09-21T09:54:13.934 INFO:tasks.rados.rados.0.smithi076.stdout:3851: oid 44 version 840 is already newer than 839 2024-09-21T09:54:13.935 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 840 (ObjNum 1455 snap 484 seq_num 1455) dirty exists 2024-09-21T09:54:13.935 INFO:tasks.rados.rados.0.smithi076.stdout:3851: left oid 44 (ObjNum 1455 snap 484 seq_num 1455) 2024-09-21T09:54:13.935 INFO:tasks.rados.rados.0.smithi076.stdout:3852: expect (ObjNum 1188 snap 399 seq_num 1188) 2024-09-21T09:54:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:13 smithi191 ceph-mon[102206]: pgmap v1027: 89 pgs: 89 active+clean; 255 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 686 KiB/s wr, 15 op/s 2024-09-21T09:54:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:13 smithi191 ceph-mon[102206]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:54:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:13 smithi191 ceph-mon[102206]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:54:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-mon[106482]: pgmap v1027: 89 pgs: 89 active+clean; 255 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 686 KiB/s wr, 15 op/s 2024-09-21T09:54:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-mon[106482]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:54:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-mon[106482]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:54:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-mon[110963]: pgmap v1027: 89 pgs: 89 active+clean; 255 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 686 KiB/s wr, 15 op/s 2024-09-21T09:54:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-mon[110963]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-21T09:54:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:13 smithi076 ceph-mon[110963]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-21T09:54:14.829 INFO:tasks.rados.rados.0.smithi076.stdout:3846: done (7 left) 2024-09-21T09:54:14.829 INFO:tasks.rados.rados.0.smithi076.stdout:3847: done (6 left) 2024-09-21T09:54:14.829 INFO:tasks.rados.rados.0.smithi076.stdout:3848: done (5 left) 2024-09-21T09:54:14.829 INFO:tasks.rados.rados.0.smithi076.stdout:3849: done (4 left) 2024-09-21T09:54:14.829 INFO:tasks.rados.rados.0.smithi076.stdout:3850: done (3 left) 2024-09-21T09:54:14.829 INFO:tasks.rados.rados.0.smithi076.stdout:3851: done (2 left) 2024-09-21T09:54:14.830 INFO:tasks.rados.rados.0.smithi076.stdout:3852: done (1 left) 2024-09-21T09:54:14.830 INFO:tasks.rados.rados.0.smithi076.stdout:3853: done (0 left) 2024-09-21T09:54:14.830 INFO:tasks.rados.rados.0.smithi076.stdout:3854: write oid 44 current snap is 485 2024-09-21T09:54:14.830 INFO:tasks.rados.rados.0.smithi076.stdout:3854: seq_num 1456 ranges {470992=585967,1551547=773499} 2024-09-21T09:54:14.834 INFO:tasks.rados.rados.0.smithi076.stdout:3854: writing smithi07685715-44 from 470992 to 1056959 tid 1 2024-09-21T09:54:14.842 INFO:tasks.rados.rados.0.smithi076.stdout:3854: writing smithi07685715-44 from 1551547 to 2325046 tid 2 2024-09-21T09:54:14.842 INFO:tasks.rados.rados.0.smithi076.stdout:3855: read oid 10 snap 483 2024-09-21T09:54:14.842 INFO:tasks.rados.rados.0.smithi076.stdout:3855: expect deleted 2024-09-21T09:54:14.842 INFO:tasks.rados.rados.0.smithi076.stdout:3856: write oid 41 current snap is 485 2024-09-21T09:54:14.842 INFO:tasks.rados.rados.0.smithi076.stdout:3854: finishing write tid 1 to smithi07685715-44 2024-09-21T09:54:14.842 INFO:tasks.rados.rados.0.smithi076.stdout:3856: seq_num 1457 ranges {487799=661216,1893676=594123} 2024-09-21T09:54:14.848 INFO:tasks.rados.rados.0.smithi076.stdout:3856: writing smithi07685715-41 from 487799 to 1149015 tid 1 2024-09-21T09:54:14.856 INFO:tasks.rados.rados.0.smithi076.stdout:3856: writing smithi07685715-41 from 1893676 to 2487799 tid 2 2024-09-21T09:54:14.856 INFO:tasks.rados.rados.0.smithi076.stdout:3857: delete oid 37 current snap is 485 2024-09-21T09:54:14.856 INFO:tasks.rados.rados.0.smithi076.stdout:3854: finishing write tid 2 to smithi07685715-44 2024-09-21T09:54:14.856 INFO:tasks.rados.rados.0.smithi076.stdout:3854: finishing write tid 3 to smithi07685715-44 2024-09-21T09:54:14.856 INFO:tasks.rados.rados.0.smithi076.stdout:3854: finishing write tid 4 to smithi07685715-44 2024-09-21T09:54:14.856 INFO:tasks.rados.rados.0.smithi076.stdout:3854: oid 44 updating version 0 to 843 2024-09-21T09:54:14.857 INFO:tasks.rados.rados.0.smithi076.stdout:3854: oid 44 updating version 843 to 844 2024-09-21T09:54:14.857 INFO:tasks.rados.rados.0.smithi076.stdout:3854: oid 44 version 844 is already newer than 842 2024-09-21T09:54:14.857 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 844 (ObjNum 1456 snap 485 seq_num 1456) dirty exists 2024-09-21T09:54:14.857 INFO:tasks.rados.rados.0.smithi076.stdout:3854: left oid 44 (ObjNum 1456 snap 485 seq_num 1456) 2024-09-21T09:54:14.857 INFO:tasks.rados.rados.0.smithi076.stdout:3856: finishing write tid 1 to smithi07685715-41 2024-09-21T09:54:14.858 INFO:tasks.rados.rados.0.smithi076.stdout:3854: done (3 left) 2024-09-21T09:54:14.858 INFO:tasks.rados.rados.0.smithi076.stdout:3855: done (2 left) 2024-09-21T09:54:14.858 INFO:tasks.rados.rados.0.smithi076.stdout:3857: done (1 left) 2024-09-21T09:54:14.858 INFO:tasks.rados.rados.0.smithi076.stdout:3858: delete oid 21 current snap is 485 2024-09-21T09:54:14.860 INFO:tasks.rados.rados.0.smithi076.stdout:3856: finishing write tid 2 to smithi07685715-41 2024-09-21T09:54:14.860 INFO:tasks.rados.rados.0.smithi076.stdout:3856: finishing write tid 3 to smithi07685715-41 2024-09-21T09:54:14.860 INFO:tasks.rados.rados.0.smithi076.stdout:3856: finishing write tid 4 to smithi07685715-41 2024-09-21T09:54:14.860 INFO:tasks.rados.rados.0.smithi076.stdout:3856: oid 41 updating version 0 to 2019 2024-09-21T09:54:14.860 INFO:tasks.rados.rados.0.smithi076.stdout:3856: oid 41 updating version 2019 to 2020 2024-09-21T09:54:14.860 INFO:tasks.rados.rados.0.smithi076.stdout:3856: oid 41 version 2020 is already newer than 2018 2024-09-21T09:54:14.861 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 2020 (ObjNum 1457 snap 485 seq_num 1457) dirty exists 2024-09-21T09:54:14.861 INFO:tasks.rados.rados.0.smithi076.stdout:3856: left oid 41 (ObjNum 1457 snap 485 seq_num 1457) 2024-09-21T09:54:14.861 INFO:tasks.rados.rados.0.smithi076.stdout:3856: done (1 left) 2024-09-21T09:54:14.861 INFO:tasks.rados.rados.0.smithi076.stdout:3858: done (0 left) 2024-09-21T09:54:14.861 INFO:tasks.rados.rados.0.smithi076.stdout:3859: delete oid 47 current snap is 485 2024-09-21T09:54:14.863 INFO:tasks.rados.rados.0.smithi076.stdout:3859: done (0 left) 2024-09-21T09:54:14.864 INFO:tasks.rados.rados.0.smithi076.stdout:3860: snap_remove snap 476 2024-09-21T09:54:15.832 INFO:tasks.rados.rados.0.smithi076.stdout:3860: done (0 left) 2024-09-21T09:54:15.832 INFO:tasks.rados.rados.0.smithi076.stdout:3861: write oid 30 current snap is 485 2024-09-21T09:54:15.832 INFO:tasks.rados.rados.0.smithi076.stdout:3861: seq_num 1461 ranges {555027=678565,1727604=640848} 2024-09-21T09:54:15.839 INFO:tasks.rados.rados.0.smithi076.stdout:3861: writing smithi07685715-30 from 555027 to 1233592 tid 1 2024-09-21T09:54:15.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: writing smithi07685715-30 from 1727604 to 2368452 tid 2 2024-09-21T09:54:15.846 INFO:tasks.rados.rados.0.smithi076.stdout:3862: snap_remove snap 484 2024-09-21T09:54:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:15 smithi191 ceph-mon[102206]: pgmap v1030: 89 pgs: 89 active+clean; 250 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 483 KiB/s wr, 7 op/s 2024-09-21T09:54:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:15 smithi191 ceph-mon[102206]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:54:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:15 smithi076 ceph-mon[106482]: pgmap v1030: 89 pgs: 89 active+clean; 250 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 483 KiB/s wr, 7 op/s 2024-09-21T09:54:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:15 smithi076 ceph-mon[106482]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:54:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:15 smithi076 ceph-mon[110963]: pgmap v1030: 89 pgs: 89 active+clean; 250 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 483 KiB/s wr, 7 op/s 2024-09-21T09:54:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:15 smithi076 ceph-mon[110963]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-21T09:54:16.845 INFO:tasks.rados.rados.0.smithi076.stdout:3862: done (1 left) 2024-09-21T09:54:16.845 INFO:tasks.rados.rados.0.smithi076.stdout:3863: rollback oid 43 current snap is 485 2024-09-21T09:54:16.845 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 469 2024-09-21T09:54:16.845 INFO:tasks.rados.rados.0.smithi076.stdout:3861: finishing write tid 1 to smithi07685715-30 2024-09-21T09:54:16.845 INFO:tasks.rados.rados.0.smithi076.stdout:3861: finishing write tid 2 to smithi07685715-30 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: finishing write tid 3 to smithi07685715-30 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: finishing write tid 4 to smithi07685715-30 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: oid 30 updating version 0 to 2922 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: oid 30 version 2922 is already newer than 2921 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: oid 30 updating version 2922 to 2923 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2923 (ObjNum 1461 snap 485 seq_num 1461) dirty exists 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: left oid 30 (ObjNum 1461 snap 485 seq_num 1461) 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3861: done (1 left) 2024-09-21T09:54:16.846 INFO:tasks.rados.rados.0.smithi076.stdout:3864: snap_create 2024-09-21T09:54:16.850 INFO:tasks.rados.rados.0.smithi076.stdout:3863: finishing rollback tid 0 to smithi07685715-43 2024-09-21T09:54:16.851 INFO:tasks.rados.rados.0.smithi076.stdout:3863: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:54:16.851 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 867 (ObjNum 33 snap 0 seq_num 864018544) dirty dne 2024-09-21T09:54:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:16 smithi191 ceph-mon[102206]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:54:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:16 smithi076 ceph-mon[106482]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:54:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:16 smithi076 ceph-mon[110963]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-21T09:54:17.844 INFO:tasks.rados.rados.0.smithi076.stdout:3863: done (1 left) 2024-09-21T09:54:17.845 INFO:tasks.rados.rados.0.smithi076.stdout:3864: done (0 left) 2024-09-21T09:54:17.845 INFO:tasks.rados.rados.0.smithi076.stdout:3865: snap_create 2024-09-21T09:54:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:17 smithi191 ceph-mon[102206]: pgmap v1033: 89 pgs: 89 active+clean; 256 MiB data, 905 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:54:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:17 smithi191 ceph-mon[102206]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:54:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:17 smithi076 ceph-mon[106482]: pgmap v1033: 89 pgs: 89 active+clean; 256 MiB data, 905 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:54:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:17 smithi076 ceph-mon[106482]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:54:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:17 smithi076 ceph-mon[110963]: pgmap v1033: 89 pgs: 89 active+clean; 256 MiB data, 905 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-09-21T09:54:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:17 smithi076 ceph-mon[110963]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-21T09:54:18.828 INFO:tasks.rados.rados.0.smithi076.stdout:3865: done (0 left) 2024-09-21T09:54:18.828 INFO:tasks.rados.rados.0.smithi076.stdout:3866: write oid 3 current snap is 487 2024-09-21T09:54:18.829 INFO:tasks.rados.rados.0.smithi076.stdout:3866: seq_num 1462 ranges {571834=753814,1786086=784506} 2024-09-21T09:54:18.835 INFO:tasks.rados.rados.0.smithi076.stdout:3866: writing smithi07685715-3 from 571834 to 1325648 tid 1 2024-09-21T09:54:18.842 INFO:tasks.rados.rados.0.smithi076.stdout:3866: writing smithi07685715-3 from 1786086 to 2570592 tid 2 2024-09-21T09:54:18.842 INFO:tasks.rados.rados.0.smithi076.stdout:3867: snap_remove snap 481 2024-09-21T09:54:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:18 smithi191 ceph-mon[102206]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:54:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:18 smithi191 ceph-mon[102206]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:54:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:18 smithi076 ceph-mon[106482]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:54:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:18 smithi076 ceph-mon[106482]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:54:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:18 smithi076 ceph-mon[110963]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-21T09:54:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:18 smithi076 ceph-mon[110963]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-21T09:54:19.829 INFO:tasks.rados.rados.0.smithi076.stdout:3867: done (1 left) 2024-09-21T09:54:19.829 INFO:tasks.rados.rados.0.smithi076.stdout:3868: snap_remove snap 408 2024-09-21T09:54:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:19 smithi191 ceph-mon[102206]: pgmap v1036: 89 pgs: 89 active+clean; 256 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:54:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:19 smithi191 ceph-mon[102206]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:54:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:19 smithi076 ceph-mon[106482]: pgmap v1036: 89 pgs: 89 active+clean; 256 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:54:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:19 smithi076 ceph-mon[106482]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:54:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:19 smithi076 ceph-mon[110963]: pgmap v1036: 89 pgs: 89 active+clean; 256 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-09-21T09:54:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:19 smithi076 ceph-mon[110963]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-21T09:54:20.830 INFO:tasks.rados.rados.0.smithi076.stdout:3866: finishing write tid 1 to smithi07685715-3 2024-09-21T09:54:20.830 INFO:tasks.rados.rados.0.smithi076.stdout:3866: finishing write tid 2 to smithi07685715-3 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: finishing write tid 3 to smithi07685715-3 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: finishing write tid 4 to smithi07685715-3 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: oid 3 updating version 0 to 326 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: oid 3 version 326 is already newer than 325 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: oid 3 updating version 326 to 327 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 327 (ObjNum 1462 snap 487 seq_num 1462) dirty exists 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: left oid 3 (ObjNum 1462 snap 487 seq_num 1462) 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3866: done (1 left) 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3868: done (0 left) 2024-09-21T09:54:20.831 INFO:tasks.rados.rados.0.smithi076.stdout:3869: delete oid 24 current snap is 487 2024-09-21T09:54:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:3869: done (0 left) 2024-09-21T09:54:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:3870: rollback oid 39 current snap is 487 2024-09-21T09:54:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 39 to 469 2024-09-21T09:54:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:3871: write oid 50 current snap is 487 2024-09-21T09:54:20.834 INFO:tasks.rados.rados.0.smithi076.stdout:3871: seq_num 1464 ranges {605448=504312,1903050=702398} 2024-09-21T09:54:20.841 INFO:tasks.rados.rados.0.smithi076.stdout:3871: writing smithi07685715-50 from 605448 to 1109760 tid 1 2024-09-21T09:54:20.850 INFO:tasks.rados.rados.0.smithi076.stdout:3871: writing smithi07685715-50 from 1903050 to 2605448 tid 2 2024-09-21T09:54:20.851 INFO:tasks.rados.rados.0.smithi076.stdout:3872: write oid 37 current snap is 487 2024-09-21T09:54:20.851 INFO:tasks.rados.rados.0.smithi076.stdout:3870: finishing rollback tid 0 to smithi07685715-39 2024-09-21T09:54:20.851 INFO:tasks.rados.rados.0.smithi076.stdout:3872: seq_num 1465 ranges {622255=579561,1845179=531833} 2024-09-21T09:54:20.857 INFO:tasks.rados.rados.0.smithi076.stdout:3872: writing smithi07685715-37 from 622255 to 1201816 tid 1 2024-09-21T09:54:20.864 INFO:tasks.rados.rados.0.smithi076.stdout:3872: writing smithi07685715-37 from 1845179 to 2377012 tid 2 2024-09-21T09:54:20.864 INFO:tasks.rados.rados.0.smithi076.stdout:3873: snap_remove snap 479 2024-09-21T09:54:21.861 INFO:tasks.rados.rados.0.smithi076.stdout:3873: done (3 left) 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3874: snap_create 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: finishing write tid 1 to smithi07685715-50 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3870: finishing rollback tid 1 to smithi07685715-39 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 39 v 2960 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: finishing write tid 2 to smithi07685715-50 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: finishing write tid 3 to smithi07685715-50 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: finishing write tid 4 to smithi07685715-50 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: oid 50 updating version 0 to 1218 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: oid 50 version 1218 is already newer than 1216 2024-09-21T09:54:21.862 INFO:tasks.rados.rados.0.smithi076.stdout:3871: oid 50 version 1218 is already newer than 1217 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 50 v 1218 (ObjNum 1464 snap 487 seq_num 1464) dirty exists 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3871: left oid 50 (ObjNum 1464 snap 487 seq_num 1464) 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: finishing write tid 1 to smithi07685715-37 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: finishing write tid 2 to smithi07685715-37 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: finishing write tid 3 to smithi07685715-37 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: finishing write tid 4 to smithi07685715-37 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: oid 37 updating version 0 to 1223 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: oid 37 updating version 1223 to 1224 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: oid 37 version 1224 is already newer than 1222 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 37 v 1224 (ObjNum 1465 snap 487 seq_num 1465) dirty exists 2024-09-21T09:54:21.863 INFO:tasks.rados.rados.0.smithi076.stdout:3872: left oid 37 (ObjNum 1465 snap 487 seq_num 1465) 2024-09-21T09:54:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:21 smithi191 ceph-mon[102206]: pgmap v1039: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 256 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 339 KiB/s wr, 3 op/s 2024-09-21T09:54:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:21 smithi191 ceph-mon[102206]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:54:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:21 smithi076 ceph-mon[106482]: pgmap v1039: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 256 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 339 KiB/s wr, 3 op/s 2024-09-21T09:54:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:21 smithi076 ceph-mon[106482]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:54:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:21 smithi076 ceph-mon[110963]: pgmap v1039: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 256 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 339 KiB/s wr, 3 op/s 2024-09-21T09:54:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:21 smithi076 ceph-mon[110963]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-21T09:54:22.866 INFO:tasks.rados.rados.0.smithi076.stdout:3870: done (3 left) 2024-09-21T09:54:22.866 INFO:tasks.rados.rados.0.smithi076.stdout:3871: done (2 left) 2024-09-21T09:54:22.866 INFO:tasks.rados.rados.0.smithi076.stdout:3872: done (1 left) 2024-09-21T09:54:22.867 INFO:tasks.rados.rados.0.smithi076.stdout:3874: done (0 left) 2024-09-21T09:54:22.867 INFO:tasks.rados.rados.0.smithi076.stdout:3875: snap_create 2024-09-21T09:54:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:22 smithi076 ceph-mon[106482]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:54:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:22 smithi076 ceph-mon[110963]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:54:23.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:22 smithi191 ceph-mon[102206]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-21T09:54:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:54:23] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:54:23.829 INFO:tasks.rados.rados.0.smithi076.stdout:3875: done (0 left) 2024-09-21T09:54:23.829 INFO:tasks.rados.rados.0.smithi076.stdout:3876: snap_create 2024-09-21T09:54:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-mon[110963]: pgmap v1042: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 214 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 942 KiB/s wr, 4 op/s 2024-09-21T09:54:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-mon[110963]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:54:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-mon[110963]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:54:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-mon[106482]: pgmap v1042: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 214 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 942 KiB/s wr, 4 op/s 2024-09-21T09:54:24.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-mon[106482]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:54:24.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:23 smithi076 ceph-mon[106482]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:54:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:23 smithi191 ceph-mon[102206]: pgmap v1042: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 214 MiB data, 875 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 942 KiB/s wr, 4 op/s 2024-09-21T09:54:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:23 smithi191 ceph-mon[102206]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-21T09:54:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:23 smithi191 ceph-mon[102206]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-21T09:54:24.829 INFO:tasks.rados.rados.0.smithi076.stdout:3876: done (0 left) 2024-09-21T09:54:24.829 INFO:tasks.rados.rados.0.smithi076.stdout:3877: write oid 5 current snap is 490 2024-09-21T09:54:24.830 INFO:tasks.rados.rados.0.smithi076.stdout:3877: seq_num 1466 ranges {639062=654810,1903661=735401} 2024-09-21T09:54:24.836 INFO:tasks.rados.rados.0.smithi076.stdout:3877: writing smithi07685715-5 from 639062 to 1293872 tid 1 2024-09-21T09:54:24.844 INFO:tasks.rados.rados.0.smithi076.stdout:3877: writing smithi07685715-5 from 1903661 to 2639062 tid 2 2024-09-21T09:54:24.844 INFO:tasks.rados.rados.0.smithi076.stdout:3878: write oid 16 current snap is 490 2024-09-21T09:54:24.844 INFO:tasks.rados.rados.0.smithi076.stdout:3878: seq_num 1467 ranges {655869=730059,1962143=535502} 2024-09-21T09:54:24.852 INFO:tasks.rados.rados.0.smithi076.stdout:3878: writing smithi07685715-16 from 655869 to 1385928 tid 1 2024-09-21T09:54:24.858 INFO:tasks.rados.rados.0.smithi076.stdout:3878: writing smithi07685715-16 from 1962143 to 2497645 tid 2 2024-09-21T09:54:24.858 INFO:tasks.rados.rados.0.smithi076.stdout:3879: write oid 46 current snap is 490 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: finishing write tid 1 to smithi07685715-5 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: finishing write tid 2 to smithi07685715-5 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: finishing write tid 3 to smithi07685715-5 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: finishing write tid 4 to smithi07685715-5 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: oid 5 updating version 0 to 2973 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: oid 5 version 2973 is already newer than 2972 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: oid 5 updating version 2973 to 2974 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 2974 (ObjNum 1466 snap 490 seq_num 1466) dirty exists 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3877: left oid 5 (ObjNum 1466 snap 490 seq_num 1466) 2024-09-21T09:54:24.859 INFO:tasks.rados.rados.0.smithi076.stdout:3879: seq_num 1468 ranges {672676=521661,1736978=795513} 2024-09-21T09:54:24.865 INFO:tasks.rados.rados.0.smithi076.stdout:3879: writing smithi07685715-46 from 672676 to 1194337 tid 1 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3879: writing smithi07685715-46 from 1736978 to 2532491 tid 2 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3877: done (2 left) 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3880: read oid 37 snap -1 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3880: expect (ObjNum 1465 snap 487 seq_num 1465) 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3881: delete oid 6 current snap is 490 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: finishing write tid 1 to smithi07685715-16 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: finishing write tid 2 to smithi07685715-16 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: finishing write tid 3 to smithi07685715-16 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: finishing write tid 4 to smithi07685715-16 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: oid 16 updating version 0 to 2044 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: oid 16 updating version 2044 to 2045 2024-09-21T09:54:24.873 INFO:tasks.rados.rados.0.smithi076.stdout:3878: oid 16 updating version 2045 to 2046 2024-09-21T09:54:24.874 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 16 v 2046 (ObjNum 1467 snap 490 seq_num 1467) dirty exists 2024-09-21T09:54:24.874 INFO:tasks.rados.rados.0.smithi076.stdout:3878: left oid 16 (ObjNum 1467 snap 490 seq_num 1467) 2024-09-21T09:54:24.874 INFO:tasks.rados.rados.0.smithi076.stdout:3879: finishing write tid 1 to smithi07685715-46 2024-09-21T09:54:24.875 INFO:tasks.rados.rados.0.smithi076.stdout:3878: done (3 left) 2024-09-21T09:54:24.875 INFO:tasks.rados.rados.0.smithi076.stdout:3881: done (2 left) 2024-09-21T09:54:24.876 INFO:tasks.rados.rados.0.smithi076.stdout:3882: read oid 3 snap -1 2024-09-21T09:54:24.876 INFO:tasks.rados.rados.0.smithi076.stdout:3882: expect (ObjNum 1462 snap 487 seq_num 1462) 2024-09-21T09:54:24.876 INFO:tasks.rados.rados.0.smithi076.stdout:3883: write oid 15 current snap is 490 2024-09-21T09:54:24.876 INFO:tasks.rados.rados.0.smithi076.stdout:3883: seq_num 1470 ranges {706290=672159,2137589=568701} 2024-09-21T09:54:24.883 INFO:tasks.rados.rados.0.smithi076.stdout:3883: writing smithi07685715-15 from 706290 to 1378449 tid 1 2024-09-21T09:54:24.890 INFO:tasks.rados.rados.0.smithi076.stdout:3883: writing smithi07685715-15 from 2137589 to 2706290 tid 2 2024-09-21T09:54:24.891 INFO:tasks.rados.rados.0.smithi076.stdout:3884: delete oid 5 current snap is 490 2024-09-21T09:54:24.891 INFO:tasks.rados.rados.0.smithi076.stdout:3880: expect (ObjNum 1465 snap 487 seq_num 1465) 2024-09-21T09:54:24.977 INFO:tasks.rados.rados.0.smithi076.stdout:3879: finishing write tid 2 to smithi07685715-46 2024-09-21T09:54:24.977 INFO:tasks.rados.rados.0.smithi076.stdout:3879: finishing write tid 3 to smithi07685715-46 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:3879: finishing write tid 4 to smithi07685715-46 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:3879: oid 46 updating version 0 to 2975 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:3879: oid 46 updating version 2975 to 2976 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:3879: oid 46 updating version 2976 to 2977 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 46 v 2977 (ObjNum 1468 snap 490 seq_num 1468) dirty exists 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:3879: left oid 46 (ObjNum 1468 snap 490 seq_num 1468) 2024-09-21T09:54:24.978 INFO:tasks.rados.rados.0.smithi076.stdout:3882: expect (ObjNum 1462 snap 487 seq_num 1462) 2024-09-21T09:54:25.054 INFO:tasks.rados.rados.0.smithi076.stdout:3883: finishing write tid 1 to smithi07685715-15 2024-09-21T09:54:25.054 INFO:tasks.rados.rados.0.smithi076.stdout:3883: finishing write tid 2 to smithi07685715-15 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: finishing write tid 3 to smithi07685715-15 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: finishing write tid 4 to smithi07685715-15 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: oid 15 updating version 0 to 824 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: oid 15 updating version 824 to 825 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: oid 15 updating version 825 to 826 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 15 v 826 (ObjNum 1470 snap 490 seq_num 1470) dirty exists 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: left oid 15 (ObjNum 1470 snap 490 seq_num 1470) 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3879: done (4 left) 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3880: done (3 left) 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3882: done (2 left) 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3883: done (1 left) 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3884: done (0 left) 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3885: write oid 47 current snap is 490 2024-09-21T09:54:25.055 INFO:tasks.rados.rados.0.smithi076.stdout:3885: seq_num 1472 ranges {739904=422657,1854553=402851} 2024-09-21T09:54:25.060 INFO:tasks.rados.rados.0.smithi076.stdout:3885: writing smithi07685715-47 from 739904 to 1162561 tid 1 2024-09-21T09:54:25.067 INFO:tasks.rados.rados.0.smithi076.stdout:3885: writing smithi07685715-47 from 1854553 to 2257404 tid 2 2024-09-21T09:54:25.067 INFO:tasks.rados.rados.0.smithi076.stdout:3886: read oid 49 snap -1 2024-09-21T09:54:25.067 INFO:tasks.rados.rados.0.smithi076.stdout:3886: expect (ObjNum 1390 snap 464 seq_num 1390) 2024-09-21T09:54:25.067 INFO:tasks.rados.rados.0.smithi076.stdout:3887: write oid 23 current snap is 490 2024-09-21T09:54:25.067 INFO:tasks.rados.rados.0.smithi076.stdout:3885: finishing write tid 1 to smithi07685715-47 2024-09-21T09:54:25.067 INFO:tasks.rados.rados.0.smithi076.stdout:3887: seq_num 1473 ranges {756711=497906,1913035=546509} 2024-09-21T09:54:25.073 INFO:tasks.rados.rados.0.smithi076.stdout:3887: writing smithi07685715-23 from 756711 to 1254617 tid 1 2024-09-21T09:54:25.080 INFO:tasks.rados.rados.0.smithi076.stdout:3887: writing smithi07685715-23 from 1913035 to 2459544 tid 2 2024-09-21T09:54:25.080 INFO:tasks.rados.rados.0.smithi076.stdout:3888: write oid 44 current snap is 490 2024-09-21T09:54:25.080 INFO:tasks.rados.rados.0.smithi076.stdout:3888: seq_num 1474 ranges {773518=573155,1855164=406520} 2024-09-21T09:54:25.088 INFO:tasks.rados.rados.0.smithi076.stdout:3888: writing smithi07685715-44 from 773518 to 1346673 tid 1 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3888: writing smithi07685715-44 from 1855164 to 2261684 tid 2 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3889: snap_create 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: finishing write tid 2 to smithi07685715-47 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: finishing write tid 3 to smithi07685715-47 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: finishing write tid 4 to smithi07685715-47 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: oid 47 updating version 0 to 2950 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: oid 47 version 2950 is already newer than 2949 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: oid 47 version 2950 is already newer than 2948 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 2950 (ObjNum 1472 snap 490 seq_num 1472) dirty exists 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3885: left oid 47 (ObjNum 1472 snap 490 seq_num 1472) 2024-09-21T09:54:25.094 INFO:tasks.rados.rados.0.smithi076.stdout:3886: expect (ObjNum 1390 snap 464 seq_num 1390) 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: finishing write tid 1 to smithi07685715-23 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: finishing write tid 2 to smithi07685715-23 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: finishing write tid 3 to smithi07685715-23 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: finishing write tid 4 to smithi07685715-23 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: oid 23 updating version 0 to 2952 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: oid 23 updating version 2952 to 2953 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: oid 23 updating version 2953 to 2954 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 23 v 2954 (ObjNum 1473 snap 490 seq_num 1473) dirty exists 2024-09-21T09:54:25.271 INFO:tasks.rados.rados.0.smithi076.stdout:3887: left oid 23 (ObjNum 1473 snap 490 seq_num 1473) 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: finishing write tid 1 to smithi07685715-44 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: finishing write tid 2 to smithi07685715-44 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: finishing write tid 3 to smithi07685715-44 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: finishing write tid 4 to smithi07685715-44 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: oid 44 updating version 0 to 859 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: oid 44 version 859 is already newer than 858 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: oid 44 updating version 859 to 860 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 860 (ObjNum 1474 snap 490 seq_num 1474) dirty exists 2024-09-21T09:54:25.272 INFO:tasks.rados.rados.0.smithi076.stdout:3888: left oid 44 (ObjNum 1474 snap 490 seq_num 1474) 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3885: done (4 left) 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3886: done (3 left) 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3887: done (2 left) 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3888: done (1 left) 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3889: done (0 left) 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3890: read oid 50 snap -1 2024-09-21T09:54:25.873 INFO:tasks.rados.rados.0.smithi076.stdout:3890: expect (ObjNum 1464 snap 487 seq_num 1464) 2024-09-21T09:54:25.874 INFO:tasks.rados.rados.0.smithi076.stdout:3891: write oid 13 current snap is 491 2024-09-21T09:54:25.874 INFO:tasks.rados.rados.0.smithi076.stdout:3891: seq_num 1475 ranges {790325=764757,2029999=550178} 2024-09-21T09:54:25.882 INFO:tasks.rados.rados.0.smithi076.stdout:3891: writing smithi07685715-13 from 790325 to 1555082 tid 1 2024-09-21T09:54:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:3891: writing smithi07685715-13 from 2029999 to 2580177 tid 2 2024-09-21T09:54:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:3892: read oid 30 snap 482 2024-09-21T09:54:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:3892: expect (ObjNum 1419 snap 473 seq_num 1419) 2024-09-21T09:54:25.887 INFO:tasks.rados.rados.0.smithi076.stdout:3893: read oid 5 snap -1 2024-09-21T09:54:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:3893: expect deleted 2024-09-21T09:54:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:3894: delete oid 12 current snap is 491 2024-09-21T09:54:25.888 INFO:tasks.rados.rados.0.smithi076.stdout:3890: expect (ObjNum 1464 snap 487 seq_num 1464) 2024-09-21T09:54:25.972 INFO:tasks.rados.rados.0.smithi076.stdout:3891: finishing write tid 1 to smithi07685715-13 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: finishing write tid 2 to smithi07685715-13 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: finishing write tid 3 to smithi07685715-13 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: finishing write tid 4 to smithi07685715-13 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: oid 13 updating version 0 to 809 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: oid 13 version 809 is already newer than 808 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: oid 13 updating version 809 to 810 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 13 v 810 (ObjNum 1475 snap 491 seq_num 1475) dirty exists 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3891: left oid 13 (ObjNum 1475 snap 491 seq_num 1475) 2024-09-21T09:54:25.973 INFO:tasks.rados.rados.0.smithi076.stdout:3892: expect (ObjNum 1419 snap 473 seq_num 1419) 2024-09-21T09:54:26.096 INFO:tasks.rados.rados.0.smithi076.stdout:3890: done (4 left) 2024-09-21T09:54:26.096 INFO:tasks.rados.rados.0.smithi076.stdout:3891: done (3 left) 2024-09-21T09:54:26.096 INFO:tasks.rados.rados.0.smithi076.stdout:3892: done (2 left) 2024-09-21T09:54:26.096 INFO:tasks.rados.rados.0.smithi076.stdout:3893: done (1 left) 2024-09-21T09:54:26.096 INFO:tasks.rados.rados.0.smithi076.stdout:3894: done (0 left) 2024-09-21T09:54:26.096 INFO:tasks.rados.rados.0.smithi076.stdout:3895: snap_create 2024-09-21T09:54:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:25 smithi191 ceph-mon[102206]: pgmap v1045: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 198 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 942 KiB/s wr, 4 op/s 2024-09-21T09:54:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:25 smithi191 ceph-mon[102206]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:54:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:25 smithi076 ceph-mon[106482]: pgmap v1045: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 198 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 942 KiB/s wr, 4 op/s 2024-09-21T09:54:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:25 smithi076 ceph-mon[106482]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:54:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:25 smithi076 ceph-mon[110963]: pgmap v1045: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 198 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 942 KiB/s wr, 4 op/s 2024-09-21T09:54:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:25 smithi076 ceph-mon[110963]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-21T09:54:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:26.877 INFO:tasks.rados.rados.0.smithi076.stdout:3895: done (0 left) 2024-09-21T09:54:26.877 INFO:tasks.rados.rados.0.smithi076.stdout:3896: read oid 3 snap -1 2024-09-21T09:54:26.878 INFO:tasks.rados.rados.0.smithi076.stdout:3896: expect (ObjNum 1462 snap 487 seq_num 1462) 2024-09-21T09:54:26.878 INFO:tasks.rados.rados.0.smithi076.stdout:3897: delete oid 34 current snap is 492 2024-09-21T09:54:26.881 INFO:tasks.rados.rados.0.smithi076.stdout:3897: done (1 left) 2024-09-21T09:54:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:3898: rollback oid 17 current snap is 492 2024-09-21T09:54:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 491 2024-09-21T09:54:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:3899: write oid 38 current snap is 492 2024-09-21T09:54:26.882 INFO:tasks.rados.rados.0.smithi076.stdout:3899: seq_num 1478 ranges {440746=590504,1689092=697505} 2024-09-21T09:54:26.887 INFO:tasks.rados.rados.0.smithi076.stdout:3899: writing smithi07685715-38 from 440746 to 1031250 tid 1 2024-09-21T09:54:26.895 INFO:tasks.rados.rados.0.smithi076.stdout:3899: writing smithi07685715-38 from 1689092 to 2386597 tid 2 2024-09-21T09:54:26.895 INFO:tasks.rados.rados.0.smithi076.stdout:3900: read oid 35 snap -1 2024-09-21T09:54:26.895 INFO:tasks.rados.rados.0.smithi076.stdout:3900: expect (ObjNum 1431 snap 476 seq_num 1431) 2024-09-21T09:54:26.895 INFO:tasks.rados.rados.0.smithi076.stdout:3901: delete oid 30 current snap is 492 2024-09-21T09:54:26.895 INFO:tasks.rados.rados.0.smithi076.stdout:3896: expect (ObjNum 1462 snap 487 seq_num 1462) 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3898: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2362 (ObjNum 129 snap 0 seq_num 864206992) dirty dne 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3899: finishing write tid 1 to smithi07685715-38 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3899: finishing write tid 2 to smithi07685715-38 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3899: finishing write tid 3 to smithi07685715-38 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3899: finishing write tid 4 to smithi07685715-38 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3899: oid 38 updating version 0 to 2391 2024-09-21T09:54:26.973 INFO:tasks.rados.rados.0.smithi076.stdout:3899: oid 38 version 2391 is already newer than 2390 2024-09-21T09:54:26.974 INFO:tasks.rados.rados.0.smithi076.stdout:3899: oid 38 version 2391 is already newer than 2389 2024-09-21T09:54:26.974 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 38 v 2391 (ObjNum 1478 snap 492 seq_num 1478) dirty exists 2024-09-21T09:54:26.974 INFO:tasks.rados.rados.0.smithi076.stdout:3899: left oid 38 (ObjNum 1478 snap 492 seq_num 1478) 2024-09-21T09:54:26.974 INFO:tasks.rados.rados.0.smithi076.stdout:3900: expect (ObjNum 1431 snap 476 seq_num 1431) 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3896: done (4 left) 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3898: done (3 left) 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3899: done (2 left) 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3900: done (1 left) 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3901: done (0 left) 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3902: write oid 42 current snap is 492 2024-09-21T09:54:27.054 INFO:tasks.rados.rados.0.smithi076.stdout:3902: seq_num 1480 ranges {474360=741002,1806056=701174} 2024-09-21T09:54:27.060 INFO:tasks.rados.rados.0.smithi076.stdout:3902: writing smithi07685715-42 from 474360 to 1215362 tid 1 2024-09-21T09:54:27.067 INFO:tasks.rados.rados.0.smithi076.stdout:3902: writing smithi07685715-42 from 1806056 to 2507230 tid 2 2024-09-21T09:54:27.067 INFO:tasks.rados.rados.0.smithi076.stdout:3902: finishing write tid 1 to smithi07685715-42 2024-09-21T09:54:27.068 INFO:tasks.rados.rados.0.smithi076.stdout:3903: delete oid 29 current snap is 492 2024-09-21T09:54:27.071 INFO:tasks.rados.rados.0.smithi076.stdout:3903: done (1 left) 2024-09-21T09:54:27.071 INFO:tasks.rados.rados.0.smithi076.stdout:3904: write oid 27 current snap is 492 2024-09-21T09:54:27.071 INFO:tasks.rados.rados.0.smithi076.stdout:3904: seq_num 1482 ranges {507974=491500,1406667=704843,2548287=359687} 2024-09-21T09:54:27.076 INFO:tasks.rados.rados.0.smithi076.stdout:3904: writing smithi07685715-27 from 507974 to 999474 tid 1 2024-09-21T09:54:27.082 INFO:tasks.rados.rados.0.smithi076.stdout:3904: writing smithi07685715-27 from 1406667 to 2111510 tid 2 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3904: writing smithi07685715-27 from 2548287 to 2907974 tid 3 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3905: read oid 38 snap -1 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3905: expect (ObjNum 1478 snap 492 seq_num 1478) 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3906: write oid 24 current snap is 492 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3902: finishing write tid 2 to smithi07685715-42 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3902: finishing write tid 3 to smithi07685715-42 2024-09-21T09:54:27.087 INFO:tasks.rados.rados.0.smithi076.stdout:3902: finishing write tid 4 to smithi07685715-42 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3902: oid 42 updating version 0 to 1024 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3902: oid 42 version 1024 is already newer than 1023 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3902: oid 42 version 1024 is already newer than 1022 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 42 v 1024 (ObjNum 1480 snap 492 seq_num 1480) dirty exists 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3902: left oid 42 (ObjNum 1480 snap 492 seq_num 1480) 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3904: finishing write tid 1 to smithi07685715-27 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3904: finishing write tid 2 to smithi07685715-27 2024-09-21T09:54:27.088 INFO:tasks.rados.rados.0.smithi076.stdout:3906: seq_num 1483 ranges {524781=683102,1981502=564854} 2024-09-21T09:54:27.093 INFO:tasks.rados.rados.0.smithi076.stdout:3906: writing smithi07685715-24 from 524781 to 1207883 tid 1 2024-09-21T09:54:27.101 INFO:tasks.rados.rados.0.smithi076.stdout:3906: writing smithi07685715-24 from 1981502 to 2546356 tid 2 2024-09-21T09:54:27.101 INFO:tasks.rados.rados.0.smithi076.stdout:3902: done (3 left) 2024-09-21T09:54:27.101 INFO:tasks.rados.rados.0.smithi076.stdout:3907: delete oid 31 current snap is 492 2024-09-21T09:54:27.101 INFO:tasks.rados.rados.0.smithi076.stdout:3904: finishing write tid 3 to smithi07685715-27 2024-09-21T09:54:27.101 INFO:tasks.rados.rados.0.smithi076.stdout:3904: finishing write tid 4 to smithi07685715-27 2024-09-21T09:54:27.101 INFO:tasks.rados.rados.0.smithi076.stdout:3905: expect (ObjNum 1478 snap 492 seq_num 1478) 2024-09-21T09:54:27.271 INFO:tasks.rados.rados.0.smithi076.stdout:3904: finishing write tid 5 to smithi07685715-27 2024-09-21T09:54:27.271 INFO:tasks.rados.rados.0.smithi076.stdout:3904: oid 27 updating version 0 to 2051 2024-09-21T09:54:27.271 INFO:tasks.rados.rados.0.smithi076.stdout:3904: oid 27 version 2051 is already newer than 2050 2024-09-21T09:54:27.271 INFO:tasks.rados.rados.0.smithi076.stdout:3904: oid 27 version 2051 is already newer than 2048 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3904: oid 27 version 2051 is already newer than 2049 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 27 v 2051 (ObjNum 1482 snap 492 seq_num 1482) dirty exists 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3904: left oid 27 (ObjNum 1482 snap 492 seq_num 1482) 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: finishing write tid 1 to smithi07685715-24 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: finishing write tid 2 to smithi07685715-24 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: finishing write tid 3 to smithi07685715-24 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3904: done (3 left) 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3905: done (2 left) 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3907: done (1 left) 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3908: snap_create 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: finishing write tid 4 to smithi07685715-24 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: oid 24 updating version 0 to 409 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: oid 24 updating version 409 to 410 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:3906: oid 24 updating version 410 to 411 2024-09-21T09:54:27.272 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 24 v 411 (ObjNum 1483 snap 492 seq_num 1483) dirty exists 2024-09-21T09:54:27.273 INFO:tasks.rados.rados.0.smithi076.stdout:3906: left oid 24 (ObjNum 1483 snap 492 seq_num 1483) 2024-09-21T09:54:27.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:26 smithi191 ceph-mon[102206]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:54:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:26 smithi076 ceph-mon[106482]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:54:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:26 smithi076 ceph-mon[110963]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-21T09:54:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3906: done (1 left) 2024-09-21T09:54:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3908: done (0 left) 2024-09-21T09:54:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3909: read oid 41 snap 489 2024-09-21T09:54:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3909: expect (ObjNum 1457 snap 485 seq_num 1457) 2024-09-21T09:54:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3910: write oid 25 current snap is 493 2024-09-21T09:54:27.878 INFO:tasks.rados.rados.0.smithi076.stdout:3910: seq_num 1485 ranges {558395=433600,1698466=568523,2863262=95133} 2024-09-21T09:54:27.883 INFO:tasks.rados.rados.0.smithi076.stdout:3910: writing smithi07685715-25 from 558395 to 991995 tid 1 2024-09-21T09:54:27.890 INFO:tasks.rados.rados.0.smithi076.stdout:3910: writing smithi07685715-25 from 1698466 to 2266989 tid 2 2024-09-21T09:54:27.894 INFO:tasks.rados.rados.0.smithi076.stdout:3910: writing smithi07685715-25 from 2863262 to 2958395 tid 3 2024-09-21T09:54:27.894 INFO:tasks.rados.rados.0.smithi076.stdout:3911: snap_remove snap 490 2024-09-21T09:54:28.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:27 smithi191 ceph-mon[102206]: pgmap v1048: 89 pgs: 89 active+clean; 185 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 19 op/s 2024-09-21T09:54:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:27 smithi191 ceph-mon[102206]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:54:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:27 smithi076 ceph-mon[106482]: pgmap v1048: 89 pgs: 89 active+clean; 185 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 19 op/s 2024-09-21T09:54:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:27 smithi076 ceph-mon[106482]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:54:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:27 smithi076 ceph-mon[110963]: pgmap v1048: 89 pgs: 89 active+clean; 185 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.6 MiB/s wr, 19 op/s 2024-09-21T09:54:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:27 smithi076 ceph-mon[110963]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-21T09:54:28.830 INFO:tasks.rados.rados.0.smithi076.stdout:3911: done (2 left) 2024-09-21T09:54:28.830 INFO:tasks.rados.rados.0.smithi076.stdout:3912: write oid 2 current snap is 493 2024-09-21T09:54:28.830 INFO:tasks.rados.rados.0.smithi076.stdout:3912: seq_num 1486 ranges {575202=508849,1640595=712181} 2024-09-21T09:54:28.836 INFO:tasks.rados.rados.0.smithi076.stdout:3912: writing smithi07685715-2 from 575202 to 1084051 tid 1 2024-09-21T09:54:28.844 INFO:tasks.rados.rados.0.smithi076.stdout:3912: writing smithi07685715-2 from 1640595 to 2352776 tid 2 2024-09-21T09:54:28.844 INFO:tasks.rados.rados.0.smithi076.stdout:3913: read oid 39 snap -1 2024-09-21T09:54:28.844 INFO:tasks.rados.rados.0.smithi076.stdout:3913: expect deleted 2024-09-21T09:54:28.844 INFO:tasks.rados.rados.0.smithi076.stdout:3914: snap_remove snap 471 2024-09-21T09:54:29.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:28 smithi191 ceph-mon[102206]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:54:29.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:28 smithi191 ceph-mon[102206]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-21T09:54:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:28 smithi076 ceph-mon[106482]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:54:29.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:28 smithi076 ceph-mon[106482]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-21T09:54:29.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:28 smithi076 ceph-mon[110963]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-21T09:54:29.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:28 smithi076 ceph-mon[110963]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-21T09:54:29.833 INFO:tasks.rados.rados.0.smithi076.stdout:3914: done (4 left) 2024-09-21T09:54:29.834 INFO:tasks.rados.rados.0.smithi076.stdout:3915: read oid 3 snap 478 2024-09-21T09:54:29.834 INFO:tasks.rados.rados.0.smithi076.stdout:3915: expect (ObjNum 1411 snap 470 seq_num 1411) 2024-09-21T09:54:29.834 INFO:tasks.rados.rados.0.smithi076.stdout:3916: read oid 46 snap -1 2024-09-21T09:54:29.834 INFO:tasks.rados.rados.0.smithi076.stdout:3916: expect (ObjNum 1468 snap 490 seq_num 1468) 2024-09-21T09:54:29.834 INFO:tasks.rados.rados.0.smithi076.stdout:3917: write oid 18 current snap is 493 2024-09-21T09:54:29.834 INFO:tasks.rados.rados.0.smithi076.stdout:3917: seq_num 1487 ranges {592009=584098,1699077=572192,2801755=190254} 2024-09-21T09:54:29.841 INFO:tasks.rados.rados.0.smithi076.stdout:3917: writing smithi07685715-18 from 592009 to 1176107 tid 1 2024-09-21T09:54:29.848 INFO:tasks.rados.rados.0.smithi076.stdout:3917: writing smithi07685715-18 from 1699077 to 2271269 tid 2 2024-09-21T09:54:29.853 INFO:tasks.rados.rados.0.smithi076.stdout:3917: writing smithi07685715-18 from 2801755 to 2992009 tid 3 2024-09-21T09:54:29.853 INFO:tasks.rados.rados.0.smithi076.stdout:3918: delete oid 38 current snap is 493 2024-09-21T09:54:29.854 INFO:tasks.rados.rados.0.smithi076.stdout:3910: finishing write tid 1 to smithi07685715-25 2024-09-21T09:54:29.854 INFO:tasks.rados.rados.0.smithi076.stdout:3910: finishing write tid 2 to smithi07685715-25 2024-09-21T09:54:29.854 INFO:tasks.rados.rados.0.smithi076.stdout:3909: expect (ObjNum 1457 snap 485 seq_num 1457) 2024-09-21T09:54:29.961 INFO:tasks.rados.rados.0.smithi076.stdout:3910: finishing write tid 3 to smithi07685715-25 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: finishing write tid 4 to smithi07685715-25 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: finishing write tid 5 to smithi07685715-25 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: oid 25 updating version 0 to 900 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: oid 25 version 900 is already newer than 899 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: oid 25 updating version 900 to 901 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: oid 25 updating version 901 to 902 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 25 v 902 (ObjNum 1485 snap 493 seq_num 1485) dirty exists 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3910: left oid 25 (ObjNum 1485 snap 493 seq_num 1485) 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3912: finishing write tid 1 to smithi07685715-2 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3912: finishing write tid 2 to smithi07685715-2 2024-09-21T09:54:29.962 INFO:tasks.rados.rados.0.smithi076.stdout:3912: finishing write tid 3 to smithi07685715-2 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3912: finishing write tid 4 to smithi07685715-2 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3912: oid 2 updating version 0 to 813 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3912: oid 2 updating version 813 to 814 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3912: oid 2 version 814 is already newer than 812 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 814 (ObjNum 1486 snap 493 seq_num 1486) dirty exists 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3912: left oid 2 (ObjNum 1486 snap 493 seq_num 1486) 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3909: done (7 left) 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3910: done (6 left) 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3912: done (5 left) 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3913: done (4 left) 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3918: done (3 left) 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3919: rollback oid 2 current snap is 493 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 487 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3917: finishing write tid 1 to smithi07685715-18 2024-09-21T09:54:29.963 INFO:tasks.rados.rados.0.smithi076.stdout:3917: finishing write tid 2 to smithi07685715-18 2024-09-21T09:54:29.964 INFO:tasks.rados.rados.0.smithi076.stdout:3915: expect (ObjNum 1411 snap 470 seq_num 1411) 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: finishing write tid 3 to smithi07685715-18 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: finishing write tid 4 to smithi07685715-18 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: finishing write tid 5 to smithi07685715-18 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: oid 18 updating version 0 to 1026 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: oid 18 updating version 1026 to 1028 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: oid 18 version 1028 is already newer than 1027 2024-09-21T09:54:30.076 INFO:tasks.rados.rados.0.smithi076.stdout:3917: oid 18 version 1028 is already newer than 1025 2024-09-21T09:54:30.077 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 1028 (ObjNum 1487 snap 493 seq_num 1487) dirty exists 2024-09-21T09:54:30.077 INFO:tasks.rados.rados.0.smithi076.stdout:3917: left oid 18 (ObjNum 1487 snap 493 seq_num 1487) 2024-09-21T09:54:30.077 INFO:tasks.rados.rados.0.smithi076.stdout:3916: expect (ObjNum 1468 snap 490 seq_num 1468) 2024-09-21T09:54:30.159 INFO:tasks.rados.rados.0.smithi076.stdout:3919: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:54:30.159 INFO:tasks.rados.rados.0.smithi076.stdout:3919: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:54:30.159 INFO:tasks.rados.rados.0.smithi076.stdout:3919: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 819 (ObjNum 1403 snap 466 seq_num 1403) dirty exists 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3915: done (3 left) 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3916: done (2 left) 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3917: done (1 left) 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3919: done (0 left) 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3920: read oid 13 snap -1 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3920: expect (ObjNum 1475 snap 491 seq_num 1475) 2024-09-21T09:54:30.160 INFO:tasks.rados.rados.0.smithi076.stdout:3921: snap_remove snap 486 2024-09-21T09:54:30.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:29 smithi191 ceph-mon[102206]: pgmap v1051: 89 pgs: 89 active+clean; 186 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:54:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:29 smithi191 ceph-mon[102206]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-21T09:54:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:29 smithi076 ceph-mon[106482]: pgmap v1051: 89 pgs: 89 active+clean; 186 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:54:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:29 smithi076 ceph-mon[106482]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-21T09:54:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:29 smithi076 ceph-mon[110963]: pgmap v1051: 89 pgs: 89 active+clean; 186 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-09-21T09:54:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:29 smithi076 ceph-mon[110963]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-21T09:54:30.897 INFO:tasks.rados.rados.0.smithi076.stdout:3921: done (1 left) 2024-09-21T09:54:30.897 INFO:tasks.rados.rados.0.smithi076.stdout:3922: snap_remove snap 485 2024-09-21T09:54:31.903 INFO:tasks.rados.rados.0.smithi076.stdout:3920: expect (ObjNum 1475 snap 491 seq_num 1475) 2024-09-21T09:54:31.996 INFO:tasks.rados.rados.0.smithi076.stdout:3920: done (1 left) 2024-09-21T09:54:31.997 INFO:tasks.rados.rados.0.smithi076.stdout:3922: done (0 left) 2024-09-21T09:54:31.997 INFO:tasks.rados.rados.0.smithi076.stdout:3923: snap_remove snap 482 2024-09-21T09:54:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:31 smithi191 ceph-mon[102206]: pgmap v1054: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 197 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T09:54:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:31 smithi191 ceph-mon[102206]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-21T09:54:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:31 smithi076 ceph-mon[106482]: pgmap v1054: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 197 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T09:54:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:31 smithi076 ceph-mon[106482]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-21T09:54:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:31 smithi076 ceph-mon[110963]: pgmap v1054: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 197 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T09:54:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:31 smithi076 ceph-mon[110963]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-21T09:54:32.912 INFO:tasks.rados.rados.0.smithi076.stdout:3923: done (0 left) 2024-09-21T09:54:32.913 INFO:tasks.rados.rados.0.smithi076.stdout:3924: rollback oid 21 current snap is 493 2024-09-21T09:54:32.913 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 21 to 489 2024-09-21T09:54:32.913 INFO:tasks.rados.rados.0.smithi076.stdout:3925: snap_remove snap 469 2024-09-21T09:54:33.250 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:32 smithi076 ceph-mon[106482]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-21T09:54:33.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:32 smithi076 ceph-mon[110963]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-21T09:54:33.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:32 smithi191 ceph-mon[102206]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-21T09:54:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:54:33] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.51.0" 2024-09-21T09:54:33.832 INFO:tasks.rados.rados.0.smithi076.stdout:3925: done (1 left) 2024-09-21T09:54:33.832 INFO:tasks.rados.rados.0.smithi076.stdout:3926: read oid 50 snap 489 2024-09-21T09:54:33.832 INFO:tasks.rados.rados.0.smithi076.stdout:3926: expect (ObjNum 1464 snap 487 seq_num 1464) 2024-09-21T09:54:33.832 INFO:tasks.rados.rados.0.smithi076.stdout:3927: snap_create 2024-09-21T09:54:33.833 INFO:tasks.rados.rados.0.smithi076.stdout:3924: finishing rollback tid 1 to smithi07685715-21 2024-09-21T09:54:33.833 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 2945 (ObjNum 129 snap 0 seq_num 864675200) dirty dne 2024-09-21T09:54:33.849 INFO:tasks.rados.rados.0.smithi076.stdout:3926: expect (ObjNum 1464 snap 487 seq_num 1464) 2024-09-21T09:54:34.274 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-mon[106482]: pgmap v1057: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 198 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-21T09:54:34.274 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-mon[106482]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-21T09:54:34.275 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-mon[106482]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-21T09:54:34.275 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-mon[110963]: pgmap v1057: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 198 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-21T09:54:34.275 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-mon[110963]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-21T09:54:34.275 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:33 smithi076 ceph-mon[110963]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-21T09:54:34.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:33 smithi191 ceph-mon[102206]: pgmap v1057: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 198 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.4 MiB/s wr, 22 op/s 2024-09-21T09:54:34.420 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:33 smithi191 ceph-mon[102206]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-21T09:54:34.420 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:33 smithi191 ceph-mon[102206]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-21T09:54:34.832 INFO:tasks.rados.rados.0.smithi076.stdout:3924: done (2 left) 2024-09-21T09:54:34.833 INFO:tasks.rados.rados.0.smithi076.stdout:3926: done (1 left) 2024-09-21T09:54:34.833 INFO:tasks.rados.rados.0.smithi076.stdout:3927: done (0 left) 2024-09-21T09:54:34.833 INFO:tasks.rados.rados.0.smithi076.stdout:3928: snap_remove snap 493 2024-09-21T09:54:35.839 INFO:tasks.rados.rados.0.smithi076.stdout:3928: done (0 left) 2024-09-21T09:54:35.839 INFO:tasks.rados.rados.0.smithi076.stdout:3929: write oid 9 current snap is 494 2024-09-21T09:54:35.839 INFO:tasks.rados.rados.0.smithi076.stdout:3929: seq_num 1489 ranges {625623=734596,1816041=459508,2740248=285375} 2024-09-21T09:54:35.846 INFO:tasks.rados.rados.0.smithi076.stdout:3929: writing smithi07685715-9 from 625623 to 1360219 tid 1 2024-09-21T09:54:35.851 INFO:tasks.rados.rados.0.smithi076.stdout:3929: writing smithi07685715-9 from 1816041 to 2275549 tid 2 2024-09-21T09:54:35.856 INFO:tasks.rados.rados.0.smithi076.stdout:3929: writing smithi07685715-9 from 2740248 to 3025623 tid 3 2024-09-21T09:54:35.856 INFO:tasks.rados.rados.0.smithi076.stdout:3930: rollback oid 5 current snap is 494 2024-09-21T09:54:35.856 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 5 to 488 2024-09-21T09:54:35.856 INFO:tasks.rados.rados.0.smithi076.stdout:3929: finishing write tid 1 to smithi07685715-9 2024-09-21T09:54:35.856 INFO:tasks.rados.rados.0.smithi076.stdout:3929: finishing write tid 2 to smithi07685715-9 2024-09-21T09:54:35.857 INFO:tasks.rados.rados.0.smithi076.stdout:3931: delete oid 39 current snap is 494 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: finishing write tid 3 to smithi07685715-9 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: finishing write tid 4 to smithi07685715-9 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: finishing write tid 5 to smithi07685715-9 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: oid 9 updating version 0 to 1419 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: oid 9 version 1419 is already newer than 1416 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: oid 9 version 1419 is already newer than 1418 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: oid 9 version 1419 is already newer than 1417 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 9 v 1419 (ObjNum 1489 snap 494 seq_num 1489) dirty exists 2024-09-21T09:54:35.859 INFO:tasks.rados.rados.0.smithi076.stdout:3929: left oid 9 (ObjNum 1489 snap 494 seq_num 1489) 2024-09-21T09:54:35.862 INFO:tasks.rados.rados.0.smithi076.stdout:3930: finishing rollback tid 1 to smithi07685715-5 2024-09-21T09:54:35.862 INFO:tasks.rados.rados.0.smithi076.stdout:3930: finishing rollback tid 2 to smithi07685715-5 2024-09-21T09:54:35.863 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 5 v 3011 (ObjNum 1370 snap 459 seq_num 1370) dirty exists 2024-09-21T09:54:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3929: done (2 left) 2024-09-21T09:54:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3930: done (1 left) 2024-09-21T09:54:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3931: done (0 left) 2024-09-21T09:54:35.865 INFO:tasks.rados.rados.0.smithi076.stdout:3932: snap_remove snap 488 2024-09-21T09:54:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:35 smithi191 ceph-mon[102206]: pgmap v1060: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 198 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 403 KiB/s wr, 13 op/s 2024-09-21T09:54:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:35 smithi191 ceph-mon[102206]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-21T09:54:36.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:35 smithi076 ceph-mon[106482]: pgmap v1060: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 198 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 403 KiB/s wr, 13 op/s 2024-09-21T09:54:36.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:35 smithi076 ceph-mon[106482]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-21T09:54:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:35 smithi076 ceph-mon[110963]: pgmap v1060: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 198 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 403 KiB/s wr, 13 op/s 2024-09-21T09:54:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:35 smithi076 ceph-mon[110963]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-21T09:54:36.848 INFO:tasks.rados.rados.0.smithi076.stdout:3932: done (0 left) 2024-09-21T09:54:36.848 INFO:tasks.rados.rados.0.smithi076.stdout:3933: snap_create 2024-09-21T09:54:37.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:36 smithi191 ceph-mon[102206]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-21T09:54:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:36 smithi076 ceph-mon[106482]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-21T09:54:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:36 smithi076 ceph-mon[110963]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-21T09:54:37.851 INFO:tasks.rados.rados.0.smithi076.stdout:3933: done (0 left) 2024-09-21T09:54:37.851 INFO:tasks.rados.rados.0.smithi076.stdout:3934: write oid 33 current snap is 495 2024-09-21T09:54:37.851 INFO:tasks.rados.rados.0.smithi076.stdout:3934: seq_num 1491 ranges {659237=601447,1933005=463177} 2024-09-21T09:54:37.858 INFO:tasks.rados.rados.0.smithi076.stdout:3934: writing smithi07685715-33 from 659237 to 1260684 tid 1 2024-09-21T09:54:37.865 INFO:tasks.rados.rados.0.smithi076.stdout:3934: writing smithi07685715-33 from 1933005 to 2396182 tid 2 2024-09-21T09:54:37.865 INFO:tasks.rados.rados.0.smithi076.stdout:3935: snap_create 2024-09-21T09:54:37.865 INFO:tasks.rados.rados.0.smithi076.stdout:3934: finishing write tid 1 to smithi07685715-33 2024-09-21T09:54:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:3934: finishing write tid 2 to smithi07685715-33 2024-09-21T09:54:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:3934: finishing write tid 3 to smithi07685715-33 2024-09-21T09:54:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:3934: finishing write tid 4 to smithi07685715-33 2024-09-21T09:54:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:3934: oid 33 updating version 0 to 916 2024-09-21T09:54:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:3934: oid 33 version 916 is already newer than 915 2024-09-21T09:54:37.868 INFO:tasks.rados.rados.0.smithi076.stdout:3934: oid 33 version 916 is already newer than 914 2024-09-21T09:54:37.869 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 916 (ObjNum 1491 snap 495 seq_num 1491) dirty exists 2024-09-21T09:54:37.869 INFO:tasks.rados.rados.0.smithi076.stdout:3934: left oid 33 (ObjNum 1491 snap 495 seq_num 1491) 2024-09-21T09:54:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:37 smithi191 ceph-mon[102206]: pgmap v1063: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 202 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 209 KiB/s wr, 7 op/s 2024-09-21T09:54:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:37 smithi191 ceph-mon[102206]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-21T09:54:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:37 smithi076 ceph-mon[106482]: pgmap v1063: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 202 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 209 KiB/s wr, 7 op/s 2024-09-21T09:54:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:37 smithi076 ceph-mon[106482]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-21T09:54:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:37 smithi076 ceph-mon[110963]: pgmap v1063: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 202 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 209 KiB/s wr, 7 op/s 2024-09-21T09:54:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:37 smithi076 ceph-mon[110963]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-21T09:54:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3934: done (1 left) 2024-09-21T09:54:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3935: done (0 left) 2024-09-21T09:54:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3936: read oid 13 snap -1 2024-09-21T09:54:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3936: expect (ObjNum 1475 snap 491 seq_num 1475) 2024-09-21T09:54:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3937: write oid 45 current snap is 496 2024-09-21T09:54:38.836 INFO:tasks.rados.rados.0.smithi076.stdout:3937: seq_num 1492 ranges {676044=676696,1991487=723188} 2024-09-21T09:54:38.843 INFO:tasks.rados.rados.0.smithi076.stdout:3937: writing smithi07685715-45 from 676044 to 1352740 tid 1 2024-09-21T09:54:38.851 INFO:tasks.rados.rados.0.smithi076.stdout:3937: writing smithi07685715-45 from 1991487 to 2714675 tid 2 2024-09-21T09:54:38.851 INFO:tasks.rados.rados.0.smithi076.stdout:3938: snap_remove snap 495 2024-09-21T09:54:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:38 smithi191 ceph-mon[102206]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-21T09:54:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:38 smithi191 ceph-mon[102206]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-21T09:54:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:38 smithi076 ceph-mon[106482]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-21T09:54:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:38 smithi076 ceph-mon[106482]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-21T09:54:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:38 smithi076 ceph-mon[110963]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-21T09:54:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:38 smithi076 ceph-mon[110963]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-21T09:54:39.857 INFO:tasks.rados.rados.0.smithi076.stdout:3938: done (2 left) 2024-09-21T09:54:39.857 INFO:tasks.rados.rados.0.smithi076.stdout:3939: write oid 22 current snap is 496 2024-09-21T09:54:39.857 INFO:tasks.rados.rados.0.smithi076.stdout:3939: seq_num 1493 ranges {692851=751945,2049969=466846} 2024-09-21T09:54:39.862 INFO:tasks.rados.rados.0.smithi076.stdout:3939: writing smithi07685715-22 from 692851 to 1444796 tid 1 2024-09-21T09:54:39.869 INFO:tasks.rados.rados.0.smithi076.stdout:3939: writing smithi07685715-22 from 2049969 to 2516815 tid 2 2024-09-21T09:54:39.869 INFO:tasks.rados.rados.0.smithi076.stdout:3940: rollback oid 30 current snap is 496 2024-09-21T09:54:39.869 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 30 to 475 2024-09-21T09:54:39.869 INFO:tasks.rados.rados.0.smithi076.stdout:3936: expect (ObjNum 1475 snap 491 seq_num 1475) 2024-09-21T09:54:39.962 INFO:tasks.rados.rados.0.smithi076.stdout:3937: finishing write tid 1 to smithi07685715-45 2024-09-21T09:54:39.962 INFO:tasks.rados.rados.0.smithi076.stdout:3937: finishing write tid 2 to smithi07685715-45 2024-09-21T09:54:39.963 INFO:tasks.rados.rados.0.smithi076.stdout:3937: finishing write tid 3 to smithi07685715-45 2024-09-21T09:54:39.963 INFO:tasks.rados.rados.0.smithi076.stdout:3937: finishing write tid 4 to smithi07685715-45 2024-09-21T09:54:39.963 INFO:tasks.rados.rados.0.smithi076.stdout:3937: oid 45 updating version 0 to 837 2024-09-21T09:54:39.964 INFO:tasks.rados.rados.0.smithi076.stdout:3937: oid 45 updating version 837 to 838 2024-09-21T09:54:39.964 INFO:tasks.rados.rados.0.smithi076.stdout:3937: oid 45 version 838 is already newer than 836 2024-09-21T09:54:39.964 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 45 v 838 (ObjNum 1492 snap 496 seq_num 1492) dirty exists 2024-09-21T09:54:39.964 INFO:tasks.rados.rados.0.smithi076.stdout:3937: left oid 45 (ObjNum 1492 snap 496 seq_num 1492) 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3936: done (3 left) 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3937: done (2 left) 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3941: snap_create 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3939: finishing write tid 1 to smithi07685715-22 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3939: finishing write tid 2 to smithi07685715-22 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3940: finishing rollback tid 1 to smithi07685715-30 2024-09-21T09:54:39.965 INFO:tasks.rados.rados.0.smithi076.stdout:3939: finishing write tid 3 to smithi07685715-22 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:3940: finishing rollback tid 2 to smithi07685715-30 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 2990 (ObjNum 1419 snap 473 seq_num 1419) dirty exists 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:3939: finishing write tid 4 to smithi07685715-22 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:3939: oid 22 updating version 0 to 2086 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:3939: oid 22 updating version 2086 to 2087 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:3939: oid 22 version 2087 is already newer than 2085 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 22 v 2087 (ObjNum 1493 snap 496 seq_num 1493) dirty exists 2024-09-21T09:54:39.966 INFO:tasks.rados.rados.0.smithi076.stdout:3939: left oid 22 (ObjNum 1493 snap 496 seq_num 1493) 2024-09-21T09:54:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:39 smithi191 ceph-mon[102206]: pgmap v1066: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 203 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 446 KiB/s wr, 7 op/s 2024-09-21T09:54:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:39 smithi191 ceph-mon[102206]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-21T09:54:40.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:39 smithi076 ceph-mon[106482]: pgmap v1066: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 203 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 446 KiB/s wr, 7 op/s 2024-09-21T09:54:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:39 smithi076 ceph-mon[106482]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-21T09:54:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:39 smithi076 ceph-mon[110963]: pgmap v1066: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 203 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 446 KiB/s wr, 7 op/s 2024-09-21T09:54:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:39 smithi076 ceph-mon[110963]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-21T09:54:40.867 INFO:tasks.rados.rados.0.smithi076.stdout:3939: done (2 left) 2024-09-21T09:54:40.867 INFO:tasks.rados.rados.0.smithi076.stdout:3940: done (1 left) 2024-09-21T09:54:40.867 INFO:tasks.rados.rados.0.smithi076.stdout:3941: done (0 left) 2024-09-21T09:54:40.867 INFO:tasks.rados.rados.0.smithi076.stdout:3942: write oid 20 current snap is 497 2024-09-21T09:54:40.867 INFO:tasks.rados.rados.0.smithi076.stdout:3942: seq_num 1494 ranges {709658=427194,1592098=726857,2761010=348648} 2024-09-21T09:54:40.873 INFO:tasks.rados.rados.0.smithi076.stdout:3942: writing smithi07685715-20 from 709658 to 1136852 tid 1 2024-09-21T09:54:40.880 INFO:tasks.rados.rados.0.smithi076.stdout:3942: writing smithi07685715-20 from 1592098 to 2318955 tid 2 2024-09-21T09:54:40.885 INFO:tasks.rados.rados.0.smithi076.stdout:3942: writing smithi07685715-20 from 2761010 to 3109658 tid 3 2024-09-21T09:54:40.885 INFO:tasks.rados.rados.0.smithi076.stdout:3943: write oid 34 current snap is 497 2024-09-21T09:54:40.885 INFO:tasks.rados.rados.0.smithi076.stdout:3942: finishing write tid 1 to smithi07685715-20 2024-09-21T09:54:40.885 INFO:tasks.rados.rados.0.smithi076.stdout:3942: finishing write tid 2 to smithi07685715-20 2024-09-21T09:54:40.885 INFO:tasks.rados.rados.0.smithi076.stdout:3943: seq_num 1495 ranges {726465=502443,1650580=470515,2788433=338032} 2024-09-21T09:54:40.892 INFO:tasks.rados.rados.0.smithi076.stdout:3943: writing smithi07685715-34 from 726465 to 1228908 tid 1 2024-09-21T09:54:40.897 INFO:tasks.rados.rados.0.smithi076.stdout:3943: writing smithi07685715-34 from 1650580 to 2121095 tid 2 2024-09-21T09:54:40.903 INFO:tasks.rados.rados.0.smithi076.stdout:3943: writing smithi07685715-34 from 2788433 to 3126465 tid 3 2024-09-21T09:54:40.903 INFO:tasks.rados.rados.0.smithi076.stdout:3944: read oid 46 snap -1 2024-09-21T09:54:40.903 INFO:tasks.rados.rados.0.smithi076.stdout:3944: expect (ObjNum 1468 snap 490 seq_num 1468) 2024-09-21T09:54:40.903 INFO:tasks.rados.rados.0.smithi076.stdout:3945: snap_create 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: finishing write tid 3 to smithi07685715-20 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: finishing write tid 4 to smithi07685715-20 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: finishing write tid 5 to smithi07685715-20 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: oid 20 updating version 0 to 1256 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: oid 20 updating version 1256 to 1257 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: oid 20 updating version 1257 to 1259 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: oid 20 version 1259 is already newer than 1258 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 20 v 1259 (ObjNum 1494 snap 497 seq_num 1494) dirty exists 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3942: left oid 20 (ObjNum 1494 snap 497 seq_num 1494) 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3943: finishing write tid 1 to smithi07685715-34 2024-09-21T09:54:40.904 INFO:tasks.rados.rados.0.smithi076.stdout:3943: finishing write tid 2 to smithi07685715-34 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: finishing write tid 3 to smithi07685715-34 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: finishing write tid 4 to smithi07685715-34 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: finishing write tid 5 to smithi07685715-34 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: oid 34 updating version 0 to 875 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: oid 34 version 875 is already newer than 873 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: oid 34 updating version 875 to 876 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: oid 34 version 876 is already newer than 874 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 34 v 876 (ObjNum 1495 snap 497 seq_num 1495) dirty exists 2024-09-21T09:54:40.906 INFO:tasks.rados.rados.0.smithi076.stdout:3943: left oid 34 (ObjNum 1495 snap 497 seq_num 1495) 2024-09-21T09:54:40.912 INFO:tasks.rados.rados.0.smithi076.stdout:3944: expect (ObjNum 1468 snap 490 seq_num 1468) 2024-09-21T09:54:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:41.874 INFO:tasks.rados.rados.0.smithi076.stdout:3942: done (3 left) 2024-09-21T09:54:41.874 INFO:tasks.rados.rados.0.smithi076.stdout:3943: done (2 left) 2024-09-21T09:54:41.874 INFO:tasks.rados.rados.0.smithi076.stdout:3944: done (1 left) 2024-09-21T09:54:41.874 INFO:tasks.rados.rados.0.smithi076.stdout:3945: done (0 left) 2024-09-21T09:54:41.875 INFO:tasks.rados.rados.0.smithi076.stdout:3946: snap_create 2024-09-21T09:54:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:41 smithi076 ceph-mon[106482]: pgmap v1069: 89 pgs: 89 active+clean; 207 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 963 KiB/s wr, 5 op/s 2024-09-21T09:54:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:41 smithi076 ceph-mon[106482]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-21T09:54:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:41 smithi076 ceph-mon[110963]: pgmap v1069: 89 pgs: 89 active+clean; 207 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 963 KiB/s wr, 5 op/s 2024-09-21T09:54:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:41 smithi076 ceph-mon[110963]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-21T09:54:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:41 smithi191 ceph-mon[102206]: pgmap v1069: 89 pgs: 89 active+clean; 207 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 963 KiB/s wr, 5 op/s 2024-09-21T09:54:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:41 smithi191 ceph-mon[102206]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-21T09:54:42.885 INFO:tasks.rados.rados.0.smithi076.stdout:3946: done (0 left) 2024-09-21T09:54:42.885 INFO:tasks.rados.rados.0.smithi076.stdout:3947: snap_remove snap 483 2024-09-21T09:54:43.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:42 smithi076 ceph-mon[106482]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-21T09:54:43.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:42 smithi076 ceph-mon[110963]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-21T09:54:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:42 smithi191 ceph-mon[102206]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-21T09:54:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:54:43] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-21T09:54:43.834 INFO:tasks.rados.rados.0.smithi076.stdout:3947: done (0 left) 2024-09-21T09:54:43.834 INFO:tasks.rados.rados.0.smithi076.stdout:3948: write oid 6 current snap is 499 2024-09-21T09:54:43.834 INFO:tasks.rados.rados.0.smithi076.stdout:3948: seq_num 1496 ranges {743272=577692,2109062=730526} 2024-09-21T09:54:43.842 INFO:tasks.rados.rados.0.smithi076.stdout:3948: writing smithi07685715-6 from 743272 to 1320964 tid 1 2024-09-21T09:54:43.851 INFO:tasks.rados.rados.0.smithi076.stdout:3948: writing smithi07685715-6 from 2109062 to 2839588 tid 2 2024-09-21T09:54:43.852 INFO:tasks.rados.rados.0.smithi076.stdout:3949: write oid 2 current snap is 499 2024-09-21T09:54:43.852 INFO:tasks.rados.rados.0.smithi076.stdout:3948: finishing write tid 1 to smithi07685715-6 2024-09-21T09:54:43.852 INFO:tasks.rados.rados.0.smithi076.stdout:3949: seq_num 1497 ranges {760079=652941,2167544=474184} 2024-09-21T09:54:43.859 INFO:tasks.rados.rados.0.smithi076.stdout:3949: writing smithi07685715-2 from 760079 to 1413020 tid 1 2024-09-21T09:54:43.866 INFO:tasks.rados.rados.0.smithi076.stdout:3949: writing smithi07685715-2 from 2167544 to 2641728 tid 2 2024-09-21T09:54:43.866 INFO:tasks.rados.rados.0.smithi076.stdout:3950: snap_remove snap 466 2024-09-21T09:54:44.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:43 smithi191 ceph-mon[102206]: pgmap v1072: 89 pgs: 89 active+clean; 217 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:54:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:43 smithi191 ceph-mon[102206]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-21T09:54:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:43 smithi191 ceph-mon[102206]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-21T09:54:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-mon[106482]: pgmap v1072: 89 pgs: 89 active+clean; 217 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:54:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-mon[106482]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-21T09:54:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-mon[106482]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-21T09:54:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-mon[110963]: pgmap v1072: 89 pgs: 89 active+clean; 217 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T09:54:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-mon[110963]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-21T09:54:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:43 smithi076 ceph-mon[110963]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3950: done (2 left) 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3951: snap_remove snap 498 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3948: finishing write tid 2 to smithi07685715-6 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3948: finishing write tid 3 to smithi07685715-6 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3948: finishing write tid 4 to smithi07685715-6 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3948: oid 6 updating version 0 to 890 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3948: oid 6 updating version 890 to 891 2024-09-21T09:54:44.836 INFO:tasks.rados.rados.0.smithi076.stdout:3948: oid 6 version 891 is already newer than 889 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 6 v 891 (ObjNum 1496 snap 499 seq_num 1496) dirty exists 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3948: left oid 6 (ObjNum 1496 snap 499 seq_num 1496) 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: finishing write tid 1 to smithi07685715-2 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: finishing write tid 2 to smithi07685715-2 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: finishing write tid 3 to smithi07685715-2 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: finishing write tid 4 to smithi07685715-2 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: oid 2 updating version 0 to 834 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: oid 2 updating version 834 to 835 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: oid 2 version 835 is already newer than 833 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 835 (ObjNum 1497 snap 499 seq_num 1497) dirty exists 2024-09-21T09:54:44.837 INFO:tasks.rados.rados.0.smithi076.stdout:3949: left oid 2 (ObjNum 1497 snap 499 seq_num 1497) 2024-09-21T09:54:45.843 INFO:tasks.rados.rados.0.smithi076.stdout:3948: done (2 left) 2024-09-21T09:54:45.843 INFO:tasks.rados.rados.0.smithi076.stdout:3949: done (1 left) 2024-09-21T09:54:45.843 INFO:tasks.rados.rados.0.smithi076.stdout:3951: done (0 left) 2024-09-21T09:54:45.843 INFO:tasks.rados.rados.0.smithi076.stdout:3952: snap_remove snap 487 2024-09-21T09:54:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:45 smithi191 ceph-mon[102206]: pgmap v1075: 89 pgs: 89 active+clean; 217 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 985 KiB/s wr, 6 op/s 2024-09-21T09:54:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:45 smithi191 ceph-mon[102206]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-21T09:54:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:45 smithi076 ceph-mon[106482]: pgmap v1075: 89 pgs: 89 active+clean; 217 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 985 KiB/s wr, 6 op/s 2024-09-21T09:54:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:45 smithi076 ceph-mon[106482]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-21T09:54:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:45 smithi076 ceph-mon[110963]: pgmap v1075: 89 pgs: 89 active+clean; 217 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 985 KiB/s wr, 6 op/s 2024-09-21T09:54:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:45 smithi076 ceph-mon[110963]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-21T09:54:46.850 INFO:tasks.rados.rados.0.smithi076.stdout:3952: done (0 left) 2024-09-21T09:54:46.850 INFO:tasks.rados.rados.0.smithi076.stdout:3953: delete oid 30 current snap is 499 2024-09-21T09:54:46.859 INFO:tasks.rados.rados.0.smithi076.stdout:3953: done (0 left) 2024-09-21T09:54:46.859 INFO:tasks.rados.rados.0.smithi076.stdout:3954: write oid 40 current snap is 499 2024-09-21T09:54:46.859 INFO:tasks.rados.rados.0.smithi076.stdout:3954: seq_num 1499 ranges {793693=519792,1884508=477853,2898125=295568} 2024-09-21T09:54:46.866 INFO:tasks.rados.rados.0.smithi076.stdout:3954: writing smithi07685715-40 from 793693 to 1313485 tid 1 2024-09-21T09:54:46.872 INFO:tasks.rados.rados.0.smithi076.stdout:3954: writing smithi07685715-40 from 1884508 to 2362361 tid 2 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3954: writing smithi07685715-40 from 2898125 to 3193693 tid 3 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3955: read oid 20 snap -1 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3955: expect (ObjNum 1494 snap 497 seq_num 1494) 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3954: finishing write tid 1 to smithi07685715-40 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3954: finishing write tid 2 to smithi07685715-40 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3956: read oid 23 snap -1 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3956: expect (ObjNum 1473 snap 490 seq_num 1473) 2024-09-21T09:54:46.877 INFO:tasks.rados.rados.0.smithi076.stdout:3957: read oid 8 snap -1 2024-09-21T09:54:46.878 INFO:tasks.rados.rados.0.smithi076.stdout:3957: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:46.878 INFO:tasks.rados.rados.0.smithi076.stdout:3958: read oid 24 snap -1 2024-09-21T09:54:46.878 INFO:tasks.rados.rados.0.smithi076.stdout:3958: expect (ObjNum 1483 snap 492 seq_num 1483) 2024-09-21T09:54:46.878 INFO:tasks.rados.rados.0.smithi076.stdout:3959: snap_remove snap 494 2024-09-21T09:54:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:46 smithi191 ceph-mon[102206]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-21T09:54:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:46 smithi076 ceph-mon[106482]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-21T09:54:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:46 smithi076 ceph-mon[110963]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-21T09:54:47.856 INFO:tasks.rados.rados.0.smithi076.stdout:3959: done (5 left) 2024-09-21T09:54:47.856 INFO:tasks.rados.rados.0.smithi076.stdout:3960: read oid 2 snap -1 2024-09-21T09:54:47.856 INFO:tasks.rados.rados.0.smithi076.stdout:3960: expect (ObjNum 1497 snap 499 seq_num 1497) 2024-09-21T09:54:47.856 INFO:tasks.rados.rados.0.smithi076.stdout:3961: delete oid 12 current snap is 499 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: finishing write tid 3 to smithi07685715-40 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: finishing write tid 4 to smithi07685715-40 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: finishing write tid 5 to smithi07685715-40 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: oid 40 updating version 0 to 2109 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: oid 40 version 2109 is already newer than 2107 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: oid 40 updating version 2109 to 2110 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: oid 40 version 2110 is already newer than 2108 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 40 v 2110 (ObjNum 1499 snap 499 seq_num 1499) dirty exists 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3954: left oid 40 (ObjNum 1499 snap 499 seq_num 1499) 2024-09-21T09:54:47.857 INFO:tasks.rados.rados.0.smithi076.stdout:3958: expect (ObjNum 1483 snap 492 seq_num 1483) 2024-09-21T09:54:47.954 INFO:tasks.rados.rados.0.smithi076.stdout:3956: expect (ObjNum 1473 snap 490 seq_num 1473) 2024-09-21T09:54:48.076 INFO:tasks.rados.rados.0.smithi076.stdout:3955: expect (ObjNum 1494 snap 497 seq_num 1494) 2024-09-21T09:54:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:47 smithi191 ceph-mon[102206]: pgmap v1078: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 206 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 595 KiB/s wr, 2 op/s 2024-09-21T09:54:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:47 smithi191 ceph-mon[102206]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-21T09:54:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:47 smithi076 ceph-mon[106482]: pgmap v1078: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 206 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 595 KiB/s wr, 2 op/s 2024-09-21T09:54:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:47 smithi076 ceph-mon[106482]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-21T09:54:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:47 smithi076 ceph-mon[110963]: pgmap v1078: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 206 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 595 KiB/s wr, 2 op/s 2024-09-21T09:54:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:47 smithi076 ceph-mon[110963]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-21T09:54:48.267 INFO:tasks.rados.rados.0.smithi076.stdout:3957: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:48.382 INFO:tasks.rados.rados.0.smithi076.stdout:3960: expect (ObjNum 1497 snap 499 seq_num 1497) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3954: done (6 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3955: done (5 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3956: done (4 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3957: done (3 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3958: done (2 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3960: done (1 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3961: done (0 left) 2024-09-21T09:54:48.525 INFO:tasks.rados.rados.0.smithi076.stdout:3962: rollback oid 43 current snap is 499 2024-09-21T09:54:48.526 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 43 to 489 2024-09-21T09:54:48.526 INFO:tasks.rados.rados.0.smithi076.stdout:3963: delete oid 7 current snap is 499 2024-09-21T09:54:48.526 INFO:tasks.rados.rados.0.smithi076.stdout:3962: finishing rollback tid 1 to smithi07685715-43 2024-09-21T09:54:48.526 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:54:48.528 INFO:tasks.rados.rados.0.smithi076.stdout:3962: done (1 left) 2024-09-21T09:54:48.528 INFO:tasks.rados.rados.0.smithi076.stdout:3963: done (0 left) 2024-09-21T09:54:48.528 INFO:tasks.rados.rados.0.smithi076.stdout:3964: read oid 44 snap -1 2024-09-21T09:54:48.528 INFO:tasks.rados.rados.0.smithi076.stdout:3964: expect (ObjNum 1474 snap 490 seq_num 1474) 2024-09-21T09:54:48.528 INFO:tasks.rados.rados.0.smithi076.stdout:3965: write oid 30 current snap is 499 2024-09-21T09:54:48.528 INFO:tasks.rados.rados.0.smithi076.stdout:3965: seq_num 1502 ranges {444114=745539,1943601=625180} 2024-09-21T09:54:48.534 INFO:tasks.rados.rados.0.smithi076.stdout:3965: writing smithi07685715-30 from 444114 to 1189653 tid 1 2024-09-21T09:54:48.542 INFO:tasks.rados.rados.0.smithi076.stdout:3965: writing smithi07685715-30 from 1943601 to 2568781 tid 2 2024-09-21T09:54:48.542 INFO:tasks.rados.rados.0.smithi076.stdout:3966: read oid 18 snap -1 2024-09-21T09:54:48.542 INFO:tasks.rados.rados.0.smithi076.stdout:3966: expect (ObjNum 1487 snap 493 seq_num 1487) 2024-09-21T09:54:48.542 INFO:tasks.rados.rados.0.smithi076.stdout:3967: snap_create 2024-09-21T09:54:48.543 INFO:tasks.rados.rados.0.smithi076.stdout:3964: expect (ObjNum 1474 snap 490 seq_num 1474) 2024-09-21T09:54:48.736 INFO:tasks.rados.rados.0.smithi076.stdout:3965: finishing write tid 1 to smithi07685715-30 2024-09-21T09:54:48.737 INFO:tasks.rados.rados.0.smithi076.stdout:3966: expect (ObjNum 1487 snap 493 seq_num 1487) 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:3965: finishing write tid 2 to smithi07685715-30 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:3965: finishing write tid 3 to smithi07685715-30 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:3965: finishing write tid 4 to smithi07685715-30 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:3965: oid 30 updating version 0 to 3011 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:3965: oid 30 version 3011 is already newer than 3009 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:3965: oid 30 version 3011 is already newer than 3010 2024-09-21T09:54:48.831 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 30 v 3011 (ObjNum 1502 snap 499 seq_num 1502) dirty exists 2024-09-21T09:54:48.832 INFO:tasks.rados.rados.0.smithi076.stdout:3965: left oid 30 (ObjNum 1502 snap 499 seq_num 1502) 2024-09-21T09:54:48.835 INFO:tasks.rados.rados.0.smithi076.stdout:3964: done (3 left) 2024-09-21T09:54:48.835 INFO:tasks.rados.rados.0.smithi076.stdout:3965: done (2 left) 2024-09-21T09:54:48.835 INFO:tasks.rados.rados.0.smithi076.stdout:3966: done (1 left) 2024-09-21T09:54:48.835 INFO:tasks.rados.rados.0.smithi076.stdout:3967: done (0 left) 2024-09-21T09:54:48.835 INFO:tasks.rados.rados.0.smithi076.stdout:3968: write oid 43 current snap is 500 2024-09-21T09:54:48.835 INFO:tasks.rados.rados.0.smithi076.stdout:3968: seq_num 1503 ranges {460921=420788,1602083=485191,2491464=621953} 2024-09-21T09:54:48.839 INFO:tasks.rados.rados.0.smithi076.stdout:3968: writing smithi07685715-43 from 460921 to 881709 tid 1 2024-09-21T09:54:48.846 INFO:tasks.rados.rados.0.smithi076.stdout:3968: writing smithi07685715-43 from 1602083 to 2087274 tid 2 2024-09-21T09:54:48.852 INFO:tasks.rados.rados.0.smithi076.stdout:3968: writing smithi07685715-43 from 2491464 to 3113417 tid 3 2024-09-21T09:54:48.852 INFO:tasks.rados.rados.0.smithi076.stdout:3969: write oid 18 current snap is 500 2024-09-21T09:54:48.852 INFO:tasks.rados.rados.0.smithi076.stdout:3968: finishing write tid 1 to smithi07685715-43 2024-09-21T09:54:48.852 INFO:tasks.rados.rados.0.smithi076.stdout:3968: finishing write tid 2 to smithi07685715-43 2024-09-21T09:54:48.853 INFO:tasks.rados.rados.0.smithi076.stdout:3969: seq_num 1504 ranges {477728=496037,1660565=628849,2918887=358841} 2024-09-21T09:54:48.858 INFO:tasks.rados.rados.0.smithi076.stdout:3969: writing smithi07685715-18 from 477728 to 973765 tid 1 2024-09-21T09:54:48.865 INFO:tasks.rados.rados.0.smithi076.stdout:3969: writing smithi07685715-18 from 1660565 to 2289414 tid 2 2024-09-21T09:54:48.870 INFO:tasks.rados.rados.0.smithi076.stdout:3969: writing smithi07685715-18 from 2918887 to 3277728 tid 3 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3970: write oid 3 current snap is 500 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: finishing write tid 3 to smithi07685715-43 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: finishing write tid 4 to smithi07685715-43 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: finishing write tid 5 to smithi07685715-43 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: oid 43 updating version 0 to 879 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: oid 43 updating version 879 to 880 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: oid 43 version 880 is already newer than 877 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: oid 43 version 880 is already newer than 878 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 43 v 880 (ObjNum 1503 snap 500 seq_num 1503) dirty exists 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3968: left oid 43 (ObjNum 1503 snap 500 seq_num 1503) 2024-09-21T09:54:48.871 INFO:tasks.rados.rados.0.smithi076.stdout:3970: seq_num 1505 ranges {494535=571286,1719047=488860,2946310=348225} 2024-09-21T09:54:48.876 INFO:tasks.rados.rados.0.smithi076.stdout:3970: writing smithi07685715-3 from 494535 to 1065821 tid 1 2024-09-21T09:54:48.882 INFO:tasks.rados.rados.0.smithi076.stdout:3970: writing smithi07685715-3 from 1719047 to 2207907 tid 2 2024-09-21T09:54:48.888 INFO:tasks.rados.rados.0.smithi076.stdout:3970: writing smithi07685715-3 from 2946310 to 3294535 tid 3 2024-09-21T09:54:48.888 INFO:tasks.rados.rados.0.smithi076.stdout:3968: done (2 left) 2024-09-21T09:54:48.888 INFO:tasks.rados.rados.0.smithi076.stdout:3971: read oid 4 snap 491 2024-09-21T09:54:48.889 INFO:tasks.rados.rados.0.smithi076.stdout:3971: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:54:48.889 INFO:tasks.rados.rados.0.smithi076.stdout:3972: read oid 20 snap 496 2024-09-21T09:54:48.889 INFO:tasks.rados.rados.0.smithi076.stdout:3972: expect (ObjNum 1452 snap 483 seq_num 1452) 2024-09-21T09:54:48.889 INFO:tasks.rados.rados.0.smithi076.stdout:3973: snap_remove snap 475 2024-09-21T09:54:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:48 smithi191 ceph-mon[102206]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-21T09:54:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:48 smithi191 ceph-mon[102206]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-21T09:54:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:48 smithi076 ceph-mon[110963]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-21T09:54:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:48 smithi076 ceph-mon[110963]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-21T09:54:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:48 smithi076 ceph-mon[106482]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-21T09:54:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:48 smithi076 ceph-mon[106482]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-21T09:54:49.870 INFO:tasks.rados.rados.0.smithi076.stdout:3969: finishing write tid 1 to smithi07685715-18 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: finishing write tid 2 to smithi07685715-18 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: finishing write tid 3 to smithi07685715-18 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: finishing write tid 4 to smithi07685715-18 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: finishing write tid 5 to smithi07685715-18 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: oid 18 updating version 0 to 1039 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: oid 18 updating version 1039 to 1041 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: oid 18 version 1041 is already newer than 1040 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: oid 18 version 1041 is already newer than 1038 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 18 v 1041 (ObjNum 1504 snap 500 seq_num 1504) dirty exists 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3969: left oid 18 (ObjNum 1504 snap 500 seq_num 1504) 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3970: finishing write tid 1 to smithi07685715-3 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3970: finishing write tid 2 to smithi07685715-3 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3970: finishing write tid 3 to smithi07685715-3 2024-09-21T09:54:49.871 INFO:tasks.rados.rados.0.smithi076.stdout:3970: finishing write tid 4 to smithi07685715-3 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3970: finishing write tid 5 to smithi07685715-3 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3970: oid 3 updating version 0 to 337 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3970: oid 3 version 337 is already newer than 336 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3970: oid 3 updating version 337 to 338 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3970: oid 3 version 338 is already newer than 335 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 3 v 338 (ObjNum 1505 snap 500 seq_num 1505) dirty exists 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3970: left oid 3 (ObjNum 1505 snap 500 seq_num 1505) 2024-09-21T09:54:49.872 INFO:tasks.rados.rados.0.smithi076.stdout:3971: expect (ObjNum 1190 snap 399 seq_num 1190) 2024-09-21T09:54:49.939 INFO:tasks.rados.rados.0.smithi076.stdout:3972: expect (ObjNum 1452 snap 483 seq_num 1452) 2024-09-21T09:54:50.073 INFO:tasks.rados.rados.0.smithi076.stdout:3969: done (4 left) 2024-09-21T09:54:50.073 INFO:tasks.rados.rados.0.smithi076.stdout:3970: done (3 left) 2024-09-21T09:54:50.073 INFO:tasks.rados.rados.0.smithi076.stdout:3971: done (2 left) 2024-09-21T09:54:50.074 INFO:tasks.rados.rados.0.smithi076.stdout:3972: done (1 left) 2024-09-21T09:54:50.074 INFO:tasks.rados.rados.0.smithi076.stdout:3973: done (0 left) 2024-09-21T09:54:50.074 INFO:tasks.rados.rados.0.smithi076.stdout:3974: snap_create 2024-09-21T09:54:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:49 smithi191 ceph-mon[102206]: pgmap v1081: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 206 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 595 KiB/s wr, 2 op/s 2024-09-21T09:54:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:49 smithi076 ceph-mon[106482]: pgmap v1081: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 206 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 595 KiB/s wr, 2 op/s 2024-09-21T09:54:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:49 smithi076 ceph-mon[110963]: pgmap v1081: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 206 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 595 KiB/s wr, 2 op/s 2024-09-21T09:54:50.878 INFO:tasks.rados.rados.0.smithi076.stdout:3974: done (0 left) 2024-09-21T09:54:50.878 INFO:tasks.rados.rados.0.smithi076.stdout:3975: write oid 44 current snap is 501 2024-09-21T09:54:50.878 INFO:tasks.rados.rados.0.smithi076.stdout:3975: seq_num 1506 ranges {511342=762888,1777529=632518,2973733=337609} 2024-09-21T09:54:50.884 INFO:tasks.rados.rados.0.smithi076.stdout:3975: writing smithi07685715-44 from 511342 to 1274230 tid 1 2024-09-21T09:54:50.891 INFO:tasks.rados.rados.0.smithi076.stdout:3975: writing smithi07685715-44 from 1777529 to 2410047 tid 2 2024-09-21T09:54:50.896 INFO:tasks.rados.rados.0.smithi076.stdout:3975: writing smithi07685715-44 from 2973733 to 3311342 tid 3 2024-09-21T09:54:50.896 INFO:tasks.rados.rados.0.smithi076.stdout:3976: rollback oid 33 current snap is 501 2024-09-21T09:54:50.897 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 33 to 492 2024-09-21T09:54:50.897 INFO:tasks.rados.rados.0.smithi076.stdout:3975: finishing write tid 1 to smithi07685715-44 2024-09-21T09:54:50.897 INFO:tasks.rados.rados.0.smithi076.stdout:3975: finishing write tid 2 to smithi07685715-44 2024-09-21T09:54:50.897 INFO:tasks.rados.rados.0.smithi076.stdout:3977: snap_remove snap 496 2024-09-21T09:54:51.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:50 smithi191 ceph-mon[102206]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-21T09:54:51.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:50 smithi076 ceph-mon[106482]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-21T09:54:51.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:50 smithi076 ceph-mon[110963]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi076.stdout:3977: done (2 left) 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi076.stdout:3978: read oid 5 snap -1 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi076.stdout:3978: expect (ObjNum 1370 snap 459 seq_num 1370) 2024-09-21T09:54:51.884 INFO:tasks.rados.rados.0.smithi076.stdout:3979: snap_create 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: finishing write tid 3 to smithi07685715-44 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: finishing write tid 4 to smithi07685715-44 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: finishing write tid 5 to smithi07685715-44 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: oid 44 updating version 0 to 905 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: oid 44 updating version 905 to 906 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: oid 44 version 906 is already newer than 903 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: oid 44 version 906 is already newer than 904 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 44 v 906 (ObjNum 1506 snap 501 seq_num 1506) dirty exists 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3975: left oid 44 (ObjNum 1506 snap 501 seq_num 1506) 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3976: finishing rollback tid 0 to smithi07685715-33 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3976: finishing rollback tid 1 to smithi07685715-33 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:3976: finishing rollback tid 2 to smithi07685715-33 2024-09-21T09:54:51.885 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 33 v 932 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2024-09-21T09:54:51.897 INFO:tasks.rados.rados.0.smithi076.stdout:3978: expect (ObjNum 1370 snap 459 seq_num 1370) 2024-09-21T09:54:52.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:51 smithi191 ceph-mon[102206]: pgmap v1084: 89 pgs: 89 active+clean; 200 MiB data, 805 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 363 KiB/s wr, 8 op/s 2024-09-21T09:54:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:51 smithi191 ceph-mon[102206]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-21T09:54:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:51 smithi076 ceph-mon[106482]: pgmap v1084: 89 pgs: 89 active+clean; 200 MiB data, 805 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 363 KiB/s wr, 8 op/s 2024-09-21T09:54:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:51 smithi076 ceph-mon[106482]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-21T09:54:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:51 smithi076 ceph-mon[110963]: pgmap v1084: 89 pgs: 89 active+clean; 200 MiB data, 805 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 363 KiB/s wr, 8 op/s 2024-09-21T09:54:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:51 smithi076 ceph-mon[110963]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-21T09:54:52.889 INFO:tasks.rados.rados.0.smithi076.stdout:3975: done (3 left) 2024-09-21T09:54:52.889 INFO:tasks.rados.rados.0.smithi076.stdout:3976: done (2 left) 2024-09-21T09:54:52.890 INFO:tasks.rados.rados.0.smithi076.stdout:3978: done (1 left) 2024-09-21T09:54:52.890 INFO:tasks.rados.rados.0.smithi076.stdout:3979: done (0 left) 2024-09-21T09:54:52.890 INFO:tasks.rados.rados.0.smithi076.stdout:3980: write oid 29 current snap is 502 2024-09-21T09:54:52.890 INFO:tasks.rados.rados.0.smithi076.stdout:3980: seq_num 1507 ranges {528149=438137,1436011=492529,2601156=667041} 2024-09-21T09:54:52.894 INFO:tasks.rados.rados.0.smithi076.stdout:3980: writing smithi07685715-29 from 528149 to 966286 tid 1 2024-09-21T09:54:52.900 INFO:tasks.rados.rados.0.smithi076.stdout:3980: writing smithi07685715-29 from 1436011 to 1928540 tid 2 2024-09-21T09:54:52.907 INFO:tasks.rados.rados.0.smithi076.stdout:3980: writing smithi07685715-29 from 2601156 to 3268197 tid 3 2024-09-21T09:54:52.907 INFO:tasks.rados.rados.0.smithi076.stdout:3981: snap_create 2024-09-21T09:54:52.907 INFO:tasks.rados.rados.0.smithi076.stdout:3980: finishing write tid 1 to smithi07685715-29 2024-09-21T09:54:52.908 INFO:tasks.rados.rados.0.smithi076.stdout:3980: finishing write tid 2 to smithi07685715-29 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: finishing write tid 3 to smithi07685715-29 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: finishing write tid 4 to smithi07685715-29 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: finishing write tid 5 to smithi07685715-29 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: oid 29 updating version 0 to 1287 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: oid 29 version 1287 is already newer than 1286 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: oid 29 version 1287 is already newer than 1285 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:3980: oid 29 version 1287 is already newer than 1284 2024-09-21T09:54:52.911 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 29 v 1287 (ObjNum 1507 snap 502 seq_num 1507) dirty exists 2024-09-21T09:54:52.912 INFO:tasks.rados.rados.0.smithi076.stdout:3980: left oid 29 (ObjNum 1507 snap 502 seq_num 1507) 2024-09-21T09:54:53.250 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:52 smithi076 ceph-mon[106482]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-21T09:54:53.250 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:52 smithi076 ceph-mon[110963]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-21T09:54:53.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:52 smithi191 ceph-mon[102206]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-21T09:54:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:54:53] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-21T09:54:53.836 INFO:tasks.rados.rados.0.smithi076.stdout:3980: done (1 left) 2024-09-21T09:54:53.836 INFO:tasks.rados.rados.0.smithi076.stdout:3981: done (0 left) 2024-09-21T09:54:53.836 INFO:tasks.rados.rados.0.smithi076.stdout:3982: snap_create 2024-09-21T09:54:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:53 smithi191 ceph-mon[102206]: pgmap v1087: 89 pgs: 89 active+clean; 200 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.1 MiB/s wr, 28 op/s 2024-09-21T09:54:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:53 smithi191 ceph-mon[102206]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-21T09:54:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:53 smithi191 ceph-mon[102206]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-21T09:54:54.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-mon[106482]: pgmap v1087: 89 pgs: 89 active+clean; 200 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.1 MiB/s wr, 28 op/s 2024-09-21T09:54:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-mon[106482]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-21T09:54:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-mon[106482]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-21T09:54:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-mon[110963]: pgmap v1087: 89 pgs: 89 active+clean; 200 MiB data, 807 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.1 MiB/s wr, 28 op/s 2024-09-21T09:54:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-mon[110963]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-21T09:54:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:53 smithi076 ceph-mon[110963]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-21T09:54:54.838 INFO:tasks.rados.rados.0.smithi076.stdout:3982: done (0 left) 2024-09-21T09:54:54.838 INFO:tasks.rados.rados.0.smithi076.stdout:3983: read oid 8 snap -1 2024-09-21T09:54:54.839 INFO:tasks.rados.rados.0.smithi076.stdout:3983: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:54.839 INFO:tasks.rados.rados.0.smithi076.stdout:3984: read oid 38 snap -1 2024-09-21T09:54:54.839 INFO:tasks.rados.rados.0.smithi076.stdout:3984: expect deleted 2024-09-21T09:54:54.839 INFO:tasks.rados.rados.0.smithi076.stdout:3985: rollback oid 26 current snap is 504 2024-09-21T09:54:54.839 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 26 to 489 2024-09-21T09:54:54.839 INFO:tasks.rados.rados.0.smithi076.stdout:3986: snap_create 2024-09-21T09:54:54.844 INFO:tasks.rados.rados.0.smithi076.stdout:3985: finishing rollback tid 0 to smithi07685715-26 2024-09-21T09:54:54.844 INFO:tasks.rados.rados.0.smithi076.stdout:3985: finishing rollback tid 1 to smithi07685715-26 2024-09-21T09:54:54.844 INFO:tasks.rados.rados.0.smithi076.stdout:3985: finishing rollback tid 2 to smithi07685715-26 2024-09-21T09:54:54.844 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 26 v 1438 (ObjNum 1424 snap 474 seq_num 1424) dirty exists 2024-09-21T09:54:54.851 INFO:tasks.rados.rados.0.smithi076.stdout:3983: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:55.842 INFO:tasks.rados.rados.0.smithi076.stdout:3983: done (3 left) 2024-09-21T09:54:55.842 INFO:tasks.rados.rados.0.smithi076.stdout:3984: done (2 left) 2024-09-21T09:54:55.842 INFO:tasks.rados.rados.0.smithi076.stdout:3985: done (1 left) 2024-09-21T09:54:55.842 INFO:tasks.rados.rados.0.smithi076.stdout:3986: done (0 left) 2024-09-21T09:54:55.842 INFO:tasks.rados.rados.0.smithi076.stdout:3987: read oid 35 snap -1 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3987: expect (ObjNum 1431 snap 476 seq_num 1431) 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3988: read oid 28 snap -1 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3988: expect (ObjNum 1439 snap 479 seq_num 1439) 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3989: read oid 8 snap 499 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3989: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3990: write oid 36 current snap is 505 2024-09-21T09:54:55.843 INFO:tasks.rados.rados.0.smithi076.stdout:3990: seq_num 1508 ranges {544956=513386,1494493=636187,2628579=594666} 2024-09-21T09:54:55.848 INFO:tasks.rados.rados.0.smithi076.stdout:3990: writing smithi07685715-36 from 544956 to 1058342 tid 1 2024-09-21T09:54:55.854 INFO:tasks.rados.rados.0.smithi076.stdout:3990: writing smithi07685715-36 from 1494493 to 2130680 tid 2 2024-09-21T09:54:55.861 INFO:tasks.rados.rados.0.smithi076.stdout:3990: writing smithi07685715-36 from 2628579 to 3223245 tid 3 2024-09-21T09:54:55.861 INFO:tasks.rados.rados.0.smithi076.stdout:3991: write oid 21 current snap is 505 2024-09-21T09:54:55.861 INFO:tasks.rados.rados.0.smithi076.stdout:3991: seq_num 1509 ranges {561763=588635,1552975=496198,2656002=405938} 2024-09-21T09:54:55.869 INFO:tasks.rados.rados.0.smithi076.stdout:3991: writing smithi07685715-21 from 561763 to 1150398 tid 1 2024-09-21T09:54:55.873 INFO:tasks.rados.rados.0.smithi076.stdout:3991: writing smithi07685715-21 from 1552975 to 2049173 tid 2 2024-09-21T09:54:55.881 INFO:tasks.rados.rados.0.smithi076.stdout:3991: writing smithi07685715-21 from 2656002 to 3061940 tid 3 2024-09-21T09:54:55.881 INFO:tasks.rados.rados.0.smithi076.stdout:3992: rollback oid 41 current snap is 505 2024-09-21T09:54:55.881 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 41 to 497 2024-09-21T09:54:55.881 INFO:tasks.rados.rados.0.smithi076.stdout:3987: expect (ObjNum 1431 snap 476 seq_num 1431) 2024-09-21T09:54:55.963 INFO:tasks.rados.rados.0.smithi076.stdout:3988: expect (ObjNum 1439 snap 479 seq_num 1439) 2024-09-21T09:54:56.064 INFO:tasks.rados.rados.0.smithi076.stdout:3989: expect (ObjNum 1400 snap 466 seq_num 1400) 2024-09-21T09:54:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:55 smithi191 ceph-mon[102206]: pgmap v1090: 89 pgs: 89 active+clean; 202 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:54:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:55 smithi191 ceph-mon[102206]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-21T09:54:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:55 smithi076 ceph-mon[110963]: pgmap v1090: 89 pgs: 89 active+clean; 202 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:54:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:55 smithi076 ceph-mon[110963]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-21T09:54:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:55 smithi076 ceph-mon[106482]: pgmap v1090: 89 pgs: 89 active+clean; 202 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.7 MiB/s wr, 20 op/s 2024-09-21T09:54:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:55 smithi076 ceph-mon[106482]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-21T09:54:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: finishing write tid 1 to smithi07685715-36 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: finishing write tid 2 to smithi07685715-36 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: finishing write tid 3 to smithi07685715-36 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: finishing write tid 4 to smithi07685715-36 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: finishing write tid 5 to smithi07685715-36 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: oid 36 updating version 0 to 1229 2024-09-21T09:54:56.180 INFO:tasks.rados.rados.0.smithi076.stdout:3990: oid 36 version 1229 is already newer than 1228 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3990: oid 36 version 1229 is already newer than 1226 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3990: oid 36 version 1229 is already newer than 1227 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 36 v 1229 (ObjNum 1508 snap 505 seq_num 1508) dirty exists 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3990: left oid 36 (ObjNum 1508 snap 505 seq_num 1508) 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3991: finishing write tid 1 to smithi07685715-21 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3987: done (5 left) 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3988: done (4 left) 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3989: done (3 left) 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3990: done (2 left) 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3993: write oid 47 current snap is 505 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3991: finishing write tid 2 to smithi07685715-21 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3991: finishing write tid 3 to smithi07685715-21 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3991: finishing write tid 4 to smithi07685715-21 2024-09-21T09:54:56.181 INFO:tasks.rados.rados.0.smithi076.stdout:3992: finishing rollback tid 0 to smithi07685715-41 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3991: finishing write tid 5 to smithi07685715-21 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3991: oid 21 updating version 0 to 3046 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3991: oid 21 version 3046 is already newer than 3044 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3991: oid 21 version 3046 is already newer than 3043 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3991: oid 21 version 3046 is already newer than 3045 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 21 v 3046 (ObjNum 1509 snap 505 seq_num 1509) dirty exists 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3991: left oid 21 (ObjNum 1509 snap 505 seq_num 1509) 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3992: finishing rollback tid 1 to smithi07685715-41 2024-09-21T09:54:56.182 INFO:tasks.rados.rados.0.smithi076.stdout:3993: seq_num 1510 ranges {578570=663884,1895104=639856,2967072=411498} 2024-09-21T09:54:56.186 INFO:tasks.rados.rados.0.smithi076.stdout:3993: writing smithi07685715-47 from 578570 to 1242454 tid 1 2024-09-21T09:54:56.193 INFO:tasks.rados.rados.0.smithi076.stdout:3993: writing smithi07685715-47 from 1895104 to 2534960 tid 2 2024-09-21T09:54:56.199 INFO:tasks.rados.rados.0.smithi076.stdout:3993: writing smithi07685715-47 from 2967072 to 3378570 tid 3 2024-09-21T09:54:56.199 INFO:tasks.rados.rados.0.smithi076.stdout:3991: done (2 left) 2024-09-21T09:54:56.199 INFO:tasks.rados.rados.0.smithi076.stdout:3994: rollback oid 2 current snap is 505 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 2 to 499 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:3992: finishing rollback tid 2 to smithi07685715-41 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 41 v 2126 (ObjNum 1457 snap 485 seq_num 1457) dirty exists 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:3993: finishing write tid 1 to smithi07685715-47 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:3993: finishing write tid 2 to smithi07685715-47 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:3992: done (2 left) 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:3995: rollback oid 17 current snap is 505 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:rollback oid 17 to 478 2024-09-21T09:54:56.200 INFO:tasks.rados.rados.0.smithi076.stdout:3996: delete oid 25 current snap is 505 2024-09-21T09:54:56.201 INFO:tasks.rados.rados.0.smithi076.stdout:3996: done (3 left) 2024-09-21T09:54:56.201 INFO:tasks.rados.rados.0.smithi076.stdout:3997: delete oid 28 current snap is 505 2024-09-21T09:54:56.202 INFO:tasks.rados.rados.0.smithi076.stdout:3995: finishing rollback tid 1 to smithi07685715-17 2024-09-21T09:54:56.203 INFO:tasks.rados.rados.0.smithi076.stdout:3995: finishing rollback tid 2 to smithi07685715-17 2024-09-21T09:54:56.203 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 17 v 2431 (ObjNum 1337 snap 450 seq_num 1337) dirty exists 2024-09-21T09:54:56.206 INFO:tasks.rados.rados.0.smithi076.stdout:3994: finishing rollback tid 0 to smithi07685715-2 2024-09-21T09:54:56.206 INFO:tasks.rados.rados.0.smithi076.stdout:3994: finishing rollback tid 1 to smithi07685715-2 2024-09-21T09:54:56.206 INFO:tasks.rados.rados.0.smithi076.stdout:3994: finishing rollback tid 2 to smithi07685715-2 2024-09-21T09:54:56.206 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 2 v 851 (ObjNum 1497 snap 499 seq_num 1497) dirty exists 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: finishing write tid 3 to smithi07685715-47 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: finishing write tid 4 to smithi07685715-47 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: finishing write tid 5 to smithi07685715-47 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: oid 47 updating version 0 to 3026 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: oid 47 version 3026 is already newer than 3024 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: oid 47 version 3026 is already newer than 3023 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: oid 47 version 3026 is already newer than 3025 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:update_object_version oid 47 v 3026 (ObjNum 1510 snap 505 seq_num 1510) dirty exists 2024-09-21T09:54:56.207 INFO:tasks.rados.rados.0.smithi076.stdout:3993: left oid 47 (ObjNum 1510 snap 505 seq_num 1510) 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3993: done (3 left) 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3994: done (2 left) 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3995: done (1 left) 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3997: done (0 left) 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3998: read oid 12 snap -1 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3998: expect deleted 2024-09-21T09:54:56.208 INFO:tasks.rados.rados.0.smithi076.stdout:3999: read oid 28 snap -1 2024-09-21T09:54:56.209 INFO:tasks.rados.rados.0.smithi076.stdout:3999: expect deleted 2024-09-21T09:54:56.209 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 2 2024-09-21T09:54:56.209 INFO:tasks.rados.rados.0.smithi076.stdout:3998: done (1 left) 2024-09-21T09:54:56.209 INFO:tasks.rados.rados.0.smithi076.stdout: waiting on 1 2024-09-21T09:54:56.209 INFO:tasks.rados.rados.0.smithi076.stdout:3999: done (0 left) 2024-09-21T09:54:56.210 INFO:tasks.rados.rados.0.smithi076.stderr:0 errors. 2024-09-21T09:54:56.210 INFO:tasks.rados.rados.0.smithi076.stderr: 2024-09-21T09:54:56.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:54:56.416 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:54:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:56 smithi191 ceph-mon[102206]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-21T09:54:57.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:56 smithi076 ceph-mon[106482]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-21T09:54:57.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:56 smithi076 ceph-mon[110963]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-21T09:54:58.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:57 smithi076 ceph-mon[106482]: pgmap v1093: 89 pgs: 89 active+clean; 212 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:54:58.018 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:57 smithi076 ceph-mon[110963]: pgmap v1093: 89 pgs: 89 active+clean; 212 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:54:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:57 smithi191 ceph-mon[102206]: pgmap v1093: 89 pgs: 89 active+clean; 212 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-21T09:54:58.438 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:54:58.438 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":1081,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:54:55.833480+0000","last_up_change":"2024-09-21T09:50:43.849706+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-21T09:37:50.213404+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1081","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":638,"snap_epoch":1081,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":717,"up_from":730,"up_thru":829,"down_at":718,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6803","nonce":724858944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6805","nonce":724858944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6809","nonce":724858944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6807","nonce":724858944}]},"public_addr":"172.21.15.76:6803/724858944","cluster_addr":"172.21.15.76:6805/724858944","heartbeat_back_addr":"172.21.15.76:6809/724858944","heartbeat_front_addr":"172.21.15.76:6807/724858944","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":750,"up_from":763,"up_thru":829,"down_at":751,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6811","nonce":46966950}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6813","nonce":46966950}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6817","nonce":46966950}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6815","nonce":46966950}]},"public_addr":"172.21.15.76:6811/46966950","cluster_addr":"172.21.15.76:6813/46966950","heartbeat_back_addr":"172.21.15.76:6817/46966950","heartbeat_front_addr":"172.21.15.76:6815/46966950","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":783,"up_from":796,"up_thru":829,"down_at":784,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6819","nonce":3488608517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6821","nonce":3488608517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6825","nonce":3488608517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6823","nonce":3488608517}]},"public_addr":"172.21.15.76:6819/3488608517","cluster_addr":"172.21.15.76:6821/3488608517","heartbeat_back_addr":"172.21.15.76:6825/3488608517","heartbeat_front_addr":"172.21.15.76:6823/3488608517","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":816,"up_from":829,"up_thru":829,"down_at":817,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6827","nonce":523580128}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6829","nonce":523580128}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6833","nonce":523580128}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6831","nonce":523580128}]},"public_addr":"172.21.15.76:6827/523580128","cluster_addr":"172.21.15.76:6829/523580128","heartbeat_back_addr":"172.21.15.76:6833/523580128","heartbeat_front_addr":"172.21.15.76:6831/523580128","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":817,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":717},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":750},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":783},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":816},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/3937727837":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6801/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6800/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6835/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1469488034":"2024-09-22T09:42:40.197673+0000","172.21.15.76:0/3117602468":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/1830630483":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1931405138":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/4144106685":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6834/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1616224256":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/1280485976":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6801/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/4175697025":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.191:0/739062149":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/3392089630":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6800/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/3270182435":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/3629581092":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1868415942":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":609,"length":1},{"begin":630,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:54:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1864537806' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:54:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:58 smithi076 ceph-mon[106482]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-21T09:54:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1864537806' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:54:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:58 smithi076 ceph-mon[110963]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-21T09:54:59.150 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:54:59.346 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:54:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1864537806' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:54:59.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:58 smithi191 ceph-mon[102206]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-21T09:55:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:54:59 smithi076 ceph-mon[106482]: pgmap v1094: 89 pgs: 89 active+clean; 212 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-09-21T09:55:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:54:59 smithi076 ceph-mon[110963]: pgmap v1094: 89 pgs: 89 active+clean; 212 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-09-21T09:55:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:54:59 smithi191 ceph-mon[102206]: pgmap v1094: 89 pgs: 89 active+clean; 212 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 1.1 MiB/s wr, 13 op/s 2024-09-21T09:55:01.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:00 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:01.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:00 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:01.205 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:01.215 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:55:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:00 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:01 smithi191 ceph-mon[102206]: pgmap v1096: 89 pgs: 89 active+clean; 214 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:55:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:01 smithi191 ceph-mon[102206]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:01 smithi076 ceph-mon[106482]: pgmap v1096: 89 pgs: 89 active+clean; 214 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:55:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:01 smithi076 ceph-mon[106482]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:01 smithi076 ceph-mon[110963]: pgmap v1096: 89 pgs: 89 active+clean; 214 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.0 MiB/s wr, 12 op/s 2024-09-21T09:55:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:01 smithi076 ceph-mon[110963]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:02.542 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1096,"stamp":"2024-09-21T09:55:00.387834+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":224313354,"num_objects":127,"num_object_clones":55,"num_object_copies":381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":127,"num_whiteouts":8,"num_read":7405,"num_read_kb":7551259,"num_write":5906,"num_write_kb":1358569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":49812180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22266,"ondisk_log_size":22266,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":845680,"kb_used_data":451196,"kb_used_omap":195,"kb_used_meta":394172,"kb_avail":749115536,"statfs":{"total":767960285184,"available":767094308864,"internally_reserved":0,"allocated":462024704,"data_stored":529283603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":403632260},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":12847516,"num_objects":6,"num_object_clones":6,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":-1,"num_read":48,"num_read_kb":56047,"num_write":29,"num_write_kb":6399,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001287"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1259,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871731+0000","last_change":"2024-09-21T09:50:44.870931+0000","last_active":"2024-09-21T09:54:55.871731+0000","last_peered":"2024-09-21T09:54:55.871731+0000","last_clean":"2024-09-21T09:54:55.871731+0000","last_became_active":"2024-09-21T09:50:44.870528+0000","last_became_peered":"2024-09-21T09:50:44.870528+0000","last_unstale":"2024-09-21T09:54:55.871731+0000","last_undegraded":"2024-09-21T09:54:55.871731+0000","last_fullsized":"2024-09-21T09:54:55.871731+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:16.312101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1007,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835781+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:54:54.835781+0000","last_peered":"2024-09-21T09:54:54.835781+0000","last_clean":"2024-09-21T09:54:54.835781+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:54:54.835781+0000","last_undegraded":"2024-09-21T09:54:54.835781+0000","last_fullsized":"2024-09-21T09:54:54.835781+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1020,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839416+0000","last_change":"2024-09-21T09:50:44.866151+0000","last_active":"2024-09-21T09:54:54.839416+0000","last_peered":"2024-09-21T09:54:54.839416+0000","last_clean":"2024-09-21T09:54:54.839416+0000","last_became_active":"2024-09-21T09:50:44.865810+0000","last_became_peered":"2024-09-21T09:50:44.865810+0000","last_unstale":"2024-09-21T09:54:54.839416+0000","last_undegraded":"2024-09-21T09:54:54.839416+0000","last_fullsized":"2024-09-21T09:54:54.839416+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:31.175969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1337,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.880744+0000","last_change":"2024-09-21T09:50:44.876550+0000","last_active":"2024-09-21T09:54:55.880744+0000","last_peered":"2024-09-21T09:54:55.880744+0000","last_clean":"2024-09-21T09:54:55.880744+0000","last_became_active":"2024-09-21T09:50:44.876246+0000","last_became_peered":"2024-09-21T09:50:44.876246+0000","last_unstale":"2024-09-21T09:54:55.880744+0000","last_undegraded":"2024-09-21T09:54:55.880744+0000","last_fullsized":"2024-09-21T09:54:55.880744+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:54.983152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1337,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.880519+0000","last_change":"2024-09-21T09:50:44.872174+0000","last_active":"2024-09-21T09:54:55.880519+0000","last_peered":"2024-09-21T09:54:55.880519+0000","last_clean":"2024-09-21T09:54:55.880519+0000","last_became_active":"2024-09-21T09:50:44.871882+0000","last_became_peered":"2024-09-21T09:50:44.871882+0000","last_unstale":"2024-09-21T09:54:55.880519+0000","last_undegraded":"2024-09-21T09:54:55.880519+0000","last_fullsized":"2024-09-21T09:54:55.880519+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:57.860279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":990,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835679+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:54:54.835679+0000","last_peered":"2024-09-21T09:54:54.835679+0000","last_clean":"2024-09-21T09:54:54.835679+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:54:54.835679+0000","last_undegraded":"2024-09-21T09:54:54.835679+0000","last_fullsized":"2024-09-21T09:54:54.835679+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1259,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871308+0000","last_change":"2024-09-21T09:50:44.871672+0000","last_active":"2024-09-21T09:54:55.871308+0000","last_peered":"2024-09-21T09:54:55.871308+0000","last_clean":"2024-09-21T09:54:55.871308+0000","last_became_active":"2024-09-21T09:50:44.870133+0000","last_became_peered":"2024-09-21T09:50:44.870133+0000","last_unstale":"2024-09-21T09:54:55.871308+0000","last_undegraded":"2024-09-21T09:54:55.871308+0000","last_fullsized":"2024-09-21T09:54:55.871308+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:27.196429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1017,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.348494+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:54:55.348494+0000","last_peered":"2024-09-21T09:54:55.348494+0000","last_clean":"2024-09-21T09:54:55.348494+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:54:55.348494+0000","last_undegraded":"2024-09-21T09:54:55.348494+0000","last_fullsized":"2024-09-21T09:54:55.348494+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"1077'342","reported_seq":3501,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835549+0000","last_change":"2024-09-21T09:54:52.704645+0000","last_active":"2024-09-21T09:54:54.835549+0000","last_peered":"2024-09-21T09:54:54.835549+0000","last_clean":"2024-09-21T09:54:54.835549+0000","last_became_active":"2024-09-21T09:37:51.983020+0000","last_became_peered":"2024-09-21T09:37:51.983020+0000","last_unstale":"2024-09-21T09:54:54.835549+0000","last_undegraded":"2024-09-21T09:54:54.835549+0000","last_fullsized":"2024-09-21T09:54:54.835549+0000","mapping_epoch":74,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":75,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":342,"log_dups_size":0,"ondisk_log_size":342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:08:00.680405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002237092,"stat_sum":{"num_bytes":8069358,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":98,"num_read_kb":99570,"num_write":94,"num_write_kb":25143,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.f","version":"71'1","reported_seq":1007,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.836228+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:54:54.836228+0000","last_peered":"2024-09-21T09:54:54.836228+0000","last_clean":"2024-09-21T09:54:54.836228+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:54:54.836228+0000","last_undegraded":"2024-09-21T09:54:54.836228+0000","last_fullsized":"2024-09-21T09:54:54.836228+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1006,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839021+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:54:54.839021+0000","last_peered":"2024-09-21T09:54:54.839021+0000","last_clean":"2024-09-21T09:54:54.839021+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:54:54.839021+0000","last_undegraded":"2024-09-21T09:54:54.839021+0000","last_fullsized":"2024-09-21T09:54:54.839021+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"1080'1438","reported_seq":5097,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.851876+0000","last_change":"2024-09-21T09:54:52.704879+0000","last_active":"2024-09-21T09:54:58.851876+0000","last_peered":"2024-09-21T09:54:58.851876+0000","last_clean":"2024-09-21T09:54:58.851876+0000","last_became_active":"2024-09-21T09:50:11.810732+0000","last_became_peered":"2024-09-21T09:50:11.810732+0000","last_unstale":"2024-09-21T09:54:58.851876+0000","last_undegraded":"2024-09-21T09:54:58.851876+0000","last_fullsized":"2024-09-21T09:54:58.851876+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":1438,"log_dups_size":0,"ondisk_log_size":1438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:00:44.134833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067772100000000005,"stat_sum":{"num_bytes":14556215,"num_objects":6,"num_object_clones":3,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":1,"num_read":432,"num_read_kb":443405,"num_write":373,"num_write_kb":89228,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":13113291,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"276","length":"1"}]},{"pgid":"3.e","version":"71'2","reported_seq":1008,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.836167+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:54:54.836167+0000","last_peered":"2024-09-21T09:54:54.836167+0000","last_clean":"2024-09-21T09:54:54.836167+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:54:54.836167+0000","last_undegraded":"2024-09-21T09:54:54.836167+0000","last_fullsized":"2024-09-21T09:54:54.836167+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1042,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.451851+0000","last_change":"2024-09-21T09:50:44.867113+0000","last_active":"2024-09-21T09:54:55.451851+0000","last_peered":"2024-09-21T09:54:55.451851+0000","last_clean":"2024-09-21T09:54:55.451851+0000","last_became_active":"2024-09-21T09:50:44.866691+0000","last_became_peered":"2024-09-21T09:50:44.866691+0000","last_unstale":"2024-09-21T09:54:55.451851+0000","last_undegraded":"2024-09-21T09:54:55.451851+0000","last_fullsized":"2024-09-21T09:54:55.451851+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:56.642995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"1075'1236","reported_seq":4846,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.852113+0000","last_change":"2024-09-21T09:54:52.705235+0000","last_active":"2024-09-21T09:54:58.852113+0000","last_peered":"2024-09-21T09:54:58.852113+0000","last_clean":"2024-09-21T09:54:58.852113+0000","last_became_active":"2024-09-21T09:50:11.811037+0000","last_became_peered":"2024-09-21T09:50:11.811037+0000","last_unstale":"2024-09-21T09:54:58.852113+0000","last_undegraded":"2024-09-21T09:54:58.852113+0000","last_fullsized":"2024-09-21T09:54:58.852113+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":1236,"log_dups_size":0,"ondisk_log_size":1236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:54.820603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075150300000000005,"stat_sum":{"num_bytes":7643539,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":1,"num_read":348,"num_read_kb":325388,"num_write":326,"num_write_kb":80950,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"276","length":"1"}]},{"pgid":"3.d","version":"71'1","reported_seq":992,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.836109+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:54:54.836109+0000","last_peered":"2024-09-21T09:54:54.836109+0000","last_clean":"2024-09-21T09:54:54.836109+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:54:54.836109+0000","last_undegraded":"2024-09-21T09:54:54.836109+0000","last_fullsized":"2024-09-21T09:54:54.836109+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1009,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.348496+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:54:55.348496+0000","last_peered":"2024-09-21T09:54:55.348496+0000","last_clean":"2024-09-21T09:54:55.348496+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:54:55.348496+0000","last_undegraded":"2024-09-21T09:54:55.348496+0000","last_fullsized":"2024-09-21T09:54:55.348496+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"1075'856","reported_seq":4537,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.851995+0000","last_change":"2024-09-21T09:54:52.704854+0000","last_active":"2024-09-21T09:54:58.851995+0000","last_peered":"2024-09-21T09:54:58.851995+0000","last_clean":"2024-09-21T09:54:58.851995+0000","last_became_active":"2024-09-21T09:50:44.879702+0000","last_became_peered":"2024-09-21T09:50:44.879702+0000","last_unstale":"2024-09-21T09:54:58.851995+0000","last_undegraded":"2024-09-21T09:54:58.851995+0000","last_fullsized":"2024-09-21T09:54:58.851995+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":856,"log_dups_size":0,"ondisk_log_size":856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:26:49.495239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045944999999999997,"stat_sum":{"num_bytes":8956683,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":270,"num_read_kb":276203,"num_write":227,"num_write_kb":53805,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"276","length":"1"}]},{"pgid":"3.c","version":"0'0","reported_seq":1022,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839347+0000","last_change":"2024-09-21T09:50:44.866100+0000","last_active":"2024-09-21T09:54:54.839347+0000","last_peered":"2024-09-21T09:54:54.839347+0000","last_clean":"2024-09-21T09:54:54.839347+0000","last_became_active":"2024-09-21T09:50:44.865710+0000","last_became_peered":"2024-09-21T09:50:44.865710+0000","last_unstale":"2024-09-21T09:54:54.839347+0000","last_undegraded":"2024-09-21T09:54:54.839347+0000","last_fullsized":"2024-09-21T09:54:54.839347+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:16.761723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1339,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.881346+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:54:55.881346+0000","last_peered":"2024-09-21T09:54:55.881346+0000","last_clean":"2024-09-21T09:54:55.881346+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:54:55.881346+0000","last_undegraded":"2024-09-21T09:54:55.881346+0000","last_fullsized":"2024-09-21T09:54:55.881346+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"1081'851","reported_seq":4540,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.852497+0000","last_change":"2024-09-21T09:54:52.706839+0000","last_active":"2024-09-21T09:54:58.852497+0000","last_peered":"2024-09-21T09:54:58.852497+0000","last_clean":"2024-09-21T09:54:58.852497+0000","last_became_active":"2024-09-21T09:50:11.807830+0000","last_became_peered":"2024-09-21T09:50:11.807830+0000","last_unstale":"2024-09-21T09:54:58.852497+0000","last_undegraded":"2024-09-21T09:54:58.852497+0000","last_fullsized":"2024-09-21T09:54:58.852497+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":851,"log_dups_size":0,"ondisk_log_size":851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:34.491567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0024287929999999998,"stat_sum":{"num_bytes":9323720,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":296,"num_read_kb":284510,"num_write":224,"num_write_kb":51695,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":4733589,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"276","length":"1"}]},{"pgid":"3.b","version":"71'2","reported_seq":1271,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871270+0000","last_change":"2024-09-21T09:50:44.876433+0000","last_active":"2024-09-21T09:54:55.871270+0000","last_peered":"2024-09-21T09:54:55.871270+0000","last_clean":"2024-09-21T09:54:55.871270+0000","last_became_active":"2024-09-21T09:50:44.875979+0000","last_became_peered":"2024-09-21T09:50:44.875979+0000","last_unstale":"2024-09-21T09:54:55.871270+0000","last_undegraded":"2024-09-21T09:54:55.871270+0000","last_fullsized":"2024-09-21T09:54:55.871270+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:29.774368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1027,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452052+0000","last_change":"2024-09-21T09:50:44.866099+0000","last_active":"2024-09-21T09:54:55.452052+0000","last_peered":"2024-09-21T09:54:55.452052+0000","last_clean":"2024-09-21T09:54:55.452052+0000","last_became_active":"2024-09-21T09:50:44.865811+0000","last_became_peered":"2024-09-21T09:50:44.865811+0000","last_unstale":"2024-09-21T09:54:55.452052+0000","last_undegraded":"2024-09-21T09:54:55.452052+0000","last_fullsized":"2024-09-21T09:54:55.452052+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:25.833506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"1075'882","reported_seq":4683,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841606+0000","last_change":"2024-09-21T09:54:51.886291+0000","last_active":"2024-09-21T09:54:55.841606+0000","last_peered":"2024-09-21T09:54:55.841606+0000","last_clean":"2024-09-21T09:54:55.841606+0000","last_became_active":"2024-09-21T09:49:38.838805+0000","last_became_peered":"2024-09-21T09:49:38.838805+0000","last_unstale":"2024-09-21T09:54:55.841606+0000","last_undegraded":"2024-09-21T09:54:55.841606+0000","last_fullsized":"2024-09-21T09:54:55.841606+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":882,"log_dups_size":0,"ondisk_log_size":882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:18:45.255999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000655683,"stat_sum":{"num_bytes":10337361,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":443,"num_read_kb":543710,"num_write":217,"num_write_kb":50081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":5991297,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.a","version":"0'0","reported_seq":1010,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.348713+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:54:55.348713+0000","last_peered":"2024-09-21T09:54:55.348713+0000","last_clean":"2024-09-21T09:54:55.348713+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:54:55.348713+0000","last_undegraded":"2024-09-21T09:54:55.348713+0000","last_fullsized":"2024-09-21T09:54:55.348713+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1042,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452228+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:54:55.452228+0000","last_peered":"2024-09-21T09:54:55.452228+0000","last_clean":"2024-09-21T09:54:55.452228+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:54:55.452228+0000","last_undegraded":"2024-09-21T09:54:55.452228+0000","last_fullsized":"2024-09-21T09:54:55.452228+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"1081'1229","reported_seq":4386,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.864495+0000","last_change":"2024-09-21T09:54:51.882369+0000","last_active":"2024-09-21T09:54:55.864495+0000","last_peered":"2024-09-21T09:54:55.864495+0000","last_clean":"2024-09-21T09:54:55.864495+0000","last_became_active":"2024-09-21T09:37:51.981968+0000","last_became_peered":"2024-09-21T09:37:51.981968+0000","last_unstale":"2024-09-21T09:54:55.864495+0000","last_undegraded":"2024-09-21T09:54:55.864495+0000","last_fullsized":"2024-09-21T09:54:55.864495+0000","mapping_epoch":74,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":75,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":1229,"log_dups_size":0,"ondisk_log_size":1229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:51.006245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044026200000000002,"stat_sum":{"num_bytes":10031080,"num_objects":4,"num_object_clones":1,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":485,"num_read_kb":526138,"num_write":303,"num_write_kb":74033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.9","version":"0'0","reported_seq":1028,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452163+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:54:55.452163+0000","last_peered":"2024-09-21T09:54:55.452163+0000","last_clean":"2024-09-21T09:54:55.452163+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:54:55.452163+0000","last_undegraded":"2024-09-21T09:54:55.452163+0000","last_fullsized":"2024-09-21T09:54:55.452163+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1027,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452351+0000","last_change":"2024-09-21T09:49:38.835119+0000","last_active":"2024-09-21T09:54:55.452351+0000","last_peered":"2024-09-21T09:54:55.452351+0000","last_clean":"2024-09-21T09:54:55.452351+0000","last_became_active":"2024-09-21T09:49:38.834858+0000","last_became_peered":"2024-09-21T09:49:38.834858+0000","last_unstale":"2024-09-21T09:54:55.452351+0000","last_undegraded":"2024-09-21T09:54:55.452351+0000","last_fullsized":"2024-09-21T09:54:55.452351+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:59.812483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"1078'1287","reported_seq":4440,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452190+0000","last_change":"2024-09-21T09:54:51.886736+0000","last_active":"2024-09-21T09:54:55.452190+0000","last_peered":"2024-09-21T09:54:55.452190+0000","last_clean":"2024-09-21T09:54:55.452190+0000","last_became_active":"2024-09-21T09:49:05.815183+0000","last_became_peered":"2024-09-21T09:49:05.815183+0000","last_unstale":"2024-09-21T09:54:55.452190+0000","last_undegraded":"2024-09-21T09:54:55.452190+0000","last_fullsized":"2024-09-21T09:54:55.452190+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":1287,"log_dups_size":0,"ondisk_log_size":1287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:46.494481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0036515089999999998,"stat_sum":{"num_bytes":17099881,"num_objects":7,"num_object_clones":4,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":394,"num_read_kb":398726,"num_write":338,"num_write_kb":83776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":956920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.8","version":"0'0","reported_seq":1261,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871433+0000","last_change":"2024-09-21T09:50:44.875119+0000","last_active":"2024-09-21T09:54:55.871433+0000","last_peered":"2024-09-21T09:54:55.871433+0000","last_clean":"2024-09-21T09:54:55.871433+0000","last_became_active":"2024-09-21T09:50:44.873783+0000","last_became_peered":"2024-09-21T09:50:44.873783+0000","last_unstale":"2024-09-21T09:54:55.871433+0000","last_undegraded":"2024-09-21T09:54:55.871433+0000","last_fullsized":"2024-09-21T09:54:55.871433+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:28.277953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1311,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841676+0000","last_change":"2024-09-21T09:50:44.873954+0000","last_active":"2024-09-21T09:54:55.841676+0000","last_peered":"2024-09-21T09:54:55.841676+0000","last_clean":"2024-09-21T09:54:55.841676+0000","last_became_active":"2024-09-21T09:50:44.873527+0000","last_became_peered":"2024-09-21T09:50:44.873527+0000","last_unstale":"2024-09-21T09:54:55.841676+0000","last_undegraded":"2024-09-21T09:54:55.841676+0000","last_fullsized":"2024-09-21T09:54:55.841676+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:53.842361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.4","version":"1077'908","reported_seq":4058,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835830+0000","last_change":"2024-09-21T09:54:52.704894+0000","last_active":"2024-09-21T09:54:54.835830+0000","last_peered":"2024-09-21T09:54:54.835830+0000","last_clean":"2024-09-21T09:54:54.835830+0000","last_became_active":"2024-09-21T09:50:11.807688+0000","last_became_peered":"2024-09-21T09:50:11.807688+0000","last_unstale":"2024-09-21T09:54:54.835830+0000","last_undegraded":"2024-09-21T09:54:54.835830+0000","last_fullsized":"2024-09-21T09:54:54.835830+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":908,"log_dups_size":0,"ondisk_log_size":908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:07.421644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0024520990000000001,"stat_sum":{"num_bytes":14231199,"num_objects":6,"num_object_clones":4,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":255,"num_read_kb":266119,"num_write":232,"num_write_kb":56280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"1.0","version":"361'149","reported_seq":1336,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835599+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:54:54.835599+0000","last_peered":"2024-09-21T09:54:54.835599+0000","last_clean":"2024-09-21T09:54:54.835599+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:54:54.835599+0000","last_undegraded":"2024-09-21T09:54:54.835599+0000","last_fullsized":"2024-09-21T09:54:54.835599+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1036,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839280+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:54:54.839280+0000","last_peered":"2024-09-21T09:54:54.839280+0000","last_clean":"2024-09-21T09:54:54.839280+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:54:54.839280+0000","last_undegraded":"2024-09-21T09:54:54.839280+0000","last_fullsized":"2024-09-21T09:54:54.839280+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1248,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871807+0000","last_change":"2024-09-21T09:50:44.867188+0000","last_active":"2024-09-21T09:54:55.871807+0000","last_peered":"2024-09-21T09:54:55.871807+0000","last_clean":"2024-09-21T09:54:55.871807+0000","last_became_active":"2024-09-21T09:50:44.866818+0000","last_became_peered":"2024-09-21T09:50:44.866818+0000","last_unstale":"2024-09-21T09:54:55.871807+0000","last_undegraded":"2024-09-21T09:54:55.871807+0000","last_fullsized":"2024-09-21T09:54:55.871807+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:17:33.499249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":994,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.348674+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:54:55.348674+0000","last_peered":"2024-09-21T09:54:55.348674+0000","last_clean":"2024-09-21T09:54:55.348674+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:54:55.348674+0000","last_undegraded":"2024-09-21T09:54:55.348674+0000","last_fullsized":"2024-09-21T09:54:55.348674+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"1081'3028","reported_seq":6055,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:56.203807+0000","last_change":"2024-09-21T09:54:51.886635+0000","last_active":"2024-09-21T09:54:56.203807+0000","last_peered":"2024-09-21T09:54:56.203807+0000","last_clean":"2024-09-21T09:54:56.203807+0000","last_became_active":"2024-09-21T09:49:38.840141+0000","last_became_peered":"2024-09-21T09:49:38.840141+0000","last_unstale":"2024-09-21T09:54:56.203807+0000","last_undegraded":"2024-09-21T09:54:56.203807+0000","last_fullsized":"2024-09-21T09:54:56.203807+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":3028,"log_dups_size":0,"ondisk_log_size":3028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:56.557142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0044986610000000002,"stat_sum":{"num_bytes":35414305,"num_objects":14,"num_object_clones":8,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":1,"num_read":984,"num_read_kb":1059050,"num_write":743,"num_write_kb":175536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":7238433,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":1034,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835538+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:54:54.835538+0000","last_peered":"2024-09-21T09:54:54.835538+0000","last_clean":"2024-09-21T09:54:54.835538+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:54:54.835538+0000","last_undegraded":"2024-09-21T09:54:54.835538+0000","last_fullsized":"2024-09-21T09:54:54.835538+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1337,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.879764+0000","last_change":"2024-09-21T09:50:44.872670+0000","last_active":"2024-09-21T09:54:55.879764+0000","last_peered":"2024-09-21T09:54:55.879764+0000","last_clean":"2024-09-21T09:54:55.879764+0000","last_became_active":"2024-09-21T09:50:44.872391+0000","last_became_peered":"2024-09-21T09:50:44.872391+0000","last_unstale":"2024-09-21T09:54:55.879764+0000","last_undegraded":"2024-09-21T09:54:55.879764+0000","last_fullsized":"2024-09-21T09:54:55.879764+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:19.739055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1349,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.880341+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:54:55.880341+0000","last_peered":"2024-09-21T09:54:55.880341+0000","last_clean":"2024-09-21T09:54:55.880341+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:54:55.880341+0000","last_undegraded":"2024-09-21T09:54:55.880341+0000","last_fullsized":"2024-09-21T09:54:55.880341+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"1077'1043","reported_seq":4645,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.852372+0000","last_change":"2024-09-21T09:54:52.707309+0000","last_active":"2024-09-21T09:54:58.852372+0000","last_peered":"2024-09-21T09:54:58.852372+0000","last_clean":"2024-09-21T09:54:58.852372+0000","last_became_active":"2024-09-21T09:50:11.809952+0000","last_became_peered":"2024-09-21T09:50:11.809952+0000","last_unstale":"2024-09-21T09:54:58.852372+0000","last_undegraded":"2024-09-21T09:54:58.852372+0000","last_fullsized":"2024-09-21T09:54:58.852372+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":1043,"log_dups_size":0,"ondisk_log_size":1043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:29.540292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002714335,"stat_sum":{"num_bytes":7350096,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":246,"num_read_kb":230268,"num_write":256,"num_write_kb":62475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"276","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":1318,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.851861+0000","last_change":"2024-09-21T09:50:44.880197+0000","last_active":"2024-09-21T09:54:58.851861+0000","last_peered":"2024-09-21T09:54:58.851861+0000","last_clean":"2024-09-21T09:54:58.851861+0000","last_became_active":"2024-09-21T09:50:44.879899+0000","last_became_peered":"2024-09-21T09:50:44.879899+0000","last_unstale":"2024-09-21T09:54:58.851861+0000","last_undegraded":"2024-09-21T09:54:58.851861+0000","last_fullsized":"2024-09-21T09:54:58.851861+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:23.596846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1311,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841054+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:54:55.841054+0000","last_peered":"2024-09-21T09:54:55.841054+0000","last_clean":"2024-09-21T09:54:55.841054+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:54:55.841054+0000","last_undegraded":"2024-09-21T09:54:55.841054+0000","last_fullsized":"2024-09-21T09:54:55.841054+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1319,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841422+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:54:55.841422+0000","last_peered":"2024-09-21T09:54:55.841422+0000","last_clean":"2024-09-21T09:54:55.841422+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:54:55.841422+0000","last_undegraded":"2024-09-21T09:54:55.841422+0000","last_fullsized":"2024-09-21T09:54:55.841422+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"1075'419","reported_seq":4231,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.880078+0000","last_change":"2024-09-21T09:54:51.886554+0000","last_active":"2024-09-21T09:54:55.880078+0000","last_peered":"2024-09-21T09:54:55.880078+0000","last_clean":"2024-09-21T09:54:55.880078+0000","last_became_active":"2024-09-21T09:49:38.830243+0000","last_became_peered":"2024-09-21T09:49:38.830243+0000","last_unstale":"2024-09-21T09:54:55.880078+0000","last_undegraded":"2024-09-21T09:54:55.880078+0000","last_fullsized":"2024-09-21T09:54:55.880078+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":419,"log_dups_size":0,"ondisk_log_size":419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:27.979831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057788999999999996,"stat_sum":{"num_bytes":6807528,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":95,"num_read_kb":111702,"num_write":114,"num_write_kb":29661,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2277168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"2.2","version":"65'2","reported_seq":1037,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839219+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:54:54.839219+0000","last_peered":"2024-09-21T09:54:54.839219+0000","last_clean":"2024-09-21T09:54:54.839219+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:54:54.839219+0000","last_undegraded":"2024-09-21T09:54:54.839219+0000","last_fullsized":"2024-09-21T09:54:54.839219+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1028,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452072+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:54:55.452072+0000","last_peered":"2024-09-21T09:54:55.452072+0000","last_clean":"2024-09-21T09:54:55.452072+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:54:55.452072+0000","last_undegraded":"2024-09-21T09:54:55.452072+0000","last_fullsized":"2024-09-21T09:54:55.452072+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1322,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841292+0000","last_change":"2024-09-21T09:50:44.879616+0000","last_active":"2024-09-21T09:54:55.841292+0000","last_peered":"2024-09-21T09:54:55.841292+0000","last_clean":"2024-09-21T09:54:55.841292+0000","last_became_active":"2024-09-21T09:50:44.879237+0000","last_became_peered":"2024-09-21T09:50:44.879237+0000","last_unstale":"2024-09-21T09:54:55.841292+0000","last_undegraded":"2024-09-21T09:54:55.841292+0000","last_fullsized":"2024-09-21T09:54:55.841292+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:21.338486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"1081'2431","reported_seq":6065,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:56.201246+0000","last_change":"2024-09-21T09:54:51.889891+0000","last_active":"2024-09-21T09:54:56.201246+0000","last_peered":"2024-09-21T09:54:56.201246+0000","last_clean":"2024-09-21T09:54:56.201246+0000","last_became_active":"2024-09-21T09:49:05.807856+0000","last_became_peered":"2024-09-21T09:49:05.807856+0000","last_unstale":"2024-09-21T09:54:56.201246+0000","last_undegraded":"2024-09-21T09:54:56.201246+0000","last_fullsized":"2024-09-21T09:54:56.201246+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":2431,"log_dups_size":0,"ondisk_log_size":2431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:12:52.478790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0042698759999999997,"stat_sum":{"num_bytes":13656894,"num_objects":8,"num_object_clones":5,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":2,"num_read":712,"num_read_kb":759940,"num_write":603,"num_write_kb":139628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":3294082,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.5","version":"0'0","reported_seq":1037,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839518+0000","last_change":"2024-09-21T09:50:44.870337+0000","last_active":"2024-09-21T09:54:54.839518+0000","last_peered":"2024-09-21T09:54:54.839518+0000","last_clean":"2024-09-21T09:54:54.839518+0000","last_became_active":"2024-09-21T09:50:44.870057+0000","last_became_peered":"2024-09-21T09:50:44.870057+0000","last_unstale":"2024-09-21T09:54:54.839518+0000","last_undegraded":"2024-09-21T09:54:54.839518+0000","last_fullsized":"2024-09-21T09:54:54.839518+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:22.969900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1333,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841297+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:54:55.841297+0000","last_peered":"2024-09-21T09:54:55.841297+0000","last_clean":"2024-09-21T09:54:55.841297+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:54:55.841297+0000","last_undegraded":"2024-09-21T09:54:55.841297+0000","last_fullsized":"2024-09-21T09:54:55.841297+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1300,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841190+0000","last_change":"2024-09-21T09:49:38.833392+0000","last_active":"2024-09-21T09:54:55.841190+0000","last_peered":"2024-09-21T09:54:55.841190+0000","last_clean":"2024-09-21T09:54:55.841190+0000","last_became_active":"2024-09-21T09:49:38.832610+0000","last_became_peered":"2024-09-21T09:49:38.832610+0000","last_unstale":"2024-09-21T09:54:55.841190+0000","last_undegraded":"2024-09-21T09:54:55.841190+0000","last_fullsized":"2024-09-21T09:54:55.841190+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:26.844310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"1081'2126","reported_seq":5172,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.883289+0000","last_change":"2024-09-21T09:54:51.891613+0000","last_active":"2024-09-21T09:54:55.883289+0000","last_peered":"2024-09-21T09:54:55.883289+0000","last_clean":"2024-09-21T09:54:55.883289+0000","last_became_active":"2024-09-21T09:49:05.803716+0000","last_became_peered":"2024-09-21T09:49:05.803716+0000","last_unstale":"2024-09-21T09:54:55.883289+0000","last_undegraded":"2024-09-21T09:54:55.883289+0000","last_fullsized":"2024-09-21T09:54:55.883289+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":2126,"log_dups_size":0,"ondisk_log_size":2126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:12.934470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0033937009999999998,"stat_sum":{"num_bytes":21748300,"num_objects":12,"num_object_clones":7,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":617,"num_read_kb":618409,"num_write":579,"num_write_kb":139555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1106847,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":1310,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841451+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:54:55.841451+0000","last_peered":"2024-09-21T09:54:55.841451+0000","last_clean":"2024-09-21T09:54:55.841451+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:54:55.841451+0000","last_undegraded":"2024-09-21T09:54:55.841451+0000","last_fullsized":"2024-09-21T09:54:55.841451+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1019,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.836092+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:54:54.836092+0000","last_peered":"2024-09-21T09:54:54.836092+0000","last_clean":"2024-09-21T09:54:54.836092+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:54:54.836092+0000","last_undegraded":"2024-09-21T09:54:54.836092+0000","last_fullsized":"2024-09-21T09:54:54.836092+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1327,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.881663+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:54:55.881663+0000","last_peered":"2024-09-21T09:54:55.881663+0000","last_clean":"2024-09-21T09:54:55.881663+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:54:55.881663+0000","last_undegraded":"2024-09-21T09:54:55.881663+0000","last_fullsized":"2024-09-21T09:54:55.881663+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"1081'3046","reported_seq":6058,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.882456+0000","last_change":"2024-09-21T09:54:51.890385+0000","last_active":"2024-09-21T09:54:55.882456+0000","last_peered":"2024-09-21T09:54:55.882456+0000","last_clean":"2024-09-21T09:54:55.882456+0000","last_became_active":"2024-09-21T09:50:44.880008+0000","last_became_peered":"2024-09-21T09:50:44.880008+0000","last_unstale":"2024-09-21T09:54:55.882456+0000","last_undegraded":"2024-09-21T09:54:55.882456+0000","last_fullsized":"2024-09-21T09:54:55.882456+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":3046,"log_dups_size":0,"ondisk_log_size":3046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:05.388723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.008012323,"stat_sum":{"num_bytes":26658971,"num_objects":11,"num_object_clones":4,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":2,"num_read":1016,"num_read_kb":1217971,"num_write":765,"num_write_kb":190689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":6945917,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.7","version":"71'2","reported_seq":1271,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871707+0000","last_change":"2024-09-21T09:50:44.875761+0000","last_active":"2024-09-21T09:54:55.871707+0000","last_peered":"2024-09-21T09:54:55.871707+0000","last_clean":"2024-09-21T09:54:55.871707+0000","last_became_active":"2024-09-21T09:50:44.875333+0000","last_became_peered":"2024-09-21T09:50:44.875333+0000","last_unstale":"2024-09-21T09:54:55.871707+0000","last_undegraded":"2024-09-21T09:54:55.871707+0000","last_fullsized":"2024-09-21T09:54:55.871707+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:51.443596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1311,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841465+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:54:55.841465+0000","last_peered":"2024-09-21T09:54:55.841465+0000","last_clean":"2024-09-21T09:54:55.841465+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:54:55.841465+0000","last_undegraded":"2024-09-21T09:54:55.841465+0000","last_fullsized":"2024-09-21T09:54:55.841465+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1269,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871226+0000","last_change":"2024-09-21T09:50:44.874771+0000","last_active":"2024-09-21T09:54:55.871226+0000","last_peered":"2024-09-21T09:54:55.871226+0000","last_clean":"2024-09-21T09:54:55.871226+0000","last_became_active":"2024-09-21T09:50:44.873380+0000","last_became_peered":"2024-09-21T09:50:44.873380+0000","last_unstale":"2024-09-21T09:54:55.871226+0000","last_undegraded":"2024-09-21T09:54:55.871226+0000","last_fullsized":"2024-09-21T09:54:55.871226+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:19:11.264697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"1081'936","reported_seq":4559,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871287+0000","last_change":"2024-09-21T09:54:52.377627+0000","last_active":"2024-09-21T09:54:55.871287+0000","last_peered":"2024-09-21T09:54:55.871287+0000","last_clean":"2024-09-21T09:54:55.871287+0000","last_became_active":"2024-09-21T09:50:44.878227+0000","last_became_peered":"2024-09-21T09:50:44.878227+0000","last_unstale":"2024-09-21T09:54:55.871287+0000","last_undegraded":"2024-09-21T09:54:55.871287+0000","last_fullsized":"2024-09-21T09:54:55.871287+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":74,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:37:50.962216+0000","last_clean_scrub_stamp":"2024-09-21T09:37:50.962216+0000","objects_scrubbed":0,"log_size":936,"log_dups_size":0,"ondisk_log_size":936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:59.151627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0026352049999999998,"stat_sum":{"num_bytes":11832283,"num_objects":7,"num_object_clones":5,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":1,"num_read":322,"num_read_kb":389316,"num_write":233,"num_write_kb":53585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2383532,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"261","length":"1"},{"start":"276","length":"1"}]},{"pgid":"3.6","version":"71'1","reported_seq":1341,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.880637+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:54:55.880637+0000","last_peered":"2024-09-21T09:54:55.880637+0000","last_clean":"2024-09-21T09:54:55.880637+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:54:55.880637+0000","last_undegraded":"2024-09-21T09:54:55.880637+0000","last_fullsized":"2024-09-21T09:54:55.880637+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1023,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.348865+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:54:55.348865+0000","last_peered":"2024-09-21T09:54:55.348865+0000","last_clean":"2024-09-21T09:54:55.348865+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:54:55.348865+0000","last_undegraded":"2024-09-21T09:54:55.348865+0000","last_fullsized":"2024-09-21T09:54:55.348865+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1012,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.451961+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:54:55.451961+0000","last_peered":"2024-09-21T09:54:55.451961+0000","last_clean":"2024-09-21T09:54:55.451961+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:54:55.451961+0000","last_undegraded":"2024-09-21T09:54:55.451961+0000","last_fullsized":"2024-09-21T09:54:55.451961+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1026,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.451993+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:54:55.451993+0000","last_peered":"2024-09-21T09:54:55.451993+0000","last_clean":"2024-09-21T09:54:55.451993+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:54:55.451993+0000","last_undegraded":"2024-09-21T09:54:55.451993+0000","last_fullsized":"2024-09-21T09:54:55.451993+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1035,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452276+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:54:55.452276+0000","last_peered":"2024-09-21T09:54:55.452276+0000","last_clean":"2024-09-21T09:54:55.452276+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:54:55.452276+0000","last_undegraded":"2024-09-21T09:54:55.452276+0000","last_fullsized":"2024-09-21T09:54:55.452276+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1311,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.841074+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:54:55.841074+0000","last_peered":"2024-09-21T09:54:55.841074+0000","last_clean":"2024-09-21T09:54:55.841074+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:54:55.841074+0000","last_undegraded":"2024-09-21T09:54:55.841074+0000","last_fullsized":"2024-09-21T09:54:55.841074+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1008,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.835988+0000","last_change":"2024-09-21T09:50:44.870735+0000","last_active":"2024-09-21T09:54:54.835988+0000","last_peered":"2024-09-21T09:54:54.835988+0000","last_clean":"2024-09-21T09:54:54.835988+0000","last_became_active":"2024-09-21T09:50:44.870370+0000","last_became_peered":"2024-09-21T09:50:44.870370+0000","last_unstale":"2024-09-21T09:54:54.835988+0000","last_undegraded":"2024-09-21T09:54:54.835988+0000","last_fullsized":"2024-09-21T09:54:54.835988+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:52.476737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1021,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839521+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:54:54.839521+0000","last_peered":"2024-09-21T09:54:54.839521+0000","last_clean":"2024-09-21T09:54:54.839521+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:54:54.839521+0000","last_undegraded":"2024-09-21T09:54:54.839521+0000","last_fullsized":"2024-09-21T09:54:54.839521+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1247,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871141+0000","last_change":"2024-09-21T09:50:44.874933+0000","last_active":"2024-09-21T09:54:55.871141+0000","last_peered":"2024-09-21T09:54:55.871141+0000","last_clean":"2024-09-21T09:54:55.871141+0000","last_became_active":"2024-09-21T09:50:44.870699+0000","last_became_peered":"2024-09-21T09:50:44.870699+0000","last_unstale":"2024-09-21T09:54:55.871141+0000","last_undegraded":"2024-09-21T09:54:55.871141+0000","last_fullsized":"2024-09-21T09:54:55.871141+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:12.912457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1336,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.881052+0000","last_change":"2024-09-21T09:50:44.872004+0000","last_active":"2024-09-21T09:54:55.881052+0000","last_peered":"2024-09-21T09:54:55.881052+0000","last_clean":"2024-09-21T09:54:55.881052+0000","last_became_active":"2024-09-21T09:50:44.871706+0000","last_became_peered":"2024-09-21T09:50:44.871706+0000","last_unstale":"2024-09-21T09:54:55.881052+0000","last_undegraded":"2024-09-21T09:54:55.881052+0000","last_fullsized":"2024-09-21T09:54:55.881052+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:00.145938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1259,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871111+0000","last_change":"2024-09-21T09:50:44.874472+0000","last_active":"2024-09-21T09:54:55.871111+0000","last_peered":"2024-09-21T09:54:55.871111+0000","last_clean":"2024-09-21T09:54:55.871111+0000","last_became_active":"2024-09-21T09:50:44.872792+0000","last_became_peered":"2024-09-21T09:50:44.872792+0000","last_unstale":"2024-09-21T09:54:55.871111+0000","last_undegraded":"2024-09-21T09:54:55.871111+0000","last_fullsized":"2024-09-21T09:54:55.871111+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:52.873552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1018,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.348786+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:54:55.348786+0000","last_peered":"2024-09-21T09:54:55.348786+0000","last_clean":"2024-09-21T09:54:55.348786+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:54:55.348786+0000","last_undegraded":"2024-09-21T09:54:55.348786+0000","last_fullsized":"2024-09-21T09:54:55.348786+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1275,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871351+0000","last_change":"2024-09-21T09:50:44.875450+0000","last_active":"2024-09-21T09:54:55.871351+0000","last_peered":"2024-09-21T09:54:55.871351+0000","last_clean":"2024-09-21T09:54:55.871351+0000","last_became_active":"2024-09-21T09:50:44.874171+0000","last_became_peered":"2024-09-21T09:50:44.874171+0000","last_unstale":"2024-09-21T09:54:55.871351+0000","last_undegraded":"2024-09-21T09:54:55.871351+0000","last_fullsized":"2024-09-21T09:54:55.871351+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:37.106157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1338,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.880428+0000","last_change":"2024-09-21T09:50:44.871526+0000","last_active":"2024-09-21T09:54:55.880428+0000","last_peered":"2024-09-21T09:54:55.880428+0000","last_clean":"2024-09-21T09:54:55.880428+0000","last_became_active":"2024-09-21T09:50:44.871236+0000","last_became_peered":"2024-09-21T09:50:44.871236+0000","last_unstale":"2024-09-21T09:54:55.880428+0000","last_undegraded":"2024-09-21T09:54:55.880428+0000","last_fullsized":"2024-09-21T09:54:55.880428+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:51.749231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1310,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.840955+0000","last_change":"2024-09-21T09:50:44.872712+0000","last_active":"2024-09-21T09:54:55.840955+0000","last_peered":"2024-09-21T09:54:55.840955+0000","last_clean":"2024-09-21T09:54:55.840955+0000","last_became_active":"2024-09-21T09:50:44.872368+0000","last_became_peered":"2024-09-21T09:50:44.872368+0000","last_unstale":"2024-09-21T09:54:55.840955+0000","last_undegraded":"2024-09-21T09:54:55.840955+0000","last_fullsized":"2024-09-21T09:54:55.840955+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:16.003164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1259,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871311+0000","last_change":"2024-09-21T09:50:44.877609+0000","last_active":"2024-09-21T09:54:55.871311+0000","last_peered":"2024-09-21T09:54:55.871311+0000","last_clean":"2024-09-21T09:54:55.871311+0000","last_became_active":"2024-09-21T09:50:44.876347+0000","last_became_peered":"2024-09-21T09:50:44.876347+0000","last_unstale":"2024-09-21T09:54:55.871311+0000","last_undegraded":"2024-09-21T09:54:55.871311+0000","last_fullsized":"2024-09-21T09:54:55.871311+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:24.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1043,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.452063+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:54:55.452063+0000","last_peered":"2024-09-21T09:54:55.452063+0000","last_clean":"2024-09-21T09:54:55.452063+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:54:55.452063+0000","last_undegraded":"2024-09-21T09:54:55.452063+0000","last_fullsized":"2024-09-21T09:54:55.452063+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1294,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-09-21T09:54:58.852179+0000","last_change":"2024-09-21T09:50:44.879265+0000","last_active":"2024-09-21T09:54:58.852179+0000","last_peered":"2024-09-21T09:54:58.852179+0000","last_clean":"2024-09-21T09:54:58.852179+0000","last_became_active":"2024-09-21T09:50:44.878975+0000","last_became_peered":"2024-09-21T09:50:44.878975+0000","last_unstale":"2024-09-21T09:54:58.852179+0000","last_undegraded":"2024-09-21T09:54:58.852179+0000","last_fullsized":"2024-09-21T09:54:58.852179+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:33.954417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1329,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.881844+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:54:55.881844+0000","last_peered":"2024-09-21T09:54:55.881844+0000","last_clean":"2024-09-21T09:54:55.881844+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:54:55.881844+0000","last_undegraded":"2024-09-21T09:54:55.881844+0000","last_fullsized":"2024-09-21T09:54:55.881844+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1041,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.451957+0000","last_change":"2024-09-21T09:50:44.866370+0000","last_active":"2024-09-21T09:54:55.451957+0000","last_peered":"2024-09-21T09:54:55.451957+0000","last_clean":"2024-09-21T09:54:55.451957+0000","last_became_active":"2024-09-21T09:50:44.865582+0000","last_became_peered":"2024-09-21T09:50:44.865582+0000","last_unstale":"2024-09-21T09:54:55.451957+0000","last_undegraded":"2024-09-21T09:54:55.451957+0000","last_fullsized":"2024-09-21T09:54:55.451957+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:06:23.001592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1023,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839336+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:54:54.839336+0000","last_peered":"2024-09-21T09:54:54.839336+0000","last_clean":"2024-09-21T09:54:54.839336+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:54:54.839336+0000","last_undegraded":"2024-09-21T09:54:54.839336+0000","last_fullsized":"2024-09-21T09:54:54.839336+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1041,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.451847+0000","last_change":"2024-09-21T09:50:44.876517+0000","last_active":"2024-09-21T09:54:55.451847+0000","last_peered":"2024-09-21T09:54:55.451847+0000","last_clean":"2024-09-21T09:54:55.451847+0000","last_became_active":"2024-09-21T09:50:44.876186+0000","last_became_peered":"2024-09-21T09:50:44.876186+0000","last_unstale":"2024-09-21T09:54:55.451847+0000","last_undegraded":"2024-09-21T09:54:55.451847+0000","last_fullsized":"2024-09-21T09:54:55.451847+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:42.875674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1008,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:54.839424+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:54:54.839424+0000","last_peered":"2024-09-21T09:54:54.839424+0000","last_clean":"2024-09-21T09:54:54.839424+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:54:54.839424+0000","last_undegraded":"2024-09-21T09:54:54.839424+0000","last_fullsized":"2024-09-21T09:54:54.839424+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1265,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871519+0000","last_change":"2024-09-21T09:50:44.878358+0000","last_active":"2024-09-21T09:54:55.871519+0000","last_peered":"2024-09-21T09:54:55.871519+0000","last_clean":"2024-09-21T09:54:55.871519+0000","last_became_active":"2024-09-21T09:50:44.877700+0000","last_became_peered":"2024-09-21T09:50:44.877700+0000","last_unstale":"2024-09-21T09:54:55.871519+0000","last_undegraded":"2024-09-21T09:54:55.871519+0000","last_fullsized":"2024-09-21T09:54:55.871519+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:39:47.027086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1254,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.871571+0000","last_change":"2024-09-21T09:50:44.878043+0000","last_active":"2024-09-21T09:54:55.871571+0000","last_peered":"2024-09-21T09:54:55.871571+0000","last_clean":"2024-09-21T09:54:55.871571+0000","last_became_active":"2024-09-21T09:50:44.877686+0000","last_became_peered":"2024-09-21T09:50:44.877686+0000","last_unstale":"2024-09-21T09:54:55.871571+0000","last_undegraded":"2024-09-21T09:54:55.871571+0000","last_fullsized":"2024-09-21T09:54:55.871571+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:39.584859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1035,"reported_epoch":1080,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.451618+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:54:55.451618+0000","last_peered":"2024-09-21T09:54:55.451618+0000","last_clean":"2024-09-21T09:54:55.451618+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:54:55.451618+0000","last_undegraded":"2024-09-21T09:54:55.451618+0000","last_fullsized":"2024-09-21T09:54:55.451618+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1344,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-09-21T09:54:55.881146+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:54:55.881146+0000","last_peered":"2024-09-21T09:54:55.881146+0000","last_clean":"2024-09-21T09:54:55.881146+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:54:55.881146+0000","last_undegraded":"2024-09-21T09:54:55.881146+0000","last_fullsized":"2024-09-21T09:54:55.881146+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":223717413,"num_objects":101,"num_object_clones":55,"num_object_copies":303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":101,"num_whiteouts":8,"num_read":7013,"num_read_kb":7550425,"num_write":5627,"num_write_kb":1356120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38,"num_bytes_recovered":48041076,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":386273280,"data_stored":485431341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22058,"ondisk_log_size":22058,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234228,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":102500,"kb_used_data":45132,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93642652,"statfs":{"total":95995035648,"available":95890075648,"internally_reserved":0,"allocated":46215168,"data_stored":49277487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":58718667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495865,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":100060,"kb_used_data":42704,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93645092,"statfs":{"total":95995035648,"available":95892574208,"internally_reserved":0,"allocated":43728896,"data_stored":49082093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":58718663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757503,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172912,"kb_used_data":83620,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93572240,"statfs":{"total":95995035648,"available":95817973760,"internally_reserved":0,"allocated":85626880,"data_stored":99005717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019139,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":119420,"kb_used_data":66156,"kb_used_omap":1,"kb_used_meta":53246,"kb_avail":93625732,"statfs":{"total":95995035648,"available":95872749568,"internally_reserved":0,"allocated":67743744,"data_stored":76301575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":54524358},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888436,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68900,"kb_used_data":36884,"kb_used_omap":34,"kb_used_meta":31965,"kb_avail":93676252,"statfs":{"total":95995035648,"available":95924482048,"internally_reserved":0,"allocated":37769216,"data_stored":43677162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":32733171},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967675,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":83640,"kb_used_data":47404,"kb_used_omap":41,"kb_used_meta":36182,"kb_avail":93661512,"statfs":{"total":95995035648,"available":95909388288,"internally_reserved":0,"allocated":48541696,"data_stored":57902968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":37050783},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046913,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84732,"kb_used_data":48496,"kb_used_omap":41,"kb_used_meta":36182,"kb_avail":93660420,"statfs":{"total":95995035648,"available":95908270080,"internally_reserved":0,"allocated":49659904,"data_stored":56971471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":37051305},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126152,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113516,"kb_used_data":80800,"kb_used_omap":72,"kb_used_meta":32631,"kb_avail":93631636,"statfs":{"total":95995035648,"available":95878795264,"internally_reserved":0,"allocated":82739200,"data_stored":97065130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":33414187},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":72065024,"data_stored":89380432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":41308160,"data_stored":51633515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":38449152,"data_stored":50800314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":27693056,"data_stored":36583346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":57647104,"data_stored":69202959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":79036416,"data_stored":100516513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":33042432,"data_stored":42806352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":37031936,"data_stored":44507910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:55:02.546 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-21T09:55:02.547 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-21T09:55:02.780 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:55:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:55:03] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.51.0" 2024-09-21T09:55:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:03 smithi191 ceph-mon[102206]: pgmap v1097: 89 pgs: 89 active+clean; 214 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 847 KiB/s wr, 10 op/s 2024-09-21T09:55:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:03 smithi191 ceph-mon[102206]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-21T09:55:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:03 smithi076 ceph-mon[106482]: pgmap v1097: 89 pgs: 89 active+clean; 214 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 847 KiB/s wr, 10 op/s 2024-09-21T09:55:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:03 smithi076 ceph-mon[106482]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-21T09:55:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:03 smithi076 ceph-mon[110963]: pgmap v1097: 89 pgs: 89 active+clean; 214 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 847 KiB/s wr, 10 op/s 2024-09-21T09:55:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:03 smithi076 ceph-mon[110963]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-21T09:55:05.209 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_0' removed 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3228533840' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3228533840' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:05.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:05 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:05 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3228533840' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:05 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T09:55:05.854 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T09:55:05.854 INFO:teuthology.task.print:**** done rados/snaps-few-objects 5-workload 2024-09-21T09:55:05.854 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:05.854 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:55:05.870 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-21T09:55:05.870 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:55:05.878 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:55:05.878 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:55:05.878 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:55:06.484 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:06 smithi076 ceph-mon[106482]: pgmap v1099: 89 pgs: 89 active+clean; 214 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:55:06.484 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:06 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:55:06.485 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:06 smithi076 ceph-mon[106482]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-21T09:55:06.485 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:06 smithi076 ceph-mon[110963]: pgmap v1099: 89 pgs: 89 active+clean; 214 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:55:06.485 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:06 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:55:06.485 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:06 smithi076 ceph-mon[110963]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-21T09:55:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:06 smithi191 ceph-mon[102206]: pgmap v1099: 89 pgs: 89 active+clean; 214 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-21T09:55:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:06 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T09:55:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:06 smithi191 ceph-mon[102206]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-21T09:55:06.636 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:08 smithi076 ceph-mon[106482]: pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 635 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:08 smithi076 ceph-mon[110963]: pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 635 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:08 smithi191 ceph-mon[102206]: pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 635 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.049 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 574 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2272622982' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T09:55:09.404 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.405 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.405 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 574 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.405 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:09.405 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:09.405 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.405 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:09 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2272622982' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 574 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:55:09.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:09 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2272622982' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-21T09:55:09.826 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":40,"flags":0,"active_gid":25058,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6800","nonce":2252645974},{"type":"v1","addr":"172.21.15.76:6801","nonce":2252645974}]},"active_addr":"172.21.15.76:6801/2252645974","active_change":"2024-09-21T09:42:40.197888+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":24847,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.8","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.8","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":359,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":2604774868}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":2393091771}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":2855575936}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.76:0","nonce":383081022}]}]} 2024-09-21T09:55:09.829 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:55:09.830 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:55:09.830 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:55:10.027 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:11 smithi076 ceph-mon[106482]: pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:11 smithi076 ceph-mon[110963]: pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:11 smithi191 ceph-mon[102206]: pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:12.033 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:12.033 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":1084,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:55:05.193768+0000","last_up_change":"2024-09-21T09:50:43.849706+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":717,"up_from":730,"up_thru":829,"down_at":718,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6803","nonce":724858944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6805","nonce":724858944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6809","nonce":724858944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6807","nonce":724858944}]},"public_addr":"172.21.15.76:6803/724858944","cluster_addr":"172.21.15.76:6805/724858944","heartbeat_back_addr":"172.21.15.76:6809/724858944","heartbeat_front_addr":"172.21.15.76:6807/724858944","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":750,"up_from":763,"up_thru":829,"down_at":751,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6811","nonce":46966950}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6813","nonce":46966950}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6817","nonce":46966950}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6815","nonce":46966950}]},"public_addr":"172.21.15.76:6811/46966950","cluster_addr":"172.21.15.76:6813/46966950","heartbeat_back_addr":"172.21.15.76:6817/46966950","heartbeat_front_addr":"172.21.15.76:6815/46966950","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":783,"up_from":796,"up_thru":829,"down_at":784,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6819","nonce":3488608517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6821","nonce":3488608517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6825","nonce":3488608517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6823","nonce":3488608517}]},"public_addr":"172.21.15.76:6819/3488608517","cluster_addr":"172.21.15.76:6821/3488608517","heartbeat_back_addr":"172.21.15.76:6825/3488608517","heartbeat_front_addr":"172.21.15.76:6823/3488608517","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":816,"up_from":829,"up_thru":829,"down_at":817,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6827","nonce":523580128}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6829","nonce":523580128}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6833","nonce":523580128}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6831","nonce":523580128}]},"public_addr":"172.21.15.76:6827/523580128","cluster_addr":"172.21.15.76:6829/523580128","heartbeat_back_addr":"172.21.15.76:6833/523580128","heartbeat_front_addr":"172.21.15.76:6831/523580128","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":817,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":717},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":750},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":783},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":816},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/3937727837":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6801/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6800/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6835/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1469488034":"2024-09-22T09:42:40.197673+0000","172.21.15.76:0/3117602468":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/1830630483":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1931405138":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/4144106685":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6834/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1616224256":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/1280485976":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6801/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/4175697025":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.191:0/739062149":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/3392089630":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6800/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/3270182435":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/3629581092":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1868415942":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:55:12.649 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:55:12.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:55:12.673 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1606747635' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:12.673 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1606747635' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:12.847 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:12.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1606747635' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:13.618 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:55:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:55:13] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.51.0" 2024-09-21T09:55:13.618 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:13 smithi076 ceph-mon[110963]: pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:13.618 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:13 smithi076 ceph-mon[106482]: pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:13 smithi191 ceph-mon[102206]: pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:14.604 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:14.605 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":1084,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T09:55:05.193768+0000","last_up_change":"2024-09-21T09:50:43.849706+0000","last_in_change":"2024-09-21T09:34:31.875448+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":717,"up_from":730,"up_thru":829,"down_at":718,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6803","nonce":724858944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6805","nonce":724858944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6809","nonce":724858944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":724858944},{"type":"v1","addr":"172.21.15.76:6807","nonce":724858944}]},"public_addr":"172.21.15.76:6803/724858944","cluster_addr":"172.21.15.76:6805/724858944","heartbeat_back_addr":"172.21.15.76:6809/724858944","heartbeat_front_addr":"172.21.15.76:6807/724858944","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":750,"up_from":763,"up_thru":829,"down_at":751,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6811","nonce":46966950}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6813","nonce":46966950}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6817","nonce":46966950}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6815","nonce":46966950}]},"public_addr":"172.21.15.76:6811/46966950","cluster_addr":"172.21.15.76:6813/46966950","heartbeat_back_addr":"172.21.15.76:6817/46966950","heartbeat_front_addr":"172.21.15.76:6815/46966950","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":783,"up_from":796,"up_thru":829,"down_at":784,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6819","nonce":3488608517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6821","nonce":3488608517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6825","nonce":3488608517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6823","nonce":3488608517}]},"public_addr":"172.21.15.76:6819/3488608517","cluster_addr":"172.21.15.76:6821/3488608517","heartbeat_back_addr":"172.21.15.76:6825/3488608517","heartbeat_front_addr":"172.21.15.76:6823/3488608517","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":816,"up_from":829,"up_thru":829,"down_at":817,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6827","nonce":523580128}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6829","nonce":523580128}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6833","nonce":523580128}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":523580128},{"type":"v1","addr":"172.21.15.76:6831","nonce":523580128}]},"public_addr":"172.21.15.76:6827/523580128","cluster_addr":"172.21.15.76:6829/523580128","heartbeat_back_addr":"172.21.15.76:6833/523580128","heartbeat_front_addr":"172.21.15.76:6831/523580128","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":817,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":829,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":717},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":750},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":783},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":816},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.b","mappings":[{"from":1,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/3937727837":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6801/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6800/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6835/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1469488034":"2024-09-22T09:42:40.197673+0000","172.21.15.76:0/3117602468":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/1830630483":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1931405138":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/4144106685":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6834/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1616224256":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/1280485976":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6801/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/4175697025":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.191:0/739062149":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/3392089630":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6800/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/3270182435":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/3629581092":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1868415942":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:55:15.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:55:15.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:55:15.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:55:15.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:55:15.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:55:15.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:55:15.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:55:15.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:55:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:15 smithi076 ceph-mon[106482]: pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:15.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2057713587' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:15 smithi076 ceph-mon[110963]: pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:15.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2057713587' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:15.626 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.626 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.626 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.629 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.629 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.632 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.632 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.638 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:15 smithi191 ceph-mon[102206]: pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:15.905 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2057713587' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T09:55:17.463 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:17 smithi076 ceph-mon[106482]: pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:17.763 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:17 smithi076 ceph-mon[110963]: pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:17 smithi191 ceph-mon[102206]: pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:19.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:19 smithi076 ceph-mon[106482]: pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:19.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:19 smithi076 ceph-mon[110963]: pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:19 smithi191 ceph-mon[102206]: pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:21.706 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:21 smithi076 ceph-mon[106482]: pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:21.707 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:21 smithi076 ceph-mon[110963]: pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:21 smithi191 ceph-mon[102206]: pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:55:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:55:23] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.51.0" 2024-09-21T09:55:23.652 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:23 smithi076 ceph-mon[106482]: pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:23.652 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:23 smithi076 ceph-mon[110963]: pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:23 smithi191 ceph-mon[102206]: pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:25.621 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:25 smithi076 ceph-mon[106482]: pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:25.621 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:25 smithi076 ceph-mon[110963]: pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:25.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:25 smithi191 ceph-mon[102206]: pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:26.031 INFO:teuthology.orchestra.run.smithi076.stdout:231928234234 2024-09-21T09:55:26.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:55:26.763 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:26.763 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:26.875 INFO:teuthology.orchestra.run.smithi076.stdout:128849019145 2024-09-21T09:55:26.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:55:27.743 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:27 smithi076 ceph-mon[106482]: pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:27.744 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:27 smithi076 ceph-mon[110963]: pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:27.763 INFO:teuthology.orchestra.run.smithi076.stdout:163208757509 2024-09-21T09:55:27.764 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:55:27.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:27 smithi191 ceph-mon[102206]: pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:28.174 INFO:teuthology.orchestra.run.smithi076.stdout:3135326126158 2024-09-21T09:55:28.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:55:29.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:29 smithi191 ceph-mon[102206]: pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:29 smithi076 ceph-mon[106482]: pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:29 smithi076 ceph-mon[110963]: pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:30.713 INFO:teuthology.orchestra.run.smithi076.stdout:3277060046920 2024-09-21T09:55:30.714 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:55:31.744 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:31 smithi076 ceph-mon[110963]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:31.744 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:31 smithi076 ceph-mon[106482]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:31.763 INFO:teuthology.orchestra.run.smithi076.stdout:3418793967682 2024-09-21T09:55:31.764 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:55:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:31 smithi191 ceph-mon[102206]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:32.300 INFO:teuthology.orchestra.run.smithi076.stdout:197568495872 2024-09-21T09:55:32.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:55:32.747 INFO:teuthology.orchestra.run.smithi076.stdout:3560527888443 2024-09-21T09:55:32.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:55:32.769 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:32.772 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:32.777 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:32.777 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:32.777 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:33.590 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:33 smithi076 ceph-mon[106482]: pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:33.590 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:55:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:55:33] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.51.0" 2024-09-21T09:55:33.590 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:33 smithi076 ceph-mon[110963]: pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:33.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:33 smithi191 ceph-mon[102206]: pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:35.382 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:35.610 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:35 smithi076 ceph-mon[110963]: pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:35.610 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:35 smithi076 ceph-mon[106482]: pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:35.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:35 smithi191 ceph-mon[102206]: pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:37.719 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:37 smithi076 ceph-mon[106482]: pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:37.719 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:37 smithi076 ceph-mon[110963]: pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:37 smithi191 ceph-mon[102206]: pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:38.260 INFO:teuthology.orchestra.run.smithi076.stdout:3277060046921 2024-09-21T09:55:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/830913814' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T09:55:38.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/830913814' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T09:55:38.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/830913814' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T09:55:39.060 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:39.247 INFO:teuthology.orchestra.run.smithi076.stdout:3135326126160 2024-09-21T09:55:39.449 INFO:teuthology.orchestra.run.smithi076.stdout:128849019148 2024-09-21T09:55:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:39 smithi076 ceph-mon[106482]: pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2520194238' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T09:55:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3431007829' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T09:55:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:39 smithi076 ceph-mon[110963]: pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2520194238' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T09:55:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3431007829' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T09:55:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:39 smithi191 ceph-mon[102206]: pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2520194238' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T09:55:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3431007829' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T09:55:40.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3277060046920 got 3277060046921 for osd.1 2024-09-21T09:55:40.684 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:41.730 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:41 smithi076 ceph-mon[106482]: pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:41.730 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:41.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:41 smithi076 ceph-mon[110963]: pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:41.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:41.753 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019145 got 128849019148 for osd.4 2024-09-21T09:55:41.754 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:41 smithi191 ceph-mon[102206]: pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:43.029 INFO:teuthology.orchestra.run.smithi076.stdout:3418793967684 2024-09-21T09:55:43.505 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:55:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:55:43] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.51.0" 2024-09-21T09:55:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:43 smithi191 ceph-mon[102206]: pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3667960479' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T09:55:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:43 smithi076 ceph-mon[106482]: pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3667960479' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T09:55:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:43 smithi076 ceph-mon[110963]: pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3667960479' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T09:55:44.401 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3418793967682 got 3418793967684 for osd.2 2024-09-21T09:55:44.401 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:44.787 INFO:teuthology.orchestra.run.smithi076.stdout:231928234238 2024-09-21T09:55:45.028 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3135326126158 got 3135326126160 for osd.0 2024-09-21T09:55:45.029 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:45 smithi076 ceph-mon[110963]: pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/971882707' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T09:55:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:45 smithi076 ceph-mon[106482]: pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:45.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/971882707' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T09:55:45.745 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234234 got 231928234238 for osd.7 2024-09-21T09:55:45.745 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:45 smithi191 ceph-mon[102206]: pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/971882707' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T09:55:45.876 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:46.738 INFO:teuthology.orchestra.run.smithi076.stdout:163208757514 2024-09-21T09:55:47.351 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757509 got 163208757514 for osd.5 2024-09-21T09:55:47.351 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:47 smithi076 ceph-mon[106482]: pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3415968168' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T09:55:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:47 smithi076 ceph-mon[110963]: pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3415968168' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T09:55:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:47 smithi191 ceph-mon[102206]: pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:47 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3415968168' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T09:55:47.957 INFO:teuthology.orchestra.run.smithi076.stdout:197568495876 2024-09-21T09:55:48.543 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/424745946' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T09:55:48.543 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/424745946' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T09:55:48.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/424745946' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T09:55:49.255 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495872 got 197568495876 for osd.6 2024-09-21T09:55:49.256 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:49 smithi191 ceph-mon[102206]: pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:49.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:49 smithi076 ceph-mon[106482]: pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:49 smithi076 ceph-mon[110963]: pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:49.887 INFO:teuthology.orchestra.run.smithi076.stdout:3560527888447 2024-09-21T09:55:50.510 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3560527888443 got 3560527888447 for osd.3 2024-09-21T09:55:50.510 DEBUG:teuthology.parallel:result is None 2024-09-21T09:55:50.510 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:55:50.511 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:55:50.711 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:50.832 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3484222519' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T09:55:50.833 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3484222519' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T09:55:50.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3484222519' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T09:55:51.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:51 smithi191 ceph-mon[102206]: pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:51.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:51 smithi076 ceph-mon[106482]: pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:51.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:51 smithi076 ceph-mon[110963]: pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:52.712 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:52.713 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:55:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:55:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:55:53] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.51.0" 2024-09-21T09:55:53.442 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1124,"stamp":"2024-09-21T09:55:52.400502+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":479504,"kb_used_data":80856,"kb_used_omap":195,"kb_used_meta":398332,"kb_avail":749481712,"statfs":{"total":767960285184,"available":767469273088,"internally_reserved":0,"allocated":82796544,"data_stored":58736239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":407892100},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002952"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531316+0000","last_change":"2024-09-21T09:50:44.870931+0000","last_active":"2024-09-21T09:55:05.531316+0000","last_peered":"2024-09-21T09:55:05.531316+0000","last_clean":"2024-09-21T09:55:05.531316+0000","last_became_active":"2024-09-21T09:50:44.870528+0000","last_became_peered":"2024-09-21T09:50:44.870528+0000","last_unstale":"2024-09-21T09:55:05.531316+0000","last_undegraded":"2024-09-21T09:55:05.531316+0000","last_fullsized":"2024-09-21T09:55:05.531316+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:16.312101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1011,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531334+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:55:05.531334+0000","last_peered":"2024-09-21T09:55:05.531334+0000","last_clean":"2024-09-21T09:55:05.531334+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:55:05.531334+0000","last_undegraded":"2024-09-21T09:55:05.531334+0000","last_fullsized":"2024-09-21T09:55:05.531334+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621949+0000","last_change":"2024-09-21T09:50:44.866151+0000","last_active":"2024-09-21T09:55:04.621949+0000","last_peered":"2024-09-21T09:55:04.621949+0000","last_clean":"2024-09-21T09:55:04.621949+0000","last_became_active":"2024-09-21T09:50:44.865810+0000","last_became_peered":"2024-09-21T09:50:44.865810+0000","last_unstale":"2024-09-21T09:55:04.621949+0000","last_undegraded":"2024-09-21T09:55:04.621949+0000","last_fullsized":"2024-09-21T09:55:04.621949+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:31.175969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530607+0000","last_change":"2024-09-21T09:50:44.876550+0000","last_active":"2024-09-21T09:55:05.530607+0000","last_peered":"2024-09-21T09:55:05.530607+0000","last_clean":"2024-09-21T09:55:05.530607+0000","last_became_active":"2024-09-21T09:50:44.876246+0000","last_became_peered":"2024-09-21T09:50:44.876246+0000","last_unstale":"2024-09-21T09:55:05.530607+0000","last_undegraded":"2024-09-21T09:55:05.530607+0000","last_fullsized":"2024-09-21T09:55:05.530607+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:54.983152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531556+0000","last_change":"2024-09-21T09:50:44.872174+0000","last_active":"2024-09-21T09:55:05.531556+0000","last_peered":"2024-09-21T09:55:05.531556+0000","last_clean":"2024-09-21T09:55:05.531556+0000","last_became_active":"2024-09-21T09:50:44.871882+0000","last_became_peered":"2024-09-21T09:50:44.871882+0000","last_unstale":"2024-09-21T09:55:05.531556+0000","last_undegraded":"2024-09-21T09:55:05.531556+0000","last_fullsized":"2024-09-21T09:55:05.531556+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:57.860279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":994,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531362+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:55:05.531362+0000","last_peered":"2024-09-21T09:55:05.531362+0000","last_clean":"2024-09-21T09:55:05.531362+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:55:05.531362+0000","last_undegraded":"2024-09-21T09:55:05.531362+0000","last_fullsized":"2024-09-21T09:55:05.531362+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530717+0000","last_change":"2024-09-21T09:50:44.871672+0000","last_active":"2024-09-21T09:55:05.530717+0000","last_peered":"2024-09-21T09:55:05.530717+0000","last_clean":"2024-09-21T09:55:05.530717+0000","last_became_active":"2024-09-21T09:50:44.870133+0000","last_became_peered":"2024-09-21T09:50:44.870133+0000","last_unstale":"2024-09-21T09:55:05.530717+0000","last_undegraded":"2024-09-21T09:55:05.530717+0000","last_fullsized":"2024-09-21T09:55:05.530717+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:27.196429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1021,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.250019+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:55:05.250019+0000","last_peered":"2024-09-21T09:55:05.250019+0000","last_clean":"2024-09-21T09:55:05.250019+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:55:05.250019+0000","last_undegraded":"2024-09-21T09:55:05.250019+0000","last_fullsized":"2024-09-21T09:55:05.250019+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1010,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698425+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:55:04.698425+0000","last_peered":"2024-09-21T09:55:04.698425+0000","last_clean":"2024-09-21T09:55:04.698425+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:55:04.698425+0000","last_undegraded":"2024-09-21T09:55:04.698425+0000","last_fullsized":"2024-09-21T09:55:04.698425+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1009,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621746+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:55:04.621746+0000","last_peered":"2024-09-21T09:55:04.621746+0000","last_clean":"2024-09-21T09:55:04.621746+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:55:04.621746+0000","last_undegraded":"2024-09-21T09:55:04.621746+0000","last_fullsized":"2024-09-21T09:55:04.621746+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698562+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:55:04.698562+0000","last_peered":"2024-09-21T09:55:04.698562+0000","last_clean":"2024-09-21T09:55:04.698562+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:55:04.698562+0000","last_undegraded":"2024-09-21T09:55:04.698562+0000","last_fullsized":"2024-09-21T09:55:04.698562+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837730+0000","last_change":"2024-09-21T09:50:44.867113+0000","last_active":"2024-09-21T09:55:03.837730+0000","last_peered":"2024-09-21T09:55:03.837730+0000","last_clean":"2024-09-21T09:55:03.837730+0000","last_became_active":"2024-09-21T09:50:44.866691+0000","last_became_peered":"2024-09-21T09:50:44.866691+0000","last_unstale":"2024-09-21T09:55:03.837730+0000","last_undegraded":"2024-09-21T09:55:03.837730+0000","last_fullsized":"2024-09-21T09:55:03.837730+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:56.642995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":995,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698756+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:55:04.698756+0000","last_peered":"2024-09-21T09:55:04.698756+0000","last_clean":"2024-09-21T09:55:04.698756+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:55:04.698756+0000","last_undegraded":"2024-09-21T09:55:04.698756+0000","last_fullsized":"2024-09-21T09:55:04.698756+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1012,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016143+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:55:04.016143+0000","last_peered":"2024-09-21T09:55:04.016143+0000","last_clean":"2024-09-21T09:55:04.016143+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:55:04.016143+0000","last_undegraded":"2024-09-21T09:55:04.016143+0000","last_fullsized":"2024-09-21T09:55:04.016143+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621821+0000","last_change":"2024-09-21T09:50:44.866100+0000","last_active":"2024-09-21T09:55:04.621821+0000","last_peered":"2024-09-21T09:55:04.621821+0000","last_clean":"2024-09-21T09:55:04.621821+0000","last_became_active":"2024-09-21T09:50:44.865710+0000","last_became_peered":"2024-09-21T09:50:44.865710+0000","last_unstale":"2024-09-21T09:55:04.621821+0000","last_undegraded":"2024-09-21T09:55:04.621821+0000","last_fullsized":"2024-09-21T09:55:04.621821+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:16.761723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1346,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530753+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:55:05.530753+0000","last_peered":"2024-09-21T09:55:05.530753+0000","last_clean":"2024-09-21T09:55:05.530753+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:55:05.530753+0000","last_undegraded":"2024-09-21T09:55:05.530753+0000","last_fullsized":"2024-09-21T09:55:05.530753+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:50:44.876433+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:50:44.875979+0000","last_became_peered":"2024-09-21T09:50:44.875979+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:29.774368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837899+0000","last_change":"2024-09-21T09:50:44.866099+0000","last_active":"2024-09-21T09:55:03.837899+0000","last_peered":"2024-09-21T09:55:03.837899+0000","last_clean":"2024-09-21T09:55:03.837899+0000","last_became_active":"2024-09-21T09:50:44.865811+0000","last_became_peered":"2024-09-21T09:50:44.865811+0000","last_unstale":"2024-09-21T09:55:03.837899+0000","last_undegraded":"2024-09-21T09:55:03.837899+0000","last_fullsized":"2024-09-21T09:55:03.837899+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:25.833506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1013,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016235+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:55:04.016235+0000","last_peered":"2024-09-21T09:55:04.016235+0000","last_clean":"2024-09-21T09:55:04.016235+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:55:04.016235+0000","last_undegraded":"2024-09-21T09:55:04.016235+0000","last_fullsized":"2024-09-21T09:55:04.016235+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838059+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:55:03.838059+0000","last_peered":"2024-09-21T09:55:03.838059+0000","last_clean":"2024-09-21T09:55:03.838059+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:55:03.838059+0000","last_undegraded":"2024-09-21T09:55:03.838059+0000","last_fullsized":"2024-09-21T09:55:03.838059+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837860+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:55:03.837860+0000","last_peered":"2024-09-21T09:55:03.837860+0000","last_clean":"2024-09-21T09:55:03.837860+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:55:03.837860+0000","last_undegraded":"2024-09-21T09:55:03.837860+0000","last_fullsized":"2024-09-21T09:55:03.837860+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838274+0000","last_change":"2024-09-21T09:49:38.835119+0000","last_active":"2024-09-21T09:55:03.838274+0000","last_peered":"2024-09-21T09:55:03.838274+0000","last_clean":"2024-09-21T09:55:03.838274+0000","last_became_active":"2024-09-21T09:49:38.834858+0000","last_became_peered":"2024-09-21T09:49:38.834858+0000","last_unstale":"2024-09-21T09:55:03.838274+0000","last_undegraded":"2024-09-21T09:55:03.838274+0000","last_fullsized":"2024-09-21T09:55:03.838274+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:59.812483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1268,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531996+0000","last_change":"2024-09-21T09:50:44.875119+0000","last_active":"2024-09-21T09:55:05.531996+0000","last_peered":"2024-09-21T09:55:05.531996+0000","last_clean":"2024-09-21T09:55:05.531996+0000","last_became_active":"2024-09-21T09:50:44.873783+0000","last_became_peered":"2024-09-21T09:50:44.873783+0000","last_unstale":"2024-09-21T09:55:05.531996+0000","last_undegraded":"2024-09-21T09:55:05.531996+0000","last_fullsized":"2024-09-21T09:55:05.531996+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:28.277953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206713+0000","last_change":"2024-09-21T09:50:44.873954+0000","last_active":"2024-09-21T09:55:05.206713+0000","last_peered":"2024-09-21T09:55:05.206713+0000","last_clean":"2024-09-21T09:55:05.206713+0000","last_became_active":"2024-09-21T09:50:44.873527+0000","last_became_peered":"2024-09-21T09:50:44.873527+0000","last_unstale":"2024-09-21T09:55:05.206713+0000","last_undegraded":"2024-09-21T09:55:05.206713+0000","last_fullsized":"2024-09-21T09:55:05.206713+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:53.842361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1340,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698022+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:55:04.698022+0000","last_peered":"2024-09-21T09:55:04.698022+0000","last_clean":"2024-09-21T09:55:04.698022+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:55:04.698022+0000","last_undegraded":"2024-09-21T09:55:04.698022+0000","last_fullsized":"2024-09-21T09:55:04.698022+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.622014+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:55:04.622014+0000","last_peered":"2024-09-21T09:55:04.622014+0000","last_clean":"2024-09-21T09:55:04.622014+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:55:04.622014+0000","last_undegraded":"2024-09-21T09:55:04.622014+0000","last_fullsized":"2024-09-21T09:55:04.622014+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531943+0000","last_change":"2024-09-21T09:50:44.867188+0000","last_active":"2024-09-21T09:55:05.531943+0000","last_peered":"2024-09-21T09:55:05.531943+0000","last_clean":"2024-09-21T09:55:05.531943+0000","last_became_active":"2024-09-21T09:50:44.866818+0000","last_became_peered":"2024-09-21T09:50:44.866818+0000","last_unstale":"2024-09-21T09:55:05.531943+0000","last_undegraded":"2024-09-21T09:55:05.531943+0000","last_fullsized":"2024-09-21T09:55:05.531943+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:17:33.499249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":998,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249772+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:55:05.249772+0000","last_peered":"2024-09-21T09:55:05.249772+0000","last_clean":"2024-09-21T09:55:05.249772+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:55:05.249772+0000","last_undegraded":"2024-09-21T09:55:05.249772+0000","last_fullsized":"2024-09-21T09:55:05.249772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1038,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1344,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530372+0000","last_change":"2024-09-21T09:50:44.872670+0000","last_active":"2024-09-21T09:55:05.530372+0000","last_peered":"2024-09-21T09:55:05.530372+0000","last_clean":"2024-09-21T09:55:05.530372+0000","last_became_active":"2024-09-21T09:50:44.872391+0000","last_became_peered":"2024-09-21T09:50:44.872391+0000","last_unstale":"2024-09-21T09:55:05.530372+0000","last_undegraded":"2024-09-21T09:55:05.530372+0000","last_fullsized":"2024-09-21T09:55:05.530372+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:19.739055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1356,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531407+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:55:05.531407+0000","last_peered":"2024-09-21T09:55:05.531407+0000","last_clean":"2024-09-21T09:55:05.531407+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:55:05.531407+0000","last_undegraded":"2024-09-21T09:55:05.531407+0000","last_fullsized":"2024-09-21T09:55:05.531407+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1322,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530523+0000","last_change":"2024-09-21T09:50:44.880197+0000","last_active":"2024-09-21T09:55:05.530523+0000","last_peered":"2024-09-21T09:55:05.530523+0000","last_clean":"2024-09-21T09:55:05.530523+0000","last_became_active":"2024-09-21T09:50:44.879899+0000","last_became_peered":"2024-09-21T09:50:44.879899+0000","last_unstale":"2024-09-21T09:55:05.530523+0000","last_undegraded":"2024-09-21T09:55:05.530523+0000","last_fullsized":"2024-09-21T09:55:05.530523+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:23.596846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1318,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206097+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:55:05.206097+0000","last_peered":"2024-09-21T09:55:05.206097+0000","last_clean":"2024-09-21T09:55:05.206097+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:55:05.206097+0000","last_undegraded":"2024-09-21T09:55:05.206097+0000","last_fullsized":"2024-09-21T09:55:05.206097+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206384+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:55:05.206384+0000","last_peered":"2024-09-21T09:55:05.206384+0000","last_clean":"2024-09-21T09:55:05.206384+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:55:05.206384+0000","last_undegraded":"2024-09-21T09:55:05.206384+0000","last_fullsized":"2024-09-21T09:55:05.206384+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621861+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:55:04.621861+0000","last_peered":"2024-09-21T09:55:04.621861+0000","last_clean":"2024-09-21T09:55:04.621861+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:55:04.621861+0000","last_undegraded":"2024-09-21T09:55:04.621861+0000","last_fullsized":"2024-09-21T09:55:04.621861+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1032,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206860+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:55:05.206860+0000","last_peered":"2024-09-21T09:55:05.206860+0000","last_clean":"2024-09-21T09:55:05.206860+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:55:05.206860+0000","last_undegraded":"2024-09-21T09:55:05.206860+0000","last_fullsized":"2024-09-21T09:55:05.206860+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1329,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205848+0000","last_change":"2024-09-21T09:50:44.879616+0000","last_active":"2024-09-21T09:55:05.205848+0000","last_peered":"2024-09-21T09:55:05.205848+0000","last_clean":"2024-09-21T09:55:05.205848+0000","last_became_active":"2024-09-21T09:50:44.879237+0000","last_became_peered":"2024-09-21T09:50:44.879237+0000","last_unstale":"2024-09-21T09:55:05.205848+0000","last_undegraded":"2024-09-21T09:55:05.205848+0000","last_fullsized":"2024-09-21T09:55:05.205848+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:21.338486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531861+0000","last_change":"2024-09-21T09:50:44.870337+0000","last_active":"2024-09-21T09:55:05.531861+0000","last_peered":"2024-09-21T09:55:05.531861+0000","last_clean":"2024-09-21T09:55:05.531861+0000","last_became_active":"2024-09-21T09:50:44.870057+0000","last_became_peered":"2024-09-21T09:50:44.870057+0000","last_unstale":"2024-09-21T09:55:05.531861+0000","last_undegraded":"2024-09-21T09:55:05.531861+0000","last_fullsized":"2024-09-21T09:55:05.531861+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:22.969900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1340,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205939+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:55:05.205939+0000","last_peered":"2024-09-21T09:55:05.205939+0000","last_clean":"2024-09-21T09:55:05.205939+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:55:05.205939+0000","last_undegraded":"2024-09-21T09:55:05.205939+0000","last_fullsized":"2024-09-21T09:55:05.205939+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1307,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205656+0000","last_change":"2024-09-21T09:49:38.833392+0000","last_active":"2024-09-21T09:55:05.205656+0000","last_peered":"2024-09-21T09:55:05.205656+0000","last_clean":"2024-09-21T09:55:05.205656+0000","last_became_active":"2024-09-21T09:49:38.832610+0000","last_became_peered":"2024-09-21T09:49:38.832610+0000","last_unstale":"2024-09-21T09:55:05.205656+0000","last_undegraded":"2024-09-21T09:55:05.205656+0000","last_fullsized":"2024-09-21T09:55:05.205656+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:26.844310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206109+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:55:05.206109+0000","last_peered":"2024-09-21T09:55:05.206109+0000","last_clean":"2024-09-21T09:55:05.206109+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:55:05.206109+0000","last_undegraded":"2024-09-21T09:55:05.206109+0000","last_fullsized":"2024-09-21T09:55:05.206109+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698801+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:55:04.698801+0000","last_peered":"2024-09-21T09:55:04.698801+0000","last_clean":"2024-09-21T09:55:04.698801+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:55:04.698801+0000","last_undegraded":"2024-09-21T09:55:04.698801+0000","last_fullsized":"2024-09-21T09:55:04.698801+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1334,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531537+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:55:05.531537+0000","last_peered":"2024-09-21T09:55:05.531537+0000","last_clean":"2024-09-21T09:55:05.531537+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:55:05.531537+0000","last_undegraded":"2024-09-21T09:55:05.531537+0000","last_fullsized":"2024-09-21T09:55:05.531537+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530863+0000","last_change":"2024-09-21T09:50:44.875761+0000","last_active":"2024-09-21T09:55:05.530863+0000","last_peered":"2024-09-21T09:55:05.530863+0000","last_clean":"2024-09-21T09:55:05.530863+0000","last_became_active":"2024-09-21T09:50:44.875333+0000","last_became_peered":"2024-09-21T09:50:44.875333+0000","last_unstale":"2024-09-21T09:55:05.530863+0000","last_undegraded":"2024-09-21T09:55:05.530863+0000","last_fullsized":"2024-09-21T09:55:05.530863+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:51.443596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205905+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:55:05.205905+0000","last_peered":"2024-09-21T09:55:05.205905+0000","last_clean":"2024-09-21T09:55:05.205905+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:55:05.205905+0000","last_undegraded":"2024-09-21T09:55:05.205905+0000","last_fullsized":"2024-09-21T09:55:05.205905+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1275,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531830+0000","last_change":"2024-09-21T09:50:44.874771+0000","last_active":"2024-09-21T09:55:05.531830+0000","last_peered":"2024-09-21T09:55:05.531830+0000","last_clean":"2024-09-21T09:55:05.531830+0000","last_became_active":"2024-09-21T09:50:44.873380+0000","last_became_peered":"2024-09-21T09:50:44.873380+0000","last_unstale":"2024-09-21T09:55:05.531830+0000","last_undegraded":"2024-09-21T09:55:05.531830+0000","last_fullsized":"2024-09-21T09:55:05.531830+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:19:11.264697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1348,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531680+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:55:05.531680+0000","last_peered":"2024-09-21T09:55:05.531680+0000","last_clean":"2024-09-21T09:55:05.531680+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:55:05.531680+0000","last_undegraded":"2024-09-21T09:55:05.531680+0000","last_fullsized":"2024-09-21T09:55:05.531680+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016056+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:55:04.016056+0000","last_peered":"2024-09-21T09:55:04.016056+0000","last_clean":"2024-09-21T09:55:04.016056+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:55:04.016056+0000","last_undegraded":"2024-09-21T09:55:04.016056+0000","last_fullsized":"2024-09-21T09:55:04.016056+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1016,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206752+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:55:05.206752+0000","last_peered":"2024-09-21T09:55:05.206752+0000","last_clean":"2024-09-21T09:55:05.206752+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:55:05.206752+0000","last_undegraded":"2024-09-21T09:55:05.206752+0000","last_fullsized":"2024-09-21T09:55:05.206752+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206779+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:55:05.206779+0000","last_peered":"2024-09-21T09:55:05.206779+0000","last_clean":"2024-09-21T09:55:05.206779+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:55:05.206779+0000","last_undegraded":"2024-09-21T09:55:05.206779+0000","last_fullsized":"2024-09-21T09:55:05.206779+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206376+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:55:05.206376+0000","last_peered":"2024-09-21T09:55:05.206376+0000","last_clean":"2024-09-21T09:55:05.206376+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:55:05.206376+0000","last_undegraded":"2024-09-21T09:55:05.206376+0000","last_fullsized":"2024-09-21T09:55:05.206376+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205508+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:55:05.205508+0000","last_peered":"2024-09-21T09:55:05.205508+0000","last_clean":"2024-09-21T09:55:05.205508+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:55:05.205508+0000","last_undegraded":"2024-09-21T09:55:05.205508+0000","last_fullsized":"2024-09-21T09:55:05.205508+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698745+0000","last_change":"2024-09-21T09:50:44.870735+0000","last_active":"2024-09-21T09:55:04.698745+0000","last_peered":"2024-09-21T09:55:04.698745+0000","last_clean":"2024-09-21T09:55:04.698745+0000","last_became_active":"2024-09-21T09:50:44.870370+0000","last_became_peered":"2024-09-21T09:50:44.870370+0000","last_unstale":"2024-09-21T09:55:04.698745+0000","last_undegraded":"2024-09-21T09:55:04.698745+0000","last_fullsized":"2024-09-21T09:55:04.698745+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:52.476737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621859+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:55:04.621859+0000","last_peered":"2024-09-21T09:55:04.621859+0000","last_clean":"2024-09-21T09:55:04.621859+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:55:04.621859+0000","last_undegraded":"2024-09-21T09:55:04.621859+0000","last_fullsized":"2024-09-21T09:55:04.621859+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532139+0000","last_change":"2024-09-21T09:50:44.874933+0000","last_active":"2024-09-21T09:55:05.532139+0000","last_peered":"2024-09-21T09:55:05.532139+0000","last_clean":"2024-09-21T09:55:05.532139+0000","last_became_active":"2024-09-21T09:50:44.870699+0000","last_became_peered":"2024-09-21T09:50:44.870699+0000","last_unstale":"2024-09-21T09:55:05.532139+0000","last_undegraded":"2024-09-21T09:55:05.532139+0000","last_fullsized":"2024-09-21T09:55:05.532139+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:12.912457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1342,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535069+0000","last_change":"2024-09-21T09:50:44.872004+0000","last_active":"2024-09-21T09:55:05.535069+0000","last_peered":"2024-09-21T09:55:05.535069+0000","last_clean":"2024-09-21T09:55:05.535069+0000","last_became_active":"2024-09-21T09:50:44.871706+0000","last_became_peered":"2024-09-21T09:50:44.871706+0000","last_unstale":"2024-09-21T09:55:05.535069+0000","last_undegraded":"2024-09-21T09:55:05.535069+0000","last_fullsized":"2024-09-21T09:55:05.535069+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:00.145938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531786+0000","last_change":"2024-09-21T09:50:44.874472+0000","last_active":"2024-09-21T09:55:05.531786+0000","last_peered":"2024-09-21T09:55:05.531786+0000","last_clean":"2024-09-21T09:55:05.531786+0000","last_became_active":"2024-09-21T09:50:44.872792+0000","last_became_peered":"2024-09-21T09:50:44.872792+0000","last_unstale":"2024-09-21T09:55:05.531786+0000","last_undegraded":"2024-09-21T09:55:05.531786+0000","last_fullsized":"2024-09-21T09:55:05.531786+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:52.873552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249921+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:55:05.249921+0000","last_peered":"2024-09-21T09:55:05.249921+0000","last_clean":"2024-09-21T09:55:05.249921+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:55:05.249921+0000","last_undegraded":"2024-09-21T09:55:05.249921+0000","last_fullsized":"2024-09-21T09:55:05.249921+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1282,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531952+0000","last_change":"2024-09-21T09:50:44.875450+0000","last_active":"2024-09-21T09:55:05.531952+0000","last_peered":"2024-09-21T09:55:05.531952+0000","last_clean":"2024-09-21T09:55:05.531952+0000","last_became_active":"2024-09-21T09:50:44.874171+0000","last_became_peered":"2024-09-21T09:50:44.874171+0000","last_unstale":"2024-09-21T09:55:05.531952+0000","last_undegraded":"2024-09-21T09:55:05.531952+0000","last_fullsized":"2024-09-21T09:55:05.531952+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:37.106157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1345,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531477+0000","last_change":"2024-09-21T09:50:44.871526+0000","last_active":"2024-09-21T09:55:05.531477+0000","last_peered":"2024-09-21T09:55:05.531477+0000","last_clean":"2024-09-21T09:55:05.531477+0000","last_became_active":"2024-09-21T09:50:44.871236+0000","last_became_peered":"2024-09-21T09:50:44.871236+0000","last_unstale":"2024-09-21T09:55:05.531477+0000","last_undegraded":"2024-09-21T09:55:05.531477+0000","last_fullsized":"2024-09-21T09:55:05.531477+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:51.749231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205429+0000","last_change":"2024-09-21T09:50:44.872712+0000","last_active":"2024-09-21T09:55:05.205429+0000","last_peered":"2024-09-21T09:55:05.205429+0000","last_clean":"2024-09-21T09:55:05.205429+0000","last_became_active":"2024-09-21T09:50:44.872368+0000","last_became_peered":"2024-09-21T09:50:44.872368+0000","last_unstale":"2024-09-21T09:55:05.205429+0000","last_undegraded":"2024-09-21T09:55:05.205429+0000","last_fullsized":"2024-09-21T09:55:05.205429+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:16.003164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531019+0000","last_change":"2024-09-21T09:50:44.877609+0000","last_active":"2024-09-21T09:55:05.531019+0000","last_peered":"2024-09-21T09:55:05.531019+0000","last_clean":"2024-09-21T09:55:05.531019+0000","last_became_active":"2024-09-21T09:50:44.876347+0000","last_became_peered":"2024-09-21T09:50:44.876347+0000","last_unstale":"2024-09-21T09:55:05.531019+0000","last_undegraded":"2024-09-21T09:55:05.531019+0000","last_fullsized":"2024-09-21T09:55:05.531019+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:24.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1046,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837776+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:55:03.837776+0000","last_peered":"2024-09-21T09:55:03.837776+0000","last_clean":"2024-09-21T09:55:03.837776+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:55:03.837776+0000","last_undegraded":"2024-09-21T09:55:03.837776+0000","last_fullsized":"2024-09-21T09:55:03.837776+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531639+0000","last_change":"2024-09-21T09:50:44.879265+0000","last_active":"2024-09-21T09:55:05.531639+0000","last_peered":"2024-09-21T09:55:05.531639+0000","last_clean":"2024-09-21T09:55:05.531639+0000","last_became_active":"2024-09-21T09:50:44.878975+0000","last_became_peered":"2024-09-21T09:50:44.878975+0000","last_unstale":"2024-09-21T09:55:05.531639+0000","last_undegraded":"2024-09-21T09:55:05.531639+0000","last_fullsized":"2024-09-21T09:55:05.531639+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:33.954417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1335,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532013+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:55:05.532013+0000","last_peered":"2024-09-21T09:55:05.532013+0000","last_clean":"2024-09-21T09:55:05.532013+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:55:05.532013+0000","last_undegraded":"2024-09-21T09:55:05.532013+0000","last_fullsized":"2024-09-21T09:55:05.532013+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837921+0000","last_change":"2024-09-21T09:50:44.866370+0000","last_active":"2024-09-21T09:55:03.837921+0000","last_peered":"2024-09-21T09:55:03.837921+0000","last_clean":"2024-09-21T09:55:03.837921+0000","last_became_active":"2024-09-21T09:50:44.865582+0000","last_became_peered":"2024-09-21T09:50:44.865582+0000","last_unstale":"2024-09-21T09:55:03.837921+0000","last_undegraded":"2024-09-21T09:55:03.837921+0000","last_fullsized":"2024-09-21T09:55:03.837921+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:06:23.001592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621778+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:55:04.621778+0000","last_peered":"2024-09-21T09:55:04.621778+0000","last_clean":"2024-09-21T09:55:04.621778+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:55:04.621778+0000","last_undegraded":"2024-09-21T09:55:04.621778+0000","last_fullsized":"2024-09-21T09:55:04.621778+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837680+0000","last_change":"2024-09-21T09:50:44.876517+0000","last_active":"2024-09-21T09:55:03.837680+0000","last_peered":"2024-09-21T09:55:03.837680+0000","last_clean":"2024-09-21T09:55:03.837680+0000","last_became_active":"2024-09-21T09:50:44.876186+0000","last_became_peered":"2024-09-21T09:50:44.876186+0000","last_unstale":"2024-09-21T09:55:03.837680+0000","last_undegraded":"2024-09-21T09:55:03.837680+0000","last_fullsized":"2024-09-21T09:55:03.837680+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:42.875674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621557+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:55:04.621557+0000","last_peered":"2024-09-21T09:55:04.621557+0000","last_clean":"2024-09-21T09:55:04.621557+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:55:04.621557+0000","last_undegraded":"2024-09-21T09:55:04.621557+0000","last_fullsized":"2024-09-21T09:55:04.621557+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1272,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532459+0000","last_change":"2024-09-21T09:50:44.878358+0000","last_active":"2024-09-21T09:55:05.532459+0000","last_peered":"2024-09-21T09:55:05.532459+0000","last_clean":"2024-09-21T09:55:05.532459+0000","last_became_active":"2024-09-21T09:50:44.877700+0000","last_became_peered":"2024-09-21T09:50:44.877700+0000","last_unstale":"2024-09-21T09:55:05.532459+0000","last_undegraded":"2024-09-21T09:55:05.532459+0000","last_fullsized":"2024-09-21T09:55:05.532459+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:39:47.027086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1260,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530581+0000","last_change":"2024-09-21T09:50:44.878043+0000","last_active":"2024-09-21T09:55:05.530581+0000","last_peered":"2024-09-21T09:55:05.530581+0000","last_clean":"2024-09-21T09:55:05.530581+0000","last_became_active":"2024-09-21T09:50:44.877686+0000","last_became_peered":"2024-09-21T09:50:44.877686+0000","last_unstale":"2024-09-21T09:55:05.530581+0000","last_undegraded":"2024-09-21T09:55:05.530581+0000","last_fullsized":"2024-09-21T09:55:05.530581+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:39.584859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837534+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:55:03.837534+0000","last_peered":"2024-09-21T09:55:03.837534+0000","last_clean":"2024-09-21T09:55:03.837534+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:55:03.837534+0000","last_undegraded":"2024-09-21T09:55:03.837534+0000","last_fullsized":"2024-09-21T09:55:03.837534+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1351,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535225+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:55:05.535225+0000","last_peered":"2024-09-21T09:55:05.535225+0000","last_clean":"2024-09-21T09:55:05.535225+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:55:05.535225+0000","last_undegraded":"2024-09-21T09:55:05.535225+0000","last_fullsized":"2024-09-21T09:55:05.535225+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234240,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67844,"kb_used_data":10484,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677308,"statfs":{"total":95995035648,"available":95925563392,"internally_reserved":0,"allocated":10735616,"data_stored":7713886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":58718667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495877,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67832,"kb_used_data":10476,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677320,"statfs":{"total":95995035648,"available":95925575680,"internally_reserved":0,"allocated":10727424,"data_stored":7713759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":58718663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757515,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":99184,"kb_used_data":9888,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93645968,"statfs":{"total":95995035648,"available":95893471232,"internally_reserved":0,"allocated":10125312,"data_stored":7119217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019151,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63164,"kb_used_data":9900,"kb_used_omap":1,"kb_used_meta":53246,"kb_avail":93681988,"statfs":{"total":95995035648,"available":95930355712,"internally_reserved":0,"allocated":10137600,"data_stored":7123448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":54524358},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888447,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":41896,"kb_used_data":9880,"kb_used_omap":34,"kb_used_meta":31965,"kb_avail":93703256,"statfs":{"total":95995035648,"available":95952134144,"internally_reserved":0,"allocated":10117120,"data_stored":7118648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":32733171},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967686,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46124,"kb_used_data":9884,"kb_used_omap":41,"kb_used_meta":36182,"kb_avail":93699028,"statfs":{"total":95995035648,"available":95947804672,"internally_reserved":0,"allocated":10121216,"data_stored":7119174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":37050783},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046925,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50276,"kb_used_data":9880,"kb_used_omap":41,"kb_used_meta":40342,"kb_avail":93694876,"statfs":{"total":95995035648,"available":95943553024,"internally_reserved":0,"allocated":10117120,"data_stored":7118577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":41311145},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126163,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43184,"kb_used_data":10464,"kb_used_omap":72,"kb_used_meta":32631,"kb_avail":93701968,"statfs":{"total":95995035648,"available":95950815232,"internally_reserved":0,"allocated":10715136,"data_stored":7709530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":33414187},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:55:53.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:55:53.644 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:53.764 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:53 smithi076 ceph-mon[106482]: pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:53.764 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:53 smithi076 ceph-mon[106482]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:53.765 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:53 smithi076 ceph-mon[110963]: pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:53.765 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:53 smithi076 ceph-mon[110963]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:53 smithi191 ceph-mon[102206]: pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:53.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:53 smithi191 ceph-mon[102206]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:55.394 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:55.395 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:55:55.835 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:55 smithi076 ceph-mon[106482]: pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:55.836 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:55 smithi076 ceph-mon[110963]: pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:55 smithi191 ceph-mon[102206]: pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:56.266 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1125,"stamp":"2024-09-21T09:55:54.401000+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":479504,"kb_used_data":80856,"kb_used_omap":195,"kb_used_meta":398332,"kb_avail":749481712,"statfs":{"total":767960285184,"available":767469273088,"internally_reserved":0,"allocated":82796544,"data_stored":58736239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":407892100},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003056"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531316+0000","last_change":"2024-09-21T09:50:44.870931+0000","last_active":"2024-09-21T09:55:05.531316+0000","last_peered":"2024-09-21T09:55:05.531316+0000","last_clean":"2024-09-21T09:55:05.531316+0000","last_became_active":"2024-09-21T09:50:44.870528+0000","last_became_peered":"2024-09-21T09:50:44.870528+0000","last_unstale":"2024-09-21T09:55:05.531316+0000","last_undegraded":"2024-09-21T09:55:05.531316+0000","last_fullsized":"2024-09-21T09:55:05.531316+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:16.312101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1011,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531334+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:55:05.531334+0000","last_peered":"2024-09-21T09:55:05.531334+0000","last_clean":"2024-09-21T09:55:05.531334+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:55:05.531334+0000","last_undegraded":"2024-09-21T09:55:05.531334+0000","last_fullsized":"2024-09-21T09:55:05.531334+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621949+0000","last_change":"2024-09-21T09:50:44.866151+0000","last_active":"2024-09-21T09:55:04.621949+0000","last_peered":"2024-09-21T09:55:04.621949+0000","last_clean":"2024-09-21T09:55:04.621949+0000","last_became_active":"2024-09-21T09:50:44.865810+0000","last_became_peered":"2024-09-21T09:50:44.865810+0000","last_unstale":"2024-09-21T09:55:04.621949+0000","last_undegraded":"2024-09-21T09:55:04.621949+0000","last_fullsized":"2024-09-21T09:55:04.621949+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:31.175969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530607+0000","last_change":"2024-09-21T09:50:44.876550+0000","last_active":"2024-09-21T09:55:05.530607+0000","last_peered":"2024-09-21T09:55:05.530607+0000","last_clean":"2024-09-21T09:55:05.530607+0000","last_became_active":"2024-09-21T09:50:44.876246+0000","last_became_peered":"2024-09-21T09:50:44.876246+0000","last_unstale":"2024-09-21T09:55:05.530607+0000","last_undegraded":"2024-09-21T09:55:05.530607+0000","last_fullsized":"2024-09-21T09:55:05.530607+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:54.983152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531556+0000","last_change":"2024-09-21T09:50:44.872174+0000","last_active":"2024-09-21T09:55:05.531556+0000","last_peered":"2024-09-21T09:55:05.531556+0000","last_clean":"2024-09-21T09:55:05.531556+0000","last_became_active":"2024-09-21T09:50:44.871882+0000","last_became_peered":"2024-09-21T09:50:44.871882+0000","last_unstale":"2024-09-21T09:55:05.531556+0000","last_undegraded":"2024-09-21T09:55:05.531556+0000","last_fullsized":"2024-09-21T09:55:05.531556+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:57.860279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":994,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531362+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:55:05.531362+0000","last_peered":"2024-09-21T09:55:05.531362+0000","last_clean":"2024-09-21T09:55:05.531362+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:55:05.531362+0000","last_undegraded":"2024-09-21T09:55:05.531362+0000","last_fullsized":"2024-09-21T09:55:05.531362+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530717+0000","last_change":"2024-09-21T09:50:44.871672+0000","last_active":"2024-09-21T09:55:05.530717+0000","last_peered":"2024-09-21T09:55:05.530717+0000","last_clean":"2024-09-21T09:55:05.530717+0000","last_became_active":"2024-09-21T09:50:44.870133+0000","last_became_peered":"2024-09-21T09:50:44.870133+0000","last_unstale":"2024-09-21T09:55:05.530717+0000","last_undegraded":"2024-09-21T09:55:05.530717+0000","last_fullsized":"2024-09-21T09:55:05.530717+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:27.196429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1021,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.250019+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:55:05.250019+0000","last_peered":"2024-09-21T09:55:05.250019+0000","last_clean":"2024-09-21T09:55:05.250019+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:55:05.250019+0000","last_undegraded":"2024-09-21T09:55:05.250019+0000","last_fullsized":"2024-09-21T09:55:05.250019+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1010,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698425+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:55:04.698425+0000","last_peered":"2024-09-21T09:55:04.698425+0000","last_clean":"2024-09-21T09:55:04.698425+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:55:04.698425+0000","last_undegraded":"2024-09-21T09:55:04.698425+0000","last_fullsized":"2024-09-21T09:55:04.698425+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1009,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621746+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:55:04.621746+0000","last_peered":"2024-09-21T09:55:04.621746+0000","last_clean":"2024-09-21T09:55:04.621746+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:55:04.621746+0000","last_undegraded":"2024-09-21T09:55:04.621746+0000","last_fullsized":"2024-09-21T09:55:04.621746+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698562+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:55:04.698562+0000","last_peered":"2024-09-21T09:55:04.698562+0000","last_clean":"2024-09-21T09:55:04.698562+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:55:04.698562+0000","last_undegraded":"2024-09-21T09:55:04.698562+0000","last_fullsized":"2024-09-21T09:55:04.698562+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837730+0000","last_change":"2024-09-21T09:50:44.867113+0000","last_active":"2024-09-21T09:55:03.837730+0000","last_peered":"2024-09-21T09:55:03.837730+0000","last_clean":"2024-09-21T09:55:03.837730+0000","last_became_active":"2024-09-21T09:50:44.866691+0000","last_became_peered":"2024-09-21T09:50:44.866691+0000","last_unstale":"2024-09-21T09:55:03.837730+0000","last_undegraded":"2024-09-21T09:55:03.837730+0000","last_fullsized":"2024-09-21T09:55:03.837730+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:56.642995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":995,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698756+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:55:04.698756+0000","last_peered":"2024-09-21T09:55:04.698756+0000","last_clean":"2024-09-21T09:55:04.698756+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:55:04.698756+0000","last_undegraded":"2024-09-21T09:55:04.698756+0000","last_fullsized":"2024-09-21T09:55:04.698756+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1012,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016143+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:55:04.016143+0000","last_peered":"2024-09-21T09:55:04.016143+0000","last_clean":"2024-09-21T09:55:04.016143+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:55:04.016143+0000","last_undegraded":"2024-09-21T09:55:04.016143+0000","last_fullsized":"2024-09-21T09:55:04.016143+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621821+0000","last_change":"2024-09-21T09:50:44.866100+0000","last_active":"2024-09-21T09:55:04.621821+0000","last_peered":"2024-09-21T09:55:04.621821+0000","last_clean":"2024-09-21T09:55:04.621821+0000","last_became_active":"2024-09-21T09:50:44.865710+0000","last_became_peered":"2024-09-21T09:50:44.865710+0000","last_unstale":"2024-09-21T09:55:04.621821+0000","last_undegraded":"2024-09-21T09:55:04.621821+0000","last_fullsized":"2024-09-21T09:55:04.621821+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:16.761723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1346,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530753+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:55:05.530753+0000","last_peered":"2024-09-21T09:55:05.530753+0000","last_clean":"2024-09-21T09:55:05.530753+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:55:05.530753+0000","last_undegraded":"2024-09-21T09:55:05.530753+0000","last_fullsized":"2024-09-21T09:55:05.530753+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:50:44.876433+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:50:44.875979+0000","last_became_peered":"2024-09-21T09:50:44.875979+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:29.774368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837899+0000","last_change":"2024-09-21T09:50:44.866099+0000","last_active":"2024-09-21T09:55:03.837899+0000","last_peered":"2024-09-21T09:55:03.837899+0000","last_clean":"2024-09-21T09:55:03.837899+0000","last_became_active":"2024-09-21T09:50:44.865811+0000","last_became_peered":"2024-09-21T09:50:44.865811+0000","last_unstale":"2024-09-21T09:55:03.837899+0000","last_undegraded":"2024-09-21T09:55:03.837899+0000","last_fullsized":"2024-09-21T09:55:03.837899+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:25.833506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1013,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016235+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:55:04.016235+0000","last_peered":"2024-09-21T09:55:04.016235+0000","last_clean":"2024-09-21T09:55:04.016235+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:55:04.016235+0000","last_undegraded":"2024-09-21T09:55:04.016235+0000","last_fullsized":"2024-09-21T09:55:04.016235+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838059+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:55:03.838059+0000","last_peered":"2024-09-21T09:55:03.838059+0000","last_clean":"2024-09-21T09:55:03.838059+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:55:03.838059+0000","last_undegraded":"2024-09-21T09:55:03.838059+0000","last_fullsized":"2024-09-21T09:55:03.838059+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837860+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:55:03.837860+0000","last_peered":"2024-09-21T09:55:03.837860+0000","last_clean":"2024-09-21T09:55:03.837860+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:55:03.837860+0000","last_undegraded":"2024-09-21T09:55:03.837860+0000","last_fullsized":"2024-09-21T09:55:03.837860+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838274+0000","last_change":"2024-09-21T09:49:38.835119+0000","last_active":"2024-09-21T09:55:03.838274+0000","last_peered":"2024-09-21T09:55:03.838274+0000","last_clean":"2024-09-21T09:55:03.838274+0000","last_became_active":"2024-09-21T09:49:38.834858+0000","last_became_peered":"2024-09-21T09:49:38.834858+0000","last_unstale":"2024-09-21T09:55:03.838274+0000","last_undegraded":"2024-09-21T09:55:03.838274+0000","last_fullsized":"2024-09-21T09:55:03.838274+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:59.812483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1268,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531996+0000","last_change":"2024-09-21T09:50:44.875119+0000","last_active":"2024-09-21T09:55:05.531996+0000","last_peered":"2024-09-21T09:55:05.531996+0000","last_clean":"2024-09-21T09:55:05.531996+0000","last_became_active":"2024-09-21T09:50:44.873783+0000","last_became_peered":"2024-09-21T09:50:44.873783+0000","last_unstale":"2024-09-21T09:55:05.531996+0000","last_undegraded":"2024-09-21T09:55:05.531996+0000","last_fullsized":"2024-09-21T09:55:05.531996+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:28.277953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206713+0000","last_change":"2024-09-21T09:50:44.873954+0000","last_active":"2024-09-21T09:55:05.206713+0000","last_peered":"2024-09-21T09:55:05.206713+0000","last_clean":"2024-09-21T09:55:05.206713+0000","last_became_active":"2024-09-21T09:50:44.873527+0000","last_became_peered":"2024-09-21T09:50:44.873527+0000","last_unstale":"2024-09-21T09:55:05.206713+0000","last_undegraded":"2024-09-21T09:55:05.206713+0000","last_fullsized":"2024-09-21T09:55:05.206713+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:53.842361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1340,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698022+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:55:04.698022+0000","last_peered":"2024-09-21T09:55:04.698022+0000","last_clean":"2024-09-21T09:55:04.698022+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:55:04.698022+0000","last_undegraded":"2024-09-21T09:55:04.698022+0000","last_fullsized":"2024-09-21T09:55:04.698022+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.622014+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:55:04.622014+0000","last_peered":"2024-09-21T09:55:04.622014+0000","last_clean":"2024-09-21T09:55:04.622014+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:55:04.622014+0000","last_undegraded":"2024-09-21T09:55:04.622014+0000","last_fullsized":"2024-09-21T09:55:04.622014+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531943+0000","last_change":"2024-09-21T09:50:44.867188+0000","last_active":"2024-09-21T09:55:05.531943+0000","last_peered":"2024-09-21T09:55:05.531943+0000","last_clean":"2024-09-21T09:55:05.531943+0000","last_became_active":"2024-09-21T09:50:44.866818+0000","last_became_peered":"2024-09-21T09:50:44.866818+0000","last_unstale":"2024-09-21T09:55:05.531943+0000","last_undegraded":"2024-09-21T09:55:05.531943+0000","last_fullsized":"2024-09-21T09:55:05.531943+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:17:33.499249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":998,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249772+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:55:05.249772+0000","last_peered":"2024-09-21T09:55:05.249772+0000","last_clean":"2024-09-21T09:55:05.249772+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:55:05.249772+0000","last_undegraded":"2024-09-21T09:55:05.249772+0000","last_fullsized":"2024-09-21T09:55:05.249772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1038,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1344,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530372+0000","last_change":"2024-09-21T09:50:44.872670+0000","last_active":"2024-09-21T09:55:05.530372+0000","last_peered":"2024-09-21T09:55:05.530372+0000","last_clean":"2024-09-21T09:55:05.530372+0000","last_became_active":"2024-09-21T09:50:44.872391+0000","last_became_peered":"2024-09-21T09:50:44.872391+0000","last_unstale":"2024-09-21T09:55:05.530372+0000","last_undegraded":"2024-09-21T09:55:05.530372+0000","last_fullsized":"2024-09-21T09:55:05.530372+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:19.739055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1356,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531407+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:55:05.531407+0000","last_peered":"2024-09-21T09:55:05.531407+0000","last_clean":"2024-09-21T09:55:05.531407+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:55:05.531407+0000","last_undegraded":"2024-09-21T09:55:05.531407+0000","last_fullsized":"2024-09-21T09:55:05.531407+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1322,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530523+0000","last_change":"2024-09-21T09:50:44.880197+0000","last_active":"2024-09-21T09:55:05.530523+0000","last_peered":"2024-09-21T09:55:05.530523+0000","last_clean":"2024-09-21T09:55:05.530523+0000","last_became_active":"2024-09-21T09:50:44.879899+0000","last_became_peered":"2024-09-21T09:50:44.879899+0000","last_unstale":"2024-09-21T09:55:05.530523+0000","last_undegraded":"2024-09-21T09:55:05.530523+0000","last_fullsized":"2024-09-21T09:55:05.530523+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:23.596846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1318,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206097+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:55:05.206097+0000","last_peered":"2024-09-21T09:55:05.206097+0000","last_clean":"2024-09-21T09:55:05.206097+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:55:05.206097+0000","last_undegraded":"2024-09-21T09:55:05.206097+0000","last_fullsized":"2024-09-21T09:55:05.206097+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206384+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:55:05.206384+0000","last_peered":"2024-09-21T09:55:05.206384+0000","last_clean":"2024-09-21T09:55:05.206384+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:55:05.206384+0000","last_undegraded":"2024-09-21T09:55:05.206384+0000","last_fullsized":"2024-09-21T09:55:05.206384+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621861+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:55:04.621861+0000","last_peered":"2024-09-21T09:55:04.621861+0000","last_clean":"2024-09-21T09:55:04.621861+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:55:04.621861+0000","last_undegraded":"2024-09-21T09:55:04.621861+0000","last_fullsized":"2024-09-21T09:55:04.621861+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1032,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206860+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:55:05.206860+0000","last_peered":"2024-09-21T09:55:05.206860+0000","last_clean":"2024-09-21T09:55:05.206860+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:55:05.206860+0000","last_undegraded":"2024-09-21T09:55:05.206860+0000","last_fullsized":"2024-09-21T09:55:05.206860+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1329,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205848+0000","last_change":"2024-09-21T09:50:44.879616+0000","last_active":"2024-09-21T09:55:05.205848+0000","last_peered":"2024-09-21T09:55:05.205848+0000","last_clean":"2024-09-21T09:55:05.205848+0000","last_became_active":"2024-09-21T09:50:44.879237+0000","last_became_peered":"2024-09-21T09:50:44.879237+0000","last_unstale":"2024-09-21T09:55:05.205848+0000","last_undegraded":"2024-09-21T09:55:05.205848+0000","last_fullsized":"2024-09-21T09:55:05.205848+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:21.338486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531861+0000","last_change":"2024-09-21T09:50:44.870337+0000","last_active":"2024-09-21T09:55:05.531861+0000","last_peered":"2024-09-21T09:55:05.531861+0000","last_clean":"2024-09-21T09:55:05.531861+0000","last_became_active":"2024-09-21T09:50:44.870057+0000","last_became_peered":"2024-09-21T09:50:44.870057+0000","last_unstale":"2024-09-21T09:55:05.531861+0000","last_undegraded":"2024-09-21T09:55:05.531861+0000","last_fullsized":"2024-09-21T09:55:05.531861+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:22.969900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1340,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205939+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:55:05.205939+0000","last_peered":"2024-09-21T09:55:05.205939+0000","last_clean":"2024-09-21T09:55:05.205939+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:55:05.205939+0000","last_undegraded":"2024-09-21T09:55:05.205939+0000","last_fullsized":"2024-09-21T09:55:05.205939+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1307,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205656+0000","last_change":"2024-09-21T09:49:38.833392+0000","last_active":"2024-09-21T09:55:05.205656+0000","last_peered":"2024-09-21T09:55:05.205656+0000","last_clean":"2024-09-21T09:55:05.205656+0000","last_became_active":"2024-09-21T09:49:38.832610+0000","last_became_peered":"2024-09-21T09:49:38.832610+0000","last_unstale":"2024-09-21T09:55:05.205656+0000","last_undegraded":"2024-09-21T09:55:05.205656+0000","last_fullsized":"2024-09-21T09:55:05.205656+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:26.844310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206109+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:55:05.206109+0000","last_peered":"2024-09-21T09:55:05.206109+0000","last_clean":"2024-09-21T09:55:05.206109+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:55:05.206109+0000","last_undegraded":"2024-09-21T09:55:05.206109+0000","last_fullsized":"2024-09-21T09:55:05.206109+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698801+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:55:04.698801+0000","last_peered":"2024-09-21T09:55:04.698801+0000","last_clean":"2024-09-21T09:55:04.698801+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:55:04.698801+0000","last_undegraded":"2024-09-21T09:55:04.698801+0000","last_fullsized":"2024-09-21T09:55:04.698801+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1334,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531537+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:55:05.531537+0000","last_peered":"2024-09-21T09:55:05.531537+0000","last_clean":"2024-09-21T09:55:05.531537+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:55:05.531537+0000","last_undegraded":"2024-09-21T09:55:05.531537+0000","last_fullsized":"2024-09-21T09:55:05.531537+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530863+0000","last_change":"2024-09-21T09:50:44.875761+0000","last_active":"2024-09-21T09:55:05.530863+0000","last_peered":"2024-09-21T09:55:05.530863+0000","last_clean":"2024-09-21T09:55:05.530863+0000","last_became_active":"2024-09-21T09:50:44.875333+0000","last_became_peered":"2024-09-21T09:50:44.875333+0000","last_unstale":"2024-09-21T09:55:05.530863+0000","last_undegraded":"2024-09-21T09:55:05.530863+0000","last_fullsized":"2024-09-21T09:55:05.530863+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:51.443596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205905+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:55:05.205905+0000","last_peered":"2024-09-21T09:55:05.205905+0000","last_clean":"2024-09-21T09:55:05.205905+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:55:05.205905+0000","last_undegraded":"2024-09-21T09:55:05.205905+0000","last_fullsized":"2024-09-21T09:55:05.205905+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1275,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531830+0000","last_change":"2024-09-21T09:50:44.874771+0000","last_active":"2024-09-21T09:55:05.531830+0000","last_peered":"2024-09-21T09:55:05.531830+0000","last_clean":"2024-09-21T09:55:05.531830+0000","last_became_active":"2024-09-21T09:50:44.873380+0000","last_became_peered":"2024-09-21T09:50:44.873380+0000","last_unstale":"2024-09-21T09:55:05.531830+0000","last_undegraded":"2024-09-21T09:55:05.531830+0000","last_fullsized":"2024-09-21T09:55:05.531830+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:19:11.264697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1348,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531680+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:55:05.531680+0000","last_peered":"2024-09-21T09:55:05.531680+0000","last_clean":"2024-09-21T09:55:05.531680+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:55:05.531680+0000","last_undegraded":"2024-09-21T09:55:05.531680+0000","last_fullsized":"2024-09-21T09:55:05.531680+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016056+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:55:04.016056+0000","last_peered":"2024-09-21T09:55:04.016056+0000","last_clean":"2024-09-21T09:55:04.016056+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:55:04.016056+0000","last_undegraded":"2024-09-21T09:55:04.016056+0000","last_fullsized":"2024-09-21T09:55:04.016056+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1016,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206752+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:55:05.206752+0000","last_peered":"2024-09-21T09:55:05.206752+0000","last_clean":"2024-09-21T09:55:05.206752+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:55:05.206752+0000","last_undegraded":"2024-09-21T09:55:05.206752+0000","last_fullsized":"2024-09-21T09:55:05.206752+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206779+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:55:05.206779+0000","last_peered":"2024-09-21T09:55:05.206779+0000","last_clean":"2024-09-21T09:55:05.206779+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:55:05.206779+0000","last_undegraded":"2024-09-21T09:55:05.206779+0000","last_fullsized":"2024-09-21T09:55:05.206779+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206376+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:55:05.206376+0000","last_peered":"2024-09-21T09:55:05.206376+0000","last_clean":"2024-09-21T09:55:05.206376+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:55:05.206376+0000","last_undegraded":"2024-09-21T09:55:05.206376+0000","last_fullsized":"2024-09-21T09:55:05.206376+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205508+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:55:05.205508+0000","last_peered":"2024-09-21T09:55:05.205508+0000","last_clean":"2024-09-21T09:55:05.205508+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:55:05.205508+0000","last_undegraded":"2024-09-21T09:55:05.205508+0000","last_fullsized":"2024-09-21T09:55:05.205508+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698745+0000","last_change":"2024-09-21T09:50:44.870735+0000","last_active":"2024-09-21T09:55:04.698745+0000","last_peered":"2024-09-21T09:55:04.698745+0000","last_clean":"2024-09-21T09:55:04.698745+0000","last_became_active":"2024-09-21T09:50:44.870370+0000","last_became_peered":"2024-09-21T09:50:44.870370+0000","last_unstale":"2024-09-21T09:55:04.698745+0000","last_undegraded":"2024-09-21T09:55:04.698745+0000","last_fullsized":"2024-09-21T09:55:04.698745+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:52.476737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621859+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:55:04.621859+0000","last_peered":"2024-09-21T09:55:04.621859+0000","last_clean":"2024-09-21T09:55:04.621859+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:55:04.621859+0000","last_undegraded":"2024-09-21T09:55:04.621859+0000","last_fullsized":"2024-09-21T09:55:04.621859+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532139+0000","last_change":"2024-09-21T09:50:44.874933+0000","last_active":"2024-09-21T09:55:05.532139+0000","last_peered":"2024-09-21T09:55:05.532139+0000","last_clean":"2024-09-21T09:55:05.532139+0000","last_became_active":"2024-09-21T09:50:44.870699+0000","last_became_peered":"2024-09-21T09:50:44.870699+0000","last_unstale":"2024-09-21T09:55:05.532139+0000","last_undegraded":"2024-09-21T09:55:05.532139+0000","last_fullsized":"2024-09-21T09:55:05.532139+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:12.912457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1342,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535069+0000","last_change":"2024-09-21T09:50:44.872004+0000","last_active":"2024-09-21T09:55:05.535069+0000","last_peered":"2024-09-21T09:55:05.535069+0000","last_clean":"2024-09-21T09:55:05.535069+0000","last_became_active":"2024-09-21T09:50:44.871706+0000","last_became_peered":"2024-09-21T09:50:44.871706+0000","last_unstale":"2024-09-21T09:55:05.535069+0000","last_undegraded":"2024-09-21T09:55:05.535069+0000","last_fullsized":"2024-09-21T09:55:05.535069+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:00.145938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531786+0000","last_change":"2024-09-21T09:50:44.874472+0000","last_active":"2024-09-21T09:55:05.531786+0000","last_peered":"2024-09-21T09:55:05.531786+0000","last_clean":"2024-09-21T09:55:05.531786+0000","last_became_active":"2024-09-21T09:50:44.872792+0000","last_became_peered":"2024-09-21T09:50:44.872792+0000","last_unstale":"2024-09-21T09:55:05.531786+0000","last_undegraded":"2024-09-21T09:55:05.531786+0000","last_fullsized":"2024-09-21T09:55:05.531786+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:52.873552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249921+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:55:05.249921+0000","last_peered":"2024-09-21T09:55:05.249921+0000","last_clean":"2024-09-21T09:55:05.249921+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:55:05.249921+0000","last_undegraded":"2024-09-21T09:55:05.249921+0000","last_fullsized":"2024-09-21T09:55:05.249921+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1282,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531952+0000","last_change":"2024-09-21T09:50:44.875450+0000","last_active":"2024-09-21T09:55:05.531952+0000","last_peered":"2024-09-21T09:55:05.531952+0000","last_clean":"2024-09-21T09:55:05.531952+0000","last_became_active":"2024-09-21T09:50:44.874171+0000","last_became_peered":"2024-09-21T09:50:44.874171+0000","last_unstale":"2024-09-21T09:55:05.531952+0000","last_undegraded":"2024-09-21T09:55:05.531952+0000","last_fullsized":"2024-09-21T09:55:05.531952+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:37.106157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1345,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531477+0000","last_change":"2024-09-21T09:50:44.871526+0000","last_active":"2024-09-21T09:55:05.531477+0000","last_peered":"2024-09-21T09:55:05.531477+0000","last_clean":"2024-09-21T09:55:05.531477+0000","last_became_active":"2024-09-21T09:50:44.871236+0000","last_became_peered":"2024-09-21T09:50:44.871236+0000","last_unstale":"2024-09-21T09:55:05.531477+0000","last_undegraded":"2024-09-21T09:55:05.531477+0000","last_fullsized":"2024-09-21T09:55:05.531477+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:51.749231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205429+0000","last_change":"2024-09-21T09:50:44.872712+0000","last_active":"2024-09-21T09:55:05.205429+0000","last_peered":"2024-09-21T09:55:05.205429+0000","last_clean":"2024-09-21T09:55:05.205429+0000","last_became_active":"2024-09-21T09:50:44.872368+0000","last_became_peered":"2024-09-21T09:50:44.872368+0000","last_unstale":"2024-09-21T09:55:05.205429+0000","last_undegraded":"2024-09-21T09:55:05.205429+0000","last_fullsized":"2024-09-21T09:55:05.205429+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:16.003164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531019+0000","last_change":"2024-09-21T09:50:44.877609+0000","last_active":"2024-09-21T09:55:05.531019+0000","last_peered":"2024-09-21T09:55:05.531019+0000","last_clean":"2024-09-21T09:55:05.531019+0000","last_became_active":"2024-09-21T09:50:44.876347+0000","last_became_peered":"2024-09-21T09:50:44.876347+0000","last_unstale":"2024-09-21T09:55:05.531019+0000","last_undegraded":"2024-09-21T09:55:05.531019+0000","last_fullsized":"2024-09-21T09:55:05.531019+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:24.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1046,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837776+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:55:03.837776+0000","last_peered":"2024-09-21T09:55:03.837776+0000","last_clean":"2024-09-21T09:55:03.837776+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:55:03.837776+0000","last_undegraded":"2024-09-21T09:55:03.837776+0000","last_fullsized":"2024-09-21T09:55:03.837776+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531639+0000","last_change":"2024-09-21T09:50:44.879265+0000","last_active":"2024-09-21T09:55:05.531639+0000","last_peered":"2024-09-21T09:55:05.531639+0000","last_clean":"2024-09-21T09:55:05.531639+0000","last_became_active":"2024-09-21T09:50:44.878975+0000","last_became_peered":"2024-09-21T09:50:44.878975+0000","last_unstale":"2024-09-21T09:55:05.531639+0000","last_undegraded":"2024-09-21T09:55:05.531639+0000","last_fullsized":"2024-09-21T09:55:05.531639+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:33.954417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1335,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532013+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:55:05.532013+0000","last_peered":"2024-09-21T09:55:05.532013+0000","last_clean":"2024-09-21T09:55:05.532013+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:55:05.532013+0000","last_undegraded":"2024-09-21T09:55:05.532013+0000","last_fullsized":"2024-09-21T09:55:05.532013+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837921+0000","last_change":"2024-09-21T09:50:44.866370+0000","last_active":"2024-09-21T09:55:03.837921+0000","last_peered":"2024-09-21T09:55:03.837921+0000","last_clean":"2024-09-21T09:55:03.837921+0000","last_became_active":"2024-09-21T09:50:44.865582+0000","last_became_peered":"2024-09-21T09:50:44.865582+0000","last_unstale":"2024-09-21T09:55:03.837921+0000","last_undegraded":"2024-09-21T09:55:03.837921+0000","last_fullsized":"2024-09-21T09:55:03.837921+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:06:23.001592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621778+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:55:04.621778+0000","last_peered":"2024-09-21T09:55:04.621778+0000","last_clean":"2024-09-21T09:55:04.621778+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:55:04.621778+0000","last_undegraded":"2024-09-21T09:55:04.621778+0000","last_fullsized":"2024-09-21T09:55:04.621778+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837680+0000","last_change":"2024-09-21T09:50:44.876517+0000","last_active":"2024-09-21T09:55:03.837680+0000","last_peered":"2024-09-21T09:55:03.837680+0000","last_clean":"2024-09-21T09:55:03.837680+0000","last_became_active":"2024-09-21T09:50:44.876186+0000","last_became_peered":"2024-09-21T09:50:44.876186+0000","last_unstale":"2024-09-21T09:55:03.837680+0000","last_undegraded":"2024-09-21T09:55:03.837680+0000","last_fullsized":"2024-09-21T09:55:03.837680+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:42.875674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621557+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:55:04.621557+0000","last_peered":"2024-09-21T09:55:04.621557+0000","last_clean":"2024-09-21T09:55:04.621557+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:55:04.621557+0000","last_undegraded":"2024-09-21T09:55:04.621557+0000","last_fullsized":"2024-09-21T09:55:04.621557+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1272,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532459+0000","last_change":"2024-09-21T09:50:44.878358+0000","last_active":"2024-09-21T09:55:05.532459+0000","last_peered":"2024-09-21T09:55:05.532459+0000","last_clean":"2024-09-21T09:55:05.532459+0000","last_became_active":"2024-09-21T09:50:44.877700+0000","last_became_peered":"2024-09-21T09:50:44.877700+0000","last_unstale":"2024-09-21T09:55:05.532459+0000","last_undegraded":"2024-09-21T09:55:05.532459+0000","last_fullsized":"2024-09-21T09:55:05.532459+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:39:47.027086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1260,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530581+0000","last_change":"2024-09-21T09:50:44.878043+0000","last_active":"2024-09-21T09:55:05.530581+0000","last_peered":"2024-09-21T09:55:05.530581+0000","last_clean":"2024-09-21T09:55:05.530581+0000","last_became_active":"2024-09-21T09:50:44.877686+0000","last_became_peered":"2024-09-21T09:50:44.877686+0000","last_unstale":"2024-09-21T09:55:05.530581+0000","last_undegraded":"2024-09-21T09:55:05.530581+0000","last_fullsized":"2024-09-21T09:55:05.530581+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:39.584859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837534+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:55:03.837534+0000","last_peered":"2024-09-21T09:55:03.837534+0000","last_clean":"2024-09-21T09:55:03.837534+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:55:03.837534+0000","last_undegraded":"2024-09-21T09:55:03.837534+0000","last_fullsized":"2024-09-21T09:55:03.837534+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1351,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535225+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:55:05.535225+0000","last_peered":"2024-09-21T09:55:05.535225+0000","last_clean":"2024-09-21T09:55:05.535225+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:55:05.535225+0000","last_undegraded":"2024-09-21T09:55:05.535225+0000","last_fullsized":"2024-09-21T09:55:05.535225+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234240,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67844,"kb_used_data":10484,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677308,"statfs":{"total":95995035648,"available":95925563392,"internally_reserved":0,"allocated":10735616,"data_stored":7713886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":58718667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495877,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67832,"kb_used_data":10476,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677320,"statfs":{"total":95995035648,"available":95925575680,"internally_reserved":0,"allocated":10727424,"data_stored":7713759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":58718663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757515,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":99184,"kb_used_data":9888,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93645968,"statfs":{"total":95995035648,"available":95893471232,"internally_reserved":0,"allocated":10125312,"data_stored":7119217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019151,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63164,"kb_used_data":9900,"kb_used_omap":1,"kb_used_meta":53246,"kb_avail":93681988,"statfs":{"total":95995035648,"available":95930355712,"internally_reserved":0,"allocated":10137600,"data_stored":7123448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":54524358},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":41896,"kb_used_data":9880,"kb_used_omap":34,"kb_used_meta":31965,"kb_avail":93703256,"statfs":{"total":95995035648,"available":95952134144,"internally_reserved":0,"allocated":10117120,"data_stored":7118648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":32733171},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967686,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46124,"kb_used_data":9884,"kb_used_omap":41,"kb_used_meta":36182,"kb_avail":93699028,"statfs":{"total":95995035648,"available":95947804672,"internally_reserved":0,"allocated":10121216,"data_stored":7119174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":37050783},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046925,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50276,"kb_used_data":9880,"kb_used_omap":41,"kb_used_meta":40342,"kb_avail":93694876,"statfs":{"total":95995035648,"available":95943553024,"internally_reserved":0,"allocated":10117120,"data_stored":7118577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":41311145},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126164,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43184,"kb_used_data":10464,"kb_used_omap":72,"kb_used_meta":32631,"kb_avail":93701968,"statfs":{"total":95995035648,"available":95950815232,"internally_reserved":0,"allocated":10715136,"data_stored":7709530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":33414187},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:55:56.269 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:55:56.269 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:55:56.269 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:55:56.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:55:56.468 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:56 smithi191 ceph-mon[102206]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:56.863 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:56 smithi076 ceph-mon[106482]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:56.863 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:56.864 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:56 smithi076 ceph-mon[110963]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:55:56.864 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:57 smithi076 ceph-mon[106482]: pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:57 smithi076 ceph-mon[110963]: pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:57.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:57 smithi191 ceph-mon[102206]: pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:58.312 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:55:58.312 INFO:teuthology.orchestra.run.smithi076.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:55:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4065948054' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T09:55:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4065948054' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T09:55:58.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4065948054' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-21T09:55:58.887 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:55:58.887 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:55:58.896 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-21T09:55:58.896 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:55:58.914 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:55:58.914 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:55:58.914 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-21T09:55:58.916 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-21T09:55:58.916 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:55:58.916 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-21T09:55:58.916 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-21T09:55:58.916 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:55:58.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:55:59.113 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:55:59.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:55:59 smithi191 ceph-mon[102206]: pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:55:59 smithi076 ceph-mon[106482]: pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:55:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:55:59 smithi076 ceph-mon[110963]: pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:01.037 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:56:01.038 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:56:01.765 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:01 smithi076 ceph-mon[106482]: pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:01.766 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:01 smithi076 ceph-mon[106482]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:01.766 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:01 smithi076 ceph-mon[110963]: pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:01.766 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:01 smithi076 ceph-mon[110963]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:01.786 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1128,"stamp":"2024-09-21T09:56:00.402550+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":479504,"kb_used_data":80856,"kb_used_omap":195,"kb_used_meta":398332,"kb_avail":749481712,"statfs":{"total":767960285184,"available":767469273088,"internally_reserved":0,"allocated":82796544,"data_stored":58736239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":407892100},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003153"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531316+0000","last_change":"2024-09-21T09:50:44.870931+0000","last_active":"2024-09-21T09:55:05.531316+0000","last_peered":"2024-09-21T09:55:05.531316+0000","last_clean":"2024-09-21T09:55:05.531316+0000","last_became_active":"2024-09-21T09:50:44.870528+0000","last_became_peered":"2024-09-21T09:50:44.870528+0000","last_unstale":"2024-09-21T09:55:05.531316+0000","last_undegraded":"2024-09-21T09:55:05.531316+0000","last_fullsized":"2024-09-21T09:55:05.531316+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:16.312101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1011,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531334+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:55:05.531334+0000","last_peered":"2024-09-21T09:55:05.531334+0000","last_clean":"2024-09-21T09:55:05.531334+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:55:05.531334+0000","last_undegraded":"2024-09-21T09:55:05.531334+0000","last_fullsized":"2024-09-21T09:55:05.531334+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621949+0000","last_change":"2024-09-21T09:50:44.866151+0000","last_active":"2024-09-21T09:55:04.621949+0000","last_peered":"2024-09-21T09:55:04.621949+0000","last_clean":"2024-09-21T09:55:04.621949+0000","last_became_active":"2024-09-21T09:50:44.865810+0000","last_became_peered":"2024-09-21T09:50:44.865810+0000","last_unstale":"2024-09-21T09:55:04.621949+0000","last_undegraded":"2024-09-21T09:55:04.621949+0000","last_fullsized":"2024-09-21T09:55:04.621949+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:31.175969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530607+0000","last_change":"2024-09-21T09:50:44.876550+0000","last_active":"2024-09-21T09:55:05.530607+0000","last_peered":"2024-09-21T09:55:05.530607+0000","last_clean":"2024-09-21T09:55:05.530607+0000","last_became_active":"2024-09-21T09:50:44.876246+0000","last_became_peered":"2024-09-21T09:50:44.876246+0000","last_unstale":"2024-09-21T09:55:05.530607+0000","last_undegraded":"2024-09-21T09:55:05.530607+0000","last_fullsized":"2024-09-21T09:55:05.530607+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:54.983152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531556+0000","last_change":"2024-09-21T09:50:44.872174+0000","last_active":"2024-09-21T09:55:05.531556+0000","last_peered":"2024-09-21T09:55:05.531556+0000","last_clean":"2024-09-21T09:55:05.531556+0000","last_became_active":"2024-09-21T09:50:44.871882+0000","last_became_peered":"2024-09-21T09:50:44.871882+0000","last_unstale":"2024-09-21T09:55:05.531556+0000","last_undegraded":"2024-09-21T09:55:05.531556+0000","last_fullsized":"2024-09-21T09:55:05.531556+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:57.860279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":994,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531362+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:55:05.531362+0000","last_peered":"2024-09-21T09:55:05.531362+0000","last_clean":"2024-09-21T09:55:05.531362+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:55:05.531362+0000","last_undegraded":"2024-09-21T09:55:05.531362+0000","last_fullsized":"2024-09-21T09:55:05.531362+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530717+0000","last_change":"2024-09-21T09:50:44.871672+0000","last_active":"2024-09-21T09:55:05.530717+0000","last_peered":"2024-09-21T09:55:05.530717+0000","last_clean":"2024-09-21T09:55:05.530717+0000","last_became_active":"2024-09-21T09:50:44.870133+0000","last_became_peered":"2024-09-21T09:50:44.870133+0000","last_unstale":"2024-09-21T09:55:05.530717+0000","last_undegraded":"2024-09-21T09:55:05.530717+0000","last_fullsized":"2024-09-21T09:55:05.530717+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:27.196429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1021,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.250019+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:55:05.250019+0000","last_peered":"2024-09-21T09:55:05.250019+0000","last_clean":"2024-09-21T09:55:05.250019+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:55:05.250019+0000","last_undegraded":"2024-09-21T09:55:05.250019+0000","last_fullsized":"2024-09-21T09:55:05.250019+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1010,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698425+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:55:04.698425+0000","last_peered":"2024-09-21T09:55:04.698425+0000","last_clean":"2024-09-21T09:55:04.698425+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:55:04.698425+0000","last_undegraded":"2024-09-21T09:55:04.698425+0000","last_fullsized":"2024-09-21T09:55:04.698425+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1009,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621746+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:55:04.621746+0000","last_peered":"2024-09-21T09:55:04.621746+0000","last_clean":"2024-09-21T09:55:04.621746+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:55:04.621746+0000","last_undegraded":"2024-09-21T09:55:04.621746+0000","last_fullsized":"2024-09-21T09:55:04.621746+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698562+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:55:04.698562+0000","last_peered":"2024-09-21T09:55:04.698562+0000","last_clean":"2024-09-21T09:55:04.698562+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:55:04.698562+0000","last_undegraded":"2024-09-21T09:55:04.698562+0000","last_fullsized":"2024-09-21T09:55:04.698562+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837730+0000","last_change":"2024-09-21T09:50:44.867113+0000","last_active":"2024-09-21T09:55:03.837730+0000","last_peered":"2024-09-21T09:55:03.837730+0000","last_clean":"2024-09-21T09:55:03.837730+0000","last_became_active":"2024-09-21T09:50:44.866691+0000","last_became_peered":"2024-09-21T09:50:44.866691+0000","last_unstale":"2024-09-21T09:55:03.837730+0000","last_undegraded":"2024-09-21T09:55:03.837730+0000","last_fullsized":"2024-09-21T09:55:03.837730+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:56.642995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":995,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698756+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:55:04.698756+0000","last_peered":"2024-09-21T09:55:04.698756+0000","last_clean":"2024-09-21T09:55:04.698756+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:55:04.698756+0000","last_undegraded":"2024-09-21T09:55:04.698756+0000","last_fullsized":"2024-09-21T09:55:04.698756+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1012,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016143+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:55:04.016143+0000","last_peered":"2024-09-21T09:55:04.016143+0000","last_clean":"2024-09-21T09:55:04.016143+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:55:04.016143+0000","last_undegraded":"2024-09-21T09:55:04.016143+0000","last_fullsized":"2024-09-21T09:55:04.016143+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621821+0000","last_change":"2024-09-21T09:50:44.866100+0000","last_active":"2024-09-21T09:55:04.621821+0000","last_peered":"2024-09-21T09:55:04.621821+0000","last_clean":"2024-09-21T09:55:04.621821+0000","last_became_active":"2024-09-21T09:50:44.865710+0000","last_became_peered":"2024-09-21T09:50:44.865710+0000","last_unstale":"2024-09-21T09:55:04.621821+0000","last_undegraded":"2024-09-21T09:55:04.621821+0000","last_fullsized":"2024-09-21T09:55:04.621821+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:16.761723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1346,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530753+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:55:05.530753+0000","last_peered":"2024-09-21T09:55:05.530753+0000","last_clean":"2024-09-21T09:55:05.530753+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:55:05.530753+0000","last_undegraded":"2024-09-21T09:55:05.530753+0000","last_fullsized":"2024-09-21T09:55:05.530753+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:50:44.876433+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:50:44.875979+0000","last_became_peered":"2024-09-21T09:50:44.875979+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:29.774368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837899+0000","last_change":"2024-09-21T09:50:44.866099+0000","last_active":"2024-09-21T09:55:03.837899+0000","last_peered":"2024-09-21T09:55:03.837899+0000","last_clean":"2024-09-21T09:55:03.837899+0000","last_became_active":"2024-09-21T09:50:44.865811+0000","last_became_peered":"2024-09-21T09:50:44.865811+0000","last_unstale":"2024-09-21T09:55:03.837899+0000","last_undegraded":"2024-09-21T09:55:03.837899+0000","last_fullsized":"2024-09-21T09:55:03.837899+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:25.833506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1013,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016235+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:55:04.016235+0000","last_peered":"2024-09-21T09:55:04.016235+0000","last_clean":"2024-09-21T09:55:04.016235+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:55:04.016235+0000","last_undegraded":"2024-09-21T09:55:04.016235+0000","last_fullsized":"2024-09-21T09:55:04.016235+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838059+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:55:03.838059+0000","last_peered":"2024-09-21T09:55:03.838059+0000","last_clean":"2024-09-21T09:55:03.838059+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:55:03.838059+0000","last_undegraded":"2024-09-21T09:55:03.838059+0000","last_fullsized":"2024-09-21T09:55:03.838059+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837860+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:55:03.837860+0000","last_peered":"2024-09-21T09:55:03.837860+0000","last_clean":"2024-09-21T09:55:03.837860+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:55:03.837860+0000","last_undegraded":"2024-09-21T09:55:03.837860+0000","last_fullsized":"2024-09-21T09:55:03.837860+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838274+0000","last_change":"2024-09-21T09:49:38.835119+0000","last_active":"2024-09-21T09:55:03.838274+0000","last_peered":"2024-09-21T09:55:03.838274+0000","last_clean":"2024-09-21T09:55:03.838274+0000","last_became_active":"2024-09-21T09:49:38.834858+0000","last_became_peered":"2024-09-21T09:49:38.834858+0000","last_unstale":"2024-09-21T09:55:03.838274+0000","last_undegraded":"2024-09-21T09:55:03.838274+0000","last_fullsized":"2024-09-21T09:55:03.838274+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:59.812483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1268,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531996+0000","last_change":"2024-09-21T09:50:44.875119+0000","last_active":"2024-09-21T09:55:05.531996+0000","last_peered":"2024-09-21T09:55:05.531996+0000","last_clean":"2024-09-21T09:55:05.531996+0000","last_became_active":"2024-09-21T09:50:44.873783+0000","last_became_peered":"2024-09-21T09:50:44.873783+0000","last_unstale":"2024-09-21T09:55:05.531996+0000","last_undegraded":"2024-09-21T09:55:05.531996+0000","last_fullsized":"2024-09-21T09:55:05.531996+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:28.277953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206713+0000","last_change":"2024-09-21T09:50:44.873954+0000","last_active":"2024-09-21T09:55:05.206713+0000","last_peered":"2024-09-21T09:55:05.206713+0000","last_clean":"2024-09-21T09:55:05.206713+0000","last_became_active":"2024-09-21T09:50:44.873527+0000","last_became_peered":"2024-09-21T09:50:44.873527+0000","last_unstale":"2024-09-21T09:55:05.206713+0000","last_undegraded":"2024-09-21T09:55:05.206713+0000","last_fullsized":"2024-09-21T09:55:05.206713+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:53.842361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1340,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698022+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:55:04.698022+0000","last_peered":"2024-09-21T09:55:04.698022+0000","last_clean":"2024-09-21T09:55:04.698022+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:55:04.698022+0000","last_undegraded":"2024-09-21T09:55:04.698022+0000","last_fullsized":"2024-09-21T09:55:04.698022+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.622014+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:55:04.622014+0000","last_peered":"2024-09-21T09:55:04.622014+0000","last_clean":"2024-09-21T09:55:04.622014+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:55:04.622014+0000","last_undegraded":"2024-09-21T09:55:04.622014+0000","last_fullsized":"2024-09-21T09:55:04.622014+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531943+0000","last_change":"2024-09-21T09:50:44.867188+0000","last_active":"2024-09-21T09:55:05.531943+0000","last_peered":"2024-09-21T09:55:05.531943+0000","last_clean":"2024-09-21T09:55:05.531943+0000","last_became_active":"2024-09-21T09:50:44.866818+0000","last_became_peered":"2024-09-21T09:50:44.866818+0000","last_unstale":"2024-09-21T09:55:05.531943+0000","last_undegraded":"2024-09-21T09:55:05.531943+0000","last_fullsized":"2024-09-21T09:55:05.531943+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:17:33.499249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":998,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249772+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:55:05.249772+0000","last_peered":"2024-09-21T09:55:05.249772+0000","last_clean":"2024-09-21T09:55:05.249772+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:55:05.249772+0000","last_undegraded":"2024-09-21T09:55:05.249772+0000","last_fullsized":"2024-09-21T09:55:05.249772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1038,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1344,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530372+0000","last_change":"2024-09-21T09:50:44.872670+0000","last_active":"2024-09-21T09:55:05.530372+0000","last_peered":"2024-09-21T09:55:05.530372+0000","last_clean":"2024-09-21T09:55:05.530372+0000","last_became_active":"2024-09-21T09:50:44.872391+0000","last_became_peered":"2024-09-21T09:50:44.872391+0000","last_unstale":"2024-09-21T09:55:05.530372+0000","last_undegraded":"2024-09-21T09:55:05.530372+0000","last_fullsized":"2024-09-21T09:55:05.530372+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:19.739055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1356,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531407+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:55:05.531407+0000","last_peered":"2024-09-21T09:55:05.531407+0000","last_clean":"2024-09-21T09:55:05.531407+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:55:05.531407+0000","last_undegraded":"2024-09-21T09:55:05.531407+0000","last_fullsized":"2024-09-21T09:55:05.531407+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1322,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530523+0000","last_change":"2024-09-21T09:50:44.880197+0000","last_active":"2024-09-21T09:55:05.530523+0000","last_peered":"2024-09-21T09:55:05.530523+0000","last_clean":"2024-09-21T09:55:05.530523+0000","last_became_active":"2024-09-21T09:50:44.879899+0000","last_became_peered":"2024-09-21T09:50:44.879899+0000","last_unstale":"2024-09-21T09:55:05.530523+0000","last_undegraded":"2024-09-21T09:55:05.530523+0000","last_fullsized":"2024-09-21T09:55:05.530523+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:23.596846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1318,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206097+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:55:05.206097+0000","last_peered":"2024-09-21T09:55:05.206097+0000","last_clean":"2024-09-21T09:55:05.206097+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:55:05.206097+0000","last_undegraded":"2024-09-21T09:55:05.206097+0000","last_fullsized":"2024-09-21T09:55:05.206097+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206384+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:55:05.206384+0000","last_peered":"2024-09-21T09:55:05.206384+0000","last_clean":"2024-09-21T09:55:05.206384+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:55:05.206384+0000","last_undegraded":"2024-09-21T09:55:05.206384+0000","last_fullsized":"2024-09-21T09:55:05.206384+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621861+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:55:04.621861+0000","last_peered":"2024-09-21T09:55:04.621861+0000","last_clean":"2024-09-21T09:55:04.621861+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:55:04.621861+0000","last_undegraded":"2024-09-21T09:55:04.621861+0000","last_fullsized":"2024-09-21T09:55:04.621861+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1032,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206860+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:55:05.206860+0000","last_peered":"2024-09-21T09:55:05.206860+0000","last_clean":"2024-09-21T09:55:05.206860+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:55:05.206860+0000","last_undegraded":"2024-09-21T09:55:05.206860+0000","last_fullsized":"2024-09-21T09:55:05.206860+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1329,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205848+0000","last_change":"2024-09-21T09:50:44.879616+0000","last_active":"2024-09-21T09:55:05.205848+0000","last_peered":"2024-09-21T09:55:05.205848+0000","last_clean":"2024-09-21T09:55:05.205848+0000","last_became_active":"2024-09-21T09:50:44.879237+0000","last_became_peered":"2024-09-21T09:50:44.879237+0000","last_unstale":"2024-09-21T09:55:05.205848+0000","last_undegraded":"2024-09-21T09:55:05.205848+0000","last_fullsized":"2024-09-21T09:55:05.205848+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:21.338486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531861+0000","last_change":"2024-09-21T09:50:44.870337+0000","last_active":"2024-09-21T09:55:05.531861+0000","last_peered":"2024-09-21T09:55:05.531861+0000","last_clean":"2024-09-21T09:55:05.531861+0000","last_became_active":"2024-09-21T09:50:44.870057+0000","last_became_peered":"2024-09-21T09:50:44.870057+0000","last_unstale":"2024-09-21T09:55:05.531861+0000","last_undegraded":"2024-09-21T09:55:05.531861+0000","last_fullsized":"2024-09-21T09:55:05.531861+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:22.969900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1340,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205939+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:55:05.205939+0000","last_peered":"2024-09-21T09:55:05.205939+0000","last_clean":"2024-09-21T09:55:05.205939+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:55:05.205939+0000","last_undegraded":"2024-09-21T09:55:05.205939+0000","last_fullsized":"2024-09-21T09:55:05.205939+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1307,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205656+0000","last_change":"2024-09-21T09:49:38.833392+0000","last_active":"2024-09-21T09:55:05.205656+0000","last_peered":"2024-09-21T09:55:05.205656+0000","last_clean":"2024-09-21T09:55:05.205656+0000","last_became_active":"2024-09-21T09:49:38.832610+0000","last_became_peered":"2024-09-21T09:49:38.832610+0000","last_unstale":"2024-09-21T09:55:05.205656+0000","last_undegraded":"2024-09-21T09:55:05.205656+0000","last_fullsized":"2024-09-21T09:55:05.205656+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:26.844310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206109+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:55:05.206109+0000","last_peered":"2024-09-21T09:55:05.206109+0000","last_clean":"2024-09-21T09:55:05.206109+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:55:05.206109+0000","last_undegraded":"2024-09-21T09:55:05.206109+0000","last_fullsized":"2024-09-21T09:55:05.206109+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698801+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:55:04.698801+0000","last_peered":"2024-09-21T09:55:04.698801+0000","last_clean":"2024-09-21T09:55:04.698801+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:55:04.698801+0000","last_undegraded":"2024-09-21T09:55:04.698801+0000","last_fullsized":"2024-09-21T09:55:04.698801+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1334,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531537+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:55:05.531537+0000","last_peered":"2024-09-21T09:55:05.531537+0000","last_clean":"2024-09-21T09:55:05.531537+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:55:05.531537+0000","last_undegraded":"2024-09-21T09:55:05.531537+0000","last_fullsized":"2024-09-21T09:55:05.531537+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530863+0000","last_change":"2024-09-21T09:50:44.875761+0000","last_active":"2024-09-21T09:55:05.530863+0000","last_peered":"2024-09-21T09:55:05.530863+0000","last_clean":"2024-09-21T09:55:05.530863+0000","last_became_active":"2024-09-21T09:50:44.875333+0000","last_became_peered":"2024-09-21T09:50:44.875333+0000","last_unstale":"2024-09-21T09:55:05.530863+0000","last_undegraded":"2024-09-21T09:55:05.530863+0000","last_fullsized":"2024-09-21T09:55:05.530863+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:51.443596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205905+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:55:05.205905+0000","last_peered":"2024-09-21T09:55:05.205905+0000","last_clean":"2024-09-21T09:55:05.205905+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:55:05.205905+0000","last_undegraded":"2024-09-21T09:55:05.205905+0000","last_fullsized":"2024-09-21T09:55:05.205905+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1275,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531830+0000","last_change":"2024-09-21T09:50:44.874771+0000","last_active":"2024-09-21T09:55:05.531830+0000","last_peered":"2024-09-21T09:55:05.531830+0000","last_clean":"2024-09-21T09:55:05.531830+0000","last_became_active":"2024-09-21T09:50:44.873380+0000","last_became_peered":"2024-09-21T09:50:44.873380+0000","last_unstale":"2024-09-21T09:55:05.531830+0000","last_undegraded":"2024-09-21T09:55:05.531830+0000","last_fullsized":"2024-09-21T09:55:05.531830+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:19:11.264697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1348,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531680+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:55:05.531680+0000","last_peered":"2024-09-21T09:55:05.531680+0000","last_clean":"2024-09-21T09:55:05.531680+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:55:05.531680+0000","last_undegraded":"2024-09-21T09:55:05.531680+0000","last_fullsized":"2024-09-21T09:55:05.531680+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016056+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:55:04.016056+0000","last_peered":"2024-09-21T09:55:04.016056+0000","last_clean":"2024-09-21T09:55:04.016056+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:55:04.016056+0000","last_undegraded":"2024-09-21T09:55:04.016056+0000","last_fullsized":"2024-09-21T09:55:04.016056+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1016,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206752+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:55:05.206752+0000","last_peered":"2024-09-21T09:55:05.206752+0000","last_clean":"2024-09-21T09:55:05.206752+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:55:05.206752+0000","last_undegraded":"2024-09-21T09:55:05.206752+0000","last_fullsized":"2024-09-21T09:55:05.206752+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206779+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:55:05.206779+0000","last_peered":"2024-09-21T09:55:05.206779+0000","last_clean":"2024-09-21T09:55:05.206779+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:55:05.206779+0000","last_undegraded":"2024-09-21T09:55:05.206779+0000","last_fullsized":"2024-09-21T09:55:05.206779+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206376+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:55:05.206376+0000","last_peered":"2024-09-21T09:55:05.206376+0000","last_clean":"2024-09-21T09:55:05.206376+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:55:05.206376+0000","last_undegraded":"2024-09-21T09:55:05.206376+0000","last_fullsized":"2024-09-21T09:55:05.206376+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205508+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:55:05.205508+0000","last_peered":"2024-09-21T09:55:05.205508+0000","last_clean":"2024-09-21T09:55:05.205508+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:55:05.205508+0000","last_undegraded":"2024-09-21T09:55:05.205508+0000","last_fullsized":"2024-09-21T09:55:05.205508+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1012,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531552+0000","last_change":"2024-09-21T09:50:44.870735+0000","last_active":"2024-09-21T09:55:05.531552+0000","last_peered":"2024-09-21T09:55:05.531552+0000","last_clean":"2024-09-21T09:55:05.531552+0000","last_became_active":"2024-09-21T09:50:44.870370+0000","last_became_peered":"2024-09-21T09:50:44.870370+0000","last_unstale":"2024-09-21T09:55:05.531552+0000","last_undegraded":"2024-09-21T09:55:05.531552+0000","last_fullsized":"2024-09-21T09:55:05.531552+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:52.476737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531467+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:55:05.531467+0000","last_peered":"2024-09-21T09:55:05.531467+0000","last_clean":"2024-09-21T09:55:05.531467+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:55:05.531467+0000","last_undegraded":"2024-09-21T09:55:05.531467+0000","last_fullsized":"2024-09-21T09:55:05.531467+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532139+0000","last_change":"2024-09-21T09:50:44.874933+0000","last_active":"2024-09-21T09:55:05.532139+0000","last_peered":"2024-09-21T09:55:05.532139+0000","last_clean":"2024-09-21T09:55:05.532139+0000","last_became_active":"2024-09-21T09:50:44.870699+0000","last_became_peered":"2024-09-21T09:50:44.870699+0000","last_unstale":"2024-09-21T09:55:05.532139+0000","last_undegraded":"2024-09-21T09:55:05.532139+0000","last_fullsized":"2024-09-21T09:55:05.532139+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:12.912457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1342,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535069+0000","last_change":"2024-09-21T09:50:44.872004+0000","last_active":"2024-09-21T09:55:05.535069+0000","last_peered":"2024-09-21T09:55:05.535069+0000","last_clean":"2024-09-21T09:55:05.535069+0000","last_became_active":"2024-09-21T09:50:44.871706+0000","last_became_peered":"2024-09-21T09:50:44.871706+0000","last_unstale":"2024-09-21T09:55:05.535069+0000","last_undegraded":"2024-09-21T09:55:05.535069+0000","last_fullsized":"2024-09-21T09:55:05.535069+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:00.145938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531786+0000","last_change":"2024-09-21T09:50:44.874472+0000","last_active":"2024-09-21T09:55:05.531786+0000","last_peered":"2024-09-21T09:55:05.531786+0000","last_clean":"2024-09-21T09:55:05.531786+0000","last_became_active":"2024-09-21T09:50:44.872792+0000","last_became_peered":"2024-09-21T09:50:44.872792+0000","last_unstale":"2024-09-21T09:55:05.531786+0000","last_undegraded":"2024-09-21T09:55:05.531786+0000","last_fullsized":"2024-09-21T09:55:05.531786+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:52.873552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249921+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:55:05.249921+0000","last_peered":"2024-09-21T09:55:05.249921+0000","last_clean":"2024-09-21T09:55:05.249921+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:55:05.249921+0000","last_undegraded":"2024-09-21T09:55:05.249921+0000","last_fullsized":"2024-09-21T09:55:05.249921+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1282,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531952+0000","last_change":"2024-09-21T09:50:44.875450+0000","last_active":"2024-09-21T09:55:05.531952+0000","last_peered":"2024-09-21T09:55:05.531952+0000","last_clean":"2024-09-21T09:55:05.531952+0000","last_became_active":"2024-09-21T09:50:44.874171+0000","last_became_peered":"2024-09-21T09:50:44.874171+0000","last_unstale":"2024-09-21T09:55:05.531952+0000","last_undegraded":"2024-09-21T09:55:05.531952+0000","last_fullsized":"2024-09-21T09:55:05.531952+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:37.106157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1345,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531477+0000","last_change":"2024-09-21T09:50:44.871526+0000","last_active":"2024-09-21T09:55:05.531477+0000","last_peered":"2024-09-21T09:55:05.531477+0000","last_clean":"2024-09-21T09:55:05.531477+0000","last_became_active":"2024-09-21T09:50:44.871236+0000","last_became_peered":"2024-09-21T09:50:44.871236+0000","last_unstale":"2024-09-21T09:55:05.531477+0000","last_undegraded":"2024-09-21T09:55:05.531477+0000","last_fullsized":"2024-09-21T09:55:05.531477+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:51.749231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205429+0000","last_change":"2024-09-21T09:50:44.872712+0000","last_active":"2024-09-21T09:55:05.205429+0000","last_peered":"2024-09-21T09:55:05.205429+0000","last_clean":"2024-09-21T09:55:05.205429+0000","last_became_active":"2024-09-21T09:50:44.872368+0000","last_became_peered":"2024-09-21T09:50:44.872368+0000","last_unstale":"2024-09-21T09:55:05.205429+0000","last_undegraded":"2024-09-21T09:55:05.205429+0000","last_fullsized":"2024-09-21T09:55:05.205429+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:16.003164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531019+0000","last_change":"2024-09-21T09:50:44.877609+0000","last_active":"2024-09-21T09:55:05.531019+0000","last_peered":"2024-09-21T09:55:05.531019+0000","last_clean":"2024-09-21T09:55:05.531019+0000","last_became_active":"2024-09-21T09:50:44.876347+0000","last_became_peered":"2024-09-21T09:50:44.876347+0000","last_unstale":"2024-09-21T09:55:05.531019+0000","last_undegraded":"2024-09-21T09:55:05.531019+0000","last_fullsized":"2024-09-21T09:55:05.531019+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:24.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1046,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837776+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:55:03.837776+0000","last_peered":"2024-09-21T09:55:03.837776+0000","last_clean":"2024-09-21T09:55:03.837776+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:55:03.837776+0000","last_undegraded":"2024-09-21T09:55:03.837776+0000","last_fullsized":"2024-09-21T09:55:03.837776+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531639+0000","last_change":"2024-09-21T09:50:44.879265+0000","last_active":"2024-09-21T09:55:05.531639+0000","last_peered":"2024-09-21T09:55:05.531639+0000","last_clean":"2024-09-21T09:55:05.531639+0000","last_became_active":"2024-09-21T09:50:44.878975+0000","last_became_peered":"2024-09-21T09:50:44.878975+0000","last_unstale":"2024-09-21T09:55:05.531639+0000","last_undegraded":"2024-09-21T09:55:05.531639+0000","last_fullsized":"2024-09-21T09:55:05.531639+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:33.954417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1335,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532013+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:55:05.532013+0000","last_peered":"2024-09-21T09:55:05.532013+0000","last_clean":"2024-09-21T09:55:05.532013+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:55:05.532013+0000","last_undegraded":"2024-09-21T09:55:05.532013+0000","last_fullsized":"2024-09-21T09:55:05.532013+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837921+0000","last_change":"2024-09-21T09:50:44.866370+0000","last_active":"2024-09-21T09:55:03.837921+0000","last_peered":"2024-09-21T09:55:03.837921+0000","last_clean":"2024-09-21T09:55:03.837921+0000","last_became_active":"2024-09-21T09:50:44.865582+0000","last_became_peered":"2024-09-21T09:50:44.865582+0000","last_unstale":"2024-09-21T09:55:03.837921+0000","last_undegraded":"2024-09-21T09:55:03.837921+0000","last_fullsized":"2024-09-21T09:55:03.837921+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:06:23.001592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621778+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:55:04.621778+0000","last_peered":"2024-09-21T09:55:04.621778+0000","last_clean":"2024-09-21T09:55:04.621778+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:55:04.621778+0000","last_undegraded":"2024-09-21T09:55:04.621778+0000","last_fullsized":"2024-09-21T09:55:04.621778+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837680+0000","last_change":"2024-09-21T09:50:44.876517+0000","last_active":"2024-09-21T09:55:03.837680+0000","last_peered":"2024-09-21T09:55:03.837680+0000","last_clean":"2024-09-21T09:55:03.837680+0000","last_became_active":"2024-09-21T09:50:44.876186+0000","last_became_peered":"2024-09-21T09:50:44.876186+0000","last_unstale":"2024-09-21T09:55:03.837680+0000","last_undegraded":"2024-09-21T09:55:03.837680+0000","last_fullsized":"2024-09-21T09:55:03.837680+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:42.875674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621557+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:55:04.621557+0000","last_peered":"2024-09-21T09:55:04.621557+0000","last_clean":"2024-09-21T09:55:04.621557+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:55:04.621557+0000","last_undegraded":"2024-09-21T09:55:04.621557+0000","last_fullsized":"2024-09-21T09:55:04.621557+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1272,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532459+0000","last_change":"2024-09-21T09:50:44.878358+0000","last_active":"2024-09-21T09:55:05.532459+0000","last_peered":"2024-09-21T09:55:05.532459+0000","last_clean":"2024-09-21T09:55:05.532459+0000","last_became_active":"2024-09-21T09:50:44.877700+0000","last_became_peered":"2024-09-21T09:50:44.877700+0000","last_unstale":"2024-09-21T09:55:05.532459+0000","last_undegraded":"2024-09-21T09:55:05.532459+0000","last_fullsized":"2024-09-21T09:55:05.532459+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:39:47.027086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1260,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530581+0000","last_change":"2024-09-21T09:50:44.878043+0000","last_active":"2024-09-21T09:55:05.530581+0000","last_peered":"2024-09-21T09:55:05.530581+0000","last_clean":"2024-09-21T09:55:05.530581+0000","last_became_active":"2024-09-21T09:50:44.877686+0000","last_became_peered":"2024-09-21T09:50:44.877686+0000","last_unstale":"2024-09-21T09:55:05.530581+0000","last_undegraded":"2024-09-21T09:55:05.530581+0000","last_fullsized":"2024-09-21T09:55:05.530581+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:39.584859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837534+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:55:03.837534+0000","last_peered":"2024-09-21T09:55:03.837534+0000","last_clean":"2024-09-21T09:55:03.837534+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:55:03.837534+0000","last_undegraded":"2024-09-21T09:55:03.837534+0000","last_fullsized":"2024-09-21T09:55:03.837534+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1351,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535225+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:55:05.535225+0000","last_peered":"2024-09-21T09:55:05.535225+0000","last_clean":"2024-09-21T09:55:05.535225+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:55:05.535225+0000","last_undegraded":"2024-09-21T09:55:05.535225+0000","last_fullsized":"2024-09-21T09:55:05.535225+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234241,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67844,"kb_used_data":10484,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677308,"statfs":{"total":95995035648,"available":95925563392,"internally_reserved":0,"allocated":10735616,"data_stored":7713886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":58718667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495878,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67832,"kb_used_data":10476,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677320,"statfs":{"total":95995035648,"available":95925575680,"internally_reserved":0,"allocated":10727424,"data_stored":7713759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":58718663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757516,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":99184,"kb_used_data":9888,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93645968,"statfs":{"total":95995035648,"available":95893471232,"internally_reserved":0,"allocated":10125312,"data_stored":7119217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019152,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63164,"kb_used_data":9900,"kb_used_omap":1,"kb_used_meta":53246,"kb_avail":93681988,"statfs":{"total":95995035648,"available":95930355712,"internally_reserved":0,"allocated":10137600,"data_stored":7123448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":54524358},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":41896,"kb_used_data":9880,"kb_used_omap":34,"kb_used_meta":31965,"kb_avail":93703256,"statfs":{"total":95995035648,"available":95952134144,"internally_reserved":0,"allocated":10117120,"data_stored":7118648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":32733171},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967688,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46124,"kb_used_data":9884,"kb_used_omap":41,"kb_used_meta":36182,"kb_avail":93699028,"statfs":{"total":95995035648,"available":95947804672,"internally_reserved":0,"allocated":10121216,"data_stored":7119174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":37050783},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046926,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50276,"kb_used_data":9880,"kb_used_omap":41,"kb_used_meta":40342,"kb_avail":93694876,"statfs":{"total":95995035648,"available":95943553024,"internally_reserved":0,"allocated":10117120,"data_stored":7118577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":41311145},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126165,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43184,"kb_used_data":10464,"kb_used_omap":72,"kb_used_meta":32631,"kb_avail":93701968,"statfs":{"total":95995035648,"available":95950815232,"internally_reserved":0,"allocated":10715136,"data_stored":7709530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":33414187},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:56:01.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:56:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:01 smithi191 ceph-mon[102206]: pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:01.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:01 smithi191 ceph-mon[102206]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:01.986 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:03.361 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:56:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:56:03] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.51.0" 2024-09-21T09:56:03.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:03 smithi076 ceph-mon[106482]: pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:03.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:03 smithi076 ceph-mon[110963]: pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:03.738 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:56:03.739 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:56:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:03 smithi191 ceph-mon[102206]: pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:04.351 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1129,"stamp":"2024-09-21T09:56:02.403095+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":479504,"kb_used_data":80856,"kb_used_omap":195,"kb_used_meta":398332,"kb_avail":749481712,"statfs":{"total":767960285184,"available":767469273088,"internally_reserved":0,"allocated":82796544,"data_stored":58736239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":407892100},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003245"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531316+0000","last_change":"2024-09-21T09:50:44.870931+0000","last_active":"2024-09-21T09:55:05.531316+0000","last_peered":"2024-09-21T09:55:05.531316+0000","last_clean":"2024-09-21T09:55:05.531316+0000","last_became_active":"2024-09-21T09:50:44.870528+0000","last_became_peered":"2024-09-21T09:50:44.870528+0000","last_unstale":"2024-09-21T09:55:05.531316+0000","last_undegraded":"2024-09-21T09:55:05.531316+0000","last_fullsized":"2024-09-21T09:55:05.531316+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:08:16.312101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1011,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531334+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:55:05.531334+0000","last_peered":"2024-09-21T09:55:05.531334+0000","last_clean":"2024-09-21T09:55:05.531334+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:55:05.531334+0000","last_undegraded":"2024-09-21T09:55:05.531334+0000","last_fullsized":"2024-09-21T09:55:05.531334+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1024,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531697+0000","last_change":"2024-09-21T09:50:44.866151+0000","last_active":"2024-09-21T09:55:05.531697+0000","last_peered":"2024-09-21T09:55:05.531697+0000","last_clean":"2024-09-21T09:55:05.531697+0000","last_became_active":"2024-09-21T09:50:44.865810+0000","last_became_peered":"2024-09-21T09:50:44.865810+0000","last_unstale":"2024-09-21T09:55:05.531697+0000","last_undegraded":"2024-09-21T09:55:05.531697+0000","last_fullsized":"2024-09-21T09:55:05.531697+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:27:31.175969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530607+0000","last_change":"2024-09-21T09:50:44.876550+0000","last_active":"2024-09-21T09:55:05.530607+0000","last_peered":"2024-09-21T09:55:05.530607+0000","last_clean":"2024-09-21T09:55:05.530607+0000","last_became_active":"2024-09-21T09:50:44.876246+0000","last_became_peered":"2024-09-21T09:50:44.876246+0000","last_unstale":"2024-09-21T09:55:05.530607+0000","last_undegraded":"2024-09-21T09:55:05.530607+0000","last_fullsized":"2024-09-21T09:55:05.530607+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:54.983152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1343,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531556+0000","last_change":"2024-09-21T09:50:44.872174+0000","last_active":"2024-09-21T09:55:05.531556+0000","last_peered":"2024-09-21T09:55:05.531556+0000","last_clean":"2024-09-21T09:55:05.531556+0000","last_became_active":"2024-09-21T09:50:44.871882+0000","last_became_peered":"2024-09-21T09:50:44.871882+0000","last_unstale":"2024-09-21T09:55:05.531556+0000","last_undegraded":"2024-09-21T09:55:05.531556+0000","last_fullsized":"2024-09-21T09:55:05.531556+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:57.860279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":994,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531362+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:55:05.531362+0000","last_peered":"2024-09-21T09:55:05.531362+0000","last_clean":"2024-09-21T09:55:05.531362+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:55:05.531362+0000","last_undegraded":"2024-09-21T09:55:05.531362+0000","last_fullsized":"2024-09-21T09:55:05.531362+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530717+0000","last_change":"2024-09-21T09:50:44.871672+0000","last_active":"2024-09-21T09:55:05.530717+0000","last_peered":"2024-09-21T09:55:05.530717+0000","last_clean":"2024-09-21T09:55:05.530717+0000","last_became_active":"2024-09-21T09:50:44.870133+0000","last_became_peered":"2024-09-21T09:50:44.870133+0000","last_unstale":"2024-09-21T09:55:05.530717+0000","last_undegraded":"2024-09-21T09:55:05.530717+0000","last_fullsized":"2024-09-21T09:55:05.530717+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:27.196429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1021,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.250019+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:55:05.250019+0000","last_peered":"2024-09-21T09:55:05.250019+0000","last_clean":"2024-09-21T09:55:05.250019+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:55:05.250019+0000","last_undegraded":"2024-09-21T09:55:05.250019+0000","last_fullsized":"2024-09-21T09:55:05.250019+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1010,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698425+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:55:04.698425+0000","last_peered":"2024-09-21T09:55:04.698425+0000","last_clean":"2024-09-21T09:55:04.698425+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:55:04.698425+0000","last_undegraded":"2024-09-21T09:55:04.698425+0000","last_fullsized":"2024-09-21T09:55:04.698425+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1009,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621746+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:55:04.621746+0000","last_peered":"2024-09-21T09:55:04.621746+0000","last_clean":"2024-09-21T09:55:04.621746+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:55:04.621746+0000","last_undegraded":"2024-09-21T09:55:04.621746+0000","last_fullsized":"2024-09-21T09:55:04.621746+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698562+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:55:04.698562+0000","last_peered":"2024-09-21T09:55:04.698562+0000","last_clean":"2024-09-21T09:55:04.698562+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:55:04.698562+0000","last_undegraded":"2024-09-21T09:55:04.698562+0000","last_fullsized":"2024-09-21T09:55:04.698562+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837730+0000","last_change":"2024-09-21T09:50:44.867113+0000","last_active":"2024-09-21T09:55:03.837730+0000","last_peered":"2024-09-21T09:55:03.837730+0000","last_clean":"2024-09-21T09:55:03.837730+0000","last_became_active":"2024-09-21T09:50:44.866691+0000","last_became_peered":"2024-09-21T09:50:44.866691+0000","last_unstale":"2024-09-21T09:55:03.837730+0000","last_undegraded":"2024-09-21T09:55:03.837730+0000","last_fullsized":"2024-09-21T09:55:03.837730+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:56.642995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":995,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698756+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:55:04.698756+0000","last_peered":"2024-09-21T09:55:04.698756+0000","last_clean":"2024-09-21T09:55:04.698756+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:55:04.698756+0000","last_undegraded":"2024-09-21T09:55:04.698756+0000","last_fullsized":"2024-09-21T09:55:04.698756+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1012,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016143+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:55:04.016143+0000","last_peered":"2024-09-21T09:55:04.016143+0000","last_clean":"2024-09-21T09:55:04.016143+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:55:04.016143+0000","last_undegraded":"2024-09-21T09:55:04.016143+0000","last_fullsized":"2024-09-21T09:55:04.016143+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621821+0000","last_change":"2024-09-21T09:50:44.866100+0000","last_active":"2024-09-21T09:55:04.621821+0000","last_peered":"2024-09-21T09:55:04.621821+0000","last_clean":"2024-09-21T09:55:04.621821+0000","last_became_active":"2024-09-21T09:50:44.865710+0000","last_became_peered":"2024-09-21T09:50:44.865710+0000","last_unstale":"2024-09-21T09:55:04.621821+0000","last_undegraded":"2024-09-21T09:55:04.621821+0000","last_fullsized":"2024-09-21T09:55:04.621821+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:16.761723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1346,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530753+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:55:05.530753+0000","last_peered":"2024-09-21T09:55:05.530753+0000","last_clean":"2024-09-21T09:55:05.530753+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:55:05.530753+0000","last_undegraded":"2024-09-21T09:55:05.530753+0000","last_fullsized":"2024-09-21T09:55:05.530753+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:50:44.876433+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:50:44.875979+0000","last_became_peered":"2024-09-21T09:50:44.875979+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:55:29.774368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837899+0000","last_change":"2024-09-21T09:50:44.866099+0000","last_active":"2024-09-21T09:55:03.837899+0000","last_peered":"2024-09-21T09:55:03.837899+0000","last_clean":"2024-09-21T09:55:03.837899+0000","last_became_active":"2024-09-21T09:50:44.865811+0000","last_became_peered":"2024-09-21T09:50:44.865811+0000","last_unstale":"2024-09-21T09:55:03.837899+0000","last_undegraded":"2024-09-21T09:55:03.837899+0000","last_fullsized":"2024-09-21T09:55:03.837899+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:25.833506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1013,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016235+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:55:04.016235+0000","last_peered":"2024-09-21T09:55:04.016235+0000","last_clean":"2024-09-21T09:55:04.016235+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:55:04.016235+0000","last_undegraded":"2024-09-21T09:55:04.016235+0000","last_fullsized":"2024-09-21T09:55:04.016235+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1045,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838059+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:55:03.838059+0000","last_peered":"2024-09-21T09:55:03.838059+0000","last_clean":"2024-09-21T09:55:03.838059+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:55:03.838059+0000","last_undegraded":"2024-09-21T09:55:03.838059+0000","last_fullsized":"2024-09-21T09:55:03.838059+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1031,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837860+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:55:03.837860+0000","last_peered":"2024-09-21T09:55:03.837860+0000","last_clean":"2024-09-21T09:55:03.837860+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:55:03.837860+0000","last_undegraded":"2024-09-21T09:55:03.837860+0000","last_fullsized":"2024-09-21T09:55:03.837860+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1030,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.838274+0000","last_change":"2024-09-21T09:49:38.835119+0000","last_active":"2024-09-21T09:55:03.838274+0000","last_peered":"2024-09-21T09:55:03.838274+0000","last_clean":"2024-09-21T09:55:03.838274+0000","last_became_active":"2024-09-21T09:49:38.834858+0000","last_became_peered":"2024-09-21T09:49:38.834858+0000","last_unstale":"2024-09-21T09:55:03.838274+0000","last_undegraded":"2024-09-21T09:55:03.838274+0000","last_fullsized":"2024-09-21T09:55:03.838274+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:59.812483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1268,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531996+0000","last_change":"2024-09-21T09:50:44.875119+0000","last_active":"2024-09-21T09:55:05.531996+0000","last_peered":"2024-09-21T09:55:05.531996+0000","last_clean":"2024-09-21T09:55:05.531996+0000","last_became_active":"2024-09-21T09:50:44.873783+0000","last_became_peered":"2024-09-21T09:50:44.873783+0000","last_unstale":"2024-09-21T09:55:05.531996+0000","last_undegraded":"2024-09-21T09:55:05.531996+0000","last_fullsized":"2024-09-21T09:55:05.531996+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:49:28.277953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206713+0000","last_change":"2024-09-21T09:50:44.873954+0000","last_active":"2024-09-21T09:55:05.206713+0000","last_peered":"2024-09-21T09:55:05.206713+0000","last_clean":"2024-09-21T09:55:05.206713+0000","last_became_active":"2024-09-21T09:50:44.873527+0000","last_became_peered":"2024-09-21T09:50:44.873527+0000","last_unstale":"2024-09-21T09:55:05.206713+0000","last_undegraded":"2024-09-21T09:55:05.206713+0000","last_fullsized":"2024-09-21T09:55:05.206713+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:53.842361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1340,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698022+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:55:04.698022+0000","last_peered":"2024-09-21T09:55:04.698022+0000","last_clean":"2024-09-21T09:55:04.698022+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:55:04.698022+0000","last_undegraded":"2024-09-21T09:55:04.698022+0000","last_fullsized":"2024-09-21T09:55:04.698022+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.622014+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:55:04.622014+0000","last_peered":"2024-09-21T09:55:04.622014+0000","last_clean":"2024-09-21T09:55:04.622014+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:55:04.622014+0000","last_undegraded":"2024-09-21T09:55:04.622014+0000","last_fullsized":"2024-09-21T09:55:04.622014+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531943+0000","last_change":"2024-09-21T09:50:44.867188+0000","last_active":"2024-09-21T09:55:05.531943+0000","last_peered":"2024-09-21T09:55:05.531943+0000","last_clean":"2024-09-21T09:55:05.531943+0000","last_became_active":"2024-09-21T09:50:44.866818+0000","last_became_peered":"2024-09-21T09:50:44.866818+0000","last_unstale":"2024-09-21T09:55:05.531943+0000","last_undegraded":"2024-09-21T09:55:05.531943+0000","last_fullsized":"2024-09-21T09:55:05.531943+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:17:33.499249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":998,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249772+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:55:05.249772+0000","last_peered":"2024-09-21T09:55:05.249772+0000","last_clean":"2024-09-21T09:55:05.249772+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:55:05.249772+0000","last_undegraded":"2024-09-21T09:55:05.249772+0000","last_fullsized":"2024-09-21T09:55:05.249772+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1038,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530416+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:55:05.530416+0000","last_peered":"2024-09-21T09:55:05.530416+0000","last_clean":"2024-09-21T09:55:05.530416+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:55:05.530416+0000","last_undegraded":"2024-09-21T09:55:05.530416+0000","last_fullsized":"2024-09-21T09:55:05.530416+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1344,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530372+0000","last_change":"2024-09-21T09:50:44.872670+0000","last_active":"2024-09-21T09:55:05.530372+0000","last_peered":"2024-09-21T09:55:05.530372+0000","last_clean":"2024-09-21T09:55:05.530372+0000","last_became_active":"2024-09-21T09:50:44.872391+0000","last_became_peered":"2024-09-21T09:50:44.872391+0000","last_unstale":"2024-09-21T09:55:05.530372+0000","last_undegraded":"2024-09-21T09:55:05.530372+0000","last_fullsized":"2024-09-21T09:55:05.530372+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:19.739055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1356,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531407+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:55:05.531407+0000","last_peered":"2024-09-21T09:55:05.531407+0000","last_clean":"2024-09-21T09:55:05.531407+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:55:05.531407+0000","last_undegraded":"2024-09-21T09:55:05.531407+0000","last_fullsized":"2024-09-21T09:55:05.531407+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1322,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530523+0000","last_change":"2024-09-21T09:50:44.880197+0000","last_active":"2024-09-21T09:55:05.530523+0000","last_peered":"2024-09-21T09:55:05.530523+0000","last_clean":"2024-09-21T09:55:05.530523+0000","last_became_active":"2024-09-21T09:50:44.879899+0000","last_became_peered":"2024-09-21T09:50:44.879899+0000","last_unstale":"2024-09-21T09:55:05.530523+0000","last_undegraded":"2024-09-21T09:55:05.530523+0000","last_fullsized":"2024-09-21T09:55:05.530523+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:23.596846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1318,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206097+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:55:05.206097+0000","last_peered":"2024-09-21T09:55:05.206097+0000","last_clean":"2024-09-21T09:55:05.206097+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:55:05.206097+0000","last_undegraded":"2024-09-21T09:55:05.206097+0000","last_fullsized":"2024-09-21T09:55:05.206097+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206384+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:55:05.206384+0000","last_peered":"2024-09-21T09:55:05.206384+0000","last_clean":"2024-09-21T09:55:05.206384+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:55:05.206384+0000","last_undegraded":"2024-09-21T09:55:05.206384+0000","last_fullsized":"2024-09-21T09:55:05.206384+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621861+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:55:04.621861+0000","last_peered":"2024-09-21T09:55:04.621861+0000","last_clean":"2024-09-21T09:55:04.621861+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:55:04.621861+0000","last_undegraded":"2024-09-21T09:55:04.621861+0000","last_fullsized":"2024-09-21T09:55:04.621861+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1032,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206860+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:55:05.206860+0000","last_peered":"2024-09-21T09:55:05.206860+0000","last_clean":"2024-09-21T09:55:05.206860+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:55:05.206860+0000","last_undegraded":"2024-09-21T09:55:05.206860+0000","last_fullsized":"2024-09-21T09:55:05.206860+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1329,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205848+0000","last_change":"2024-09-21T09:50:44.879616+0000","last_active":"2024-09-21T09:55:05.205848+0000","last_peered":"2024-09-21T09:55:05.205848+0000","last_clean":"2024-09-21T09:55:05.205848+0000","last_became_active":"2024-09-21T09:50:44.879237+0000","last_became_peered":"2024-09-21T09:50:44.879237+0000","last_unstale":"2024-09-21T09:55:05.205848+0000","last_undegraded":"2024-09-21T09:55:05.205848+0000","last_fullsized":"2024-09-21T09:55:05.205848+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:21.338486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1041,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531861+0000","last_change":"2024-09-21T09:50:44.870337+0000","last_active":"2024-09-21T09:55:05.531861+0000","last_peered":"2024-09-21T09:55:05.531861+0000","last_clean":"2024-09-21T09:55:05.531861+0000","last_became_active":"2024-09-21T09:50:44.870057+0000","last_became_peered":"2024-09-21T09:50:44.870057+0000","last_unstale":"2024-09-21T09:55:05.531861+0000","last_undegraded":"2024-09-21T09:55:05.531861+0000","last_fullsized":"2024-09-21T09:55:05.531861+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:38:22.969900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1340,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205939+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:55:05.205939+0000","last_peered":"2024-09-21T09:55:05.205939+0000","last_clean":"2024-09-21T09:55:05.205939+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:55:05.205939+0000","last_undegraded":"2024-09-21T09:55:05.205939+0000","last_fullsized":"2024-09-21T09:55:05.205939+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1307,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205656+0000","last_change":"2024-09-21T09:49:38.833392+0000","last_active":"2024-09-21T09:55:05.205656+0000","last_peered":"2024-09-21T09:55:05.205656+0000","last_clean":"2024-09-21T09:55:05.205656+0000","last_became_active":"2024-09-21T09:49:38.832610+0000","last_became_peered":"2024-09-21T09:49:38.832610+0000","last_unstale":"2024-09-21T09:55:05.205656+0000","last_undegraded":"2024-09-21T09:55:05.205656+0000","last_fullsized":"2024-09-21T09:55:05.205656+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:26.844310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206109+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:55:05.206109+0000","last_peered":"2024-09-21T09:55:05.206109+0000","last_clean":"2024-09-21T09:55:05.206109+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:55:05.206109+0000","last_undegraded":"2024-09-21T09:55:05.206109+0000","last_fullsized":"2024-09-21T09:55:05.206109+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.698801+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:55:04.698801+0000","last_peered":"2024-09-21T09:55:04.698801+0000","last_clean":"2024-09-21T09:55:04.698801+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:55:04.698801+0000","last_undegraded":"2024-09-21T09:55:04.698801+0000","last_fullsized":"2024-09-21T09:55:04.698801+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1334,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531537+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:55:05.531537+0000","last_peered":"2024-09-21T09:55:05.531537+0000","last_clean":"2024-09-21T09:55:05.531537+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:55:05.531537+0000","last_undegraded":"2024-09-21T09:55:05.531537+0000","last_fullsized":"2024-09-21T09:55:05.531537+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1278,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530863+0000","last_change":"2024-09-21T09:50:44.875761+0000","last_active":"2024-09-21T09:55:05.530863+0000","last_peered":"2024-09-21T09:55:05.530863+0000","last_clean":"2024-09-21T09:55:05.530863+0000","last_became_active":"2024-09-21T09:50:44.875333+0000","last_became_peered":"2024-09-21T09:50:44.875333+0000","last_unstale":"2024-09-21T09:55:05.530863+0000","last_undegraded":"2024-09-21T09:55:05.530863+0000","last_fullsized":"2024-09-21T09:55:05.530863+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:51.443596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205905+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:55:05.205905+0000","last_peered":"2024-09-21T09:55:05.205905+0000","last_clean":"2024-09-21T09:55:05.205905+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:55:05.205905+0000","last_undegraded":"2024-09-21T09:55:05.205905+0000","last_fullsized":"2024-09-21T09:55:05.205905+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1275,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531830+0000","last_change":"2024-09-21T09:50:44.874771+0000","last_active":"2024-09-21T09:55:05.531830+0000","last_peered":"2024-09-21T09:55:05.531830+0000","last_clean":"2024-09-21T09:55:05.531830+0000","last_became_active":"2024-09-21T09:50:44.873380+0000","last_became_peered":"2024-09-21T09:50:44.873380+0000","last_unstale":"2024-09-21T09:55:05.531830+0000","last_undegraded":"2024-09-21T09:55:05.531830+0000","last_fullsized":"2024-09-21T09:55:05.531830+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:19:11.264697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1348,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531680+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:55:05.531680+0000","last_peered":"2024-09-21T09:55:05.531680+0000","last_clean":"2024-09-21T09:55:05.531680+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:55:05.531680+0000","last_undegraded":"2024-09-21T09:55:05.531680+0000","last_fullsized":"2024-09-21T09:55:05.531680+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.016056+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:55:04.016056+0000","last_peered":"2024-09-21T09:55:04.016056+0000","last_clean":"2024-09-21T09:55:04.016056+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:55:04.016056+0000","last_undegraded":"2024-09-21T09:55:04.016056+0000","last_fullsized":"2024-09-21T09:55:04.016056+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1016,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206752+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:55:05.206752+0000","last_peered":"2024-09-21T09:55:05.206752+0000","last_clean":"2024-09-21T09:55:05.206752+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:55:05.206752+0000","last_undegraded":"2024-09-21T09:55:05.206752+0000","last_fullsized":"2024-09-21T09:55:05.206752+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206779+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:55:05.206779+0000","last_peered":"2024-09-21T09:55:05.206779+0000","last_clean":"2024-09-21T09:55:05.206779+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:55:05.206779+0000","last_undegraded":"2024-09-21T09:55:05.206779+0000","last_fullsized":"2024-09-21T09:55:05.206779+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1039,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.206376+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:55:05.206376+0000","last_peered":"2024-09-21T09:55:05.206376+0000","last_clean":"2024-09-21T09:55:05.206376+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:55:05.206376+0000","last_undegraded":"2024-09-21T09:55:05.206376+0000","last_fullsized":"2024-09-21T09:55:05.206376+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205508+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:55:05.205508+0000","last_peered":"2024-09-21T09:55:05.205508+0000","last_clean":"2024-09-21T09:55:05.205508+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:55:05.205508+0000","last_undegraded":"2024-09-21T09:55:05.205508+0000","last_fullsized":"2024-09-21T09:55:05.205508+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1012,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531552+0000","last_change":"2024-09-21T09:50:44.870735+0000","last_active":"2024-09-21T09:55:05.531552+0000","last_peered":"2024-09-21T09:55:05.531552+0000","last_clean":"2024-09-21T09:55:05.531552+0000","last_became_active":"2024-09-21T09:50:44.870370+0000","last_became_peered":"2024-09-21T09:50:44.870370+0000","last_unstale":"2024-09-21T09:55:05.531552+0000","last_undegraded":"2024-09-21T09:55:05.531552+0000","last_fullsized":"2024-09-21T09:55:05.531552+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:52.476737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1025,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531467+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:55:05.531467+0000","last_peered":"2024-09-21T09:55:05.531467+0000","last_clean":"2024-09-21T09:55:05.531467+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:55:05.531467+0000","last_undegraded":"2024-09-21T09:55:05.531467+0000","last_fullsized":"2024-09-21T09:55:05.531467+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1254,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532139+0000","last_change":"2024-09-21T09:50:44.874933+0000","last_active":"2024-09-21T09:55:05.532139+0000","last_peered":"2024-09-21T09:55:05.532139+0000","last_clean":"2024-09-21T09:55:05.532139+0000","last_became_active":"2024-09-21T09:50:44.870699+0000","last_became_peered":"2024-09-21T09:50:44.870699+0000","last_unstale":"2024-09-21T09:55:05.532139+0000","last_undegraded":"2024-09-21T09:55:05.532139+0000","last_fullsized":"2024-09-21T09:55:05.532139+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:12.912457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1342,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535069+0000","last_change":"2024-09-21T09:50:44.872004+0000","last_active":"2024-09-21T09:55:05.535069+0000","last_peered":"2024-09-21T09:55:05.535069+0000","last_clean":"2024-09-21T09:55:05.535069+0000","last_became_active":"2024-09-21T09:50:44.871706+0000","last_became_peered":"2024-09-21T09:50:44.871706+0000","last_unstale":"2024-09-21T09:55:05.535069+0000","last_undegraded":"2024-09-21T09:55:05.535069+0000","last_fullsized":"2024-09-21T09:55:05.535069+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:00.145938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531786+0000","last_change":"2024-09-21T09:50:44.874472+0000","last_active":"2024-09-21T09:55:05.531786+0000","last_peered":"2024-09-21T09:55:05.531786+0000","last_clean":"2024-09-21T09:55:05.531786+0000","last_became_active":"2024-09-21T09:50:44.872792+0000","last_became_peered":"2024-09-21T09:50:44.872792+0000","last_unstale":"2024-09-21T09:55:05.531786+0000","last_undegraded":"2024-09-21T09:55:05.531786+0000","last_fullsized":"2024-09-21T09:55:05.531786+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:52.873552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.249921+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:55:05.249921+0000","last_peered":"2024-09-21T09:55:05.249921+0000","last_clean":"2024-09-21T09:55:05.249921+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:55:05.249921+0000","last_undegraded":"2024-09-21T09:55:05.249921+0000","last_fullsized":"2024-09-21T09:55:05.249921+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1282,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531952+0000","last_change":"2024-09-21T09:50:44.875450+0000","last_active":"2024-09-21T09:55:05.531952+0000","last_peered":"2024-09-21T09:55:05.531952+0000","last_clean":"2024-09-21T09:55:05.531952+0000","last_became_active":"2024-09-21T09:50:44.874171+0000","last_became_peered":"2024-09-21T09:50:44.874171+0000","last_unstale":"2024-09-21T09:55:05.531952+0000","last_undegraded":"2024-09-21T09:55:05.531952+0000","last_fullsized":"2024-09-21T09:55:05.531952+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:37.106157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1345,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531477+0000","last_change":"2024-09-21T09:50:44.871526+0000","last_active":"2024-09-21T09:55:05.531477+0000","last_peered":"2024-09-21T09:55:05.531477+0000","last_clean":"2024-09-21T09:55:05.531477+0000","last_became_active":"2024-09-21T09:50:44.871236+0000","last_became_peered":"2024-09-21T09:50:44.871236+0000","last_unstale":"2024-09-21T09:55:05.531477+0000","last_undegraded":"2024-09-21T09:55:05.531477+0000","last_fullsized":"2024-09-21T09:55:05.531477+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:51.749231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.205429+0000","last_change":"2024-09-21T09:50:44.872712+0000","last_active":"2024-09-21T09:55:05.205429+0000","last_peered":"2024-09-21T09:55:05.205429+0000","last_clean":"2024-09-21T09:55:05.205429+0000","last_became_active":"2024-09-21T09:50:44.872368+0000","last_became_peered":"2024-09-21T09:50:44.872368+0000","last_unstale":"2024-09-21T09:55:05.205429+0000","last_undegraded":"2024-09-21T09:55:05.205429+0000","last_fullsized":"2024-09-21T09:55:05.205429+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:16.003164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1266,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531019+0000","last_change":"2024-09-21T09:50:44.877609+0000","last_active":"2024-09-21T09:55:05.531019+0000","last_peered":"2024-09-21T09:55:05.531019+0000","last_clean":"2024-09-21T09:55:05.531019+0000","last_became_active":"2024-09-21T09:50:44.876347+0000","last_became_peered":"2024-09-21T09:50:44.876347+0000","last_unstale":"2024-09-21T09:55:05.531019+0000","last_undegraded":"2024-09-21T09:55:05.531019+0000","last_fullsized":"2024-09-21T09:55:05.531019+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:24.344138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1046,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837776+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:55:03.837776+0000","last_peered":"2024-09-21T09:55:03.837776+0000","last_clean":"2024-09-21T09:55:03.837776+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:55:03.837776+0000","last_undegraded":"2024-09-21T09:55:03.837776+0000","last_fullsized":"2024-09-21T09:55:03.837776+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.531639+0000","last_change":"2024-09-21T09:50:44.879265+0000","last_active":"2024-09-21T09:55:05.531639+0000","last_peered":"2024-09-21T09:55:05.531639+0000","last_clean":"2024-09-21T09:55:05.531639+0000","last_became_active":"2024-09-21T09:50:44.878975+0000","last_became_peered":"2024-09-21T09:50:44.878975+0000","last_unstale":"2024-09-21T09:55:05.531639+0000","last_undegraded":"2024-09-21T09:55:05.531639+0000","last_fullsized":"2024-09-21T09:55:05.531639+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:19:33.954417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1335,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532013+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:55:05.532013+0000","last_peered":"2024-09-21T09:55:05.532013+0000","last_clean":"2024-09-21T09:55:05.532013+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:55:05.532013+0000","last_undegraded":"2024-09-21T09:55:05.532013+0000","last_fullsized":"2024-09-21T09:55:05.532013+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837921+0000","last_change":"2024-09-21T09:50:44.866370+0000","last_active":"2024-09-21T09:55:03.837921+0000","last_peered":"2024-09-21T09:55:03.837921+0000","last_clean":"2024-09-21T09:55:03.837921+0000","last_became_active":"2024-09-21T09:50:44.865582+0000","last_became_peered":"2024-09-21T09:50:44.865582+0000","last_unstale":"2024-09-21T09:55:03.837921+0000","last_undegraded":"2024-09-21T09:55:03.837921+0000","last_fullsized":"2024-09-21T09:55:03.837921+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:06:23.001592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1026,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621778+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:55:04.621778+0000","last_peered":"2024-09-21T09:55:04.621778+0000","last_clean":"2024-09-21T09:55:04.621778+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:55:04.621778+0000","last_undegraded":"2024-09-21T09:55:04.621778+0000","last_fullsized":"2024-09-21T09:55:04.621778+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1044,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837680+0000","last_change":"2024-09-21T09:50:44.876517+0000","last_active":"2024-09-21T09:55:03.837680+0000","last_peered":"2024-09-21T09:55:03.837680+0000","last_clean":"2024-09-21T09:55:03.837680+0000","last_became_active":"2024-09-21T09:50:44.876186+0000","last_became_peered":"2024-09-21T09:50:44.876186+0000","last_unstale":"2024-09-21T09:55:03.837680+0000","last_undegraded":"2024-09-21T09:55:03.837680+0000","last_fullsized":"2024-09-21T09:55:03.837680+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:42.875674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1011,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:04.621557+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:55:04.621557+0000","last_peered":"2024-09-21T09:55:04.621557+0000","last_clean":"2024-09-21T09:55:04.621557+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:55:04.621557+0000","last_undegraded":"2024-09-21T09:55:04.621557+0000","last_fullsized":"2024-09-21T09:55:04.621557+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1272,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.532459+0000","last_change":"2024-09-21T09:50:44.878358+0000","last_active":"2024-09-21T09:55:05.532459+0000","last_peered":"2024-09-21T09:55:05.532459+0000","last_clean":"2024-09-21T09:55:05.532459+0000","last_became_active":"2024-09-21T09:50:44.877700+0000","last_became_peered":"2024-09-21T09:50:44.877700+0000","last_unstale":"2024-09-21T09:55:05.532459+0000","last_undegraded":"2024-09-21T09:55:05.532459+0000","last_fullsized":"2024-09-21T09:55:05.532459+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:39:47.027086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1260,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.530581+0000","last_change":"2024-09-21T09:50:44.878043+0000","last_active":"2024-09-21T09:55:05.530581+0000","last_peered":"2024-09-21T09:55:05.530581+0000","last_clean":"2024-09-21T09:55:05.530581+0000","last_became_active":"2024-09-21T09:50:44.877686+0000","last_became_peered":"2024-09-21T09:50:44.877686+0000","last_unstale":"2024-09-21T09:55:05.530581+0000","last_undegraded":"2024-09-21T09:55:05.530581+0000","last_fullsized":"2024-09-21T09:55:05.530581+0000","mapping_epoch":829,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":830,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:39.584859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-09-21T09:55:03.837534+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:55:03.837534+0000","last_peered":"2024-09-21T09:55:03.837534+0000","last_clean":"2024-09-21T09:55:03.837534+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:55:03.837534+0000","last_undegraded":"2024-09-21T09:55:03.837534+0000","last_fullsized":"2024-09-21T09:55:03.837534+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1351,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-09-21T09:55:05.535225+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:55:05.535225+0000","last_peered":"2024-09-21T09:55:05.535225+0000","last_clean":"2024-09-21T09:55:05.535225+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:55:05.535225+0000","last_undegraded":"2024-09-21T09:55:05.535225+0000","last_fullsized":"2024-09-21T09:55:05.535225+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234242,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67844,"kb_used_data":10484,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677308,"statfs":{"total":95995035648,"available":95925563392,"internally_reserved":0,"allocated":10735616,"data_stored":7713886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":58718667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495879,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67832,"kb_used_data":10476,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677320,"statfs":{"total":95995035648,"available":95925575680,"internally_reserved":0,"allocated":10727424,"data_stored":7713759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":58718663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757517,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":99184,"kb_used_data":9888,"kb_used_omap":1,"kb_used_meta":89278,"kb_avail":93645968,"statfs":{"total":95995035648,"available":95893471232,"internally_reserved":0,"allocated":10125312,"data_stored":7119217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":91421126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019153,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63164,"kb_used_data":9900,"kb_used_omap":1,"kb_used_meta":53246,"kb_avail":93681988,"statfs":{"total":95995035648,"available":95930355712,"internally_reserved":0,"allocated":10137600,"data_stored":7123448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":54524358},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888449,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":41896,"kb_used_data":9880,"kb_used_omap":34,"kb_used_meta":31965,"kb_avail":93703256,"statfs":{"total":95995035648,"available":95952134144,"internally_reserved":0,"allocated":10117120,"data_stored":7118648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":32733171},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967688,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46124,"kb_used_data":9884,"kb_used_omap":41,"kb_used_meta":36182,"kb_avail":93699028,"statfs":{"total":95995035648,"available":95947804672,"internally_reserved":0,"allocated":10121216,"data_stored":7119174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":37050783},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046927,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50276,"kb_used_data":9880,"kb_used_omap":41,"kb_used_meta":40342,"kb_avail":93694876,"statfs":{"total":95995035648,"available":95943553024,"internally_reserved":0,"allocated":10117120,"data_stored":7118577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":41311145},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126165,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43184,"kb_used_data":10464,"kb_used_omap":72,"kb_used_meta":32631,"kb_avail":93701968,"statfs":{"total":95995035648,"available":95950815232,"internally_reserved":0,"allocated":10715136,"data_stored":7709530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":33414187},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:56:04.354 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:56:04.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump 2024-09-21T09:56:04.557 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:04.984 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:04 smithi076 ceph-mon[106482]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:04.984 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:04 smithi076 ceph-mon[110963]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:04 smithi191 ceph-mon[102206]: from='client.34605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:56:06.005 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:05 smithi076 ceph-mon[106482]: pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:06.005 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:05 smithi076 ceph-mon[110963]: pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:05 smithi191 ceph-mon[102206]: pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:06.404 INFO:teuthology.orchestra.run.smithi076.stdout:epoch 1084 2024-09-21T09:56:06.404 INFO:teuthology.orchestra.run.smithi076.stdout:fsid e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:created 2024-09-21T09:29:53.043145+0000 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:modified 2024-09-21T09:55:05.193768+0000 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:crush_version 22 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:full_ratio 0.95 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:backfillfull_ratio 0.9 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:nearfull_ratio 0.85 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:require_min_compat_client luminous 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:min_compat_client luminous 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:require_osd_release reef 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:stretch_mode_enabled false 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-21T09:56:06.405 INFO:teuthology.orchestra.run.smithi076.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 65 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:max_osd 8 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0 up in weight 1 up_from 730 up_thru 829 down_at 718 last_clean_interval [8,717) [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944] [v2:172.21.15.76:6804/724858944,v1:172.21.15.76:6805/724858944] exists,up e79ed056-c2aa-44c5-915a-5680ca7c081e 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 up in weight 1 up_from 763 up_thru 829 down_at 751 last_clean_interval [13,750) [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950] [v2:172.21.15.76:6812/46966950,v1:172.21.15.76:6813/46966950] exists,up 1de58204-a1cd-4ff3-96f7-e03b83aa0041 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2 up in weight 1 up_from 796 up_thru 829 down_at 784 last_clean_interval [18,783) [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517] [v2:172.21.15.76:6820/3488608517,v1:172.21.15.76:6821/3488608517] exists,up f5de7797-a6a1-4301-8efc-5fe9a50da8cb 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 up in weight 1 up_from 829 up_thru 829 down_at 817 last_clean_interval [25,816) [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128] [v2:172.21.15.76:6828/523580128,v1:172.21.15.76:6829/523580128] exists,up 357ff6e9-4d95-42ac-8a3f-a9f3c4a48835 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4 up in weight 1 up_from 30 up_thru 829 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433] [v2:172.21.15.191:6802/2797067433,v1:172.21.15.191:6803/2797067433] exists,up c25d7b87-1abd-4472-af56-e090a83c5835 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5 up in weight 1 up_from 38 up_thru 829 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840] [v2:172.21.15.191:6810/4066883840,v1:172.21.15.191:6811/4066883840] exists,up 68bcb809-42fe-4390-ae82-4ab0f2f9c54e 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6 up in weight 1 up_from 46 up_thru 817 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952] [v2:172.21.15.191:6818/2601453952,v1:172.21.15.191:6819/2601453952] exists,up 0592317d-31bb-4c86-81c9-5f134a98cba0 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7 up in weight 1 up_from 54 up_thru 829 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071] [v2:172.21.15.191:6826/3899643071,v1:172.21.15.191:6827/3899643071] exists,up 08ee88fa-c070-487e-96c8-d85878cc1249 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:pg_upmap_items 4.b [1,5] 2024-09-21T09:56:06.406 INFO:teuthology.orchestra.run.smithi076.stdout:pg_upmap_items 4.18 [1,2] 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:pg_upmap_items 4.1f [1,2] 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3129788088 expires 2024-09-22T09:30:15.293339+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:6835/2769276191 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6800/2955791868 expires 2024-09-22T09:30:38.364115+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/659358277 expires 2024-09-22T09:30:38.364115+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6801/771074100 expires 2024-09-22T09:35:18.636475+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6800/771074100 expires 2024-09-22T09:35:18.636475+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3280887082 expires 2024-09-22T09:35:18.636475+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/4005379925 expires 2024-09-22T09:35:18.636475+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6800/2928923484 expires 2024-09-22T09:42:40.197673+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:0/1830630483 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6801/2928923484 expires 2024-09-22T09:42:40.197673+0000 2024-09-21T09:56:06.407 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3937727837 expires 2024-09-22T09:42:40.197673+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:0/1931405138 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6801/606566558 expires 2024-09-22T09:30:15.293339+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/4144106685 expires 2024-09-22T09:42:40.197673+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:6834/2769276191 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/720143062 expires 2024-09-22T09:30:15.293339+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/1946495829 expires 2024-09-22T09:30:15.293339+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/1616224256 expires 2024-09-22T09:39:37.220289+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:0/1868415942 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3270182435 expires 2024-09-22T09:42:40.197673+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:0/3629581092 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/1280485976 expires 2024-09-22T09:39:37.220289+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3625371489 expires 2024-09-22T09:35:18.636475+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6801/23865699 expires 2024-09-22T09:39:37.220289+0000 2024-09-21T09:56:06.408 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/4089488545 expires 2024-09-22T09:35:18.636475+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:0/4175697025 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6800/606566558 expires 2024-09-22T09:30:15.293339+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6801/2955791868 expires 2024-09-22T09:30:38.364115+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.191:0/739062149 expires 2024-09-22T09:40:06.660926+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/2990785964 expires 2024-09-22T09:30:38.364115+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3392089630 expires 2024-09-22T09:39:37.220289+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:6800/23865699 expires 2024-09-22T09:39:37.220289+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/200756880 expires 2024-09-22T09:30:38.364115+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/3117602468 expires 2024-09-22T09:39:37.220289+0000 2024-09-21T09:56:06.409 INFO:teuthology.orchestra.run.smithi076.stdout:blocklist 172.21.15.76:0/1469488034 expires 2024-09-22T09:42:40.197673+0000 2024-09-21T09:56:07.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1641865835' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T09:56:07.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1641865835' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T09:56:07.214 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 730 up_thru 829 down_at 718 last_clean_interval [8,717) [v2:172.21.15.76:6802/724858944,v1:172.21.15.76:6803/724858944] [v2:172.21.15.76:6804/724858944,v1:172.21.15.76:6805/724858944] exists,up e79ed056-c2aa-44c5-915a-5680ca7c081e', 'osd.1 up in weight 1 up_from 763 up_thru 829 down_at 751 last_clean_interval [13,750) [v2:172.21.15.76:6810/46966950,v1:172.21.15.76:6811/46966950] [v2:172.21.15.76:6812/46966950,v1:172.21.15.76:6813/46966950] exists,up 1de58204-a1cd-4ff3-96f7-e03b83aa0041', 'osd.2 up in weight 1 up_from 796 up_thru 829 down_at 784 last_clean_interval [18,783) [v2:172.21.15.76:6818/3488608517,v1:172.21.15.76:6819/3488608517] [v2:172.21.15.76:6820/3488608517,v1:172.21.15.76:6821/3488608517] exists,up f5de7797-a6a1-4301-8efc-5fe9a50da8cb', 'osd.3 up in weight 1 up_from 829 up_thru 829 down_at 817 last_clean_interval [25,816) [v2:172.21.15.76:6826/523580128,v1:172.21.15.76:6827/523580128] [v2:172.21.15.76:6828/523580128,v1:172.21.15.76:6829/523580128] exists,up 357ff6e9-4d95-42ac-8a3f-a9f3c4a48835', 'osd.4 up in weight 1 up_from 30 up_thru 829 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6800/2797067433,v1:172.21.15.191:6801/2797067433] [v2:172.21.15.191:6802/2797067433,v1:172.21.15.191:6803/2797067433] exists,up c25d7b87-1abd-4472-af56-e090a83c5835', 'osd.5 up in weight 1 up_from 38 up_thru 829 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6808/4066883840,v1:172.21.15.191:6809/4066883840] [v2:172.21.15.191:6810/4066883840,v1:172.21.15.191:6811/4066883840] exists,up 68bcb809-42fe-4390-ae82-4ab0f2f9c54e', 'osd.6 up in weight 1 up_from 46 up_thru 817 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6816/2601453952,v1:172.21.15.191:6817/2601453952] [v2:172.21.15.191:6818/2601453952,v1:172.21.15.191:6819/2601453952] exists,up 0592317d-31bb-4c86-81c9-5f134a98cba0', 'osd.7 up in weight 1 up_from 54 up_thru 829 down_at 0 last_clean_interval [0,0) [v2:172.21.15.191:6824/3899643071,v1:172.21.15.191:6825/3899643071] [v2:172.21.15.191:6826/3899643071,v1:172.21.15.191:6827/3899643071] exists,up 08ee88fa-c070-487e-96c8-d85878cc1249'] 2024-09-21T09:56:07.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon mon.a config show 2024-09-21T09:56:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1641865835' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-21T09:56:07.414 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:07 smithi076 ceph-mon[110963]: pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:07 smithi076 ceph-mon[106482]: pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:08.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:07 smithi191 ceph-mon[102206]: pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:09.065 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:56:09.065 INFO:teuthology.orchestra.run.smithi076.stdout: "name": "mon.a", 2024-09-21T09:56:09.065 INFO:teuthology.orchestra.run.smithi076.stdout: "cluster": "ceph", 2024-09-21T09:56:09.065 INFO:teuthology.orchestra.run.smithi076.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "admin_socket_mode": "", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_client_required": "cephx, none", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_cluster_required": "cephx", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_debug": "true", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_service_required": "cephx", 2024-09-21T09:56:09.066 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "auth_supported": "", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_aio": "true", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_aio_poll_ms": "250", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_aio_reap_max": "16", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_async_discard_threads": "0", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_block_size": "4096", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_debug_aio": "false", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_debug_inflight_ios": "false", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_enable_discard": "false", 2024-09-21T09:56:09.067 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_flock_retry": "3", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_inject_crash": "0", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_ioring": "false", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_ioring_hipri": "false", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-21T09:56:09.068 INFO:teuthology.orchestra.run.smithi076.stdout: "bdev_type": "", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_alloc_size": "1048576", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_allocator": "hybrid", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_buffered_io": "true", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_check_for_zeros": "false", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_compact_log_sync": "false", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-21T09:56:09.069 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_max_log_runway": "4194304", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_max_prefetch": "1048576", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_min_flush_size": "524288", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_min_log_runway": "1048576", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_replay_recovery": "false", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluefs_sync_write": "false", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-21T09:56:09.070 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_allocation_from_file": "true", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_allocator": "hybrid", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_async_db_compaction": "true", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bdev_label_multi": "true", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-21T09:56:09.071 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_create": "true", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_db_create": "false", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_db_path": "", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_db_size": "0", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_path": "", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_preallocate_file": "false", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_size": "107374182400", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_wal_create": "false", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_wal_path": "", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_block_wal_size": "100663296", 2024-09-21T09:56:09.072 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bluefs": "true", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_autotune": "true", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-21T09:56:09.073 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_size": "0", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cache_type": "2q", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_clone_cow": "true", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-21T09:56:09.074 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_mode": "none", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_csum_type": "crc32c", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-21T09:56:09.075 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-21T09:56:09.076 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_freelist": "false", 2024-09-21T09:56:09.076 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-21T09:56:09.077 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_small_allocations": "0", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_default_buffered_read": "true", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_default_buffered_write": "false", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-21T09:56:09.078 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fail_eio": "false", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-21T09:56:09.079 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_on_mount": "false", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_on_umount": "false", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-21T09:56:09.080 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_ignore_data_csum": "false", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_log_op_age": "5.000000", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_max_alloc_size": "0", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_max_blob_size": "0", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-21T09:56:09.081 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_max_deferred_txc": "32", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_min_alloc_size": "0", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_nid_prealloc": "1024", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-21T09:56:09.082 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-21T09:56:09.083 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_retry_disk_reads": "3", 2024-09-21T09:56:09.083 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_rocksdb_cf": "true", 2024-09-21T09:56:09.083 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-21T09:56:09.083 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_spdk_mem": "512", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-21T09:56:09.084 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-21T09:56:09.085 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-21T09:56:09.085 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_tracing": "false", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-21T09:56:09.086 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-21T09:56:09.087 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-21T09:56:09.087 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-21T09:56:09.087 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-21T09:56:09.087 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-21T09:56:09.087 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-21T09:56:09.087 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_write_v2": "false", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_write_v2_random": "false", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "bluestore_zero_block_detection": "false", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-21T09:56:09.088 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-21T09:56:09.089 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-21T09:56:09.089 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-21T09:56:09.089 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-21T09:56:09.089 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-21T09:56:09.089 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-21T09:56:09.089 INFO:teuthology.orchestra.run.smithi076.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_cluster_require_signatures": "false", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_cluster_require_version": "2", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_require_signatures": "false", 2024-09-21T09:56:09.090 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_require_version": "2", 2024-09-21T09:56:09.091 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_service_require_signatures": "false", 2024-09-21T09:56:09.091 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_service_require_version": "2", 2024-09-21T09:56:09.091 INFO:teuthology.orchestra.run.smithi076.stdout: "cephx_sign_messages": "true", 2024-09-21T09:56:09.091 INFO:teuthology.orchestra.run.smithi076.stdout: "chdir": "", 2024-09-21T09:56:09.091 INFO:teuthology.orchestra.run.smithi076.stdout: "client_acl_type": "", 2024-09-21T09:56:09.092 INFO:teuthology.orchestra.run.smithi076.stdout: "client_asio_thread_count": "2", 2024-09-21T09:56:09.092 INFO:teuthology.orchestra.run.smithi076.stdout: "client_cache_mid": "0.750000", 2024-09-21T09:56:09.092 INFO:teuthology.orchestra.run.smithi076.stdout: "client_cache_size": "16384", 2024-09-21T09:56:09.092 INFO:teuthology.orchestra.run.smithi076.stdout: "client_caps_release_delay": "5", 2024-09-21T09:56:09.092 INFO:teuthology.orchestra.run.smithi076.stdout: "client_check_pool_perm": "true", 2024-09-21T09:56:09.092 INFO:teuthology.orchestra.run.smithi076.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-21T09:56:09.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_debug_force_sync_read": "false", 2024-09-21T09:56:09.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_debug_getattr_caps": "false", 2024-09-21T09:56:09.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_debug_inject_features": "", 2024-09-21T09:56:09.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_debug_inject_tick_delay": "0", 2024-09-21T09:56:09.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-21T09:56:09.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_die_on_failed_remount": "false", 2024-09-21T09:56:09.094 INFO:teuthology.orchestra.run.smithi076.stdout: "client_dirsize_rbytes": "true", 2024-09-21T09:56:09.094 INFO:teuthology.orchestra.run.smithi076.stdout: "client_force_lazyio": "false", 2024-09-21T09:56:09.094 INFO:teuthology.orchestra.run.smithi076.stdout: "client_fs": "", 2024-09-21T09:56:09.094 INFO:teuthology.orchestra.run.smithi076.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-21T09:56:09.094 INFO:teuthology.orchestra.run.smithi076.stdout: "client_inject_release_failure": "false", 2024-09-21T09:56:09.094 INFO:teuthology.orchestra.run.smithi076.stdout: "client_max_inline_size": "4096", 2024-09-21T09:56:09.095 INFO:teuthology.orchestra.run.smithi076.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-21T09:56:09.095 INFO:teuthology.orchestra.run.smithi076.stdout: "client_mds_namespace": "", 2024-09-21T09:56:09.095 INFO:teuthology.orchestra.run.smithi076.stdout: "client_metadata": "", 2024-09-21T09:56:09.095 INFO:teuthology.orchestra.run.smithi076.stdout: "client_mount_gid": "-1", 2024-09-21T09:56:09.095 INFO:teuthology.orchestra.run.smithi076.stdout: "client_mount_timeout": "300", 2024-09-21T09:56:09.095 INFO:teuthology.orchestra.run.smithi076.stdout: "client_mount_uid": "-1", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_mountpoint": "/", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_notify_timeout": "10", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_oc": "true", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_oc_max_dirty": "104857600", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_oc_max_objects": "1000", 2024-09-21T09:56:09.096 INFO:teuthology.orchestra.run.smithi076.stdout: "client_oc_size": "209715200", 2024-09-21T09:56:09.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_oc_target_dirty": "8388608", 2024-09-21T09:56:09.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_permissions": "true", 2024-09-21T09:56:09.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_quota": "true", 2024-09-21T09:56:09.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_quota_df": "true", 2024-09-21T09:56:09.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_readahead_max_bytes": "0", 2024-09-21T09:56:09.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_readahead_max_periods": "4", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_readahead_min": "131072", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_reconnect_stale": "false", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_shutdown_timeout": "30", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_snapdir": ".snap", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_tick_interval": "1", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_trace": "", 2024-09-21T09:56:09.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_try_dentry_invalidate": "false", 2024-09-21T09:56:09.099 INFO:teuthology.orchestra.run.smithi076.stdout: "client_use_faked_inos": "false", 2024-09-21T09:56:09.099 INFO:teuthology.orchestra.run.smithi076.stdout: "client_use_random_mds": "false", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_graylog": "false", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_graylog_port": "12201", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_monitors": "default=true", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_syslog": "false", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "clog_to_syslog_level": "info", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "cluster_addr": "-", 2024-09-21T09:56:09.100 INFO:teuthology.orchestra.run.smithi076.stdout: "cluster_network": "", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "cluster_network_interface": "", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "compressor_zlib_isal": "false", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "compressor_zlib_level": "5", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "compressor_zlib_winsize": "-15", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "compressor_zstd_level": "1", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_alien_op_num_threads": "6", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-21T09:56:09.101 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_osd_stat_interval": "0", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_seastar_cpu_cores": "", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "crimson_seastar_num_threads": "0", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "crush_location": "", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "crush_location_hook": "", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "crush_location_hook_timeout": "10", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "daemonize": "false", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "daos_pool": "tank", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-21T09:56:09.102 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_alienstore": "0/5", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_allow_any_pool_priority": "false", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_asok": "1/5", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_asok_assert_abort": "false", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_asserts_on_shutdown": "true", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_auth": "1/5", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_bdev": "1/3", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_bluefs": "1/5", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_bluestore": "1/5", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_buffer": "0/1", 2024-09-21T09:56:09.103 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_ceph_dedup": "0/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_ceph_exporter": "1/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_cephfs_mirror": "0/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_cephsqlite": "0/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_client": "0/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_compressor": "1/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_context": "0/1", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_crimson_interrupt": "0/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_crush": "1/1", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_crypto": "1/5", 2024-09-21T09:56:09.104 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_cyanstore": "0/5", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_deliberately_leak_memory": "false", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_disable_randomized_ping": "false", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_dpdk": "1/5", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_eventtrace": "1/5", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_filer": "0/1", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_filestore": "1/3", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_finisher": "1/1", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_fuse": "1/5", 2024-09-21T09:56:09.105 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_heartbeat_testing_span": "0", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_heartbeatmap": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_javaclient": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_journal": "1/3", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_journaler": "0/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_kstore": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_lockdep": "0/1", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mclock": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds_balancer": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds_locker": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds_log": "1/5", 2024-09-21T09:56:09.106 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds_log_expire": "1/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds_migrator": "1/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mds_quiesce": "3/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_memstore": "1/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mgr": "2/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mgrc": "1/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_mon": "20/20", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_monc": "0/10", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_ms": "1/1", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_none": "0/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_objclass": "0/5", 2024-09-21T09:56:09.107 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_objectcacher": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_objecter": "0/1", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_optracker": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_osd": "1/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_paxos": "20/20", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_perfcounter": "1/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_prioritycache": "1/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rados": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rbd": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rbd_mirror": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rbd_pwl": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rbd_replay": "0/5", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_refs": "0/0", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_reserver": "1/1", 2024-09-21T09:56:09.108 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_access": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_datacache": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_dbstore": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_flight": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_notification": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rgw_sync": "1/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_rocksdb": "4/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore": "0/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_backref": "0/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_cache": "0/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_cleaner": "0/5", 2024-09-21T09:56:09.109 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_device": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_epm": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_journal": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_lba": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_odata": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_omap": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_onode": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_t": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_seastore_tm": "0/5", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_striper": "0/1", 2024-09-21T09:56:09.110 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_test": "0/5", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_throttle": "1/1", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_timer": "0/1", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_tp": "0/5", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "debug_trace": "1/5", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "device_failure_prediction_mode": "none", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "err_to_graylog": "false", 2024-09-21T09:56:09.111 INFO:teuthology.orchestra.run.smithi076.stdout: "err_to_journald": "false", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "err_to_stderr": "true", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "err_to_syslog": "false", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event_tracing": "false", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_addr": "0.0.0.0", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_cert_file": "", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_http_port": "9926", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_key_file": "", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_prio_limit": "5", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_sort_metrics": "true", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "exporter_stats_period": "5", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "fake_statfs_for_testing": "0", 2024-09-21T09:56:09.112 INFO:teuthology.orchestra.run.smithi076.stdout: "fatal_signal_handlers": "true", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filer_max_purge_ops": "10", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filer_max_truncate_ops": "128", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_apply_finisher_threads": "1", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_blackhole": "false", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_btrfs_clone_range": "true", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_btrfs_snap": "true", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_caller_concurrency": "10", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_collect_device_partition_information": "true", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_commit_timeout": "600.000000", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_debug_inject_read_err": "false", 2024-09-21T09:56:09.113 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_debug_omap_check": "false", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_debug_verify_split": "false", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_dump_file": "", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fadvise": "true", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fail_eio": "true", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fd_cache_shards": "16", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fd_cache_size": "128", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fiemap": "false", 2024-09-21T09:56:09.114 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fiemap_threshold": "4096", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_inject_stall": "0", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_journal_parallel": "false", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_journal_trailing": "false", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_journal_writeahead": "false", 2024-09-21T09:56:09.115 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_kill_at": "0", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattrs": "0", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-21T09:56:09.116 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_xattr_value_size": "0", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_merge_threshold": "-10", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_odsync_write": "false", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_omap_backend": "rocksdb", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_omap_backend_path": "", 2024-09-21T09:56:09.117 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_op_thread_timeout": "60", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_op_threads": "2", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_punch_hole": "false", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-21T09:56:09.118 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_queue_max_ops": "50", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-21T09:56:09.119 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_seek_data_hole": "false", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_sloppy_crc": "false", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_splice": "false", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_split_multiple": "2", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_split_rand_factor": "20", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_update_to": "1000", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-21T09:56:09.120 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_enable": "true", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-21T09:56:09.121 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_xfs_extsize": "false", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "filestore_zfs_snap": "false", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fio_dir": "/tmp/fio", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fsid": "e97bd93e-77fb-11ef-baf6-efdc52797490", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_allow_other": "true", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_atomic_o_trunc": "true", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_big_writes": "true", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_debug": "false", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_default_permissions": "false", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_disable_pagecache": "false", 2024-09-21T09:56:09.122 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_max_write": "0", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_multithreaded": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_require_active_mds": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_set_user_groups": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_splice_move": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_splice_read": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_splice_write": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "fuse_use_invalidate_cb": "true", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "gss_target_name": "ceph", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "heartbeat_file": "", 2024-09-21T09:56:09.123 INFO:teuthology.orchestra.run.smithi076.stdout: "heartbeat_inject_failure": "0", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "heartbeat_interval": "5", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "host": "smithi076", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-21T09:56:09.124 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "inject_early_sigterm": "false", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "jaeger_agent_port": "6799", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "jaeger_tracing_enable": "false", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_aio": "true", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_align_min_size": "65536", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_block_align": "true", 2024-09-21T09:56:09.125 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_block_size": "4096", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_dio": "true", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_discard": "false", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_force_aio": "false", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_ignore_corruption": "false", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_max_write_bytes": "10485760", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_max_write_entries": "100", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_replay_from": "0", 2024-09-21T09:56:09.126 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_write_header_frequency": "0", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journal_zero_on_create": "false", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journaler_prefetch_periods": "10", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journaler_prezero_periods": "5", 2024-09-21T09:56:09.127 INFO:teuthology.orchestra.run.smithi076.stdout: "journaler_write_head_interval": "15", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "key": "", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "keyfile": "", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_backend": "rocksdb", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_default_stripe_size": "65536", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_fsck_on_mount": "false", 2024-09-21T09:56:09.128 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_max_bytes": "67108864", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_max_ops": "512", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_nid_prealloc": "1024", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_onode_map_size": "1024", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_sync_submit_transaction": "false", 2024-09-21T09:56:09.129 INFO:teuthology.orchestra.run.smithi076.stdout: "kstore_sync_transaction": "false", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "librados_thread_count": "2", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "lockdep": "false", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "lockdep_force_backtrace": "false", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "log_coarse_timestamps": "true", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "log_flush_on_exit": "false", 2024-09-21T09:56:09.130 INFO:teuthology.orchestra.run.smithi076.stdout: "log_graylog_host": "127.0.0.1", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_graylog_port": "12201", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_max_new": "1000", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_max_recent": "10000", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_stderr_prefix": "", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_stop_at_utilization": "0.970000", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_to_file": "true", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_to_graylog": "false", 2024-09-21T09:56:09.131 INFO:teuthology.orchestra.run.smithi076.stdout: "log_to_journald": "false", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "log_to_stderr": "false", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "log_to_syslog": "false", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "max_rotating_auth_attempts": "10", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_action_on_write_error": "1", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_alternate_name_max": "8192", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_asio_thread_count": "2", 2024-09-21T09:56:09.132 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_export_pin": "true", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_fragment_dirs": "true", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_fragment_interval": "5", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_interval": "10", 2024-09-21T09:56:09.133 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_max": "-1", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_max_until": "-1", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_merge_size": "50", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_midchunk": "0.300000", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_min_start": "0.200000", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_minchunk": "0.001000", 2024-09-21T09:56:09.134 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_mode": "0", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_need_max": "1.200000", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_need_min": "0.800000", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_overload_epochs": "2", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_split_bits": "3", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-21T09:56:09.135 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_split_size": "10000", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_target_decay": "10.000000", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_beacon_grace": "15.000000", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_beacon_interval": "4.000000", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-21T09:56:09.136 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_mid": "0.700000", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_quiesce_delay": "0", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_release_free_interval": "10", 2024-09-21T09:56:09.137 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_reservation": "0.050000", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_trim_interval": "1", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cache_trim_threshold": "262144", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-21T09:56:09.138 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_client_prealloc_inos": "1000", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_connect_bootstrapping": "false", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_damage_table_max_entries": "10000", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_debug_auth_pins": "false", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_debug_frag": "false", 2024-09-21T09:56:09.139 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_debug_scatterstat": "false", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_debug_subtrees": "false", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_decay_halflife": "5.000000", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_default_dir_hash": "2", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_defer_session_stale": "true", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_deny_all_reconnect": "false", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dir_keys_per_op": "16384", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dir_max_commit_size": "10", 2024-09-21T09:56:09.140 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dir_max_entries": "0", 2024-09-21T09:56:09.141 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dir_prefetch": "true", 2024-09-21T09:56:09.141 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-21T09:56:09.141 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-21T09:56:09.141 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dump_cache_on_map": "false", 2024-09-21T09:56:09.141 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-21T09:56:09.141 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-21T09:56:09.142 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_early_reply": "true", 2024-09-21T09:56:09.142 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_enable_op_tracker": "true", 2024-09-21T09:56:09.142 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_enforce_unique_name": "true", 2024-09-21T09:56:09.142 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-21T09:56:09.142 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_export_ephemeral_random": "true", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-21T09:56:09.143 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_health_summarize_threshold": "10", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_inject_health_dummy": "false", 2024-09-21T09:56:09.144 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_inject_migrator_session_race": "false", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_join_fs": "", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_journal_format": "1", 2024-09-21T09:56:09.145 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_create_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_dirfrag_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_export_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_import_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_journal_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_journal_expire_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_journal_replay_at": "0", 2024-09-21T09:56:09.146 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_link_at": "0", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_mdstable_at": "0", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_openc_at": "0", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_rename_at": "0", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_kill_shutdown_at": "0", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_event_large_threshold": "524288", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_events_per_segment": "1024", 2024-09-21T09:56:09.147 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_max_events": "-1", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_max_segments": "128", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_pause": "false", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_segment_size": "0", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-21T09:56:09.148 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_trim_threshold": "128", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_log_warn_factor": "2.000000", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_caps_per_client": "1048576", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_completed_flushes": "100000", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_completed_requests": "100000", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_export_size": "20971520", 2024-09-21T09:56:09.149 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_file_recover": "32", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_purge_files": "64", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_purge_ops": "8192", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_max_snaps_per_dir": "100", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_metrics_update_interval": "2", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_min_caps_per_client": "100", 2024-09-21T09:56:09.150 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_min_caps_working_set": "10000", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_numa_node": "-1", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_op_complaint_time": "30.000000", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_op_history_duration": "600", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_op_history_size": "20", 2024-09-21T09:56:09.151 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_op_history_slow_op_size": "20", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_op_log_threshold": "5", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_ping_grace": "15", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_ping_interval": "5", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_recall_max_caps": "30000", 2024-09-21T09:56:09.152 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_recall_warning_threshold": "262144", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_replay_interval": "1.000000", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-21T09:56:09.153 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_root_ino_gid": "0", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_root_ino_uid": "0", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_scrub_stats_review_period": "1", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-21T09:56:09.154 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_shutdown_check": "0", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_skip_ino": "0", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_snap_max_uid": "4294967294", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_snap_min_uid": "0", 2024-09-21T09:56:09.155 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_snap_rstat": "false", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_standby_replay_damaged": "false", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_symlink_recovery": "true", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_thrash_exports": "0", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_thrash_fragments": "0", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_tick_interval": "5.000000", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_valgrind_exit": "false", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_verify_backtrace": "1", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_verify_scatter": "false", 2024-09-21T09:56:09.156 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "mds_wipe_sessions": "false", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "mempool_debug": "false", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "memstore_device_bytes": "1073741824", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "memstore_page_set": "false", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "memstore_page_size": "65536", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_client_bytes": "134217728", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_client_messages": "512", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-21T09:56:09.157 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_disabled_modules": "", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_enable_op_tracker": "true", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_max_pg_creating": "1024", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_max_pg_num_change": "128", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_mds_bytes": "134217728", 2024-09-21T09:56:09.158 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_mds_messages": "128", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_mon_bytes": "134217728", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_mon_messages": "128", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_op_history_duration": "600", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_op_history_size": "20", 2024-09-21T09:56:09.159 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_op_log_threshold": "5", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_osd_bytes": "536870912", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_osd_messages": "8192", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_pool": "true", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-21T09:56:09.160 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_standby_modules": "true", 2024-09-21T09:56:09.161 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_stats_period": "5", 2024-09-21T09:56:09.161 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_stats_threshold": "5", 2024-09-21T09:56:09.161 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_tick_period": "2", 2024-09-21T09:56:09.161 INFO:teuthology.orchestra.run.smithi076.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-21T09:56:09.161 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_allow_pool_delete": "true", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_allow_pool_size_one": "false", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_auth_validate_all_caps": "true", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_bytes": "104857600", 2024-09-21T09:56:09.162 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_directed_command_retry": "2", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_hunt_parallel": "3", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_log_interval": "1.000000", 2024-09-21T09:56:09.163 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_ping_interval": "10.000000", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_client_target_rank": "-1", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-21T09:56:09.164 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_level": "debug", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_file": "true", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_journald": "true", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-21T09:56:09.165 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_compact_on_bootstrap": "false", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_compact_on_start": "false", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_compact_on_trim": "true", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-21T09:56:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_cpu_threads": "4", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_crush_min_required_version": "hammer", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_daemon_bytes": "419430400", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_data_avail_crit": "5", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_data_avail_warn": "5", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_data_size_warn": "16106127360", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-21T09:56:09.167 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_dump_json": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_dump_transactions": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_extra_checks": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_no_require_reef": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_no_require_squid": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-21T09:56:09.168 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_down_mkfs_grace": "60", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_down_uptime_grace": "60", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_election_default_strategy": "1", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_election_timeout": "5.000000", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_elector_ping_divisor": "2", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-21T09:56:09.169 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_enable_op_tracker": "true", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_fake_pool_delete": "false", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_force_quorum_join": "false", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_globalid_prealloc": "10000", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_health_detail_to_clog": "true", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_health_log_update_period": "5", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_health_max_detail": "50", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_health_to_clog": "true", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_health_to_clog_interval": "600", 2024-09-21T09:56:09.170 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_host": "[v2:172.21.15.76:3300/0,v1:172.21.15.76:6789/0] [v2:172.21.15.191:3300/0,v1:172.21.15.191:6789/0] [v2:172.21.15.76:3301/0,v1:172.21.15.76:6790/0]", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_host_override": "", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_initial_members": "", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-21T09:56:09.171 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_lease": "5.000000", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_log_full_interval": "50", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_log_max": "10000", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_log_max_summary": "50", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-21T09:56:09.172 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_log_epochs": "500", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_osd": "10000", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_pg_per_osd": "10000", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_pool_pg_num": "65536", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-21T09:56:09.173 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mds_force_trim_to": "0", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mds_skip_sanity": "true", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_memory_autotune": "true", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_memory_target": "2147483648", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mgr_beacon_grace": "30", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-21T09:56:09.174 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mgr_digest_period": "5", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mgr_inactive_grace": "60", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_min_osdmap_epochs": "500", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_op_complaint_time": "30", 2024-09-21T09:56:09.175 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_op_history_duration": "600", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_op_history_size": "20", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_op_history_slow_op_size": "20", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_op_log_threshold": "5", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-21T09:56:09.176 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_auto_mark_in": "false", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_cache_size": "500", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-21T09:56:09.177 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_down_out_interval": "600", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_force_trim_to": "0", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-21T09:56:09.178 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_min_down_reporters": "2", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-21T09:56:09.179 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-21T09:56:09.180 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-21T09:56:09.180 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-21T09:56:09.180 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-21T09:56:09.180 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_report_timeout": "900", 2024-09-21T09:56:09.181 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-21T09:56:09.181 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-21T09:56:09.181 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-21T09:56:09.181 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-21T09:56:09.181 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pg_stuck_threshold": "60", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-21T09:56:09.182 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_probe_timeout": "2.000000", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_reweight_max_change": "0.050000", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_reweight_max_osds": "4", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_scrub_interval": "86400", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_scrub_max_keys": "100", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_scrub_timeout": "300", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_session_timeout": "300", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_smart_report_timeout": "5", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_stat_smooth_intervals": "6", 2024-09-21T09:56:09.183 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_stretch_pool_min_size": "2", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_stretch_pool_size": "4", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_sync_debug": "false", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_sync_provider_kill_at": "0", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_sync_requester_kill_at": "0", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_sync_timeout": "60.000000", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_target_pg_per_osd": "100", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_tick_interval": "5", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_timecheck_interval": "300.000000", 2024-09-21T09:56:09.184 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_use_min_delay_socket": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_older_version_delay": "604800", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_misplaced": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_older_version": "true", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-21T09:56:09.185 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "monmap": "", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-21T09:56:09.186 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "motr_tracing_enabled": "false", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_op_threads": "3", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_cm": "false", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_device_name": "", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_dscp": "96", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_local_gid": "", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-21T09:56:09.187 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_port_num": "1", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_sl": "3", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_support_srq": "true", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_rdma_type": "ib", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_async_reap_threshold": "5", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_before_connect": "false", 2024-09-21T09:56:09.188 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_ipv4": "true", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_ipv6": "false", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_msgr1": "true", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_msgr2": "true", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_port_max": "7568", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_port_min": "6800", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_retry_count": "3", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_bind_retry_delay": "5", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_blackhole_client": "false", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_blackhole_mds": "false", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_blackhole_mgr": "false", 2024-09-21T09:56:09.189 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_blackhole_mon": "false", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_blackhole_osd": "false", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_client_mode": "crc secure", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_cluster_mode": "crc secure", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_cluster_type": "", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_compress_secure": "false", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_connection_idle_timeout": "900", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_connection_ready_timeout": "10", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_crc_data": "true", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_crc_header": "true", 2024-09-21T09:56:09.190 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_die_on_bad_msg": "false", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_die_on_bug": "true", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_die_on_old_message": "true", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_die_on_skipped_message": "false", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_coremask": "0xF", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_enable_tso": "true", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-21T09:56:09.191 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_hugepages": "", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_lro": "true", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_memory_channel": "4", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_pmd": "", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_port_id": "0", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_dump_on_send": "false", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_initial_backoff": "0.200000", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_inject_delay_max": "1.000000", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-21T09:56:09.192 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_inject_delay_type": "", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_inject_network_congestion": "0", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_inject_socket_failures": "0", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_learn_addr_from_peer": "true", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_max_accept_failures": "4", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_max_backoff": "15.000000", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_mon_client_mode": "secure crc", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_mon_service_mode": "secure crc", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_osd_compress_min_size": "1024", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_osd_compress_mode": "none", 2024-09-21T09:56:09.193 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_pq_min_cost": "65536", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_public_type": "", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_service_mode": "crc secure", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_tcp_listen_backlog": "512", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_tcp_nodelay": "true", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_tcp_rcvbuf": "0", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "ms_type": "async+posix", 2024-09-21T09:56:09.194 INFO:teuthology.orchestra.run.smithi076.stdout: "no_config_file": "false", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_completion_locks_per_session": "32", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_inflight_ops": "1024", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_tick_interval": "5.000000", 2024-09-21T09:56:09.195 INFO:teuthology.orchestra.run.smithi076.stdout: "objecter_timeout": "10.000000", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "objectstore_blackhole": "false", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "openssl_engine_opts": "", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_delay_time": "5.000000", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_hist_halflife": "1000", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_max_low_ops": "2", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_max_ops": "4", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_agent_slop": "0.020000", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_asio_thread_count": "2", 2024-09-21T09:56:09.196 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_async_recovery_min_cost": "100", 2024-09-21T09:56:09.197 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-21T09:56:09.197 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-21T09:56:09.197 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_backfill_scan_max": "512", 2024-09-21T09:56:09.197 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_backfill_scan_min": "64", 2024-09-21T09:56:09.197 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_backoff_on_degraded": "false", 2024-09-21T09:56:09.197 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_backoff_on_peering": "false", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_backoff_on_unfound": "true", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_beacon_report_interval": "300", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_bench_duration": "30", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_bench_max_block_size": "67108864", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-21T09:56:09.198 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_blkin_trace_all": "false", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_check_for_log_corruption": "false", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_class_update_on_start": "true", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_client_message_cap": "256", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_client_message_size_cap": "524288000", 2024-09-21T09:56:09.199 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_client_op_priority": "63", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_client_watch_timeout": "30", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_command_max_records": "256", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_compact_on_start": "false", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_crush_update_on_start": "true", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_crush_update_weight_set": "true", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-21T09:56:09.200 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_feed_pullee": "-1", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_misdirected_ops": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_no_acting_change": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_no_purge_strays": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_op_order": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-21T09:56:09.201 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_shutdown": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_trim_objects": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_verify_snaps": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_keys": "1024", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-21T09:56:09.202 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_stride": "524288", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_default_notify_timeout": "30", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_delete_sleep": "0.000000", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_discard_disconnected_ops": "true", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_ec_partial_reads": "true", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_enable_op_tracker": "true", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-21T09:56:09.203 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_fast_info": "true", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_fast_shutdown": "true", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_function_tracing": "false", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_grace": "20", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_interval": "6", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_min_peers": "10", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_min_size": "2000", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_stale": "600", 2024-09-21T09:56:09.204 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_hit_set_max_size": "100000", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_hit_set_min_size": "1000", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_journal_size": "5120", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_kill_backfill_at": "0", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_map_cache_size": "50", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_map_dedup": "true", 2024-09-21T09:56:09.205 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_map_message_max": "40", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_map_share_max_epochs": "40", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_attr_name_len": "100", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_attr_size": "0", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_backfills": "1", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_markdown_count": "5", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_markdown_period": "600", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_object_name_len": "2048", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_object_namespace_len": "256", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_object_size": "134217728", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_pg_blocked_by": "16", 2024-09-21T09:56:09.206 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_pg_log_entries": "10000", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_pgls": "1024", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_push_cost": "8388608", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_push_objects": "10", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_scrubs": "3", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_trimming_pgs": "2", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_write_op_reply_len": "64", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_max_write_size": "90", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-21T09:56:09.207 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_profile": "balanced", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-21T09:56:09.208 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_base": "805306368", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_cache_min": "134217728", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_target": "4294967296", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_target_autotune": "false", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_min_pg_log_entries": "250", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_min_recovery_priority": "0", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mon_report_interval": "5", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-21T09:56:09.209 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_num_cache_shards": "32", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_num_op_tracker_shard": "32", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_numa_auto_affinity": "true", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_numa_node": "-1", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_numa_prefer_iface": "true", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_objecter_finishers": "1", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_objectstore": "bluestore", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_objectstore_fuse": "false", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_objectstore_tracing": "false", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_complaint_time": "30.000000", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_history_duration": "600", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_history_size": "20", 2024-09-21T09:56:09.210 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_history_slow_op_size": "20", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_log_threshold": "5", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_num_shards": "0", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_num_shards_hdd": "5", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_num_shards_ssd": "8", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_pq_min_cost": "65536", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_queue_cut_off": "high", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-21T09:56:09.211 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_op_thread_timeout": "15", 2024-09-21T09:56:09.212 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_open_classes_on_start": "true", 2024-09-21T09:56:09.212 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:08 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:09.212 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:08 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_os_flags": "0", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_peering_op_priority": "255", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_delete_cost": "1048576", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_delete_priority": "5", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_log_trim_max": "10000", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_log_trim_min": "100", 2024-09-21T09:56:09.213 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_crimson": "false", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-21T09:56:09.214 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_flags": "0", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_min_size": "0", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_pg_num": "32", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_pgp_num": "0", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_read_ratio": "70", 2024-09-21T09:56:09.215 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_size": "3", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_default_type": "replicated", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_push_per_object_cost": "1000", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recover_clone_overlap": "true", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_cost": "20971520", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_max_active": "0", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-21T09:56:09.216 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_max_single_start": "1", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_op_priority": "3", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_priority": "5", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_sleep": "0.000000", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_repair_during_recovery": "false", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_requested_scrub_priority": "5", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-21T09:56:09.217 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_auto_repair": "false", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_begin_hour": "0", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_begin_week_day": "0", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_chunk_max": "15", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_chunk_min": "5", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_cost": "52428800", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_during_recovery": "false", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_end_hour": "0", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_end_week_day": "0", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_event_cost": "4096", 2024-09-21T09:56:09.218 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_invalid_stats": "true", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_max_preemptions": "5", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_priority": "5", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_retry_after_noscrub": "60", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_retry_delay": "30", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_retry_new_interval": "10", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_retry_pg_state": "60", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_retry_trimming": "10", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_scrub_sleep": "0.000000", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-21T09:56:09.219 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_skip_data_digest": "false", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_smart_report_timeout": "5", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_snap_trim_cost": "1048576", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_snap_trim_priority": "5", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_target_transaction_size": "30", 2024-09-21T09:56:09.220 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_tracing": "false", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_use_stale_snap": "false", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "osdc_blkin_trace_all": "false", 2024-09-21T09:56:09.221 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_kill_at": "0", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_max_join_drift": "10", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_min": "500", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_min_wait": "0.050000", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_propose_interval": "1.000000", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_service_trim_max": "500", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_service_trim_min": "250", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_stash_full_interval": "25", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_trim_max": "500", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "paxos_trim_min": "250", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "perf": "true", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "pid_file": "", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-21T09:56:09.222 INFO:teuthology.orchestra.run.smithi076.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "public_addr": "-", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "public_addrv": "", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "public_bind_addr": "-", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "public_network": "172.21.0.0/20", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "public_network_interface": "", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "qat_compressor_busy_polling": "false", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "qat_compressor_enabled": "false", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "qat_compressor_session_max_number": "256", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "rados_mon_op_timeout": "0", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "rados_osd_op_timeout": "0", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "rados_tracing": "false", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_atime_update_interval": "60", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-21T09:56:09.223 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_balance_parent_reads": "false", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_balance_snap_reads": "false", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_blkin_trace_all": "false", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache": "true", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_policy": "writearound", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_size": "33554432", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-21T09:56:09.224 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_clone_copy_on_read": "false", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_compression_hint": "none", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_concurrent_management_ops": "10", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_clone_format": "auto", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_data_pool": "", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_features": "61", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_format": "2", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_map_options": "", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_order": "22", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_pool": "rbd", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_stripe_count": "0", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_default_stripe_unit": "0", 2024-09-21T09:56:09.225 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_enable_alloc_hint": "true", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_io_scheduler": "simple", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-21T09:56:09.226 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_order": "24", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_pool": "", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_journal_splay_width": "4", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_localize_parent_reads": "false", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_localize_snap_reads": "false", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-21T09:56:09.227 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-21T09:56:09.228 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_mtime_update_interval": "60", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_non_blocking_aio": "true", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_op_thread_timeout": "60", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_op_threads": "1", 2024-09-21T09:56:09.229 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_parent_cache_enabled": "false", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_plugins": "", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_bps_burst": "0", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_bps_limit": "0", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_exclude_ops": "0", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_iops_burst": "0", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_iops_limit": "0", 2024-09-21T09:56:09.230 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_read_from_replica_policy": "default", 2024-09-21T09:56:09.231 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_skip_partial_discard": "true", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_tracing": "false", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_validate_names": "true", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rbd_validate_pool": "true", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "restapi_base_url": "", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "restapi_log_level": "", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_acl_grants_max_num": "100", 2024-09-21T09:56:09.232 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_admin_entry": "admin", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_asio_assert_yielding": "false", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_backend_store": "rados", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_barbican_url": "", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_beast_enable_async": "true", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_counters_cache": "false", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-21T09:56:09.233 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_cache_enabled": "true", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_cache_expiry_interval": "900", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_cache_lru_size": "25000", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_config_store": "rados", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_content_length_compat": "false", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_copy_obj_progress": "true", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_cors_rules_max_num": "100", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_cross_domain_policy": "", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-21T09:56:09.234 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_addr": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_password": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_kmip_username": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_require_ssl": "true", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-21T09:56:09.235 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_addr": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_auth": "token", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_namespace": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_prefix": "", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-21T09:56:09.236 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_token_file": "", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_curl_buffersize": "524288", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_curl_low_speed_time": "300", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-21T09:56:09.237 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_log_changes_size": "1000", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_log_num_shards": "128", 2024-09-21T09:56:09.238 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_log_window": "30", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-21T09:56:09.239 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_disable_s3select": "false", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-21T09:56:09.240 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dns_name": "", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dns_s3website_name": "", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dynamic_resharding": "true", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_gc_threads": "true", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_lc_threads": "true", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_mdsearch": "true", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_ops_log": "false", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_quota_threads": "true", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_static_website": "false", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enable_usage_log": "false", 2024-09-21T09:56:09.241 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_enforce_swift_acls": "true", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_exit_timeout_secs": "120", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_expose_bucket": "false", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_extended_http_attrs": "", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_filter": "none", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_frontends": "beast port=7480", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_max_deferred": "50", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_max_objs": "32", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-21T09:56:09.242 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_gc_processor_period": "3600", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_init_timeout": "300", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_kafka_connection_idle": "300", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-21T09:56:09.243 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_domain": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_password": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_password_path": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_project": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_tenant": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_token": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_token_path": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_admin_user": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_barbican_domain": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_barbican_password": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_barbican_project": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_barbican_user": "", 2024-09-21T09:56:09.244 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_url": "", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_debug_interval": "-1", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_lock_max_time": "90", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_max_objs": "32", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_max_rules": "1000", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_max_worker": "3", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lc_thread_delay": "0", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-21T09:56:09.245 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ldap_dnattr": "uid", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ldap_searchfilter": "", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ldap_uri": "", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_log_http_headers": "", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_log_object_name_utc": "false", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-21T09:56:09.246 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_attr_name_len": "0", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_attr_size": "0", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_chunk_size": "4194304", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_listing_results": "5000", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_notify_retries": "10", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_put_param_size": "1048576", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_put_size": "5368709120", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_max_slo_entries": "1000", 2024-09-21T09:56:09.247 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_md_log_max_shards": "64", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_mp_lock_max_time": "600", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_max_gc": "300", 2024-09-21T09:56:09.248 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_num_async_rados_threads": "32", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_num_control_oids": "8", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_numa_node": "-1", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-21T09:56:09.249 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_objexp_chunk_size": "100", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_objexp_gc_interval": "600", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_op_thread_timeout": "600", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_op_tracing": "false", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_opa_token": "", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_opa_url": "", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_opa_verify_ssl": "true", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-21T09:56:09.250 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ops_log_rados": "false", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_ops_log_socket_path": "", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_period_push_interval": "2.000000", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_posix_cache_lanes": "3", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_posix_cache_partitions": "3", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-21T09:56:09.251 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_print_continue": "true", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_rados_tracing": "false", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_realm": "", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_realm_id": "", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_region": "", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-21T09:56:09.252 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_request_uri": "", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_batch_size": "64", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_max_aio": "128", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_num_logs": "16", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_progress_judge_interval": "120", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_progress_judge_ratio": "0.500000", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshard_thread_interval": "600", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_reshardlog_threshold": "30000", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_resolve_cname": "false", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_run_sync_thread": "true", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-21T09:56:09.253 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_scheduler_type": "throttler", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_script_uri": "", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_service_provider_name": "", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_client_id": "", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_client_secret": "", 2024-09-21T09:56:09.254 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_entry": "sts", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_key": "sts", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_min_session_duration": "900", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sts_token_introspection_url": "", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_account_in_url": "false", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_auth_entry": "auth", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_auth_url": "", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_custom_header": "", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_need_stats": "true", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_tenant_name": "", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_token_expiration": "86400", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_url": "", 2024-09-21T09:56:09.255 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_url_prefix": "swift", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_lease_period": "120", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-21T09:56:09.256 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_thread_pool_size": "512", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_comment": "", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_createby": "", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_encoding": "", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_flag": "false", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_origin": "", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_torrent_tracker": "", 2024-09-21T09:56:09.257 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_trust_forwarded_https": "false", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_usage_max_shards": "32", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_usage_max_user_shards": "1", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_use_opa_authz": "false", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_counters_cache": "false", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_max_buckets": "1000", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_policies_max_num": "100", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-21T09:56:09.258 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_user_unique_email": "true", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_verify_ssl": "true", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_zone": "", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_zone_id": "", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_zonegroup": "", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_zonegroup_id": "", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgwlc_auto_session_clear": "true", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_block_size": "4096", 2024-09-21T09:56:09.259 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cache_size": "536870912", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-21T09:56:09.260 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_index_type": "binary_search", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_partition_filters": "false", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_perf": "false", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "run_dir": "/var/run/ceph", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_block_create": "true", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_cache_lru_size": "67108864", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_default_max_object_size": "16777216", 2024-09-21T09:56:09.261 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_device_size": "53687091200", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_full_integrity_check": "false", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_journal_batch_capacity": "16", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_main_device_type": "SSD", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-21T09:56:09.262 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "seastore_segment_size": "67108864", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "set_keepcaps": "false", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "setgroup": "ceph", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "setuser": "ceph", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "setuser_match_path": "", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "thp": "false", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "threadpool_default_timeout": "60", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "throttler_perf_counter": "true", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "tmp_dir": "/tmp", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-21T09:56:09.263 INFO:teuthology.orchestra.run.smithi076.stdout: "uadk_compressor_enabled": "false", 2024-09-21T09:56:09.264 INFO:teuthology.orchestra.run.smithi076.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-21T09:56:09.264 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:56:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:08 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:10.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:09 smithi076 ceph-mon[106482]: pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:10.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:09 smithi076 ceph-mon[110963]: pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:10.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-21T09:56:10.188 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:09 smithi191 ceph-mon[102206]: pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:10.849 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:11.142 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:11.254 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:11.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:11.992 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:11 smithi076 ceph-mon[106482]: pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:11.992 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:11 smithi076 ceph-mon[110963]: pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:11 smithi191 ceph-mon[102206]: pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:12.528 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a: {} 2024-09-21T09:56:12.528 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b: {} 2024-09-21T09:56:12.528 INFO:teuthology.orchestra.run.smithi076.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T09:56:12.528 INFO:teuthology.orchestra.run.smithi076.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T09:56:12.532 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c: {} 2024-09-21T09:56:12.532 INFO:teuthology.orchestra.run.smithi076.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T09:56:13.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/178343378' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T09:56:13.116 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/178343378' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T09:56:13.135 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-21T09:56:13.135 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-21T09:56:13.135 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T09:56:13.137 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T09:56:13.137 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-21T09:56:13.137 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_1 16 2024-09-21T09:56:13.138 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-21T09:56:13.138 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:56:13.138 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:56:13.138 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:56:13.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 3 2024-09-21T09:56:13.139 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-21T09:56:13.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:13.139 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-21T09:56:13.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:56:13.140 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-21T09:56:13.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T09:56:13.140 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-21T09:56:13.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:56:13.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:13.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:13.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:13.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:13.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/178343378' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T09:56:13.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:56:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:56:13] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.51.0" 2024-09-21T09:56:13.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:13.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:13.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:13.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:13.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:13.632 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:13.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:13.750 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:13.751 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:13.752 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:13.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:13.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:13.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:13.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:13.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:14.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:14.035 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:14.121 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:14.122 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:14.136 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:14.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:14.211 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:14 smithi076 ceph-mon[106482]: pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:14.211 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:14.211 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:14.211 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:14 smithi076 ceph-mon[110963]: pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:14.211 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:14.212 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:14.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:14.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:14 smithi191 ceph-mon[102206]: pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:14.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:14.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:14.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.530 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:14.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:14.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:14.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:14.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:14.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:14.899 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:14.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:14.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:15.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:15.023 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:15.125 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:15.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:15.247 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:15.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:15.370 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:15.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:15.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:15.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:15.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:15.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:15.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:15.763 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:15.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:15.887 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:15.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:15.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:16.011 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:16.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:16.135 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:16.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:16.258 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:16.290 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:16 smithi076 ceph-mon[106482]: pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:16.290 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:16 smithi076 ceph-mon[110963]: pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:16.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:16 smithi191 ceph-mon[102206]: pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:16.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:16.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:16.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:16.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:16.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:16.632 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:16.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:16.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:16.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:16.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:16.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:16.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:17.004 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:17.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:17.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:17.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:17.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:17.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:17.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:17.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:17.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:17.497 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:17.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:17.621 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:17.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:17.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:17.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:17.869 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:17.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:17.993 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:17.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:18.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:18.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:18.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:18.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:18.240 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:18.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:18.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:18 smithi191 ceph-mon[102206]: pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:18.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:18 smithi076 ceph-mon[106482]: pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:18 smithi076 ceph-mon[110963]: pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:18.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:18.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:18.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:18.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:18.713 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:18.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:18.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:18.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:18.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.863 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:18.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:18.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:18.987 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:18.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:19.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:19.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:19.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:19.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:19.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:19.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:19.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:19.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:19.462 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:19.484 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:19.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:19.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:19.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:19.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:19.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:19.860 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:19.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:19.987 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:19.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:20.088 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:20.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:20.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:20.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:20.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:20 smithi076 ceph-mon[106482]: pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:20.288 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:20 smithi076 ceph-mon[110963]: pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:20.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:20.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:20 smithi191 ceph-mon[102206]: pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:20.371 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:56:20.371 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T09:56:20.371 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T09:56:20.371 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:56:20.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:20.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:20.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:20.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:20.713 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:20.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:20.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:20.863 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:20.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:20.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:20.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:20.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:20.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T09:56:21.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:21.113 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:21.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:21.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:21.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:21.365 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:21.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:21.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:21.493 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:21.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:21.566 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:56:21.566 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T09:56:21.582 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:56:21.583 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T09:56:21.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:21.599 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:56:21.599 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:21.615 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:56:21.615 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T09:56:21.621 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:21.632 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:56:21.632 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:21.649 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:56:21.649 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:21.666 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:56:21.666 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:21.683 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:56:21.683 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:21.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:21.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:21.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:21.873 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:21.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:21.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:21.975 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:22.001 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:22.103 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:22.124 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:22.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:22.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:22.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:22.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:22.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[106482]: pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:22.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2725547876' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:22.297 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:22.297 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:22.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[110963]: pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:22.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2725547876' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:22.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:22.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:22.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:22.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:22 smithi191 ceph-mon[102206]: pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2725547876' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:22.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:22.478 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:22.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:22.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:22.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:22.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:22.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:22.746 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:22.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:22.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:22.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:22.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:22.996 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:22.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:22.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:23.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:23.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:23.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:23.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:56:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:56:23] "GET /metrics HTTP/1.1" 200 33901 "" "Prometheus/2.51.0" 2024-09-21T09:56:23.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:23.502 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:23.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:23.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:23.720 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:23.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:23.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:23.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:23.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:23.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:23.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:23.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:24.007 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:24.046 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.3. 2024-09-21T09:56:24.046 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T09:56:24.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T09:56:23.929+0000 7f240edc2640 -1 mon.a@0(leader).osd e1084 definitely_dead 0 2024-09-21T09:56:24.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[106482]: pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2214104071' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2379197187' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[106482]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[110963]: pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2214104071' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2379197187' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-21T09:56:24.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:24 smithi076 ceph-mon[110963]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-21T09:56:24.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:24.133 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:24.235 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:24.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:24.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:24 smithi191 ceph-mon[102206]: pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 468 MiB used, 715 GiB / 715 GiB avail 2024-09-21T09:56:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2214104071' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:24 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:56:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2379197187' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-21T09:56:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:24 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["3"]} : dispatch 2024-09-21T09:56:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:24 smithi191 ceph-mon[102206]: Client client.admin marked osd.3 out, while it was still marked up 2024-09-21T09:56:24.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:24.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:24.506 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:24.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:24.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:24.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:24.736 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:24.759 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:24.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:24.882 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:24.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:24.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:24.984 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:25.010 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:25.042 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_1' created 2024-09-21T09:56:25.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:25.133 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:25.235 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:25.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:25.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:56:25.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-21T09:56:25.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: osdmap e1085: 8 total, 8 up, 7 in 2024-09-21T09:56:25.243 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1051195347' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: osdmap e1085: 8 total, 8 up, 7 in 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:25.244 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1051195347' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T09:56:25.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:25.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:25.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:25.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:56:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2024-09-21T09:56:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: osdmap e1085: 8 total, 8 up, 7 in 2024-09-21T09:56:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1051195347' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-21T09:56:25.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:25.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:25.512 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:25.535 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:56:25.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:25.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:25.736 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:25.757 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:25.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-21T09:56:25.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:25.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:25.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:25.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:26.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:26.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:26.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[106482]: pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1051195347' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[106482]: osdmap e1086: 8 total, 8 up, 7 in 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[110963]: pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1051195347' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T09:56:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[110963]: osdmap e1086: 8 total, 8 up, 7 in 2024-09-21T09:56:26.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:26.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:26.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:26.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:26.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:26.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:26.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:26 smithi191 ceph-mon[102206]: pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail 2024-09-21T09:56:26.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1051195347' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-21T09:56:26.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:26 smithi191 ceph-mon[102206]: osdmap e1086: 8 total, 8 up, 7 in 2024-09-21T09:56:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:26.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:26.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:26.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:26.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:26.608 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:26.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:26.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:26.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:26.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:26.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:26.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:26.880 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:26.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:26.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:26.983 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:27.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:27.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:27.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:27.197 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:27 smithi076 ceph-mon[110963]: osdmap e1087: 8 total, 8 up, 7 in 2024-09-21T09:56:27.198 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:27 smithi076 ceph-mon[106482]: osdmap e1087: 8 total, 8 up, 7 in 2024-09-21T09:56:27.221 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:27.235 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:27.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:27.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:27.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:27.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:27 smithi191 ceph-mon[102206]: osdmap e1087: 8 total, 8 up, 7 in 2024-09-21T09:56:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.388 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:27.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:27.489 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.513 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:27.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:27.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:27.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:27.737 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:27.759 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:27.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:27.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:27.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:27.984 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:28.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:28.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:28.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:28.134 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:28.210 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:28.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:28.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:28.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:28.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[106482]: pgmap v1144: 89 pgs: 5 unknown, 3 peering, 81 active+clean; 582 KiB data, 430 MiB used, 625 GiB / 626 GiB avail; 14 B/s, 0 objects/s recovering 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[106482]: osdmap e1088: 8 total, 8 up, 7 in 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[110963]: pgmap v1144: 89 pgs: 5 unknown, 3 peering, 81 active+clean; 582 KiB data, 430 MiB used, 625 GiB / 626 GiB avail; 14 B/s, 0 objects/s recovering 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:56:28.293 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:28 smithi076 ceph-mon[110963]: osdmap e1088: 8 total, 8 up, 7 in 2024-09-21T09:56:28.324 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:56:28.324 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:56:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:28 smithi191 ceph-mon[102206]: pgmap v1144: 89 pgs: 5 unknown, 3 peering, 81 active+clean; 582 KiB data, 430 MiB used, 625 GiB / 626 GiB avail; 14 B/s, 0 objects/s recovering 2024-09-21T09:56:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:28 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T09:56:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:28 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:56:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:28 smithi191 ceph-mon[102206]: osdmap e1088: 8 total, 8 up, 7 in 2024-09-21T09:56:28.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:28.389 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:28.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:28.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:28.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:28.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:28.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:28.741 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:28.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:28.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:28.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:28.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:28.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:28.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:29.017 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:29.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:29.145 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:29.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:29.247 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:29.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:29.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:29.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:29.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:29.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:29.522 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:29.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:29.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:29.713 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T09:56:29.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:29.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:29.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:29.874 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:29.896 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:29.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:29.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:30.023 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:30.082 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:30.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:30.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:30.251 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:30.264 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:56:30.264 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T09:56:30.264 INFO:tasks.thrashosds.thrasher:Adding osd 3 2024-09-21T09:56:30.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 3 2024-09-21T09:56:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:30.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:30.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:30 smithi191 ceph-mon[102206]: pgmap v1146: 89 pgs: 3 unknown, 3 peering, 83 active+clean; 582 KiB data, 430 MiB used, 625 GiB / 626 GiB avail; 20 B/s, 0 objects/s recovering 2024-09-21T09:56:30.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:30 smithi076 ceph-mon[106482]: pgmap v1146: 89 pgs: 3 unknown, 3 peering, 83 active+clean; 582 KiB data, 430 MiB used, 625 GiB / 626 GiB avail; 20 B/s, 0 objects/s recovering 2024-09-21T09:56:30.365 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:30 smithi076 ceph-mon[110963]: pgmap v1146: 89 pgs: 3 unknown, 3 peering, 83 active+clean; 582 KiB data, 430 MiB used, 625 GiB / 626 GiB avail; 20 B/s, 0 objects/s recovering 2024-09-21T09:56:30.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:30.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:30.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:30.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:30.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:30.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:30.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:30.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:30.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:30.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:30.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:30.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:30.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:30.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:30.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:31.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:31.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:31.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.149 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:31.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:31.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:31.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:31.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:31 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-09-21T09:56:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:31 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-09-21T09:56:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:31 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2024-09-21T09:56:31.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:31.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:31.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:31.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:31.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.710 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:31.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:31.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:31.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:31.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:31.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:31.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:31.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:32.069 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.096 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:32.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:32.198 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.221 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:32.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:32.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:32.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.349 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:32.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:32 smithi191 ceph-mon[102206]: pgmap v1147: 89 pgs: 89 active+clean; 582 KiB data, 429 MiB used, 625 GiB / 626 GiB avail; 14 B/s, 0 objects/s recovering 2024-09-21T09:56:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:32 smithi076 ceph-mon[106482]: pgmap v1147: 89 pgs: 89 active+clean; 582 KiB data, 429 MiB used, 625 GiB / 626 GiB avail; 14 B/s, 0 objects/s recovering 2024-09-21T09:56:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:32 smithi076 ceph-mon[110963]: pgmap v1147: 89 pgs: 89 active+clean; 582 KiB data, 429 MiB used, 625 GiB / 626 GiB avail; 14 B/s, 0 objects/s recovering 2024-09-21T09:56:32.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:32.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:32.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:32.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.599 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:32.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:32.670 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:56:32.671 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:32.679 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:56:32.679 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:32.695 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:56:32.695 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:32.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:32.711 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:56:32.711 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:32.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:32.728 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:56:32.729 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:32.745 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:56:32.746 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:32.762 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:56:32.762 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:32.778 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:56:32.778 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:32.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.846 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:32.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:32.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:32.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:32.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:33.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:33.092 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:33.105 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:33.116 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-21T09:56:33.116 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T09:56:33.194 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:33.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:33.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:33.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:33.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:33 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4279236024' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:56:33.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:33 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3226839343' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:33.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:33 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2242952343' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4279236024' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:56:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3226839343' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2242952343' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:33.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:56:33] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.51.0" 2024-09-21T09:56:33.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4279236024' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:56:33.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3226839343' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:33.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:33 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2242952343' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T09:56:33.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:33.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:33.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:33.598 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:33.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:33.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:33.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:33.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:33.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:33.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:33.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:33.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:34.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:34.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:34.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:34.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:34.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:34.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:34.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:34.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[106482]: pgmap v1148: 89 pgs: 89 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail; 12 B/s, 0 objects/s recovering 2024-09-21T09:56:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4279236024' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:56:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2242952343' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:56:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[106482]: osdmap e1089: 8 total, 8 up, 7 in 2024-09-21T09:56:34.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[110963]: pgmap v1148: 89 pgs: 89 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail; 12 B/s, 0 objects/s recovering 2024-09-21T09:56:34.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:34.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4279236024' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:56:34.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2242952343' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:56:34.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:34 smithi076 ceph-mon[110963]: osdmap e1089: 8 total, 8 up, 7 in 2024-09-21T09:56:34.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:34.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:34.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:34.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:34 smithi191 ceph-mon[102206]: pgmap v1148: 89 pgs: 89 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail; 12 B/s, 0 objects/s recovering 2024-09-21T09:56:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:34 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:34 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4279236024' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:56:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:34 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2242952343' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:56:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:34 smithi191 ceph-mon[102206]: osdmap e1089: 8 total, 8 up, 7 in 2024-09-21T09:56:34.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:34.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:34.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:34.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:34.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:34.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:34.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:34.980 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:34.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:35.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:35.103 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:35.205 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:35.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:35.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:35.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:35.333 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:35 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:56:35.333 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:35 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:56:35.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:35.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:35.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:35.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:35.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:56:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:35.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:35.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:35 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:56:35.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:35.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:35.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:35.861 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:35.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:35.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:35.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:35.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:6715 5.3 5:c4e656cc:::smithi07685715-38:head [delete] snapc 1d6=[1d6,1d5,1d4,1d3,1d2,1d1,1cf,1ce,1cc,1c6,1a3] ondisk+write+peerstat_old+known_if_redirected+supports_pool_eio e745)", 2024-09-21T09:56:36.070 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:49:19.882331+0000", 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 436.16035063200002, 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015872234999999998, 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 6715 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.071 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.882331+0000", 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.882331+0000", 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.882333+0000", 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8369999999999999e-06 2024-09-21T09:56:36.072 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.882340+0000", 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9090000000000003e-06 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.882355+0000", 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4477e-05 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.073 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.882365+0000", 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0966e-05 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.884524+0000", 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021582860000000001 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.884622+0000", 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7913999999999998e-05 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.074 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.884813+0000", 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019159099999999999 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.886050+0000", 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012367389999999999 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.891120+0000", 2024-09-21T09:56:36.075 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050704870000000003 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.898065+0000", 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069446439999999998 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.898131+0000", 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5432999999999995e-05 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.076 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:49:19.898204+0000", 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2952000000000002e-05 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:7371 5.3 5:c90ace77:::smithi07685715-7:head [sparse-read 0~0,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e796)", 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:50:10.795245+0000", 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 385.247436628, 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017276732, 2024-09-21T09:56:36.077 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "reached pg", 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7371 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795245+0000", 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.078 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795245+0000", 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795248+0000", 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.542e-06 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.079 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795257+0000", 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9579999999999996e-06 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795281+0000", 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4284e-05 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795295+0000", 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3685e-05 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.080 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795382+0000", 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6735000000000002e-05 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op must wait for map", 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795390+0000", 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1219999999999995e-06 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.810951+0000", 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015561634 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.081 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.812522+0000", 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001570772 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:7372 5.3 5:c90ace77:::smithi07685715-7:head [sparse-read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e796)", 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:50:10.795328+0000", 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 385.24735402800002, 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017044252999999999, 2024-09-21T09:56:36.082 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "reached pg", 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7372 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795328+0000", 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.083 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795328+0000", 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795328+0000", 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8800000000000002e-07 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795332+0000", 2024-09-21T09:56:36.084 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0790000000000001e-06 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795464+0000", 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013275799999999999 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795475+0000", 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0618000000000001e-05 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.085 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795559+0000", 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4439999999999998e-05 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795564+0000", 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0679999999999996e-06 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.812260+0000", 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016695437 2024-09-21T09:56:36.086 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.812372+0000", 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112265 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:7373 5.3 5:c90ace77:::smithi07685715-7:head [read 0~0,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e796)", 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:50:10.795496+0000", 2024-09-21T09:56:36.087 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 385.24718598999999, 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016983464, 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "reached pg", 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7373 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795496+0000", 2024-09-21T09:56:36.088 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795496+0000", 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795497+0000", 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9500000000000001e-07 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.089 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795502+0000", 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9189999999999998e-06 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795510+0000", 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3250000000000008e-06 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795515+0000", 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4190000000000001e-06 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.090 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795586+0000", 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0698000000000006e-05 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.795591+0000", 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6290000000000001e-06 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.091 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.812399+0000", 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016808303 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:10.812479+0000", 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0376000000000004e-05 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:7563 5.3 5:cbb1e42f:::smithi07685715-17:head [sparse-read 0~1773148] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e807)", 2024-09-21T09:56:36.092 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:50:22.056119+0000", 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 373.98656331299998, 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011738480000000001, 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7563 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.093 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056119+0000", 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056119+0000", 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056119+0000", 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9599999999999999e-07 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.094 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056122+0000", 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.14e-06 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056130+0000", 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8189999999999992e-06 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056137+0000", 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6830000000000002e-06 2024-09-21T09:56:36.095 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056243+0000", 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106513 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056250+0000", 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0489999999999998e-06 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.067653+0000", 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011402093 2024-09-21T09:56:36.096 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.067737+0000", 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4967000000000002e-05 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.067857+0000", 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011962 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.097 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:7564 5.3 5:cbb1e42f:::smithi07685715-17:head [read 0~3512894,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e807)", 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:50:22.056147+0000", 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 373.98653460700001, 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013525881, 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7564 2024-09-21T09:56:36.098 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056147+0000", 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056147+0000", 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.099 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056148+0000", 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7000000000000005e-07 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056151+0000", 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4979999999999998e-06 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056158+0000", 2024-09-21T09:56:36.100 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2519999999999996e-06 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056163+0000", 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4229999999999996e-06 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056282+0000", 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118674 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.101 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.056287+0000", 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.989e-06 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.067885+0000", 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011598186999999999 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.067961+0000", 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6129000000000002e-05 2024-09-21T09:56:36.102 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:50:22.069673+0000", 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001712159 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:7966 5.3 5:c4e656cc:::smithi07685715-38:head [write 430576~492841] snapc 20e=[20e,20d,20c,20b,20a,209,207,206,202,201,200,1fd,1fb,1fa,1f5,1f3,1ed,1ec,1e9,1e5] ondisk+write+known_if_redirected+supports_pool_eio e848)", 2024-09-21T09:56:36.103 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:02.986505+0000", 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 333.05617670399999, 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013027360999999999, 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7966 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.104 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.986505+0000", 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.986505+0000", 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.986507+0000", 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8029999999999999e-06 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.105 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.986837+0000", 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000329508 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.986850+0000", 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3451e-05 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.986858+0000", 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.746e-06 2024-09-21T09:56:36.106 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.989012+0000", 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021544060000000002 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.989113+0000", 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010066899999999999 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.989304+0000", 2024-09-21T09:56:36.107 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019158399999999999 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.991852+0000", 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025477239999999999 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.996282+0000", 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0044299040000000001 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.108 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.999410+0000", 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031280460000000002 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.999475+0000", 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4984e-05 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:02.999533+0000", 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7535999999999999e-05 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.109 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.14799.0:8014 5.7 e850/763 5:e242f207:::smithi07685715-8:head v 850'2304, mlcod=848'2300)", 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:04.779288+0000", 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 331.26339363900001, 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015105772, 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.110 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.779288+0000", 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.779288+0000", 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.779289+0000", 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4000000000000002e-07 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.111 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.779297+0000", 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1780000000000003e-06 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.779304+0000", 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2540000000000003e-06 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.779311+0000", 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5540000000000003e-06 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.780410+0000", 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001099528 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.780428+0000", 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8134e-05 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.794354+0000", 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013925796000000001 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.113 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.794394+0000", 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9888000000000002e-05 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.14799.0:8015 5.7 e850/763 5:e242f207:::smithi07685715-8:head v 850'2305, mlcod=848'2300)", 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:04.782759+0000", 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 331.25992282200002, 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011673126000000001, 2024-09-21T09:56:36.114 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.782759+0000", 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.782759+0000", 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.115 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.782760+0000", 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0500000000000003e-07 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.782766+0000", 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9259999999999997e-06 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.782775+0000", 2024-09-21T09:56:36.116 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9209999999999998e-06 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.782781+0000", 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0870000000000003e-06 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.787300+0000", 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004519688 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.117 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.787321+0000", 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0792999999999999e-05 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.794397+0000", 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0070753810000000004 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.794432+0000", 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5624999999999999e-05 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.118 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(osd.5.0:1948 5.7 e850/763 5:e413bd1f:::smithi07685715-48:1f4 v 850'2308, mlcod=848'2300)", 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:04.786095+0000", 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 331.25658657999998, 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011785334, 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.119 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786095+0000", 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786095+0000", 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786096+0000", 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0999999999999997e-07 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.120 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786102+0000", 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.947e-06 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786119+0000", 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7042e-05 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786127+0000", 2024-09-21T09:56:36.121 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4839999999999998e-06 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.793304+0000", 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0071771710000000004 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.793325+0000", 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1036e-05 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.122 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.797837+0000", 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045124070000000004 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.797881+0000", 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3436999999999998e-05 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(osd.5.0:1949 5.7 e850/763 5:e242f207:::smithi07685715-8:1f0 v 850'2310, mlcod=848'2300)", 2024-09-21T09:56:36.123 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:04.786228+0000", 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 331.25645379999997, 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012018703, 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786228+0000", 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.124 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786228+0000", 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786229+0000", 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2399999999999998e-07 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786234+0000", 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1089999999999997e-06 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.125 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786242+0000", 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2349999999999999e-06 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.786249+0000", 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4359999999999999e-06 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.794523+0000", 2024-09-21T09:56:36.126 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0082747740000000004 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.794536+0000", 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2812e-05 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.798219+0000", 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0036825199999999999 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.127 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:04.798247+0000", 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8192999999999999e-05 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8316 5.3 5:ca961e2d:::smithi07685715-14:head [delete] snapc 21a=[21a,219,218,217,216,215,211,210,209,200,1fd,1fb,1fa,1ed,1e5] ondisk+write+peerstat_old+known_if_redirected+supports_pool_eio e877)", 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:31.888710+0000", 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 304.15397209299999, 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.014900353, 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.128 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8316 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888710+0000", 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.129 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888710+0000", 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888712+0000", 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1830000000000001e-06 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888720+0000", 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5220000000000001e-06 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.130 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888742+0000", 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2323999999999998e-05 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888752+0000", 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0572e-05 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888836+0000", 2024-09-21T09:56:36.131 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3276999999999996e-05 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op must wait for map", 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.888843+0000", 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2339999999999997e-06 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.898725+0000", 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0098816070000000006 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.132 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.898831+0000", 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106612 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.899065+0000", 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023344500000000001 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.903397+0000", 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043322509999999996 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.133 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.903433+0000", 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6461000000000002e-05 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.903497+0000", 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3811000000000003e-05 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.903551+0000", 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3943999999999997e-05 2024-09-21T09:56:36.134 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:31.903610+0000", 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9110000000000002e-05 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8470 5.3 5:c4e656cc:::smithi07685715-38:head [append 0~0] snapc 221=[221,220,21f,21e,21d,21c,21b,21a,215,211,209,1fd] ondisk+write+known_if_redirected+supports_pool_eio e894)", 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:48.785137+0000", 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 287.25754526999998, 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012789844999999999, 2024-09-21T09:56:36.135 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8470 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785137+0000", 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.136 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785137+0000", 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785138+0000", 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4890000000000001e-06 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785147+0000", 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7320000000000004e-06 2024-09-21T09:56:36.137 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785159+0000", 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2497000000000001e-05 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785175+0000", 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5690000000000001e-05 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785248+0000", 2024-09-21T09:56:36.138 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2795999999999999e-05 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op must wait for map", 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785255+0000", 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9970000000000001e-06 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.791664+0000", 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0064090960000000004 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.139 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.791764+0000", 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010031600000000001 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.791973+0000", 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020867300000000001 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797215+0000", 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052418200000000003 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.140 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797267+0000", 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2648000000000001e-05 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797794+0000", 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00052630500000000002 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797859+0000", 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5389999999999996e-05 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.141 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797927+0000", 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7396000000000003e-05 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8471 5.3 5:c4e656cc:::smithi07685715-38:head [rollback 21f] snapc 221=[221,220,21f,21e,21d,21c,21b,21a,215,211,209,1fd] ondisk+write+known_if_redirected+supports_pool_eio e894)", 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:48.785199+0000", 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 287.25748279200002, 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012867689999999999, 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.142 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8471 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785199+0000", 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.143 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785199+0000", 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785200+0000", 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.82e-07 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785203+0000", 2024-09-21T09:56:36.144 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7980000000000001e-06 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785210+0000", 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4329999999999998e-06 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785215+0000", 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3179999999999997e-06 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.145 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785303+0000", 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8014000000000001e-05 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785309+0000", 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.383e-06 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.794201+0000", 2024-09-21T09:56:36.146 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0088926979999999992 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.794297+0000", 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6047999999999999e-05 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.794738+0000", 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00044058600000000003 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.147 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797935+0000", 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031970879999999998 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797967+0000", 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1708999999999997e-05 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.798007+0000", 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0337999999999999e-05 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.148 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.798038+0000", 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0679999999999998e-05 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.798067+0000", 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9014999999999999e-05 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8472 5.3 5:c4e656cc:::smithi07685715-38:head [append 0~0] snapc 221=[221,220,21f,21e,21d,21c,21b,21a,215,211,209,1fd] ondisk+write+known_if_redirected+supports_pool_eio e894)", 2024-09-21T09:56:36.149 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:51:48.785256+0000", 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 287.25742605900001, 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.014463451, 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8472 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.150 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785256+0000", 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785256+0000", 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785256+0000", 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.06e-07 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.151 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785259+0000", 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4389999999999999e-06 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785266+0000", 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0829999999999998e-06 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785271+0000", 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8790000000000002e-06 2024-09-21T09:56:36.152 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785333+0000", 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1774999999999997e-05 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.785337+0000", 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.464e-06 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.795860+0000", 2024-09-21T09:56:36.153 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010522981000000001 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.796113+0000", 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025327500000000002 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.796279+0000", 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016578599999999999 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.154 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.797978+0000", 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001699124 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.798103+0000", 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012469799999999999 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.799653+0000", 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015503820000000001 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.155 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.799690+0000", 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6859000000000001e-05 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:51:48.799719+0000", 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9200000000000002e-05 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8613 5.3 5:c90ace77:::smithi07685715-7:228 [read 0~3933102,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e916)", 2024-09-21T09:56:36.156 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:52:10.859367+0000", 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 265.18331537799997, 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012910233, 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8613 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.157 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859367+0000", 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859367+0000", 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859367+0000", 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4899999999999995e-07 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.158 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859371+0000", 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1010000000000002e-06 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859378+0000", 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2139999999999999e-06 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859384+0000", 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2310000000000004e-06 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.159 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.870562+0000", 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011177934 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.870668+0000", 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106656 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.872277+0000", 2024-09-21T09:56:36.160 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001608548 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8614 5.3 5:c90ace77:::smithi07685715-7:228 [sparse-read 0~2411154] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e916)", 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:52:10.859392+0000", 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 265.18328985900001, 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013266205999999999, 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.161 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8614 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859392+0000", 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859392+0000", 2024-09-21T09:56:36.162 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859392+0000", 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9900000000000001e-07 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859395+0000", 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4399999999999999e-06 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.163 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859400+0000", 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0869999999999997e-06 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.859404+0000", 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4700000000000004e-06 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.872321+0000", 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.01291665 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.164 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.872459+0000", 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013793400000000001 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:10.872658+0000", 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019922599999999999 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.165 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8915 5.3 5:cbb1e42f:::smithi07685715-17:head [append 0~0] snapc 23e=[23e,23d,23b,23a,237,235,232,231,22e,227,222,21e] ondisk+write+known_if_redirected+supports_pool_eio e952)", 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:52:46.813283+0000", 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 229.229399282, 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011780106, 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8915 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.166 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813283+0000", 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813283+0000", 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.167 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813284+0000", 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.826e-06 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813291+0000", 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0959999999999997e-06 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813307+0000", 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6507999999999998e-05 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.168 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813315+0000", 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0039999999999999e-06 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.814848+0000", 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015325130000000001 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.814962+0000", 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000114443 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.169 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.170 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.170 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.815172+0000", 2024-09-21T09:56:36.170 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021003499999999999 2024-09-21T09:56:36.170 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.170 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.170 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:36.171 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.823112+0000", 2024-09-21T09:56:36.171 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079396180000000007 2024-09-21T09:56:36.171 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.823620+0000", 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00050812299999999997 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.824902+0000", 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012817200000000001 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.824986+0000", 2024-09-21T09:56:36.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4772999999999996e-05 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.825063+0000", 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6446999999999995e-05 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8916 5.3 5:cbb1e42f:::smithi07685715-17:head [rollback 231] snapc 23e=[23e,23d,23b,23a,237,235,232,231,22e,227,222,21e] ondisk+write+known_if_redirected+supports_pool_eio e952)", 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:52:46.813331+0000", 2024-09-21T09:56:36.173 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 229.22935052899999, 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011906399999999999, 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8916 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813331+0000", 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813331+0000", 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813332+0000", 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5600000000000005e-07 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813339+0000", 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1999999999999997e-06 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813347+0000", 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7740000000000004e-06 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813352+0000", 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4559999999999999e-06 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.816526+0000", 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031737089999999998 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.816638+0000", 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000111426 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.816869+0000", 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023164600000000001 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.823120+0000", 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0062505319999999996 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.823663+0000", 2024-09-21T09:56:36.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00054300699999999997 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.825119+0000", 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001455895 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.825188+0000", 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8825999999999995e-05 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.825238+0000", 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0272999999999998e-05 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.14799.0:8917 5.3 5:cbb1e42f:::smithi07685715-17:head [append 0~0] snapc 23e=[23e,23d,23b,23a,237,235,232,231,22e,227,222,21e] ondisk+write+known_if_redirected+supports_pool_eio e952)", 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:52:46.813362+0000", 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 229.22932017400001, 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012766533, 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:56:36.180 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.14799", 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3625324184", 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 8917 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813362+0000", 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813362+0000", 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813362+0000", 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9200000000000001e-07 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813365+0000", 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4609999999999998e-06 2024-09-21T09:56:36.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813371+0000", 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0680000000000002e-06 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.813376+0000", 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0390000000000004e-06 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.817795+0000", 2024-09-21T09:56:36.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004419493 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.817895+0000", 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9115000000000002e-05 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.818098+0000", 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000203091 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:56:36.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.823127+0000", 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050291049999999999 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.825285+0000", 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021578869999999998 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.825418+0000", 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000133573 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.826072+0000", 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00065393299999999999 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:52:46.826128+0000", 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6175999999999999e-05 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:56:36.186 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:56:36.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:36.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:36.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:36.330 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:36.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:36 smithi076 ceph-mon[110963]: pgmap v1150: 89 pgs: 89 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:36 smithi076 ceph-mon[106482]: pgmap v1150: 89 pgs: 89 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:36.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:36.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:36.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:36.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:36.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:36 smithi191 ceph-mon[102206]: pgmap v1150: 89 pgs: 89 active+clean; 582 KiB data, 427 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:36.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:36.701 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:36.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:36.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:36.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:36.856 DEBUG:teuthology.orchestra.run.smithi076:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-21T09:56:36.862 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T09:56:36.862 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T09:56:36.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:hints = 1 2024-09-21T09:56:36.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T09:56:36.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object prefix: benchmark_data_smithi076_154645 2024-09-21T09:56:36.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:56:36.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 0 0 0 0 0 0 - 0 2024-09-21T09:56:36.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:36.966 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:36.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:37.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:37.104 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:37.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:37.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T09:56:37.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:37.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:37.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:37.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:37.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:37.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:37.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.517 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:37.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:37.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:37.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:37.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:37.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.796 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:37.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:37.846 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:37.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:37.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 1 16 4046 4030 251.766 251.875 0.00614485 0.00395798 2024-09-21T09:56:37.913 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:37.933 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:37.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:38.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:38.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.076 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:38.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:38.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:38.215 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:38.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:38.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:38.352 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:38.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:38.387 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:38 smithi076 ceph-mon[110963]: pgmap v1151: 89 pgs: 89 active+clean; 582 KiB data, 425 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:38.387 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:38 smithi076 ceph-mon[106482]: pgmap v1151: 89 pgs: 89 active+clean; 582 KiB data, 425 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:38.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:38.470 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:38 smithi191 ceph-mon[102206]: pgmap v1151: 89 pgs: 89 active+clean; 582 KiB data, 425 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:38.492 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:38.594 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:38.627 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:38.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:38.765 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:38.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:38.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:38.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:38.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:38.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:38.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 2 16 7568 7552 235.934 220.125 0.00667355 0.00419685 2024-09-21T09:56:39.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:39.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:39.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:39.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:39.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:39.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:39.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:39.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:39.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:39.434 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:39.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:39.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:39.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:39.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:39.807 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:39.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:39.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:56:39.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 3 16 10337 10321 214.977 173.062 0.0044615 0.00464401 2024-09-21T09:56:39.928 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:39.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:39.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:39.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:40.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:40.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:40.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:40.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:40.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:40.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:40.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:40.478 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:40 smithi191 ceph-mon[102206]: pgmap v1152: 89 pgs: 89 active+clean; 582 KiB data, 424 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:40.489 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:40.493 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:40 smithi076 ceph-mon[106482]: pgmap v1152: 89 pgs: 89 active+clean; 582 KiB data, 424 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:40.494 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:40 smithi076 ceph-mon[110963]: pgmap v1152: 89 pgs: 89 active+clean; 582 KiB data, 424 MiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T09:56:40.522 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:40.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:40.658 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:40.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:40.798 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:40.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:40.910 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 4 16 13467 13451 210.135 195.625 0.00451612 0.00475212 2024-09-21T09:56:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:40.932 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:40.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:41.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:41.069 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:41.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:41.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:41.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:41.344 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:41.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:41.367 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [4, 2]} : dispatch 2024-09-21T09:56:41.367 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 0]} : dispatch 2024-09-21T09:56:41.367 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-09-21T09:56:41.367 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:41.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [4, 2]} : dispatch 2024-09-21T09:56:41.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 0]} : dispatch 2024-09-21T09:56:41.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-09-21T09:56:41.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:41.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:41.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:41.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:41.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:41.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [4, 2]} : dispatch 2024-09-21T09:56:41.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 0]} : dispatch 2024-09-21T09:56:41.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-09-21T09:56:41.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:41.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:41.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:41.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:41.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:41.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:41.892 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:41.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:41.911 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 5 16 16598 16582 207.216 195.688 0.00367789 0.00481892 2024-09-21T09:56:41.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:42.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:42.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:42.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:42.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:42.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:42.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:42.342 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[106482]: pgmap v1153: 89 pgs: 89 active+clean; 101 MiB data, 871 MiB used, 625 GiB / 626 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-21T09:56:42.342 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [4, 2]}]': finished 2024-09-21T09:56:42.342 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 0]}]': finished 2024-09-21T09:56:42.342 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-09-21T09:56:42.343 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[106482]: osdmap e1090: 8 total, 8 up, 7 in 2024-09-21T09:56:42.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[110963]: pgmap v1153: 89 pgs: 89 active+clean; 101 MiB data, 871 MiB used, 625 GiB / 626 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-21T09:56:42.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [4, 2]}]': finished 2024-09-21T09:56:42.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 0]}]': finished 2024-09-21T09:56:42.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-09-21T09:56:42.343 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:42 smithi076 ceph-mon[110963]: osdmap e1090: 8 total, 8 up, 7 in 2024-09-21T09:56:42.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:42.429 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:42.530 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:42.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:42.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:42.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:42 smithi191 ceph-mon[102206]: pgmap v1153: 89 pgs: 89 active+clean; 101 MiB data, 871 MiB used, 625 GiB / 626 GiB avail; 10 MiB/s wr, 159 op/s 2024-09-21T09:56:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.8", "id": [4, 2]}]': finished 2024-09-21T09:56:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 0]}]': finished 2024-09-21T09:56:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-09-21T09:56:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:42 smithi191 ceph-mon[102206]: osdmap e1090: 8 total, 8 up, 7 in 2024-09-21T09:56:42.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:42.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:42.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:42.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:42.911 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 6 16 19544 19528 203.365 184.125 0.00312768 0.0049115 2024-09-21T09:56:42.939 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:42.987 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:42.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:42.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:43.089 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:43.124 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:43.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:43.262 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:43.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:43.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:43.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:43.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:43.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:43.564 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:43 smithi076 ceph-mon[106482]: osdmap e1091: 8 total, 8 up, 7 in 2024-09-21T09:56:43.564 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:56:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:56:43] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-21T09:56:43.564 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:43 smithi076 ceph-mon[110963]: osdmap e1091: 8 total, 8 up, 7 in 2024-09-21T09:56:43.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:43 smithi191 ceph-mon[102206]: osdmap e1091: 8 total, 8 up, 7 in 2024-09-21T09:56:43.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:43.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:43.707 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:43.752 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:56:43.752 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:43.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:43.783 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:56:43.783 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:43.810 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:56:43.810 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:43.813 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:43.840 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T09:56:43.561+0000 7f240edc2640 -1 mon.a@0(leader).osd e1091 definitely_dead 0 2024-09-21T09:56:43.840 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:56:43.840 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:43.870 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:56:43.870 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:43.900 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:56:43.900 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:43.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 7 16 22277 22261 198.713 170.812 0.00401847 0.00502622 2024-09-21T09:56:43.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:43.935 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:56:43.936 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:43.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:43.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:43.965 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:56:43.966 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:44.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:44.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:44.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:44.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:44.219 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.3. 2024-09-21T09:56:44.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:44.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:44.348 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:44.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:44.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:44.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:44.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:44.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:44 smithi191 ceph-mon[102206]: pgmap v1156: 89 pgs: 2 peering, 87 active+clean; 862 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-21T09:56:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2125861817' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-21T09:56:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3164687597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:44 smithi076 ceph-mon[106482]: pgmap v1156: 89 pgs: 2 peering, 87 active+clean; 862 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-21T09:56:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2125861817' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-21T09:56:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3164687597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:44 smithi076 ceph-mon[110963]: pgmap v1156: 89 pgs: 2 peering, 87 active+clean; 862 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-21T09:56:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2125861817' entity='client.admin' cmd={"prefix": "osd in", "ids": ["3"]} : dispatch 2024-09-21T09:56:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3164687597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:44.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:44.726 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:44.828 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:44.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.851 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:44.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:44.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 8 16 23088 23072 180.211 50.6875 0.00274081 0.00507018 2024-09-21T09:56:44.952 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:44.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:44.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:45.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:45.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:45.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:45.228 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:45.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:45.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:45.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:45.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:45.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:45.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:45.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:45.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:45.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:45.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2125861817' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-21T09:56:45.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:45 smithi191 ceph-mon[102206]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-21T09:56:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2125861817' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-21T09:56:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:45 smithi076 ceph-mon[110963]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-21T09:56:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2125861817' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2024-09-21T09:56:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:45 smithi076 ceph-mon[106482]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-21T09:56:45.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:45.730 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:45.832 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:45.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:45.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:45.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 9 16 23088 23072 160.189 0 - 0.00507018 2024-09-21T09:56:45.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:45.985 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:45.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:46.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:46.109 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:46.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:46.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:46.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:46.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:46 smithi076 ceph-mon[106482]: pgmap v1158: 89 pgs: 2 peering, 87 active+clean; 975 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 162 MiB/s wr, 2.60k op/s 2024-09-21T09:56:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:46 smithi076 ceph-mon[106482]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-21T09:56:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:46 smithi076 ceph-mon[110963]: pgmap v1158: 89 pgs: 2 peering, 87 active+clean; 975 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 162 MiB/s wr, 2.60k op/s 2024-09-21T09:56:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:46 smithi076 ceph-mon[110963]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-21T09:56:46.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:46.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:46.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:46.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:46 smithi191 ceph-mon[102206]: pgmap v1158: 89 pgs: 2 peering, 87 active+clean; 975 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 162 MiB/s wr, 2.60k op/s 2024-09-21T09:56:46.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:46 smithi191 ceph-mon[102206]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-21T09:56:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:46.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:46.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:46.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:46.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:46.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 10 16 24227 24211 151.289 35.5938 0.00859373 0.00660391 2024-09-21T09:56:46.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:46.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:47.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:47.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:47.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:47.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:47.199 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:47.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:47.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:47.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:47.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:47.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:47.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:47 smithi191 ceph-mon[102206]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-21T09:56:47.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:47 smithi191 ceph-mon[102206]: pgmap v1161: 89 pgs: 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 4 peering, 78 active+clean; 1.3 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 123 MiB/s wr, 1.98k op/s; 8557/66393 objects degraded (12.888%); 1441/66393 objects misplaced (2.170%); 21 B/s, 0 objects/s recovering 2024-09-21T09:56:47.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:47.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:47.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:47 smithi076 ceph-mon[106482]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-21T09:56:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:47 smithi076 ceph-mon[106482]: pgmap v1161: 89 pgs: 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 4 peering, 78 active+clean; 1.3 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 123 MiB/s wr, 1.98k op/s; 8557/66393 objects degraded (12.888%); 1441/66393 objects misplaced (2.170%); 21 B/s, 0 objects/s recovering 2024-09-21T09:56:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:47 smithi076 ceph-mon[110963]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-21T09:56:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:47 smithi076 ceph-mon[110963]: pgmap v1161: 89 pgs: 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 4 peering, 78 active+clean; 1.3 GiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 123 MiB/s wr, 1.98k op/s; 8557/66393 objects degraded (12.888%); 1441/66393 objects misplaced (2.170%); 21 B/s, 0 objects/s recovering 2024-09-21T09:56:47.721 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:47.761 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:56:47.761 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:56:47.762 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:47.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:47.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:47.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:47.900 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:47.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:47.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 11 16 26195 26179 148.716 123 0.0199881 0.00671553 2024-09-21T09:56:48.000 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:48.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:48.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:56:48.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:48.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:48.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:48.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:48 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 8557/66393 objects degraded (12.888%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:48 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 8557/66393 objects degraded (12.888%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:48.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:48.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:48.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T09:56:48.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:48.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:48 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 8557/66393 objects degraded (12.888%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:48.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:48.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:48.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:48.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:48.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:48.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 12 16 27966 27950 145.546 110.688 0.00514581 0.00686531 2024-09-21T09:56:48.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:48.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:48.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:49.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:49.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:49.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:49.192 INFO:tasks.thrashosds.thrasher:Added osd 3 2024-09-21T09:56:49.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:49.266 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:49.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:49.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:49.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:49 smithi076 ceph-mon[106482]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 3 peering, 78 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 10035/69321 objects degraded (14.476%); 1441/69321 objects misplaced (2.079%); 14 B/s, 0 objects/s recovering 2024-09-21T09:56:49.427 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:49 smithi076 ceph-mon[110963]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 3 peering, 78 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 10035/69321 objects degraded (14.476%); 1441/69321 objects misplaced (2.079%); 14 B/s, 0 objects/s recovering 2024-09-21T09:56:49.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:49.521 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:49.521 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:49.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.555 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:49.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:49.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:49 smithi191 ceph-mon[102206]: pgmap v1162: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 3 peering, 78 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 10035/69321 objects degraded (14.476%); 1441/69321 objects misplaced (2.079%); 14 B/s, 0 objects/s recovering 2024-09-21T09:56:49.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:49.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:49.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:49.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:49.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:49.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 13 16 29589 29573 142.153 101.438 0.00643888 0.00702887 2024-09-21T09:56:49.950 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:49.989 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:49.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:50.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:50.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:50.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:50.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:50.263 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:50.287 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T09:56:50.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3677538201' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3677538201' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:50.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:50.496 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:50.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:50.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:50.593 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3677538201' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:56:50.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:50.674 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:50.776 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:50.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:50.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 14 16 31233 31217 139.338 102.75 0.00660715 0.00716966 2024-09-21T09:56:50.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:50.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:50.945 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:50.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:51.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:51.082 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:51.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:51.220 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:51.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:51.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:51.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:51.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:51.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:51.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:51 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:51.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3677538201' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:56:51.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:51 smithi191 ceph-mon[102206]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-21T09:56:51.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:51 smithi191 ceph-mon[102206]: pgmap v1164: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 12906/72663 objects degraded (17.761%); 1441/72663 objects misplaced (1.983%); 14 B/s, 0 objects/s recovering 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3677538201' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[106482]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[106482]: pgmap v1164: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 12906/72663 objects degraded (17.761%); 1441/72663 objects misplaced (1.983%); 14 B/s, 0 objects/s recovering 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3677538201' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[110963]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-21T09:56:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:51 smithi076 ceph-mon[110963]: pgmap v1164: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+degraded+remapped, 6 activating+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s; 12906/72663 objects degraded (17.761%); 1441/72663 objects misplaced (1.983%); 14 B/s, 0 objects/s recovering 2024-09-21T09:56:51.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:51.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:51.768 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:51.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:51.907 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:51.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:51.912 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 15 15 33075 33060 137.728 115.188 0.00629486 0.00725499 2024-09-21T09:56:52.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:52.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:52.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:52.173 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:52.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:52.306 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:52.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:52.408 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:52.440 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:52.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:52.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:52.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:52.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:52.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:52.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:52.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:52.821 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:52 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 11460/94212 objects degraded (12.164%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:52.821 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:52 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 11460/94212 objects degraded (12.164%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:52.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:52 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 11460/94212 objects degraded (12.164%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:52.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 16 16 34863 34847 136.064 111.688 0.00796079 0.00734175 2024-09-21T09:56:52.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:52.985 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:52.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:53.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:53.118 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:53.220 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:53.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:53.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:56:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:56:53] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.51.0" 2024-09-21T09:56:53.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:53.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:53.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:53.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:53.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:53.711 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:56:53.712 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:56:53.712 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:53 smithi076 ceph-mon[106482]: pgmap v1165: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 11460/94212 objects degraded (12.164%); 2384/94212 objects misplaced (2.530%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:56:53.712 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:53 smithi076 ceph-mon[110963]: pgmap v1165: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 11460/94212 objects degraded (12.164%); 2384/94212 objects misplaced (2.530%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:56:53.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:53.757 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:56:53.757 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:56:53.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:53.796 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:56:53.796 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:56:53.843 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:56:53.844 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:56:53.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:53 smithi191 ceph-mon[102206]: pgmap v1165: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s; 11460/94212 objects degraded (12.164%); 2384/94212 objects misplaced (2.530%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:56:53.879 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:56:53.879 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:56:53.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:53.913 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:56:53.913 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:56:53.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 17 16 36537 36521 134.215 104.625 0.00716431 0.00744508 2024-09-21T09:56:53.917 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:53.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:53.951 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:56:53.951 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:56:53.960 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:56:53.960 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T09:56:53.960 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T09:56:53.960 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T09:56:53.960 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:56:54.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:54.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:56:54.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:56:54.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:54.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:54.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:54.193 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T09:56:54.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T09:56:54.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:54.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:54.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:54.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:54.497 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:54.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:54.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:54.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:54.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:54.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:54.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1235779333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:54.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:54.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1235779333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:54.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1235779333' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:56:54.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:54.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:54.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 18 16 38126 38110 132.276 99.3125 0.00989872 0.00755376 2024-09-21T09:56:54.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:56:55.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:55.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:55.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:55.173 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:55.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:55.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:55.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:55.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:55.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:55.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T09:56:55.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:55.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:55.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:55.558 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:55.571 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:55.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.632 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:55.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:55.707 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:55 smithi076 ceph-mon[106482]: pgmap v1166: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 11460/97548 objects degraded (11.748%); 2384/97548 objects misplaced (2.444%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T09:56:55.708 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:55 smithi076 ceph-mon[110963]: pgmap v1166: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 11460/97548 objects degraded (11.748%); 2384/97548 objects misplaced (2.444%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T09:56:55.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:55.768 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:55.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:55 smithi191 ceph-mon[102206]: pgmap v1166: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 11460/97548 objects degraded (11.748%); 2384/97548 objects misplaced (2.444%); 3.8 MiB/s, 61 objects/s recovering 2024-09-21T09:56:55.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:55.899 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:55.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:55.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:55.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 19 16 39639 39623 130.291 94.5625 0.0109354 0.00766914 2024-09-21T09:56:56.001 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:56.037 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:56.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:56.172 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:56.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:56:56.332 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:56:56.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:56.468 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:56.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:56.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:56.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:56.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:56.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:56.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:56.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:56.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:56.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:56:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:56.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T09:56:56.916394+0000 min lat: 0.00146111 max lat: 2.14015 avg lat: 0.00775228 2024-09-21T09:56:56.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:56:56.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 20 16 41279 41263 128.901 102.5 0.00712421 0.00775228 2024-09-21T09:56:56.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:56.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:56:57.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:57.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:56:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.147 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:57.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:56:57.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:57.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:57.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:57.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:57.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:57.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:57.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:57.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:57.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:57.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:57.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:57.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:57.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:57 smithi191 ceph-mon[102206]: pgmap v1167: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 11460/116376 objects degraded (9.847%); 1883/116376 objects misplaced (1.618%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T09:56:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:57 smithi076 ceph-mon[106482]: pgmap v1167: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 11460/116376 objects degraded (9.847%); 1883/116376 objects misplaced (1.618%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T09:56:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:57 smithi076 ceph-mon[110963]: pgmap v1167: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 11460/116376 objects degraded (9.847%); 1883/116376 objects misplaced (1.618%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T09:56:57.917 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 21 16 42855 42839 127.453 98.5 0.00427751 0.00783957 2024-09-21T09:56:57.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:57.976 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:57.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:58.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:58.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:58.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:58.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:58.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:58.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:58.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:58.516 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:56:58.550 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:58.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:56:58.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:58.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:58.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:58.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:58.829 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:58 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 11460/119403 objects degraded (9.598%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:58.829 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:58 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 11460/119403 objects degraded (9.598%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:58.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:58.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 22 16 44437 44421 126.147 98.875 0.0244378 0.00792065 2024-09-21T09:56:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:58.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:58.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:59.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:56:59.082 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:59.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:56:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:58 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 11460/119403 objects degraded (9.598%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:59.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:59.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:59.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:56:59.359 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:56:59.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:59.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:56:59.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:59.502 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:59.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:59.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:56:59.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:56:59.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:56:59.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:56:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:56:59.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:56:59.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:56:59 smithi076 ceph-mon[110963]: pgmap v1168: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 11460/119403 objects degraded (9.598%); 1883/119403 objects misplaced (1.577%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T09:56:59.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:56:59 smithi076 ceph-mon[106482]: pgmap v1168: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 11460/119403 objects degraded (9.598%); 1883/119403 objects misplaced (1.577%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T09:56:59.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:56:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:56:59.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:56:59.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 23 16 45951 45935 124.775 94.625 0.01328 0.0080087 2024-09-21T09:57:00.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.047 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:00.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:56:59 smithi191 ceph-mon[102206]: pgmap v1168: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 11460/119403 objects degraded (9.598%); 1883/119403 objects misplaced (1.577%); 6.3 MiB/s, 100 objects/s recovering 2024-09-21T09:57:00.148 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:00.189 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:00.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:00.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:00.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.460 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:00.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:00.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:00.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:00.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:00.608 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:00.625 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T09:57:00.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:00.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:00.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:00.833 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1737310773' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T09:57:00.833 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1737310773' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T09:57:00.850 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:00.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 24 16 47149 47133 122.696 74.875 0.00306574 0.00808539 2024-09-21T09:57:00.952 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:00.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:00.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:00.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:01.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:01.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1737310773' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T09:57:01.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:01.221 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:01.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:01.322 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:01.344 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:01.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:01.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:01.468 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:01.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:01.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:01.591 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:01.693 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:01.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:01.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:01.834 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:01.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:01.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[106482]: pgmap v1169: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 11460/123981 objects degraded (9.243%); 1883/123981 objects misplaced (1.519%); 6.2 MiB/s, 99 objects/s recovering 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1737310773' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[106482]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[106482]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[110963]: pgmap v1169: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 11460/123981 objects degraded (9.243%); 1883/123981 objects misplaced (1.519%); 6.2 MiB/s, 99 objects/s recovering 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1737310773' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[110963]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-21T09:57:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:01 smithi076 ceph-mon[110963]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-21T09:57:01.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 25 16 47533 47517 118.749 24 0.0113474 0.00841475 2024-09-21T09:57:01.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:02.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:02.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:01 smithi191 ceph-mon[102206]: pgmap v1169: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.5 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 11460/123981 objects degraded (9.243%); 1883/123981 objects misplaced (1.519%); 6.2 MiB/s, 99 objects/s recovering 2024-09-21T09:57:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1737310773' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T09:57:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:01 smithi191 ceph-mon[102206]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-21T09:57:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:01 smithi191 ceph-mon[102206]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-21T09:57:02.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:02.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:02.283 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:02.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:02.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:02.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:02.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:02.559 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:02.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:02.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:02.791 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:02.827 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:02.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:02.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 26 16 49025 49009 117.768 93.25 0.00727144 0.00848526 2024-09-21T09:57:02.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:02.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:02.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:03.065 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:03.095 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:03.197 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:03.233 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:03.335 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:03.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T09:57:03.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:03.488 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:03.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.669 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:03.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:03.670 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:57:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:57:03] "GET /metrics HTTP/1.1" 200 36346 "" "Prometheus/2.51.0" 2024-09-21T09:57:03.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:03.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:03.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:03.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 27 16 50316 50300 116.395 80.6875 0.0139292 0.00858548 2024-09-21T09:57:03.940 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:03.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:03.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:04.014 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:03 smithi076 ceph-mon[106482]: pgmap v1172: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 10026/139077 objects degraded (7.209%); 3133/139077 objects misplaced (2.253%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T09:57:04.014 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:03 smithi076 ceph-mon[110963]: pgmap v1172: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 10026/139077 objects degraded (7.209%); 3133/139077 objects misplaced (2.253%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T09:57:04.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:04.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:03 smithi191 ceph-mon[102206]: pgmap v1172: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 10026/139077 objects degraded (7.209%); 3133/139077 objects misplaced (2.253%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T09:57:04.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:04.214 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:04.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.356 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:04.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:04.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:04.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:04.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:04.615 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:04.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:04.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:04.774 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 10026/139077 objects degraded (7.209%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:04.774 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 10026/139077 objects degraded (7.209%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:04.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:04.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:04.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:04.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 28 16 51578 51562 115.055 78.875 0.0171304 0.00868476 2024-09-21T09:57:05.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:05.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 10026/139077 objects degraded (7.209%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:05.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:05.184 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:05.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:05.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:05.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:05.437 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:57:05.437 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:7905 6.4 6:2e41550e:::benchmark_data_smithi076_154645_object7904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:39.012727+0000", 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 26.420537702000001, 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.095483806000000004, 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.438 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 7905 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012727+0000", 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.439 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012727+0000", 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012728+0000", 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.088e-06 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012751+0000", 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2966e-05 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012763+0000", 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2173e-05 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012774+0000", 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1100999999999999e-05 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012824+0000", 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9215000000000002e-05 2024-09-21T09:57:05.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.012925+0000", 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000100955 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 5,6", 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013024+0000", 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9252000000000004e-05 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.442 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013553+0000", 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00052962899999999999 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107959+0000", 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.094405627000000006 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.108114+0000", 2024-09-21T09:57:05.443 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015528300000000001 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.108168+0000", 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4030999999999998e-05 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.108211+0000", 2024-09-21T09:57:05.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2485999999999999e-05 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:7903 6.b e1089/1086 6:d080dfec:::benchmark_data_smithi076_154645_object7902:head v 1089'488, mlcod=1089'486)", 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:39.013484+0000", 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 26.419781237999999, 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.094409555000000006, 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.445 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013484+0000", 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013484+0000", 2024-09-21T09:57:05.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013484+0000", 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2699999999999998e-07 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013556+0000", 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1749999999999996e-05 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.447 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013563+0000", 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0820000000000001e-06 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013568+0000", 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.335e-06 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013617+0000", 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.986e-05 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013641+0000", 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3411999999999999e-05 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107851+0000", 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.094210146999999994 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107893+0000", 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2141999999999997e-05 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.449 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:7902 6.e e1089/1086 6:7b93d3d1:::benchmark_data_smithi076_154645_object7901:head v 1089'488, mlcod=1089'485)", 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:39.013565+0000", 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 26.419699853000001, 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.095387498000000001, 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013565+0000", 2024-09-21T09:57:05.450 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013565+0000", 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013566+0000", 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7000000000000004e-07 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.451 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013669+0000", 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010288700000000001 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013676+0000", 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.272e-06 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.013680+0000", 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3819999999999997e-06 2024-09-21T09:57:05.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:05.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107664+0000", 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.093983502999999996 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107698+0000", 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4649e-05 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.108889+0000", 2024-09-21T09:57:05.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011906340000000001 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.108952+0000", 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3501000000000001e-05 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop_reply(client.34674.0:7905 6.4 e1089/1086 ondisk, result = 0)", 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:39.014297+0000", 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 26.418968062000001, 2024-09-21T09:57:05.455 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.093669833999999993, 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.014297+0000", 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.014297+0000", 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.014297+0000", 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7700000000000004e-07 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.014302+0000", 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.724e-06 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107637+0000", 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.093335193999999996 2024-09-21T09:57:05.457 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107653+0000", 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5588000000000001e-05 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107946+0000", 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00029286600000000002 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107953+0000", 2024-09-21T09:57:05.458 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3250000000000002e-06 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:39.107967+0000", 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3560000000000001e-05 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:39107 6.4 6:26a7462d:::benchmark_data_smithi076_154645_object39106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:55.575049+0000", 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 9.8582162550000003, 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.040980580000000003, 2024-09-21T09:57:05.459 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 39107 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.575049+0000", 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.460 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.575049+0000", 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.575051+0000", 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2840000000000001e-06 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.575080+0000", 2024-09-21T09:57:05.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9475000000000002e-05 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.575099+0000", 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8473000000000001e-05 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.575109+0000", 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7510000000000003e-06 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.576532+0000", 2024-09-21T09:57:05.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014235129999999999 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.576697+0000", 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000164513 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.576860+0000", 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016376499999999999 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.589909+0000", 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013048244000000001 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.589931+0000", 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2178999999999999e-05 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.614890+0000", 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.024959563000000001 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.615001+0000", 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110571 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.616029+0000", 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001028249 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:39108 6.4 6:206f3013:::benchmark_data_smithi076_154645_object39107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:55.578695+0000", 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 9.8545700479999994, 2024-09-21T09:57:05.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.038469167999999998, 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 39108 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.578695+0000", 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.466 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.578695+0000", 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.578699+0000", 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5090000000000004e-06 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.578739+0000", 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9579000000000002e-05 2024-09-21T09:57:05.467 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.578761+0000", 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2011999999999999e-05 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.578773+0000", 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2016e-05 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.580501+0000", 2024-09-21T09:57:05.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017286039999999999 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.580664+0000", 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000162075 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.580833+0000", 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016904299999999999 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.589975+0000", 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0091427509999999993 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.598533+0000", 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0085573590000000005 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.616085+0000", 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017552432999999999 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.616353+0000", 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026759700000000001 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.617164+0000", 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00081119000000000004 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.471 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:39119 6.4 6:2c93bb15:::benchmark_data_smithi076_154645_object39118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:56:55.582686+0000", 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 9.8505790859999998, 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.037535487999999999, 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 39119 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.472 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.582686+0000", 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.582686+0000", 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.582687+0000", 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6390000000000001e-06 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.582710+0000", 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2580999999999999e-05 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.582723+0000", 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3383999999999999e-05 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.582731+0000", 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.909e-06 2024-09-21T09:57:05.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.590009+0000", 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072779170000000001 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.590139+0000", 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013013599999999999 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.590305+0000", 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000165988 2024-09-21T09:57:05.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.598518+0000", 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.008212301 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.598539+0000", 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1840000000000001e-05 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.617423+0000", 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018883252 2024-09-21T09:57:05.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.617567+0000", 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014480800000000001 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:56:55.620221+0000", 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026537330000000001 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.477 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:46732 6.4 6:2ba7da88:::benchmark_data_smithi076_154645_object46731:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.474932+0000", 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 4.958332864, 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.032122256000000002, 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 46732 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.478 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.474932+0000", 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.474932+0000", 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.474933+0000", 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8100000000000003e-07 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.475003+0000", 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0573000000000003e-05 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.475013+0000", 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0095000000000001e-05 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.475020+0000", 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9700000000000002e-06 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.475182+0000", 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000161633 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.475386+0000", 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020437799999999999 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.475486+0000", 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9607999999999999e-05 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.480601+0000", 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051152400000000001 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.480668+0000", 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6682999999999998e-05 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.505928+0000", 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.025260156999999998 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.482 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.506049+0000", 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000121549 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.507054+0000", 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010047890000000001 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47384 6.4 6:2e80749b:::benchmark_data_smithi076_154645_object47383:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:01.786405+0000", 2024-09-21T09:57:05.483 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.6468593149999999, 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.042458674000000002, 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47384 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.484 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786405+0000", 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786405+0000", 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786407+0000", 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8339999999999999e-06 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786465+0000", 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7308000000000001e-05 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786476+0000", 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0984e-05 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786483+0000", 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1810000000000004e-06 2024-09-21T09:57:05.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786697+0000", 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000213892 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.786851+0000", 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015472700000000001 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.787035+0000", 2024-09-21T09:57:05.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018350899999999999 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.791215+0000", 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0041801900000000003 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.791229+0000", 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4005999999999999e-05 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.827843+0000", 2024-09-21T09:57:05.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.036613892000000002 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.827909+0000", 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5968000000000003e-05 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.828864+0000", 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00095518299999999997 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47755 6.4 6:29caeb75:::benchmark_data_smithi076_154645_object47754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:02.053478+0000", 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.3797871219999998, 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.051315566999999999, 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47755 2024-09-21T09:57:05.490 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.053478+0000", 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.053478+0000", 2024-09-21T09:57:05.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.053479+0000", 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7230000000000001e-06 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.053502+0000", 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2354e-05 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.053515+0000", 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.278e-05 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.053523+0000", 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9789999999999993e-06 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.054707+0000", 2024-09-21T09:57:05.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001184835 2024-09-21T09:57:05.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.494 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.054852+0000", 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014486100000000001 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.055008+0000", 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015620200000000001 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.057047+0000", 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002038527 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.057125+0000", 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8508000000000001e-05 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.103741+0000", 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.046615901000000001 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.103863+0000", 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000121817 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.104793+0000", 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00093008000000000001 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47764 6.4 6:2fb0671b:::benchmark_data_smithi076_154645_object47763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.497 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:02.059345+0000", 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.3739194709999998, 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.046244384999999999, 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47764 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.498 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059345+0000", 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059345+0000", 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059347+0000", 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.888e-06 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.499 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059370+0000", 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2413999999999999e-05 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059384+0000", 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3878e-05 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059392+0000", 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1280000000000008e-06 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059465+0000", 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3538e-05 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059626+0000", 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016081799999999999 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.059832+0000", 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000206267 2024-09-21T09:57:05.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.061394+0000", 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015618030000000001 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.062513+0000", 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011189080000000001 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.104839+0000", 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.042325580000000002 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.104945+0000", 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106451 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.105590+0000", 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00064471199999999998 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.503 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:48139 6.4 6:2bfaacfa:::benchmark_data_smithi076_154645_object48138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:02.332801+0000", 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.1004636579999998, 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.035719491999999999, 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 48139 2024-09-21T09:57:05.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.332801+0000", 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.332801+0000", 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.505 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.332802+0000", 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.266e-06 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.332825+0000", 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2174000000000001e-05 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.332837+0000", 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1955e-05 2024-09-21T09:57:05.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.332844+0000", 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3440000000000003e-06 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.333124+0000", 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027966500000000001 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.333313+0000", 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018953399999999999 2024-09-21T09:57:05.507 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.333466+0000", 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015288500000000001 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.338852+0000", 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053861819999999998 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.338895+0000", 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2991000000000002e-05 2024-09-21T09:57:05.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.367488+0000", 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.028593159999999999 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.367548+0000", 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9587999999999999e-05 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.368521+0000", 2024-09-21T09:57:05.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00097274799999999995 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:49144 6.1 6:83120acb:::benchmark_data_smithi076_154645_object49143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:02.991380+0000", 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 2.4418851359999998, 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.050267869999999999, 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.510 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 49144 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991380+0000", 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991380+0000", 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991383+0000", 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8619999999999999e-06 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991407+0000", 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4729e-05 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991424+0000", 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6327000000000001e-05 2024-09-21T09:57:05.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991433+0000", 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3090000000000001e-06 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991543+0000", 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110568 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991686+0000", 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014254900000000001 2024-09-21T09:57:05.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.991823+0000", 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013676699999999999 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.994013+0000", 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021905840000000002 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.514 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:02.994117+0000", 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103887 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:03.041505+0000", 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.047387437999999997 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:03.041600+0000", 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5167999999999994e-05 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:03.041648+0000", 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7682000000000002e-05 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:50428 6.4 6:2cce5ef9:::benchmark_data_smithi076_154645_object50427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:04.000269+0000", 2024-09-21T09:57:05.516 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 1.432995367, 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.034756922000000003, 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 50428 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000269+0000", 2024-09-21T09:57:05.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000269+0000", 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000272+0000", 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0710000000000002e-06 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.518 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000297+0000", 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5375e-05 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000311+0000", 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4397e-05 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000320+0000", 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7350000000000002e-06 2024-09-21T09:57:05.519 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000800+0000", 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00047981400000000001 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.000999+0000", 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019958199999999999 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.001228+0000", 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000228657 2024-09-21T09:57:05.520 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.005136+0000", 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003908206 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.010992+0000", 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0058552379999999996 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.034032+0000", 2024-09-21T09:57:05.521 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023040451999999999 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.034086+0000", 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3686000000000001e-05 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.035026+0000", 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00094070899999999997 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.522 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:51212 6.4 6:263eb6be:::benchmark_data_smithi076_154645_object51211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:04.625084+0000", 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.80818096100000003, 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.036981159, 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 51212 2024-09-21T09:57:05.523 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625084+0000", 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625084+0000", 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.524 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625086+0000", 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7379999999999999e-06 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625114+0000", 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8062999999999999e-05 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625126+0000", 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2709e-05 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.525 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625135+0000", 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2449999999999998e-06 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.625893+0000", 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00075808199999999998 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.626083+0000", 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019001200000000001 2024-09-21T09:57:05.526 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.626253+0000", 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017048500000000001 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.629620+0000", 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003367114 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.527 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.632265+0000", 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026443629999999998 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.660949+0000", 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.028683921000000001 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.661065+0000", 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116502 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.528 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.662065+0000", 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00099992500000000003 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:51231 6.4 6:25eb3556:::benchmark_data_smithi076_154645_object51230:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:04.637847+0000", 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.79541804699999996, 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.036561582000000002, 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.529 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 51231 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637847+0000", 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637847+0000", 2024-09-21T09:57:05.530 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637847+0000", 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9999999999999998e-07 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637867+0000", 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0086e-05 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.531 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637874+0000", 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0049999999999998e-06 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637879+0000", 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0669999999999999e-06 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.640586+0000", 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027065700000000002 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.532 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.640742+0000", 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015580499999999999 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.640927+0000", 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018567599999999999 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.645551+0000", 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046233050000000003 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.533 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.645589+0000", 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8152999999999999e-05 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.670687+0000", 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.02509834 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.670851+0000", 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016419000000000001 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.534 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.674408+0000", 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0035568850000000001 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:51232 6.4 6:266b8b3c:::benchmark_data_smithi076_154645_object51231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:04.637890+0000", 2024-09-21T09:57:05.535 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.79537519700000003, 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.037809042000000001, 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 51232 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.537 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637890+0000", 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637890+0000", 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637890+0000", 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7800000000000002e-07 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637925+0000", 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5296000000000002e-05 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637933+0000", 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1720000000000001e-06 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.637938+0000", 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2939999999999996e-06 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.539 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.642865+0000", 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0049268810000000001 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.643029+0000", 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016475499999999999 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.643274+0000", 2024-09-21T09:57:05.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024456599999999999 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.646017+0000", 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002742619 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.647611+0000", 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015944360000000001 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.674467+0000", 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.026855403 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.674737+0000", 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027037099999999999 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.675699+0000", 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00096177100000000005 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.542 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:51588 6.1 6:8d642ffc:::benchmark_data_smithi076_154645_object51587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:04.923776+0000", 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.50948848199999996, 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.034640308000000002, 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.543 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 51588 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.923776+0000", 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.923776+0000", 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.923777+0000", 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.057e-06 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.923798+0000", 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1109999999999999e-05 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.923810+0000", 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1233000000000001e-05 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.923817+0000", 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.43e-06 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.933553+0000", 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097353479999999996 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.933704+0000", 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015100199999999999 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.933786+0000", 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2291000000000002e-05 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.935968+0000", 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021820680000000001 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.940414+0000", 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004446106 2024-09-21T09:57:05.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.958318+0000", 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017904234000000002 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.958370+0000", 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2210999999999999e-05 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.958417+0000", 2024-09-21T09:57:05.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6217999999999997e-05 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:51589 6.1 6:822ddceb:::benchmark_data_smithi076_154645_object51588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:04.924087+0000", 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.50917771999999994, 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.034453527999999997, 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.549 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 51589 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.924087+0000", 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.924087+0000", 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.924088+0000", 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4799999999999997e-07 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.924112+0000", 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3632999999999999e-05 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.924125+0000", 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3322999999999999e-05 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.551 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.924132+0000", 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6560000000000003e-06 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.936287+0000", 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012155318 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.936418+0000", 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130667 2024-09-21T09:57:05.552 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.936578+0000", 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000160156 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.940377+0000", 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037995899999999998 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.940833+0000", 2024-09-21T09:57:05.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00045528499999999999 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.958454+0000", 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017621418 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.958498+0000", 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3942e-05 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.554 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:04.958541+0000", 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2592000000000001e-05 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:51761 6.4 6:2bfb7ebd:::benchmark_data_smithi076_154645_object51760:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1097)", 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:05.057049+0000", 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.37621547700000002, 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.032569526000000001, 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:57:05.555 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 51761 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.057049+0000", 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.057049+0000", 2024-09-21T09:57:05.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.057050+0000", 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5600000000000004e-07 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.057072+0000", 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2220000000000001e-05 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.057081+0000", 2024-09-21T09:57:05.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7979999999999995e-06 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.057088+0000", 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8650000000000003e-06 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.063077+0000", 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005988953 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:05.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.063390+0000", 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031342700000000002 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,5", 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.063808+0000", 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00041767300000000002 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.066355+0000", 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025464649999999999 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.069024+0000", 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026690110000000002 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.088558+0000", 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.019534877999999999 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.088745+0000", 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000186493 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:05.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:05.089619+0000", 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00087388699999999995 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:05.561 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:57:05.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:05.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:05.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:05.692 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:05.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:05.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:05.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:05.920 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 29 16 52783 52767 113.685 75.3125 0.0248101 0.00878989 2024-09-21T09:57:05.927 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:05.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:05.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:05.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:06.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:06.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:05 smithi191 ceph-mon[102206]: pgmap v1173: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 10026/143004 objects degraded (7.011%); 2690/143004 objects misplaced (1.881%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T09:57:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:05 smithi076 ceph-mon[106482]: pgmap v1173: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 10026/143004 objects degraded (7.011%); 2690/143004 objects misplaced (1.881%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T09:57:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:05 smithi076 ceph-mon[110963]: pgmap v1173: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s; 10026/143004 objects degraded (7.011%); 2690/143004 objects misplaced (1.881%); 4.9 MiB/s, 78 objects/s recovering 2024-09-21T09:57:06.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:06.241 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:06.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:06.381 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:06.483 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:06.516 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:06.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:06.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:06.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:06.730 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T09:57:06.753 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:06.824 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:06.920 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 30 16 54208 54192 112.864 89.0625 0.0080609 0.00885305 2024-09-21T09:57:06.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:06.976 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:06.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:07.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:07.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:07.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:07.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:07.239 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:07.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:07.375 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:07.477 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:07.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:07.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:07.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:07.682 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T09:57:07.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:07.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:07.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:07.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:07.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:07.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:07.915 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:07 smithi076 ceph-mon[106482]: pgmap v1174: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 10026/155448 objects degraded (6.450%); 2286/155448 objects misplaced (1.471%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T09:57:07.915 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:07 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2988486878' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:07.915 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:07 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:07.915 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:07 smithi076 ceph-mon[110963]: pgmap v1174: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 10026/155448 objects degraded (6.450%); 2286/155448 objects misplaced (1.471%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T09:57:07.916 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:07 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2988486878' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:07.916 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:07 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:07.919 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 31 16 55459 55443 111.745 78.1875 0.00666915 0.00894236 2024-09-21T09:57:08.015 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:08.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:08.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:07 smithi191 ceph-mon[102206]: pgmap v1174: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 10026/155448 objects degraded (6.450%); 2286/155448 objects misplaced (1.471%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T09:57:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:07 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2988486878' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:07 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:08.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:08.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:08.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:08.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:08.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:08.466 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:08.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:08.583 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:08.584 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:08.585 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:08.585 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:08.585 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:08.585 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:08.585 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:08.585 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:08.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:08.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:08.718 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:08.761 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:08.842 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:57:08.842 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:08.842 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[106482]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-21T09:57:08.843 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3628452834' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:08.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:57:08.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:08.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[110963]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-21T09:57:08.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:08 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3628452834' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:08.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:08.903 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:08.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:08.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:08.920 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 32 16 56810 56794 110.892 84.4375 0.0126593 0.00901177 2024-09-21T09:57:09.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:09.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:08 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:57:09.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:08 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:09.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:08 smithi191 ceph-mon[102206]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-21T09:57:09.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:08 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3628452834' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:09.043 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:09.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:09.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:09.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:09.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:09.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:09.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:09.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:09.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:09.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:09.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:09.610 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:09.712 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:09.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:09.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:09 smithi076 ceph-mon[106482]: pgmap v1176: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s; 10026/159378 objects degraded (6.291%); 2286/159378 objects misplaced (1.434%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T09:57:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 10026/159378 objects degraded (6.291%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:09 smithi076 ceph-mon[110963]: pgmap v1176: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s; 10026/159378 objects degraded (6.291%); 2286/159378 objects misplaced (1.434%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T09:57:09.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 10026/159378 objects degraded (6.291%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:09.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:09.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:09.927 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 33 16 58203 58187 110.17 87.0625 0.0182373 0.00907043 2024-09-21T09:57:09.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:10.021 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:10.026 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:10.064 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:09 smithi191 ceph-mon[102206]: pgmap v1176: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 95 MiB/s wr, 1.51k op/s; 10026/159378 objects degraded (6.291%); 2286/159378 objects misplaced (1.434%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T09:57:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 10026/159378 objects degraded (6.291%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:10.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:10.161 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:10.264 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.305 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:10.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:10.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:10.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:10.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:10.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:10.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:10.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:10.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:10.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:10.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:10.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:10.924 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 34 16 59509 59493 109.33 81.625 0.0109444 0.00914056 2024-09-21T09:57:10.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:11.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:11.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:11.144 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:11.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:11.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:11.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:11.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:11.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:11.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:57:11.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:11.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:11.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:11.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:11 smithi191 ceph-mon[102206]: pgmap v1177: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 10026/163122 objects degraded (6.146%); 2286/163122 objects misplaced (1.401%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T09:57:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:11.867 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:11 smithi076 ceph-mon[106482]: pgmap v1177: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 10026/163122 objects degraded (6.146%); 2286/163122 objects misplaced (1.401%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T09:57:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:11.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:11.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:11 smithi076 ceph-mon[110963]: pgmap v1177: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s; 10026/163122 objects degraded (6.146%); 2286/163122 objects misplaced (1.401%); 6.0 MiB/s, 96 objects/s recovering 2024-09-21T09:57:11.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:11.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:11.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 35 16 60854 60838 108.607 84.0625 0.0101173 0.00920101 2024-09-21T09:57:11.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:11.998 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:11.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:11.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:12.007 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:57:12.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:12.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:12.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:12.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:12.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:12.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:12.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:12.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:12.508 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:12.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:12.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:12.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:12.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:12.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:12.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:12.901 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:12.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 36 16 62126 62110 107.798 79.5 0.0159517 0.0092696 2024-09-21T09:57:12.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:12.976 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:12.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:13.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:13.109 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:13.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:13.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:13.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:13.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:13.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:13.461 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:57:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:57:13] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.51.0" 2024-09-21T09:57:13.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:13.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:13.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:13.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:13.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:13.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:13 smithi191 ceph-mon[102206]: pgmap v1178: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 10026/178107 objects degraded (5.629%); 1897/178107 objects misplaced (1.065%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T09:57:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:13 smithi076 ceph-mon[106482]: pgmap v1178: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 10026/178107 objects degraded (5.629%); 1897/178107 objects misplaced (1.065%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T09:57:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:13 smithi076 ceph-mon[110963]: pgmap v1178: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 10026/178107 objects degraded (5.629%); 1897/178107 objects misplaced (1.065%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T09:57:13.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:13.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 37 16 63372 63356 106.989 77.875 0.0109051 0.00934047 2024-09-21T09:57:13.926 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:13.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:14.028 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:14.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:14.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:14.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:14.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:14.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:14.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:14.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:14.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:14.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:14.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:14.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:14.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:14.709 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T09:57:14.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:14.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:14.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:14.861 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:14.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 10026/178107 objects degraded (5.629%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2531817769' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T09:57:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:14 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T09:57:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 10026/178107 objects degraded (5.629%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2531817769' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T09:57:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:14 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T09:57:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 10026/178107 objects degraded (5.629%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2531817769' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T09:57:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:14 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T09:57:14.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 38 16 64605 64589 106.202 77.0625 0.00641723 0.0094098 2024-09-21T09:57:14.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:15.019 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:15.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:15.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:15.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:15.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:15.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:15.443 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:15.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T09:57:15.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:15.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:15.686 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.741 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:15.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:15.842 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:15.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:15 smithi076 ceph-mon[106482]: pgmap v1179: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10026/181731 objects degraded (5.517%); 1897/181731 objects misplaced (1.044%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T09:57:15.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:15 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T09:57:15.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:15 smithi076 ceph-mon[106482]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-21T09:57:15.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:15 smithi076 ceph-mon[110963]: pgmap v1179: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10026/181731 objects degraded (5.517%); 1897/181731 objects misplaced (1.044%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T09:57:15.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:15 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T09:57:15.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:15 smithi076 ceph-mon[110963]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-21T09:57:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:15 smithi191 ceph-mon[102206]: pgmap v1179: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 10026/181731 objects degraded (5.517%); 1897/181731 objects misplaced (1.044%); 5.0 MiB/s, 79 objects/s recovering 2024-09-21T09:57:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:15 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T09:57:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:15 smithi191 ceph-mon[102206]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-21T09:57:15.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:15.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:15.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:15.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 39 16 65830 65814 105.442 76.5625 0.00631079 0.0094768 2024-09-21T09:57:15.984 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:16.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:16.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:16.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:16.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.171 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:16.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:16.203 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:16.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:16.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:16.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:16.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:16.458 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:16.560 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:16.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:16.701 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:16.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:16.835 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:16.867 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:16.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:16.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T09:57:16.920288+0000 min lat: 0.00146111 max lat: 2.14015 avg lat: 0.00954107 2024-09-21T09:57:16.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:57:16.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 40 16 67077 67061 104.754 77.9375 0.0274141 0.00954107 2024-09-21T09:57:16.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:16.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:16.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:17.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:17.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:17.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:17.254 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:17.356 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:17.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:17.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:17.520 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:17.591 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T09:57:17.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:17.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:17.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:17.781 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:17 smithi191 ceph-mon[102206]: pgmap v1181: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 10026/194895 objects degraded (5.144%); 1479/194895 objects misplaced (0.759%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T09:57:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:17 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1322335147' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:17 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:17 smithi076 ceph-mon[106482]: pgmap v1181: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 10026/194895 objects degraded (5.144%); 1479/194895 objects misplaced (0.759%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T09:57:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:17 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1322335147' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:17 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:17 smithi076 ceph-mon[110963]: pgmap v1181: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s; 10026/194895 objects degraded (5.144%); 1479/194895 objects misplaced (0.759%); 5.8 MiB/s, 92 objects/s recovering 2024-09-21T09:57:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:17 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1322335147' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:17 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:57:17.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:17.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:17.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:17.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 41 16 68029 68013 103.65 59.5 0.0055258 0.00959368 2024-09-21T09:57:18.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:18.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:18.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:18.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:18.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:18.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:18.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:18.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:18.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:18.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:18.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:18.521 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:18.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:18.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:18.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:18.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:18.792 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:18 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:18 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:57:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:18 smithi191 ceph-mon[102206]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-21T09:57:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:18 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:18 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:57:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:18 smithi076 ceph-mon[106482]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-21T09:57:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:18 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:18 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:57:18.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:18 smithi076 ceph-mon[110963]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-21T09:57:18.894 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:18.922 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 42 16 68354 68338 101.666 20.3125 0.012096 0.00983022 2024-09-21T09:57:18.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:18.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:18.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:19.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:19.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:19.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:19.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:19.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:19.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:19.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:19.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:19.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.458 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:19.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:19.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:19.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:19.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:19 smithi076 ceph-mon[106482]: pgmap v1183: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 8548/198201 objects degraded (4.313%); 2950/198201 objects misplaced (1.488%); 6.4 MiB/s, 101 objects/s recovering 2024-09-21T09:57:19.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:19 smithi076 ceph-mon[106482]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-21T09:57:19.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/198201 objects degraded (4.313%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:19.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:19 smithi076 ceph-mon[110963]: pgmap v1183: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 8548/198201 objects degraded (4.313%); 2950/198201 objects misplaced (1.488%); 6.4 MiB/s, 101 objects/s recovering 2024-09-21T09:57:19.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:19 smithi076 ceph-mon[110963]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-21T09:57:19.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/198201 objects degraded (4.313%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:19.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:19.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:19.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:19.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:19.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:19 smithi191 ceph-mon[102206]: pgmap v1183: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 8548/198201 objects degraded (4.313%); 2950/198201 objects misplaced (1.488%); 6.4 MiB/s, 101 objects/s recovering 2024-09-21T09:57:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:19 smithi191 ceph-mon[102206]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-21T09:57:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/198201 objects degraded (4.313%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:19.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 43 16 69533 69517 101.016 73.6875 0.0118011 0.00989344 2024-09-21T09:57:19.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:19.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:19.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:19.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:20.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:20.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:20.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:20.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:20.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:20.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:20.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:20.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:20.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:20.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:20.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:20.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:20.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.817 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:20.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:20.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:20.921 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 44 16 70746 70730 100.442 75.8125 0.00572951 0.00995047 2024-09-21T09:57:20.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:20.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:20.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:21.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:21.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:21.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:21.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:21.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:21.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:21.345 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:21.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:21.476 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:21.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:21.605 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:21.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:21.696 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:21 smithi076 ceph-mon[106482]: pgmap v1185: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 8548/200865 objects degraded (4.256%); 2950/200865 objects misplaced (1.469%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:57:21.696 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:21 smithi076 ceph-mon[110963]: pgmap v1185: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 8548/200865 objects degraded (4.256%); 2950/200865 objects misplaced (1.469%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:57:21.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:21.735 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:21.754 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:57:21.754 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:57:21.837 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:21 smithi191 ceph-mon[102206]: pgmap v1185: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 8548/200865 objects degraded (4.256%); 2950/200865 objects misplaced (1.469%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:57:21.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:21.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:21.922 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 45 16 71962 71946 99.899 76 0.00952421 0.010005 2024-09-21T09:57:21.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:22.019 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:22.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:22.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:22.162 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:22.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:22.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:22.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:22.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:22.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:22.413 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:22.449 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:22.493 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:57:22.494 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:22.530 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:57:22.530 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:22.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:22.564 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:57:22.564 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:22.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:22.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:22.599 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:57:22.599 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:22.629 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:57:22.630 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:22.659 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:57:22.660 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:22.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:22.689 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:57:22.689 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:22.725 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:57:22.725 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:22.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.728 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:22.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:22.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1691037878' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:22.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:22.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1691037878' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1691037878' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:22.922 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 46 16 73079 73063 99.245 69.8125 0.0116036 0.0100698 2024-09-21T09:57:22.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:22.998 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:22.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:22.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:23.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:23.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:23.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:23.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:57:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:57:23] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.51.0" 2024-09-21T09:57:23.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:23.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:23.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:57:23.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:23.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:23.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:23.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:23.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:23.806 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:23 smithi076 ceph-mon[106482]: pgmap v1186: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 8548/211962 objects degraded (4.033%); 2514/211962 objects misplaced (1.186%); 6.8 MiB/s, 109 objects/s recovering 2024-09-21T09:57:23.807 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:23 smithi076 ceph-mon[110963]: pgmap v1186: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 8548/211962 objects degraded (4.033%); 2514/211962 objects misplaced (1.186%); 6.8 MiB/s, 109 objects/s recovering 2024-09-21T09:57:23.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:23.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T09:57:23.923 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 47 16 74231 74215 98.6637 72 0.00816311 0.0101293 2024-09-21T09:57:23.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:23.991 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:23.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:24.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:23 smithi191 ceph-mon[102206]: pgmap v1186: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 8548/211962 objects degraded (4.033%); 2514/211962 objects misplaced (1.186%); 6.8 MiB/s, 109 objects/s recovering 2024-09-21T09:57:24.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:24.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:24.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:24.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:24.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:24.404 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:24.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:24.505 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:24.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:24.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:24.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:24.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:24.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:24.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:24.909 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:24.923 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 48 16 75359 75343 98.0769 70.5 0.0076172 0.0101894 2024-09-21T09:57:24.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:24.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:24.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:25.032 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/211962 objects degraded (4.033%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:25.032 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/211962 objects degraded (4.033%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:25.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:25.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.077 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:25.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/211962 objects degraded (4.033%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:25.137 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:25.170 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:25.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:25.248 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:25.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:25.394 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:25.496 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.541 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:25.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:25.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:57:25.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:25.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:25.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:25.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:25.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:25.860 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[110963]: pgmap v1187: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 8548/215274 objects degraded (3.971%); 2514/215274 objects misplaced (1.168%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[106482]: pgmap v1187: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 8548/215274 objects degraded (3.971%); 2514/215274 objects misplaced (1.168%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:25.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:25.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:25 smithi191 ceph-mon[102206]: pgmap v1187: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 8548/215274 objects degraded (3.971%); 2514/215274 objects misplaced (1.168%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T09:57:25.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:25.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:25.923 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 49 16 76329 76313 97.3126 60.625 0.00903401 0.0102704 2024-09-21T09:57:25.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:25.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:26.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.066 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:26.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:26.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.208 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:26.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:26.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:26.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:26.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:26.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:26.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:26.610 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:26.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:26.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:26.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:26.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:26.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:26.923 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 50 16 77493 77477 96.8213 72.75 0.0101812 0.0103229 2024-09-21T09:57:26.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:27.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:27.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:27.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:27.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:27.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:27.301 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:27.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:27.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:27.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:27.538 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:27.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:27.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:27.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:27.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:27.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:27.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:27.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:27 smithi076 ceph-mon[106482]: pgmap v1188: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 8548/226710 objects degraded (3.770%); 2514/226710 objects misplaced (1.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T09:57:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:27 smithi076 ceph-mon[110963]: pgmap v1188: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 8548/226710 objects degraded (3.770%); 2514/226710 objects misplaced (1.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T09:57:27.926 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 51 16 78645 78629 96.3291 72 0.0301548 0.0103746 2024-09-21T09:57:27.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:27.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:27.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:27.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:28.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:28.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:27 smithi191 ceph-mon[102206]: pgmap v1188: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 8548/226710 objects degraded (3.770%); 2514/226710 objects misplaced (1.109%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T09:57:28.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:28.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:28.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:28.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:28.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:28.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:28.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:28.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:28.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:28.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:28.658 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T09:57:28.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:28.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:28.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:28.855 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:28.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:28 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1821407246' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T09:57:28.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:28 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T09:57:28.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:28 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1821407246' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T09:57:28.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:28 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T09:57:28.926 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 52 16 79683 79667 95.7243 64.875 0.01888 0.010441 2024-09-21T09:57:28.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:28.982 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:28.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:28.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:29.084 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:29.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:28 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1821407246' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T09:57:29.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:28 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T09:57:29.128 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:29.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:29.263 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:29.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:29.399 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:29.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:29.524 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:29.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:29.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:29.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:29.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:29.671 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T09:57:29.766 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:29.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:29.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:29.889 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:29.926 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 53 16 80613 80597 95.0146 58.125 0.022092 0.0105181 2024-09-21T09:57:29.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:29.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:29.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:29 smithi191 ceph-mon[102206]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8548/229806 objects degraded (3.720%); 2263/229806 objects misplaced (0.985%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T09:57:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:29 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T09:57:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:29 smithi191 ceph-mon[102206]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-21T09:57:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/229806 objects degraded (3.720%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:29 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1054114362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:29.966 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:29 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:30.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:30.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[110963]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8548/229806 objects degraded (3.720%); 2263/229806 objects misplaced (0.985%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T09:57:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T09:57:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[110963]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-21T09:57:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/229806 objects degraded (3.720%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1054114362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[106482]: pgmap v1189: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8548/229806 objects degraded (3.720%); 2263/229806 objects misplaced (0.985%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T09:57:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T09:57:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[106482]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-21T09:57:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/229806 objects degraded (3.720%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1054114362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:29 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:57:30.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:30.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:30.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:30.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:30.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.363 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:30.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:30.464 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:30.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:30.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:30.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:30.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:30.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:30.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:30.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:30.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:30.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:30.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:30.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:30.926 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 54 16 81655 81639 94.4612 65.125 0.0097248 0.01058 2024-09-21T09:57:31.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.044 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:31.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:30 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:30 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:30 smithi191 ceph-mon[102206]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-21T09:57:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:30 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:30 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[106482]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[110963]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:30 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:31.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:31.185 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:31.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:31.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:31.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:31.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:31.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:31.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:31.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T09:57:31.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:31.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:31.744 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:31.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:31.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:31.869 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:31 smithi076 ceph-mon[106482]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 8548/232752 objects degraded (3.673%); 2263/232752 objects misplaced (0.972%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:57:31.869 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:31 smithi076 ceph-mon[110963]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 8548/232752 objects degraded (3.673%); 2263/232752 objects misplaced (0.972%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:57:31.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:31.927 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 55 16 82713 82697 93.9449 66.125 0.0218169 0.0106388 2024-09-21T09:57:31.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:31.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:31.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:32.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:32.074 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:31 smithi191 ceph-mon[102206]: pgmap v1192: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 8548/232752 objects degraded (3.673%); 2263/232752 objects misplaced (0.972%); 2.0 MiB/s, 31 objects/s recovering 2024-09-21T09:57:32.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:32.217 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:32.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:32.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:32.362 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:32.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:32.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:32.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:32.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:32.593 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.641 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:32.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:32.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:32.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:32.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:32.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:32.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:32.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:32.927 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 56 16 83816 83800 93.4983 68.9375 0.0171684 0.0106891 2024-09-21T09:57:33.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:33.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:33.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:33.175 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:33.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:33.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:33.303 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:33.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:33.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:57:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:57:33] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.51.0" 2024-09-21T09:57:33.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:33.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:33.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:33.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:33.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:33.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:33.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:33.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:33.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:33.927 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 57 16 84910 84894 93.0576 68.375 0.0325338 0.0107402 2024-09-21T09:57:33.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:33.965 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:33.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:34.009 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:33 smithi076 ceph-mon[106482]: pgmap v1193: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 8548/244755 objects degraded (3.492%); 2000/244755 objects misplaced (0.817%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:34.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:33 smithi076 ceph-mon[110963]: pgmap v1193: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 8548/244755 objects degraded (3.492%); 2000/244755 objects misplaced (0.817%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:34.067 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:34.076 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:57:34.076 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T09:57:34.076 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T09:57:34.076 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T09:57:34.076 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:57:34.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:33 smithi191 ceph-mon[102206]: pgmap v1193: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 8548/244755 objects degraded (3.492%); 2000/244755 objects misplaced (0.817%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:34.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.245 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:34.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:34.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:34.371 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:34.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:57:34.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:34.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:34.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:34.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:34.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:34.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:34.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:34.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/244755 objects degraded (3.492%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/244755 objects degraded (3.492%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:34.927 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 58 16 85782 85766 92.3929 54.5 0.0114833 0.0108181 2024-09-21T09:57:34.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:34.982 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:34.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:34.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:35.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:35.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:35.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/244755 objects degraded (3.492%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:35.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:35.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:35.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:35.358 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:35.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:35.505 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:35.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:35.575 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:35.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T09:57:35.638 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.697 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:35.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:35.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:35.820 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:35 smithi076 ceph-mon[110963]: pgmap v1194: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 8548/247587 objects degraded (3.453%); 2000/247587 objects misplaced (0.808%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:35.821 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:35 smithi076 ceph-mon[106482]: pgmap v1194: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 8548/247587 objects degraded (3.453%); 2000/247587 objects misplaced (0.808%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:35.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:35.928 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 59 16 86647 86631 91.7427 54.0625 0.00576406 0.0108928 2024-09-21T09:57:35.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:35.988 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:35.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:36.089 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:35 smithi191 ceph-mon[102206]: pgmap v1194: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 868 op/s; 8548/247587 objects degraded (3.453%); 2000/247587 objects misplaced (0.808%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:36.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:36.231 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.263 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:36.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:36.364 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:36.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:36.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:36.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:36.550 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:36.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:36.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:36.687 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:36.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.871 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:36.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:36.929 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T09:57:36.927927+0000 min lat: 0.00146111 max lat: 2.14015 avg lat: 0.0109545 2024-09-21T09:57:36.929 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:57:36.929 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 60 16 87631 87615 91.2373 61.5 0.0104332 0.0109545 2024-09-21T09:57:36.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:36.996 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:36.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:37.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:37.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:37.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:37.172 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:57:37.173 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:57:37.173 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:37.173 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:37.188 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:57:37.188 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:37.224 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:57:37.224 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:37.226 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:37.255 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:57:37.255 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:37.266 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:37.292 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:57:37.292 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:37.325 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:57:37.325 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:37.356 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:57:37.356 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:37.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:37.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:37.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:37.551 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:37.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:37.689 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:37.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:37.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:37.819 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:37.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:37.929 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 61 16 88648 88632 90.7834 63.5625 0.0355351 0.011009 2024-09-21T09:57:37.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:37.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:37.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:38.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:38.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.114 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:38.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:38.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:37 smithi191 ceph-mon[102206]: pgmap v1195: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8548/257952 objects degraded (3.314%); 2000/257952 objects misplaced (0.775%); 2.1 MiB/s, 32 objects/s recovering 2024-09-21T09:57:38.115 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2068761633' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:37 smithi076 ceph-mon[106482]: pgmap v1195: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8548/257952 objects degraded (3.314%); 2000/257952 objects misplaced (0.775%); 2.1 MiB/s, 32 objects/s recovering 2024-09-21T09:57:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2068761633' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:37 smithi076 ceph-mon[110963]: pgmap v1195: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 8548/257952 objects degraded (3.314%); 2000/257952 objects misplaced (0.775%); 2.1 MiB/s, 32 objects/s recovering 2024-09-21T09:57:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2068761633' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:38.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:38.297 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:38.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:38.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:38.427 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:38.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:38.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:38.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:38.705 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:38.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:38.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:38.867 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:38.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:38.926 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:38.930 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 62 16 89653 89637 90.3307 62.8125 0.0192921 0.0110637 2024-09-21T09:57:38.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:38.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:39.101 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:39.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:39.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:39.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:39.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:39.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:39.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:39.521 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:39.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:39.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:39.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.694 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:39.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:39.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.821 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:39.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:39 smithi076 ceph-mon[106482]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8548/260676 objects degraded (3.279%); 1763/260676 objects misplaced (0.676%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T09:57:39.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/260676 objects degraded (3.279%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:39 smithi076 ceph-mon[110963]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8548/260676 objects degraded (3.279%); 1763/260676 objects misplaced (0.676%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T09:57:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/260676 objects degraded (3.279%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:39.923 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:39.931 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 63 16 90673 90657 89.9088 63.75 0.0257461 0.0111162 2024-09-21T09:57:39.956 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:39.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:40.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:39 smithi191 ceph-mon[102206]: pgmap v1196: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8548/260676 objects degraded (3.279%); 1763/260676 objects misplaced (0.676%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T09:57:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/260676 objects degraded (3.279%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:40.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:40.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:40.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:57:40.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:40.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:40.376 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:40.421 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:40.523 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:40.555 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:40.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:40.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:40.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:40.740 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T09:57:40.780 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/212235766' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T09:57:40.780 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:40 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T09:57:40.780 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:40.780 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/212235766' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T09:57:40.780 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:40 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T09:57:40.780 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:40.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:40.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:40.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/212235766' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T09:57:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:40 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T09:57:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:40.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 64 16 91612 91596 89.4199 58.6875 0.020315 0.0111774 2024-09-21T09:57:40.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:41.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:41.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:41.152 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:41.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:41.254 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:41.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:41.393 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:41.434 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:41.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:41.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:41.578 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:41.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:41.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:41.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:41.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:41.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:41.931 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 65 16 92513 92497 88.9106 56.3125 0.0163803 0.0112415 2024-09-21T09:57:41.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:41.971 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:41.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:42.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:42.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:41 smithi191 ceph-mon[102206]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 8548/263358 objects degraded (3.246%); 1763/263358 objects misplaced (0.669%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T09:57:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:41 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T09:57:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:41 smithi191 ceph-mon[102206]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-21T09:57:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:41 smithi076 ceph-mon[106482]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 8548/263358 objects degraded (3.246%); 1763/263358 objects misplaced (0.669%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T09:57:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:41 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T09:57:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:41 smithi076 ceph-mon[106482]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-21T09:57:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:41 smithi076 ceph-mon[110963]: pgmap v1197: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 8548/263358 objects degraded (3.246%); 1763/263358 objects misplaced (0.669%); 2.9 MiB/s, 46 objects/s recovering 2024-09-21T09:57:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:41 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T09:57:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:41 smithi076 ceph-mon[110963]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-21T09:57:42.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:42.235 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:42.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:42.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:42.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.378 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:42.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:42.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:42.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:42.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:42.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:42.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:42.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:42.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:42.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:42.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:42.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:42.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:42.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:42.931 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 66 16 93543 93527 88.5389 64.375 0.00891567 0.0112885 2024-09-21T09:57:43.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:43.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:43.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:43.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:43.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:43.323 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:43.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T09:57:43.425 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:43.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:43.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:43.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:43.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:57:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:57:43] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.51.0" 2024-09-21T09:57:43.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:43.752 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:43.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:43.880 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:43.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:43.887 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:43 smithi076 ceph-mon[106482]: pgmap v1199: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 8548/274611 objects degraded (3.113%); 1487/274611 objects misplaced (0.541%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T09:57:43.887 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:43 smithi076 ceph-mon[110963]: pgmap v1199: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 8548/274611 objects degraded (3.113%); 1487/274611 objects misplaced (0.541%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T09:57:43.932 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 67 16 94538 94522 88.1442 62.1875 0.00803121 0.0113322 2024-09-21T09:57:43.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:44.016 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:44.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:43 smithi191 ceph-mon[102206]: pgmap v1199: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 8548/274611 objects degraded (3.113%); 1487/274611 objects misplaced (0.541%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T09:57:44.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:44.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:44.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:44.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:44.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:44.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:44.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:44.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:44.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:44.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:44.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:44.621 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:44.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:44.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:44.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:44.769 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:44.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:44.794 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T09:57:44.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:44.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:44.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:44.932 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 68 15 94777 94762 87.0688 15 0.021573 0.0114794 2024-09-21T09:57:45.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:45.039 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:44 smithi191 ceph-mon[102206]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-21T09:57:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/274611 objects degraded (3.113%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/840886008' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:44 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[106482]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/274611 objects degraded (3.113%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/840886008' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[110963]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/274611 objects degraded (3.113%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/840886008' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:44 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:57:45.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:45.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:45.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:45.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:45.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:45.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:45.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:45.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:45.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:45.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:45.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:45.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:45.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:45.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:45.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:45.874 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:45.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:45.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:45.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 69 16 95810 95794 86.7417 64.5 0.0213392 0.0115227 2024-09-21T09:57:45.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:46.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:45 smithi191 ceph-mon[102206]: pgmap v1201: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 8548/277389 objects degraded (3.082%); 1487/277389 objects misplaced (0.536%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:45 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:45 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:57:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:45 smithi191 ceph-mon[102206]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-21T09:57:46.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:46.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[106482]: pgmap v1201: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 8548/277389 objects degraded (3.082%); 1487/277389 objects misplaced (0.536%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:46.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:46.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:57:46.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[106482]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-21T09:57:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[110963]: pgmap v1201: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 8548/277389 objects degraded (3.082%); 1487/277389 objects misplaced (0.536%); 4.0 MiB/s, 64 objects/s recovering 2024-09-21T09:57:46.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:57:46.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:57:46.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:45 smithi076 ceph-mon[110963]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-21T09:57:46.161 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:46.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:46.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:46.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:46.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:46.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:46.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:46.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:46.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:46.702 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:46.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:46.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:46.932 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 70 16 96738 96722 86.3312 58 0.0140053 0.0115777 2024-09-21T09:57:46.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:46.985 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:46.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:47.019 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:47.021 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:57:47.022 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:57:47.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:47.118 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:47.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:47.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:47.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:47.357 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:47.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:47.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:47.497 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:47.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:47.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:47.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:47.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:47.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:47.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:47.894 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:47.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:47.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:47.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 71 16 97669 97653 85.9348 58.1875 0.0403269 0.0116302 2024-09-21T09:57:48.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:48.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:47 smithi191 ceph-mon[102206]: pgmap v1203: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 8548/285999 objects degraded (2.989%); 1141/285999 objects misplaced (0.399%); 6.5 MiB/s, 103 objects/s recovering 2024-09-21T09:57:48.127 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:47 smithi076 ceph-mon[106482]: pgmap v1203: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 8548/285999 objects degraded (2.989%); 1141/285999 objects misplaced (0.399%); 6.5 MiB/s, 103 objects/s recovering 2024-09-21T09:57:48.127 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:47 smithi076 ceph-mon[110963]: pgmap v1203: 89 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 8548/285999 objects degraded (2.989%); 1141/285999 objects misplaced (0.399%); 6.5 MiB/s, 103 objects/s recovering 2024-09-21T09:57:48.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:48.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:48.320 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:48.358 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:48.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:48.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:48.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:48.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:48.629 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:48.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:48.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:48.763 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:48.865 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:48.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:48.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:48.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 72 16 98624 98608 85.5702 59.6875 0.0313444 0.0116805 2024-09-21T09:57:49.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:49.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.054 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:49.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:49.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:49.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:49.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:49.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:49.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.338 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:49.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:49.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:49.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:49.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:49.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:49.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:49.746 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:49.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:49 smithi076 ceph-mon[106482]: pgmap v1204: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 8548/288315 objects degraded (2.965%); 1094/288315 objects misplaced (0.379%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:57:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/288315 objects degraded (2.965%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:49 smithi076 ceph-mon[110963]: pgmap v1204: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 8548/288315 objects degraded (2.965%); 1094/288315 objects misplaced (0.379%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:57:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/288315 objects degraded (2.965%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:49.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:49.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:49.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 73 16 99633 99617 85.2619 63.0625 0.0150074 0.0117228 2024-09-21T09:57:49.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:50.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:49 smithi191 ceph-mon[102206]: pgmap v1204: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 8548/288315 objects degraded (2.965%); 1094/288315 objects misplaced (0.379%); 5.4 MiB/s, 86 objects/s recovering 2024-09-21T09:57:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/288315 objects degraded (2.965%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:50.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:50.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:50.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:50.272 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:50.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:50.418 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:50.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:50.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:50.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:50.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:50.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:50.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:50.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:50.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:50.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:50.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 74 16 100602 100586 84.9281 60.5625 0.0246596 0.0117686 2024-09-21T09:57:50.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:51.022 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:51.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:51.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:51.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:51.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:51.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:51.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:51.415 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:51.425 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T09:57:51.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:51.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:51.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:51.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:51.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:51.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:51.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 75 16 101577 101561 84.6083 60.9375 0.0124964 0.0118129 2024-09-21T09:57:51.953 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:51.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:57:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:51.998 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:51.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:52.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:51 smithi076 ceph-mon[106482]: pgmap v1205: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 8548/290727 objects degraded (2.940%); 1094/290727 objects misplaced (0.376%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T09:57:52.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:51 smithi076 ceph-mon[110963]: pgmap v1205: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 8548/290727 objects degraded (2.940%); 1094/290727 objects misplaced (0.376%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T09:57:52.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:51 smithi191 ceph-mon[102206]: pgmap v1205: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 42 MiB/s wr, 671 op/s; 8548/290727 objects degraded (2.940%); 1094/290727 objects misplaced (0.376%); 3.1 MiB/s, 49 objects/s recovering 2024-09-21T09:57:52.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:52.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:52.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:52.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:52.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.417 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:52.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:52.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:52.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:52.646 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:52.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:52.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:52.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:52.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 76 16 102486 102470 84.2425 56.8125 0.0115823 0.0118651 2024-09-21T09:57:52.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:52.967 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:52.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:53.069 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:53.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:53.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:53.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.242 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:53.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:53.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:53.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:53.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:53.409 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:57:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:57:53] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.51.0" 2024-09-21T09:57:53.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.572 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:53.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:53.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:53.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:53.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:53.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:53.835 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:53.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:53.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 77 16 103295 103279 83.8052 50.5625 0.0142335 0.0119246 2024-09-21T09:57:53.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:53.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:53.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:54.084 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:54.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:54.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:54.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.258 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:54.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[106482]: pgmap v1206: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 8548/301644 objects degraded (2.834%); 786/301644 objects misplaced (0.261%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/301644 objects degraded (2.834%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[110963]: pgmap v1206: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 8548/301644 objects degraded (2.834%); 786/301644 objects misplaced (0.261%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:54.322 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:54.323 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/301644 objects degraded (2.834%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:54.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:54 smithi191 ceph-mon[102206]: pgmap v1206: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 8548/301644 objects degraded (2.834%); 786/301644 objects misplaced (0.261%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:57:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/301644 objects degraded (2.834%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:54.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:54.481 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:54.529 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:54.612 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:57:54.613 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:57:54.631 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:54.661 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:57:54.661 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:57:54.678 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:54.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:54.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:54.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:57:54.698 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:57:54.711 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:54.743 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:57:54.743 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:57:54.778 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:57:54.778 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:57:54.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:54.812 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:57:54.812 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:57:54.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:54.850 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:57:54.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:57:54.883 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:57:54.883 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:57:54.924 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:54.933 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 78 16 104014 103998 83.3066 44.9375 0.0159271 0.0119972 2024-09-21T09:57:54.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:54.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:55.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:55.090 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T09:57:55.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:55.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:55.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:55.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:55.241 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4154060364' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T09:57:55.241 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2313819391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:55.241 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4154060364' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2313819391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:55.242 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:55.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4154060364' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T09:57:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2313819391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:57:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:57:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:55.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:55.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:55.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:55.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:55.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:55.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:55.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:55.791 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:55.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:55.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:55.922 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:55.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:55.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:55.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 79 16 104965 104949 83.0045 59.4375 0.0152507 0.0120406 2024-09-21T09:57:56.024 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:56.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:56.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:56.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:56.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:56.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:56.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:56.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[106482]: pgmap v1207: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 8548/304293 objects degraded (2.809%); 786/304293 objects misplaced (0.258%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4154060364' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[106482]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[110963]: pgmap v1207: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 8548/304293 objects degraded (2.809%); 786/304293 objects misplaced (0.258%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4154060364' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[110963]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-21T09:57:56.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:56 smithi191 ceph-mon[102206]: pgmap v1207: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 8548/304293 objects degraded (2.809%); 786/304293 objects misplaced (0.258%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:57:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4154060364' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T09:57:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:56 smithi191 ceph-mon[102206]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-21T09:57:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:56.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T09:57:56.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:56.542 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:56.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:56.643 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:56.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:56.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:56.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.839 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:56.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:56.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T09:57:56.933016+0000 min lat: 0.00146111 max lat: 2.14015 avg lat: 0.0120908 2024-09-21T09:57:56.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T09:57:56.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 80 16 105863 105847 82.6685 56.125 0.0117929 0.0120908 2024-09-21T09:57:56.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:56.973 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:56.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:56.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:57.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:57.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:57.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:57.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:57.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.263 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:57.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:57.364 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:57.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:57.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:57.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:57.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:57.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:57.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:57:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:57.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:57:57.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:57.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:57.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:57.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:57:57.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:57:57.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 81 16 106736 106720 82.3213 54.5625 0.0191923 0.0121409 2024-09-21T09:57:57.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:57.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:57:58.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:58.123 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:58.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:58.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:58.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:58 smithi191 ceph-mon[102206]: pgmap v1209: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 8548/313119 objects degraded (2.730%); 464/313119 objects misplaced (0.148%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T09:57:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:58 smithi076 ceph-mon[106482]: pgmap v1209: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 8548/313119 objects degraded (2.730%); 464/313119 objects misplaced (0.148%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T09:57:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:58 smithi076 ceph-mon[110963]: pgmap v1209: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s; 8548/313119 objects degraded (2.730%); 464/313119 objects misplaced (0.148%); 4.2 MiB/s, 67 objects/s recovering 2024-09-21T09:57:58.387 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:58.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:58.521 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:58.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:58.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:58.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:58.662 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:57:58.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:58.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:58.889 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:58.931 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:58.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:58.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 82 16 107645 107629 82.0102 56.8125 0.011862 0.0121876 2024-09-21T09:57:59.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:59.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:59.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:57:59.175 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:57:59.175 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107768 6.8 e1107/1100 6:1785aa03:::benchmark_data_smithi076_154645_object107767:head v 1107'6809, mlcod=1107'6807)", 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.080142+0000", 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.030078433000000002, 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.030106452999999998, 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.080142+0000", 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.080142+0000", 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.080142+0000", 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2000000000000001e-07 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.080176+0000", 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4369999999999998e-05 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.080182+0000", 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8619999999999999e-06 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.080186+0000", 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1049999999999997e-06 2024-09-21T09:57:59.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.094095+0000", 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013908873 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.094122+0000", 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7372000000000002e-05 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107770 6.0 e1107/1093 6:0953b1cb:::benchmark_data_smithi076_154645_object107769:head v 1107'6689, mlcod=0'0)", 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.081997+0000", 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.028222849000000001, 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.028309210000000001, 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.081997+0000", 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.081997+0000", 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.081998+0000", 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8000000000000005e-07 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.082023+0000", 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4590999999999999e-05 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.082033+0000", 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.043e-05 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.082039+0000", 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1419999999999998e-06 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.094624+0000", 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012584779000000001 2024-09-21T09:57:59.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.094648+0000", 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3978999999999999e-05 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107771 6.c e1107/1093 6:37c6c6e0:::benchmark_data_smithi076_154645_object107770:head v 1107'6680, mlcod=0'0)", 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.084199+0000", 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.026020702999999999, 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.026159679000000002, 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.183 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084199+0000", 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084199+0000", 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084200+0000", 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0100000000000004e-07 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084242+0000", 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1832999999999997e-05 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084250+0000", 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.507e-06 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084256+0000", 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6690000000000003e-06 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084304+0000", 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7700000000000001e-05 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.084325+0000", 2024-09-21T09:57:59.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1234999999999999e-05 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107776 6.8 e1107/1100 6:179a077c:::benchmark_data_smithi076_154645_object107775:head v 1107'6810, mlcod=1107'6807)", 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.085570+0000", 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.024650450000000001, 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.024838015000000001, 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.085570+0000", 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.085570+0000", 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.085570+0000", 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1699999999999997e-07 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.085602+0000", 2024-09-21T09:57:59.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2113999999999998e-05 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.085614+0000", 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1725999999999999e-05 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.085621+0000", 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6819999999999997e-06 2024-09-21T09:57:59.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.104199+0000", 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018577749000000001 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.104218+0000", 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9327999999999999e-05 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107777 6.4 e1107/1105 6:2a0da330:::benchmark_data_smithi076_154645_object107776:head v 1107'6691, mlcod=1107'6688)", 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.089010+0000", 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.021210395, 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021448691999999998, 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.089010+0000", 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.089010+0000", 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.089011+0000", 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3099999999999996e-07 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.089033+0000", 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2201e-05 2024-09-21T09:57:59.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.089045+0000", 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1918e-05 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.089052+0000", 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5320000000000001e-06 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.092044+0000", 2024-09-21T09:57:59.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029918269999999999 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.092090+0000", 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5614000000000001e-05 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107780 6.c e1107/1093 6:3e8ee8db:::benchmark_data_smithi076_154645_object107779:head v 1107'6681, mlcod=0'0)", 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.095201+0000", 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.015018909, 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015308683, 2024-09-21T09:57:59.195 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095201+0000", 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095201+0000", 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095202+0000", 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8400000000000004e-07 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095256+0000", 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3817000000000003e-05 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095269+0000", 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3457e-05 2024-09-21T09:57:59.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095276+0000", 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0859999999999996e-06 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095733+0000", 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00045725300000000002 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095755+0000", 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1481e-05 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107782 6.4 e1107/1105 6:2205a5ed:::benchmark_data_smithi076_154645_object107781:head v 1107'6692, mlcod=1107'6688)", 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.095951+0000", 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.014269196, 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.014608215000000001, 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.199 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095951+0000", 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095951+0000", 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.095962+0000", 2024-09-21T09:57:59.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0788000000000001e-05 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.096058+0000", 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6277e-05 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.096069+0000", 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0759e-05 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.096075+0000", 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4339999999999997e-06 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105753+0000", 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0096778400000000001 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105770+0000", 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7010000000000001e-05 2024-09-21T09:57:59.202 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107784 6.c e1107/1093 6:30595347:::benchmark_data_smithi076_154645_object107783:head v 1107'6682, mlcod=0'0)", 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.099795+0000", 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.010425438, 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.01081384, 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.099795+0000", 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.099795+0000", 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.099795+0000", 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.27e-07 2024-09-21T09:57:59.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.099827+0000", 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1748000000000002e-05 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.099833+0000", 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3180000000000003e-06 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.099837+0000", 2024-09-21T09:57:59.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8449999999999996e-06 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105447+0000", 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056101110000000001 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105475+0000", 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7900000000000001e-05 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.206 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107787 6.7 e1107/1093 6:e742e03c:::benchmark_data_smithi076_154645_object107786:head v 1107'6625, mlcod=0'0)", 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.102455+0000", 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0077650499999999999, 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0082037440000000007, 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102455+0000", 2024-09-21T09:57:59.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102455+0000", 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102457+0000", 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6810000000000001e-06 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102496+0000", 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9243999999999999e-05 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102508+0000", 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2046e-05 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102516+0000", 2024-09-21T09:57:59.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0870000000000007e-06 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102589+0000", 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2680000000000002e-05 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.102612+0000", 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3396000000000001e-05 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.210 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107789 6.4 e1107/1105)", 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.105590+0000", 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0046303230000000004, 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051144210000000001, 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105590+0000", 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105590+0000", 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105591+0000", 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0869999999999999e-06 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105619+0000", 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.862e-05 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105630+0000", 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0373e-05 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.105636+0000", 2024-09-21T09:57:59.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2149999999999998e-06 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:107791 6.7 e1107/1093)", 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.107260+0000", 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0029596900000000001, 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034825350000000001, 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107260+0000", 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107260+0000", 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107261+0000", 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4399999999999999e-07 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107298+0000", 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6795999999999998e-05 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107313+0000", 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5330000000000001e-05 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107321+0000", 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3640000000000001e-06 2024-09-21T09:57:59.216 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:107793 6.6 6:60f373b3:::benchmark_data_smithi076_154645_object107792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:59.107689+0000", 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0025313459999999999, 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031026230000000001, 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:57:59.217 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 107793 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107689+0000", 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107689+0000", 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107691+0000", 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.875e-06 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107714+0000", 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3713000000000001e-05 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107730+0000", 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5345e-05 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107737+0000", 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6159999999999996e-06 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.107915+0000", 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000177639 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.108251+0000", 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00033649400000000001 2024-09-21T09:57:59.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:59.108660+0000", 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000409047 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 12 2024-09-21T09:57:59.221 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:57:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:59.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:57:59.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:57:59.350 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:57:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:57:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/313119 objects degraded (2.730%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:57:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/313119 objects degraded (2.730%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:59.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:57:59.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:57:59.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:57:59.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:57:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/313119 objects degraded (2.730%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:57:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.618 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:59.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:57:59.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:57:59.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:57:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:57:59.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:57:59.890 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:57:59.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:57:59.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:57:59.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 83 16 108558 108542 81.7096 57.0625 0.0176149 0.0122329 2024-09-21T09:57:59.992 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:00.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:00.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:00.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:00.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:00.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:00.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:00.297 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:00 smithi076 ceph-mon[106482]: pgmap v1210: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 8548/315657 objects degraded (2.708%); 464/315657 objects misplaced (0.147%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T09:58:00.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:00 smithi076 ceph-mon[110963]: pgmap v1210: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 8548/315657 objects degraded (2.708%); 464/315657 objects misplaced (0.147%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T09:58:00.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:00.445 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:00.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.473 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:00.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:00.574 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:00.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:00 smithi191 ceph-mon[102206]: pgmap v1210: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 8548/315657 objects degraded (2.708%); 464/315657 objects misplaced (0.147%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T09:58:00.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:00.733 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:00.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:00.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:00.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:00.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:00.934 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 84 16 109479 109463 81.4221 57.5625 0.0200151 0.0122758 2024-09-21T09:58:01.001 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T09:58:01.024 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.108 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:01.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:01.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:01.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:01.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:01.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:01.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:01.567 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:01.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:01.668 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:01.693 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:01.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:01.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:01.917 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:01.935 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 85 16 110293 110277 81.0615 50.875 0.0120809 0.0123308 2024-09-21T09:58:01.961 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:01.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:01.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:02.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:02.101 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:02.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:02 smithi076 ceph-mon[106482]: pgmap v1211: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8548/318150 objects degraded (2.687%); 464/318150 objects misplaced (0.146%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T09:58:02.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:02.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:02.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:02.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:02.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:02.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:02.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:02.479 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:02 smithi076 ceph-mon[110963]: pgmap v1211: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8548/318150 objects degraded (2.687%); 464/318150 objects misplaced (0.146%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T09:58:02.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:02.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:02.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:02 smithi191 ceph-mon[102206]: pgmap v1211: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s; 8548/318150 objects degraded (2.687%); 464/318150 objects misplaced (0.146%); 3.9 MiB/s, 62 objects/s recovering 2024-09-21T09:58:02.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:02.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:02.759 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:02.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:02.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:02.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:02.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 86 16 111166 111150 80.7534 54.5625 0.0238017 0.0123779 2024-09-21T09:58:03.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:03.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:03.139 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T09:58:03.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:03.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:03.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:03.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:03.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:03.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:03.523 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:03 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1351586415' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:03.523 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:58:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:58:03] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.51.0" 2024-09-21T09:58:03.524 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:03 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1351586415' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:03.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:03.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:03.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:03.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:03 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1351586415' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:03.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:03.702 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:03.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:03.827 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:03.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:03.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 87 16 111570 111554 80.1156 25.25 0.00481451 0.0123922 2024-09-21T09:58:03.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:03.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:04.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:04.072 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:04.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:04.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:04.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:04.346 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:04.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:04.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:04.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:04.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:04 smithi191 ceph-mon[102206]: pgmap v1212: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 8548/328059 objects degraded (2.606%); 95/328059 objects misplaced (0.029%); 4.3 MiB/s, 69 objects/s recovering 2024-09-21T09:58:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:04 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1351586415' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:04 smithi191 ceph-mon[102206]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-21T09:58:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3090248411' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T09:58:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8548/328059 objects degraded (2.606%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[106482]: pgmap v1212: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 8548/328059 objects degraded (2.606%); 95/328059 objects misplaced (0.029%); 4.3 MiB/s, 69 objects/s recovering 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1351586415' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[106482]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3090248411' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8548/328059 objects degraded (2.606%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[110963]: pgmap v1212: 89 pgs: 1 active+recovering+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 8548/328059 objects degraded (2.606%); 95/328059 objects misplaced (0.029%); 4.3 MiB/s, 69 objects/s recovering 2024-09-21T09:58:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:04.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1351586415' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:04.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[110963]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-21T09:58:04.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3090248411' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T09:58:04.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8548/328059 objects degraded (2.606%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:04.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:04.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:04.835 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:04.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:04.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:04.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:04.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 88 16 112167 112151 79.6292 37.3125 0.0541726 0.0125519 2024-09-21T09:58:04.966 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:05.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:05.029 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:05.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:05.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:05.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:05.148 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T09:58:05.244 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:05.285 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:05 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3090248411' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T09:58:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:05 smithi076 ceph-mon[106482]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-21T09:58:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:05 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3090248411' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T09:58:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:05 smithi076 ceph-mon[110963]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-21T09:58:05.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:05.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:05.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:05.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.550 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:05.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:05.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:05 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3090248411' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T09:58:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:05 smithi191 ceph-mon[102206]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-21T09:58:05.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:05.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:05.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.820 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:05.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:05.871 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T09:58:05.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:05.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 89 16 113066 113050 79.3658 56.1875 0.0163067 0.012592 2024-09-21T09:58:05.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:05.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:06.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:06.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:06.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:06.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:06.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:06 smithi076 ceph-mon[106482]: pgmap v1215: 89 pgs: 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 8548/330711 objects degraded (2.585%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T09:58:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:06 smithi076 ceph-mon[110963]: pgmap v1215: 89 pgs: 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 8548/330711 objects degraded (2.585%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T09:58:06.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:06.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:06.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:06.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:06 smithi191 ceph-mon[102206]: pgmap v1215: 89 pgs: 1 creating+peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 8548/330711 objects degraded (2.585%); 3.6 MiB/s, 57 objects/s recovering 2024-09-21T09:58:06.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:06.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:06.831 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:06.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:06.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:06.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total time run: 90.015 2024-09-21T09:58:06.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total writes made: 113903 2024-09-21T09:58:06.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Write size: 65536 2024-09-21T09:58:06.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object size: 65536 2024-09-21T09:58:06.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Bandwidth (MB/sec): 79.0861 2024-09-21T09:58:06.936 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Bandwidth: 41.9481 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max bandwidth (MB/sec): 251.875 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min bandwidth (MB/sec): 0 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average IOPS: 1265 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev IOPS: 671.176 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max IOPS: 4030 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min IOPS: 0 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average Latency(s): 0.0126391 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Latency(s): 0.0338975 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max latency(s): 2.14015 2024-09-21T09:58:06.937 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min latency(s): 0.00146111 2024-09-21T09:58:06.940 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T09:58:06.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:07.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:07.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:58:07.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:07.134 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:07.235 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:07.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:07.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:07 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4196703960' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:07 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4196703960' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:07.375 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:07.449 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:58:07.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:07.515 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:07.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:07.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:07.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:07 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4196703960' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:07.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:07.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:07.812 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:07.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:07.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:07.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:07.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:08.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:08.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:08.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:08.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:08.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:08.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:08.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:08.489 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:08.501 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:08 smithi076 ceph-mon[106482]: pgmap v1216: 89 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 7163/337473 objects degraded (2.123%); 1103/337473 objects misplaced (0.327%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T09:58:08.501 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:08 smithi076 ceph-mon[110963]: pgmap v1216: 89 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 7163/337473 objects degraded (2.123%); 1103/337473 objects misplaced (0.327%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T09:58:08.509 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:08.529 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:08.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:08 smithi191 ceph-mon[102206]: pgmap v1216: 89 pgs: 1 active+recovering+undersized+remapped, 1 creating+peering, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 7163/337473 objects degraded (2.123%); 1103/337473 objects misplaced (0.327%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T09:58:08.631 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:08.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:08.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:08.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:08.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:08.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:08.933 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:08.952 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:08.953 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:08.970 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:08.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:09.072 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:09.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:09.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:09.253 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:09.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:09.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:09.397 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:09.440 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 7163/337473 objects degraded (2.123%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:09.441 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 7163/337473 objects degraded (2.123%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:09.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:09.513 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 7163/337473 objects degraded (2.123%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:09.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:09.629 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:09.680 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:09.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:09.803 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:09.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:09.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:09.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:09.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:09.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:10.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:10.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:10.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:10.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:10.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:10.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:58:10.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:10.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:10.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:10.482 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:10 smithi076 ceph-mon[106482]: pgmap v1217: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 915 op/s; 7163/339321 objects degraded (2.111%); 1103/339321 objects misplaced (0.325%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T09:58:10.483 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:10 smithi076 ceph-mon[110963]: pgmap v1217: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 915 op/s; 7163/339321 objects degraded (2.111%); 1103/339321 objects misplaced (0.325%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T09:58:10.503 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:10.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:10.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:10.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:10 smithi191 ceph-mon[102206]: pgmap v1217: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 915 op/s; 7163/339321 objects degraded (2.111%); 1103/339321 objects misplaced (0.325%); 5.8 MiB/s, 93 objects/s recovering 2024-09-21T09:58:10.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:10.745 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:10.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:10.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:10.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:10.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:11.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:11.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:11.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:11.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T09:58:11.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.210 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:11.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:11.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:11.364 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:11.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:11.466 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:11.510 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:11.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:11.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:11.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:11.743 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:11.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:11.785 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:11 smithi191 ceph-mon[102206]: pgmap v1218: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 604 op/s; 7163/339285 objects degraded (2.111%); 1103/339285 objects misplaced (0.325%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:58:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:11 smithi076 ceph-mon[106482]: pgmap v1218: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 604 op/s; 7163/339285 objects degraded (2.111%); 1103/339285 objects misplaced (0.325%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:58:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:11 smithi076 ceph-mon[110963]: pgmap v1218: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 33 MiB/s wr, 604 op/s; 7163/339285 objects degraded (2.111%); 1103/339285 objects misplaced (0.325%); 2.9 MiB/s, 47 objects/s recovering 2024-09-21T09:58:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:11.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:11.906 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:11.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:11.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:12.008 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:12.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:12.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:12.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:12.185 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:12.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:12.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:12.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:12.442 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:12.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:12.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:12.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:12.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:12.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:12.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:12.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:12.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:12.921 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:12.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:13.023 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:13.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:13.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:13.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:13.295 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.332 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:13.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:13.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:13.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:13.509 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:58:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:58:13] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T09:58:13.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:13.615 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:13.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:13.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:13.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:13.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:13 smithi191 ceph-mon[102206]: pgmap v1219: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 109 B/s rd, 31 MiB/s wr, 845 op/s; 7163/331908 objects degraded (2.158%); 867/331908 objects misplaced (0.261%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T09:58:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:13 smithi076 ceph-mon[110963]: pgmap v1219: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 109 B/s rd, 31 MiB/s wr, 845 op/s; 7163/331908 objects degraded (2.158%); 867/331908 objects misplaced (0.261%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T09:58:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:13 smithi076 ceph-mon[106482]: pgmap v1219: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 109 B/s rd, 31 MiB/s wr, 845 op/s; 7163/331908 objects degraded (2.158%); 867/331908 objects misplaced (0.261%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T09:58:13.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:13.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:13.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:14.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:14.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:14.025 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:14.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:14.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:14.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:14.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:14.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:14.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:14.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:14.565 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T09:58:14.569 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:14.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:14.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:14.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.707 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:14.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:14.808 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:14.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:14.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 7163/331908 objects degraded (2.158%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3339668138' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T09:58:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 7163/331908 objects degraded (2.158%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3339668138' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T09:58:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 7163/331908 objects degraded (2.158%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3339668138' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T09:58:14.942 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:14.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:14.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:15.070 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:15.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:15.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:15.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:15.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:15.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:15.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:15.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:15.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:15.505 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:15.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:15.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:15.639 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:15.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:15.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:15.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:15 smithi191 ceph-mon[102206]: pgmap v1220: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 99 B/s rd, 22 MiB/s wr, 787 op/s; 7163/328497 objects degraded (2.181%); 867/328497 objects misplaced (0.264%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3339668138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T09:58:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:15 smithi191 ceph-mon[102206]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-21T09:58:15.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:15 smithi076 ceph-mon[106482]: pgmap v1220: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 99 B/s rd, 22 MiB/s wr, 787 op/s; 7163/328497 objects degraded (2.181%); 867/328497 objects misplaced (0.264%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3339668138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T09:58:15.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:15 smithi076 ceph-mon[106482]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-21T09:58:15.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:15 smithi076 ceph-mon[110963]: pgmap v1220: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 99 B/s rd, 22 MiB/s wr, 787 op/s; 7163/328497 objects degraded (2.181%); 867/328497 objects misplaced (0.264%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:15.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3339668138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T09:58:15.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:15 smithi076 ceph-mon[110963]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-21T09:58:15.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:15.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:16.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:16.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:16.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:16.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:16.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:16.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:16.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.465 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:16.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:16.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:16.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:16.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:16.737 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:16.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:16.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:16.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:16.888 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:16.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:16.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:17.020 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:17.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:17.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T09:58:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:17.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:17.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:17.315 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:17.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:17.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:17.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:17.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:17.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:17.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:17.626 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:17 smithi076 ceph-mon[106482]: pgmap v1222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 861 op/s; 7163/320247 objects degraded (2.237%); 621/320247 objects misplaced (0.194%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:17.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:17 smithi076 ceph-mon[110963]: pgmap v1222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 861 op/s; 7163/320247 objects degraded (2.237%); 621/320247 objects misplaced (0.194%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:17.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:17.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:17.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:17.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:17 smithi191 ceph-mon[102206]: pgmap v1222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 9.0 MiB/s wr, 861 op/s; 7163/320247 objects degraded (2.237%); 621/320247 objects misplaced (0.194%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:17.908 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:58:17.908 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:17.945 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:58:17.945 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:17.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:17.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:17.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:17.984 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:58:17.984 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:18.027 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:58:18.027 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:18.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:58:18.062 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:18.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:18.105 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:58:18.105 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:18.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:18.142 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:58:18.142 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:18.179 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:58:18.179 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:18.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:18.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:18.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:18.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:18.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:18.602 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T09:58:18.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:18.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:18.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:18.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4151213133' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/422225382' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:18 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4151213133' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/422225382' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:18 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4151213133' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/422225382' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:18 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:58:18.878 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:18.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:18.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:19.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:19.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:19.137 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:19.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:19.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:19.266 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:19.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:19.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:19.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:19.497 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:19.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:19.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:19.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:19.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:19.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:19 smithi191 ceph-mon[102206]: pgmap v1223: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 4.3 MiB/s wr, 884 op/s; 7163/316887 objects degraded (2.260%); 621/316887 objects misplaced (0.196%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:19 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:19 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:58:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:19 smithi191 ceph-mon[102206]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-21T09:58:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 7163/316887 objects degraded (2.260%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[106482]: pgmap v1223: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 4.3 MiB/s wr, 884 op/s; 7163/316887 objects degraded (2.260%); 621/316887 objects misplaced (0.196%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:58:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[106482]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-21T09:58:19.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 7163/316887 objects degraded (2.260%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[110963]: pgmap v1223: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 4.3 MiB/s wr, 884 op/s; 7163/316887 objects degraded (2.260%); 621/316887 objects misplaced (0.196%); 3.0 MiB/s, 48 objects/s recovering 2024-09-21T09:58:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:58:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[110963]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-21T09:58:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 7163/316887 objects degraded (2.260%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:19.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:19.949 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:19.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:19.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:20.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:20.076 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:20.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:20.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:20.217 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:20.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:20.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:20.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:20.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:20.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.492 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:20.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:20.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:20.620 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:20.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:20.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:20.854 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:20.895 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:20.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:20.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:20.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:21.022 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:21.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:21.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:21.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:21.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:21.368 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:21.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:21.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:21.537 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:21.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:21.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:21.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:21.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:21 smithi191 ceph-mon[102206]: pgmap v1225: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 7163/314511 objects degraded (2.278%); 621/314511 objects misplaced (0.197%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:21.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:21.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:21 smithi076 ceph-mon[106482]: pgmap v1225: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 7163/314511 objects degraded (2.278%); 621/314511 objects misplaced (0.197%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:21 smithi076 ceph-mon[110963]: pgmap v1225: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 7163/314511 objects degraded (2.278%); 621/314511 objects misplaced (0.197%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T09:58:21.966 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:22.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:22.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:22.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:22.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:22.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:46985 6.6 6:6e1f852b:::benchmark_data_smithi076_154645_object46984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.669767+0000", 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.513273100000006, 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96641414199999998, 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 46985 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669767+0000", 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669767+0000", 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669768+0000", 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.716e-06 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669789+0000", 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0208e-05 2024-09-21T09:58:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669800+0000", 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1205e-05 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669806+0000", 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8370000000000002e-06 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669871+0000", 2024-09-21T09:58:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5026999999999999e-05 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669881+0000", 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0001e-05 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.632690+0000", 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96280914299999998 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.632809+0000", 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011861700000000001 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.632916+0000", 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000107295 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636065+0000", 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031496459999999999 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636088+0000", 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2756999999999999e-05 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636123+0000", 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5277999999999997e-05 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636153+0000", 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9638999999999999e-05 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636181+0000", 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7773e-05 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47011 6.6 6:61779813:::benchmark_data_smithi076_154645_object47010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.686871+0000", 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.496169162000001, 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95140228800000004, 2024-09-21T09:58:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47011 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686871+0000", 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686871+0000", 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686872+0000", 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7010000000000001e-06 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686894+0000", 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1946e-05 2024-09-21T09:58:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686909+0000", 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4918e-05 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686925+0000", 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6084000000000001e-05 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.687084+0000", 2024-09-21T09:58:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015839200000000001 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.687093+0000", 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8470000000000002e-06 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.634228+0000", 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94713563499999998 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.634305+0000", 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6805999999999996e-05 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.634388+0000", 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3156000000000001e-05 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636245+0000", 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018568479999999999 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638112+0000", 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018669030000000001 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638166+0000", 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3739000000000002e-05 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638229+0000", 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3026000000000003e-05 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638273+0000", 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4286999999999998e-05 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47019 6.6 6:6051a9e9:::benchmark_data_smithi076_154645_object47018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.691688+0000", 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.491351671999993, 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95238085900000002, 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47019 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691688+0000", 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691688+0000", 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691690+0000", 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.61e-06 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691714+0000", 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4204000000000001e-05 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691729+0000", 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5175e-05 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691736+0000", 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.764e-06 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691797+0000", 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1158e-05 2024-09-21T09:58:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691806+0000", 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.166e-06 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.635098+0000", 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94329137500000004 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.635175+0000", 2024-09-21T09:58:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7442999999999997e-05 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.635267+0000", 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2093000000000001e-05 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638073+0000", 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028060369999999999 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638324+0000", 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025033400000000001 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.643961+0000", 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056372310000000004 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644024+0000", 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3589000000000004e-05 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644069+0000", 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4679999999999999e-05 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47049 6.6 6:62e62db7:::benchmark_data_smithi076_154645_object47048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.715047+0000", 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.467992753000004, 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92919887800000001, 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47049 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715047+0000", 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715047+0000", 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715049+0000", 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1090000000000001e-06 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715073+0000", 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3575999999999999e-05 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715099+0000", 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5953999999999999e-05 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715112+0000", 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2958e-05 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715214+0000", 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102301 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715776+0000", 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056210400000000001 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636260+0000", 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92048376300000001 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636333+0000", 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2977000000000003e-05 2024-09-21T09:58:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636424+0000", 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1558000000000007e-05 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638124+0000", 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0016996400000000001 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.643778+0000", 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056541059999999999 2024-09-21T09:58:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644140+0000", 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00036158099999999998 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644197+0000", 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6860000000000001e-05 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644246+0000", 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9391000000000002e-05 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47057 6.6 6:69a8df47:::benchmark_data_smithi076_154645_object47056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.716332+0000", 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.466707679999999, 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.93103870899999996, 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47057 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716332+0000", 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716332+0000", 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716334+0000", 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.364e-06 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716355+0000", 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1871000000000001e-05 2024-09-21T09:58:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716371+0000", 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5804999999999999e-05 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716378+0000", 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0079999999999996e-06 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716445+0000", 2024-09-21T09:58:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6452999999999995e-05 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716454+0000", 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7860000000000008e-06 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.637102+0000", 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92064744700000001 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.637176+0000", 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3728999999999998e-05 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.637375+0000", 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000199655 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.643917+0000", 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065415359999999997 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644327+0000", 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00041007899999999999 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647275+0000", 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029477190000000001 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647325+0000", 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9966999999999998e-05 2024-09-21T09:58:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647371+0000", 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6289999999999999e-05 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47076 6.6 6:6ee40c86:::benchmark_data_smithi076_154645_object47075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.732163+0000", 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.450877336999994, 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91528458199999996, 2024-09-21T09:58:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47076 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732163+0000", 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732163+0000", 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732164+0000", 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7120000000000001e-06 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732187+0000", 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2736000000000001e-05 2024-09-21T09:58:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732200+0000", 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2661e-05 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732208+0000", 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3990000000000004e-06 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732281+0000", 2024-09-21T09:58:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2748000000000002e-05 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732291+0000", 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0692e-05 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638367+0000", 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90607561700000006 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.640014+0000", 2024-09-21T09:58:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001647019 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.640171+0000", 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015642600000000001 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644376+0000", 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042058360000000001 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644415+0000", 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9106000000000001e-05 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647379+0000", 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029633319999999999 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647417+0000", 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7871999999999999e-05 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647447+0000", 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0426e-05 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47083 6.6 6:643c6a0e:::benchmark_data_smithi076_154645_object47082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.736858+0000", 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.446182074000006, 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91066441899999995, 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47083 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736858+0000", 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736858+0000", 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736859+0000", 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5009999999999999e-06 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736882+0000", 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2955999999999998e-05 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736898+0000", 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5270000000000001e-05 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736906+0000", 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1689999999999992e-06 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737064+0000", 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000157844 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737074+0000", 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0335e-05 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.642009+0000", 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90493507699999998 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.642168+0000", 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000158698 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.642312+0000", 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014451500000000001 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644458+0000", 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021454870000000002 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647260+0000", 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002801984 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647454+0000", 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0001939 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647491+0000", 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7478999999999998e-05 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647522+0000", 2024-09-21T09:58:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1204000000000001e-05 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47084 6.6 6:6c69cbb1:::benchmark_data_smithi076_154645_object47083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.737229+0000", 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.445810887999997, 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91724743799999997, 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47084 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737229+0000", 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737229+0000", 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737230+0000", 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.066e-06 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737274+0000", 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4033999999999999e-05 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737286+0000", 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.182e-05 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737293+0000", 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8480000000000003e-06 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737346+0000", 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3501999999999997e-05 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737356+0000", 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5380000000000008e-06 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644489+0000", 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90713360899999995 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644593+0000", 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010351700000000001 2024-09-21T09:58:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644716+0000", 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000122936 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.650977+0000", 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006261656 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.650988+0000", 2024-09-21T09:58:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0947e-05 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654373+0000", 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0033849190000000001 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654433+0000", 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9237999999999998e-05 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654476+0000", 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3807999999999998e-05 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47102 6.6 6:62017bac:::benchmark_data_smithi076_154645_object47101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.748493+0000", 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.434547175000006, 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90618986199999996, 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47102 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748493+0000", 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748493+0000", 2024-09-21T09:58:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748495+0000", 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0470000000000001e-06 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748517+0000", 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2297e-05 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748536+0000", 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8678e-05 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748544+0000", 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8720000000000002e-06 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748694+0000", 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015076900000000001 2024-09-21T09:58:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748783+0000", 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8314999999999997e-05 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647598+0000", 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89881569400000005 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647899+0000", 2024-09-21T09:58:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000300759 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.648048+0000", 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014875700000000001 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.650993+0000", 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029446709999999998 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654524+0000", 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0035316089999999998 2024-09-21T09:58:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654604+0000", 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9351000000000006e-05 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654649+0000", 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5185000000000002e-05 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654683+0000", 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3858e-05 2024-09-21T09:58:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47104 6.6 6:65f7465e:::benchmark_data_smithi076_154645_object47103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.749569+0000", 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.433470658000005, 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90523379999999998, 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47104 2024-09-21T09:58:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749569+0000", 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749569+0000", 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749571+0000", 2024-09-21T09:58:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.39e-06 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749589+0000", 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8525e-05 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749600+0000", 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0804e-05 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749608+0000", 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6709999999999999e-06 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749707+0000", 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9166000000000006e-05 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.308 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749714+0000", 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7560000000000002e-06 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.649434+0000", 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89972096499999998 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.649544+0000", 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109367 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.309 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.649674+0000", 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000129958 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654534+0000", 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004860596 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654574+0000", 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9669999999999998e-05 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.310 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654718+0000", 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014418699999999999 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654760+0000", 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1433000000000001e-05 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654803+0000", 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3312000000000001e-05 2024-09-21T09:58:22.311 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47112 6.6 6:61680d0b:::benchmark_data_smithi076_154645_object47111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.755556+0000", 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.427483565000003, 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90315495099999998, 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.312 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47112 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755556+0000", 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755556+0000", 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.313 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755558+0000", 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2470000000000001e-06 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755596+0000", 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8209e-05 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755607+0000", 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0753e-05 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.314 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755612+0000", 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6470000000000003e-06 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755663+0000", 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1007999999999997e-05 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755671+0000", 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7510000000000008e-06 2024-09-21T09:58:22.315 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.651036+0000", 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89536528599999998 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.651179+0000", 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000142874 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.651304+0000", 2024-09-21T09:58:22.316 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124945 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654540+0000", 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003235833 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658565+0000", 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040251549999999999 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658607+0000", 2024-09-21T09:58:22.317 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1953000000000003e-05 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658671+0000", 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3708999999999996e-05 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658711+0000", 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0581000000000003e-05 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.318 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47117 6.6 6:61d9a68f:::benchmark_data_smithi076_154645_object47116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.759560+0000", 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.423480061000006, 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90283510499999997, 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47117 2024-09-21T09:58:22.319 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759560+0000", 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759560+0000", 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.320 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759562+0000", 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7940000000000001e-06 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759607+0000", 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5479000000000003e-05 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759621+0000", 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3978000000000001e-05 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.321 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759628+0000", 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7769999999999997e-06 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759687+0000", 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9446e-05 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759696+0000", 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9120000000000004e-06 2024-09-21T09:58:22.322 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.652733+0000", 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89303699599999997 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.652851+0000", 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118035 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.652976+0000", 2024-09-21T09:58:22.323 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000125241 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654544+0000", 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001567613 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662253+0000", 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0077084429999999997 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.324 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662297+0000", 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4849999999999999e-05 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662349+0000", 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1956000000000002e-05 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662395+0000", 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5584999999999998e-05 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.325 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47118 6.6 6:63619ed9:::benchmark_data_smithi076_154645_object47117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.759968+0000", 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 81.423071526000001, 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90271875099999999, 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.326 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47118 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759968+0000", 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759968+0000", 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.327 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759969+0000", 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6600000000000005e-07 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759988+0000", 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9284999999999999e-05 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760000+0000", 2024-09-21T09:58:22.328 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1949e-05 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760008+0000", 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3289999999999996e-06 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760133+0000", 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012529300000000001 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.329 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760141+0000", 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3999999999999992e-06 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654839+0000", 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89469742399999996 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.655030+0000", 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019161199999999999 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.330 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.655554+0000", 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00052317800000000001 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658576+0000", 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030221060000000001 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662437+0000", 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038615670000000002 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.331 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662593+0000", 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015548499999999999 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662646+0000", 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3599e-05 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662687+0000", 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0658000000000003e-05 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.332 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111335 6.b 6:d43ac681:::benchmark_data_smithi076_154645_object111334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.158293+0000", 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 19.024747199, 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.99914799300000001, 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111335 2024-09-21T09:58:22.333 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158293+0000", 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158293+0000", 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.334 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158294+0000", 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4470000000000001e-06 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158318+0000", 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4261999999999999e-05 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158329+0000", 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.095e-05 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.335 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158339+0000", 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5019999999999998e-06 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158900+0000", 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056092700000000002 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.336 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158909+0000", 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6260000000000007e-06 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.147702+0000", 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.98879247100000001 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.338 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.147851+0000", 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014964199999999999 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.147993+0000", 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000141953 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154220+0000", 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0062264310000000002 2024-09-21T09:58:22.339 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157142+0000", 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029221329999999999 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157336+0000", 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000194153 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157403+0000", 2024-09-21T09:58:22.340 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6457000000000004e-05 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157441+0000", 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8039e-05 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111356 6.b 6:d2723c0a:::benchmark_data_smithi076_154645_object111355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.200592+0000", 2024-09-21T09:58:22.341 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 18.982447779000001, 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96187469699999995, 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111356 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.342 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200592+0000", 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200592+0000", 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200593+0000", 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.311e-06 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.343 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200614+0000", 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0341000000000001e-05 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200625+0000", 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1280999999999999e-05 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200632+0000", 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1049999999999997e-06 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.344 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200712+0000", 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0185999999999994e-05 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200732+0000", 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9264000000000001e-05 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.151228+0000", 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95049628399999997 2024-09-21T09:58:22.345 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.151368+0000", 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013990199999999999 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.151532+0000", 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000164253 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.346 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157285+0000", 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005753073 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157454+0000", 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000169276 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162351+0000", 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004897022 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.347 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162416+0000", 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4478000000000002e-05 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162467+0000", 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0921000000000003e-05 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111358 6.b 6:d2bb22f9:::benchmark_data_smithi076_154645_object111357:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.204700+0000", 2024-09-21T09:58:22.348 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 18.978339437999999, 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.958391628, 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111358 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204700+0000", 2024-09-21T09:58:22.349 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204700+0000", 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204702+0000", 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.33e-06 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.350 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204723+0000", 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1506999999999999e-05 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204734+0000", 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0671e-05 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204741+0000", 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1749999999999999e-06 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.351 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204827+0000", 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5944000000000002e-05 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204838+0000", 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1168999999999999e-05 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154255+0000", 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94941701300000003 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.352 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154360+0000", 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105165 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154460+0000", 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000100055 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157459+0000", 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029986240000000001 2024-09-21T09:58:22.353 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157512+0000", 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2787999999999997e-05 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162998+0000", 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054861119999999996 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163050+0000", 2024-09-21T09:58:22.354 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1573000000000002e-05 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163092+0000", 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2502000000000001e-05 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111371 6.b 6:d2bd86e5:::benchmark_data_smithi076_154645_object111370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.224867+0000", 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 18.958173345999999, 2024-09-21T09:58:22.355 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95080495399999998, 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111371 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224867+0000", 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.356 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224867+0000", 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224868+0000", 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8870000000000001e-06 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224894+0000", 2024-09-21T09:58:22.357 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5743999999999999e-05 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224905+0000", 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0577000000000001e-05 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224917+0000", 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2252000000000001e-05 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.358 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224995+0000", 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7725000000000005e-05 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.225007+0000", 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2738000000000001e-05 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157624+0000", 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.932616691 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.359 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157763+0000", 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000139295 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157928+0000", 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016422799999999999 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163177+0000", 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052494389999999998 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.360 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163218+0000", 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1242999999999998e-05 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175584+0000", 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012365309999999999 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175627+0000", 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3164000000000002e-05 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.361 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175671+0000", 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4660999999999998e-05 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111382 6.b 6:dca8587a:::benchmark_data_smithi076_154645_object111381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.240220+0000", 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 18.942819858, 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.93556396500000005, 2024-09-21T09:58:22.362 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111382 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240220+0000", 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.363 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240220+0000", 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240222+0000", 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5820000000000001e-06 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240253+0000", 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1544000000000002e-05 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.364 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240268+0000", 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5145e-05 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240281+0000", 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3077e-05 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240358+0000", 2024-09-21T09:58:22.365 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6327000000000003e-05 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240369+0000", 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0949e-05 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163248+0000", 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92287976999999999 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.366 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163365+0000", 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011615700000000001 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163502+0000", 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000137214 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175571+0000", 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012069623999999999 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.367 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175678+0000", 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106986 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175715+0000", 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6426e-05 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175755+0000", 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9758000000000001e-05 2024-09-21T09:58:22.368 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175784+0000", 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9405999999999999e-05 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111385 6.b 6:dec80fb3:::benchmark_data_smithi076_154645_object111384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.245780+0000", 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 18.937260305999999, 2024-09-21T09:58:22.369 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94135651099999995, 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111385 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245780+0000", 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.370 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245780+0000", 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245781+0000", 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4440000000000001e-06 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245806+0000", 2024-09-21T09:58:22.371 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4955000000000001e-05 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245816+0000", 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0173999999999999e-05 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245822+0000", 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5990000000000001e-06 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.372 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245896+0000", 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4368000000000004e-05 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245906+0000", 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4569999999999994e-06 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.168204+0000", 2024-09-21T09:58:22.373 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92229890699999995 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.168319+0000", 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000114707 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.168450+0000", 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130569 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.374 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175683+0000", 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072328649999999998 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175818+0000", 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013559100000000001 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.186999+0000", 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011180456 2024-09-21T09:58:22.375 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187089+0000", 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0130999999999995e-05 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187136+0000", 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7287999999999999e-05 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.376 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111417 6.b 6:d8513038:::benchmark_data_smithi076_154645_object111416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.286159+0000", 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 18.896880565, 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90118947800000004, 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111417 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.377 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286159+0000", 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286159+0000", 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286161+0000", 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6360000000000001e-06 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.378 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286186+0000", 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4655e-05 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286203+0000", 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7e-05 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286215+0000", 2024-09-21T09:58:22.379 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2395000000000001e-05 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286431+0000", 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021601500000000001 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286445+0000", 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4331000000000001e-05 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.380 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.171860+0000", 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88541457300000004 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.172020+0000", 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015997400000000001 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.172166+0000", 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014658600000000001 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.381 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175827+0000", 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0036610240000000001 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187185+0000", 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011357631999999999 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187222+0000", 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6575999999999997e-05 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.382 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187309+0000", 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7380999999999996e-05 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187349+0000", 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9700000000000003e-05 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:58:22.383 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:58:22.429 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:22.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:22.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:22.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:22.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:22.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:22.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:22.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:22.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:22.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:22.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:22.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:22.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:23.076 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:23.113 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:23.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:23.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:23.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:23.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:23.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:58:23.469 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:58:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:58:23] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T09:58:23.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:23.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:23.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:23.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:23.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:23.815 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:23 smithi076 ceph-mon[106482]: pgmap v1226: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 7163/305403 objects degraded (2.345%); 372/305403 objects misplaced (0.122%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:58:23.815 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:23 smithi076 ceph-mon[110963]: pgmap v1226: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 7163/305403 objects degraded (2.345%); 372/305403 objects misplaced (0.122%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:58:23.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:23.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:23 smithi191 ceph-mon[102206]: pgmap v1226: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 7163/305403 objects degraded (2.345%); 372/305403 objects misplaced (0.122%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:58:23.936 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:23.966 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:23.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:24.067 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:24.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:24.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T09:58:24.183 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:24.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:24.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:24.370 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:24.416 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:24.441 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:24.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:24.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:24.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:24.696 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:24.741 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 7163/305403 objects degraded (2.345%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:24.741 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 7163/305403 objects degraded (2.345%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:24.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:24.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:24.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 7163/305403 objects degraded (2.345%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:24.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:24.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:24.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:25.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:25.094 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:25.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:25.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:25.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:25.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:25.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:25.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:25.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:25.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:25.524 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:25.596 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:25.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:25.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:25.631 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T09:58:25.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:25.750 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:25.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:25.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[106482]: pgmap v1227: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 7163/302217 objects degraded (2.370%); 372/302217 objects misplaced (0.123%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2667031311' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T09:58:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T09:58:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[110963]: pgmap v1227: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 7163/302217 objects degraded (2.370%); 372/302217 objects misplaced (0.123%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2667031311' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T09:58:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T09:58:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:25.895 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:25.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:25.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:26.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:25 smithi191 ceph-mon[102206]: pgmap v1227: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 7163/302217 objects degraded (2.370%); 372/302217 objects misplaced (0.123%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2667031311' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T09:58:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T09:58:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:26.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:26.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:26.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:26.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:26.303 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:26.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:26.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:26.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:26.730 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:26.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:26 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T09:58:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:26 smithi191 ceph-mon[102206]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-21T09:58:26.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:26.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:26.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:27.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:27.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:27.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:26 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T09:58:27.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:26 smithi076 ceph-mon[110963]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-21T09:58:27.118 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:26 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T09:58:27.118 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:26 smithi076 ceph-mon[106482]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-21T09:58:27.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:27.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:27.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:27.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:27.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:27.461 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:27.562 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:27.596 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:27.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:27.728 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:27.743 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:27 smithi076 ceph-mon[106482]: pgmap v1229: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 7163/294462 objects degraded (2.433%); 127/294462 objects misplaced (0.043%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:58:27.744 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:27 smithi076 ceph-mon[110963]: pgmap v1229: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 7163/294462 objects degraded (2.433%); 127/294462 objects misplaced (0.043%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:58:27.757 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:27 smithi191 ceph-mon[102206]: pgmap v1229: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 934 op/s; 7163/294462 objects degraded (2.433%); 127/294462 objects misplaced (0.043%); 3.9 MiB/s, 61 objects/s recovering 2024-09-21T09:58:27.831 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:27.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:27.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:27.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:27.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:28.076 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:58:28.077 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T09:58:28.077 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T09:58:28.077 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:58:28.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:28.139 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:28.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:28.240 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:28.278 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:28.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:28.376 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T09:58:28.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:58:28.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:28.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:28.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:28.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:28.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:28.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:28.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:28.763 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:28.818 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:28.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:28.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:28.941 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:28.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:28.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:29.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:29.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T09:58:29.058 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:29.063 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:29.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:29.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:29.258 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:29.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:29.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:29.413 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:29.434 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:29.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:29.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:29.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:29.679 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T09:58:29.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:29.779 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:58:29.779 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:29.788 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:29.815 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:58:29.815 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:29.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:29.852 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:58:29.852 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:29.888 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:58:29.888 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:29.918 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:58:29.918 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[106482]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 7163/291375 objects degraded (2.458%); 127/291375 objects misplaced (0.044%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[106482]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 7163/291375 objects degraded (2.458%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3355432430' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[110963]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 7163/291375 objects degraded (2.458%); 127/291375 objects misplaced (0.044%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[110963]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 7163/291375 objects degraded (2.458%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:29.919 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3355432430' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:29.920 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:29 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:29.930 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:29.954 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:58:29.955 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:29.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:29.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:29.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:29.987 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:58:29.987 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:30.025 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:58:30.025 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:30.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:30.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:29 smithi191 ceph-mon[102206]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 7163/291375 objects degraded (2.458%); 127/291375 objects misplaced (0.044%); 3.1 MiB/s, 50 objects/s recovering 2024-09-21T09:58:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:29 smithi191 ceph-mon[102206]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-21T09:58:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 7163/291375 objects degraded (2.458%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:29 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3355432430' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:29 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:30.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:30.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:30.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:30.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:30.381 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:30.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:30.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:30.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:30.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:30.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:30.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:30.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:30.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.791 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:30.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:30.806 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[106482]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/120249446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[110963]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-21T09:58:30.807 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:30 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/120249446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:30.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:30.921 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:30.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:31.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:31.066 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:30 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:30 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:30 smithi191 ceph-mon[102206]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-21T09:58:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:30 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/120249446' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:31.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:31.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:31.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:31.328 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:31.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:31.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:31.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.591 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:31.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:31.692 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:31.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:31.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:31.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:31 smithi191 ceph-mon[102206]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 6904/289239 objects degraded (2.387%); 208/289239 objects misplaced (0.072%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:31 smithi076 ceph-mon[106482]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 6904/289239 objects degraded (2.387%); 208/289239 objects misplaced (0.072%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:31 smithi076 ceph-mon[110963]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 6904/289239 objects degraded (2.387%); 208/289239 objects misplaced (0.072%); 4.4 MiB/s, 70 objects/s recovering 2024-09-21T09:58:31.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:31.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:31.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:32.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:32.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.151 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:32.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:32.198 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:32.199 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:32.251 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:32.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:32.396 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:32.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:32.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:32.569 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:32.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:32.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:32.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:32.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:32.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:32.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:32.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:32.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T09:58:32.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:32.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:33.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:33.111 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:58:33.117 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:58:33.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:33.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:33.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:58:33.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:33.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:33.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:33.292 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:33.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:58:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:58:33] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.51.0" 2024-09-21T09:58:33.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:33.414 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:33.516 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:33.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.548 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:33.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:33.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:33.682 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:33.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:33.784 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:33.825 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:33.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:33.890 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1234,"stamp":"2024-09-21T09:58:32.462021+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6155933693,"num_objects":93950,"num_object_clones":0,"num_object_copies":281850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6904,"num_objects_misplaced":81,"num_objects_unfound":0,"num_objects_dirty":93950,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":134163,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7374,"num_bytes_recovered":484181780,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130592,"ondisk_log_size":130592,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20186084,"kb_used_data":19091528,"kb_used_omap":195,"kb_used_meta":1094076,"kb_avail":729775132,"statfs":{"total":767960285184,"available":747289735168,"internally_reserved":0,"allocated":19549724672,"data_stored":19533614386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1120333956},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-427687936,"num_objects":-6526,"num_object_clones":0,"num_object_copies":-19578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-259,"num_objects_misplaced":-291,"num_objects_unfound":0,"num_objects_dirty":-6526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6526,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":550,"num_bytes_recovered":36044800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.837678"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.658234+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T09:58:25.658234+0000","last_peered":"2024-09-21T09:58:25.658234+0000","last_clean":"2024-09-21T09:58:25.658234+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T09:58:25.658234+0000","last_undegraded":"2024-09-21T09:58:25.658234+0000","last_fullsized":"2024-09-21T09:58:25.658234+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1042,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.676664+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:58:28.676664+0000","last_peered":"2024-09-21T09:58:28.676664+0000","last_clean":"2024-09-21T09:58:28.676664+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:58:28.676664+0000","last_undegraded":"2024-09-21T09:58:28.676664+0000","last_fullsized":"2024-09-21T09:58:28.676664+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1073,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673823+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:58:28.673823+0000","last_peered":"2024-09-21T09:58:28.673823+0000","last_clean":"2024-09-21T09:58:28.673823+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:58:28.673823+0000","last_undegraded":"2024-09-21T09:58:28.673823+0000","last_fullsized":"2024-09-21T09:58:28.673823+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1417,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692817+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T09:58:28.692817+0000","last_peered":"2024-09-21T09:58:28.692817+0000","last_clean":"2024-09-21T09:58:28.692817+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T09:58:28.692817+0000","last_undegraded":"2024-09-21T09:58:28.692817+0000","last_fullsized":"2024-09-21T09:58:28.692817+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1416,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.693014+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T09:58:28.693014+0000","last_peered":"2024-09-21T09:58:28.693014+0000","last_clean":"2024-09-21T09:58:28.693014+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T09:58:28.693014+0000","last_undegraded":"2024-09-21T09:58:28.693014+0000","last_fullsized":"2024-09-21T09:58:28.693014+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1025,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678549+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:58:28.678549+0000","last_peered":"2024-09-21T09:58:28.678549+0000","last_clean":"2024-09-21T09:58:28.678549+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:58:28.678549+0000","last_undegraded":"2024-09-21T09:58:28.678549+0000","last_fullsized":"2024-09-21T09:58:28.678549+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.657636+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T09:58:25.657636+0000","last_peered":"2024-09-21T09:58:25.657636+0000","last_clean":"2024-09-21T09:58:25.657636+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T09:58:25.657636+0000","last_undegraded":"2024-09-21T09:58:25.657636+0000","last_fullsized":"2024-09-21T09:58:25.657636+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1052,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678361+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:58:28.678361+0000","last_peered":"2024-09-21T09:58:28.678361+0000","last_clean":"2024-09-21T09:58:28.678361+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:58:28.678361+0000","last_undegraded":"2024-09-21T09:58:28.678361+0000","last_fullsized":"2024-09-21T09:58:28.678361+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1114'8330","reported_seq":8367,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.381712+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T09:58:31.381712+0000","last_peered":"2024-09-21T09:58:31.381712+0000","last_clean":"2024-09-21T09:58:31.381712+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T09:58:31.381712+0000","last_undegraded":"2024-09-21T09:58:31.381712+0000","last_fullsized":"2024-09-21T09:58:31.381712+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8330,"log_dups_size":0,"ondisk_log_size":8330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378404864,"num_objects":5774,"num_object_clones":0,"num_object_copies":17322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8330,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1041,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674151+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:58:28.674151+0000","last_peered":"2024-09-21T09:58:28.674151+0000","last_clean":"2024-09-21T09:58:28.674151+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:58:28.674151+0000","last_undegraded":"2024-09-21T09:58:28.674151+0000","last_fullsized":"2024-09-21T09:58:28.674151+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1041,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673098+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:58:28.673098+0000","last_peered":"2024-09-21T09:58:28.673098+0000","last_clean":"2024-09-21T09:58:28.673098+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:58:28.673098+0000","last_undegraded":"2024-09-21T09:58:28.673098+0000","last_fullsized":"2024-09-21T09:58:28.673098+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1112'8185","reported_seq":11122,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.287185+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T09:58:28.287185+0000","last_peered":"2024-09-21T09:58:28.287185+0000","last_clean":"2024-09-21T09:58:28.287185+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T09:58:28.287185+0000","last_undegraded":"2024-09-21T09:58:28.287185+0000","last_fullsized":"2024-09-21T09:58:28.287185+0000","mapping_epoch":1108,"log_start":"1089'800","ondisk_log_start":"1089'800","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7385,"log_dups_size":0,"ondisk_log_size":7385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388694016,"num_objects":5931,"num_object_clones":0,"num_object_copies":17793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8185,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1042,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674521+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:58:28.674521+0000","last_peered":"2024-09-21T09:58:28.674521+0000","last_clean":"2024-09-21T09:58:28.674521+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:58:28.674521+0000","last_undegraded":"2024-09-21T09:58:28.674521+0000","last_fullsized":"2024-09-21T09:58:28.674521+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1095,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.675842+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T09:58:29.675842+0000","last_peered":"2024-09-21T09:58:29.675842+0000","last_clean":"2024-09-21T09:58:29.675842+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T09:58:29.675842+0000","last_undegraded":"2024-09-21T09:58:29.675842+0000","last_fullsized":"2024-09-21T09:58:29.675842+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1114'8456","reported_seq":11376,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.319529+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T09:58:31.319529+0000","last_peered":"2024-09-21T09:58:31.319529+0000","last_clean":"2024-09-21T09:58:31.319529+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T09:58:31.319529+0000","last_undegraded":"2024-09-21T09:58:31.319529+0000","last_fullsized":"2024-09-21T09:58:31.319529+0000","mapping_epoch":1100,"log_start":"1089'300","ondisk_log_start":"1089'300","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8156,"log_dups_size":0,"ondisk_log_size":8156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389021696,"num_objects":5936,"num_object_clones":0,"num_object_copies":17808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8456,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1026,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674875+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:58:28.674875+0000","last_peered":"2024-09-21T09:58:28.674875+0000","last_clean":"2024-09-21T09:58:28.674875+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:58:28.674875+0000","last_undegraded":"2024-09-21T09:58:28.674875+0000","last_fullsized":"2024-09-21T09:58:28.674875+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1044,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.743365+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:58:29.743365+0000","last_peered":"2024-09-21T09:58:29.743365+0000","last_clean":"2024-09-21T09:58:29.743365+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:58:29.743365+0000","last_undegraded":"2024-09-21T09:58:29.743365+0000","last_fullsized":"2024-09-21T09:58:29.743365+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1113'8328","reported_seq":8393,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.895460+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T09:58:28.895460+0000","last_peered":"2024-09-21T09:58:28.895460+0000","last_clean":"2024-09-21T09:58:28.895460+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T09:58:28.895460+0000","last_undegraded":"2024-09-21T09:58:28.895460+0000","last_fullsized":"2024-09-21T09:58:28.895460+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8328,"log_dups_size":0,"ondisk_log_size":8328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392691712,"num_objects":5992,"num_object_clones":0,"num_object_copies":17976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8328,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1074,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674607+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:58:28.674607+0000","last_peered":"2024-09-21T09:58:28.674607+0000","last_clean":"2024-09-21T09:58:28.674607+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:58:28.674607+0000","last_undegraded":"2024-09-21T09:58:28.674607+0000","last_fullsized":"2024-09-21T09:58:28.674607+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1405,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692420+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:58:28.692420+0000","last_peered":"2024-09-21T09:58:28.692420+0000","last_clean":"2024-09-21T09:58:28.692420+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:58:28.692420+0000","last_undegraded":"2024-09-21T09:58:28.692420+0000","last_fullsized":"2024-09-21T09:58:28.692420+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1114'8393","reported_seq":8430,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.337343+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T09:58:31.337343+0000","last_peered":"2024-09-21T09:58:31.337343+0000","last_clean":"2024-09-21T09:58:31.337343+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T09:58:31.337343+0000","last_undegraded":"2024-09-21T09:58:31.337343+0000","last_fullsized":"2024-09-21T09:58:31.337343+0000","mapping_epoch":1086,"log_start":"1089'400","ondisk_log_start":"1089'400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7993,"log_dups_size":0,"ondisk_log_size":7993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378339328,"num_objects":5773,"num_object_clones":0,"num_object_copies":17319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8393,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":54,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.650843+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T09:58:25.650843+0000","last_peered":"2024-09-21T09:58:25.650843+0000","last_clean":"2024-09-21T09:58:25.650843+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T09:58:25.650843+0000","last_undegraded":"2024-09-21T09:58:25.650843+0000","last_fullsized":"2024-09-21T09:58:25.650843+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1079,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679179+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T09:58:28.679179+0000","last_peered":"2024-09-21T09:58:28.679179+0000","last_clean":"2024-09-21T09:58:28.679179+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T09:58:28.679179+0000","last_undegraded":"2024-09-21T09:58:28.679179+0000","last_fullsized":"2024-09-21T09:58:28.679179+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1114'8370","reported_seq":8794,"reported_epoch":1114,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:30.023503+0000","last_change":"2024-09-21T09:58:28.149252+0000","last_active":"2024-09-21T09:58:30.023503+0000","last_peered":"2024-09-21T09:58:30.023503+0000","last_clean":"2024-09-21T09:56:44.221905+0000","last_became_active":"2024-09-21T09:56:46.358647+0000","last_became_peered":"2024-09-21T09:56:46.358647+0000","last_unstale":"2024-09-21T09:58:30.023503+0000","last_undegraded":"2024-09-21T09:56:46.247338+0000","last_fullsized":"2024-09-21T09:56:46.236810+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8370,"log_dups_size":0,"ondisk_log_size":8370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392560640,"num_objects":5990,"num_object_clones":0,"num_object_copies":17970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1151,"num_objects_misplaced":81,"num_objects_unfound":0,"num_objects_dirty":5990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8370,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":178,"num_bytes_recovered":11665408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":4758},{"shards":"2,4,6","objects":81}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1045,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678672+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:58:28.678672+0000","last_peered":"2024-09-21T09:58:28.678672+0000","last_clean":"2024-09-21T09:58:28.678672+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:58:28.678672+0000","last_undegraded":"2024-09-21T09:58:28.678672+0000","last_fullsized":"2024-09-21T09:58:28.678672+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1076,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679464+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:58:28.679464+0000","last_peered":"2024-09-21T09:58:28.679464+0000","last_clean":"2024-09-21T09:58:28.679464+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:58:28.679464+0000","last_undegraded":"2024-09-21T09:58:28.679464+0000","last_fullsized":"2024-09-21T09:58:28.679464+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1114'8388","reported_seq":8426,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:31.382056+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T09:58:31.382056+0000","last_peered":"2024-09-21T09:58:31.382056+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T09:58:31.382056+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8388,"log_dups_size":0,"ondisk_log_size":8388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379191296,"num_objects":5786,"num_object_clones":0,"num_object_copies":17358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8388,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":4310},{"shards":"5,6,7","objects":175}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1063,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678966+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:58:28.678966+0000","last_peered":"2024-09-21T09:58:28.678966+0000","last_clean":"2024-09-21T09:58:28.678966+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:58:28.678966+0000","last_undegraded":"2024-09-21T09:58:28.678966+0000","last_fullsized":"2024-09-21T09:58:28.678966+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1070,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679115+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T09:58:28.679115+0000","last_peered":"2024-09-21T09:58:28.679115+0000","last_clean":"2024-09-21T09:58:28.679115+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T09:58:28.679115+0000","last_undegraded":"2024-09-21T09:58:28.679115+0000","last_fullsized":"2024-09-21T09:58:28.679115+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1114'8439","reported_seq":8477,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.386514+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T09:58:31.386514+0000","last_peered":"2024-09-21T09:58:31.386514+0000","last_clean":"2024-09-21T09:58:31.386514+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T09:58:31.386514+0000","last_undegraded":"2024-09-21T09:58:31.386514+0000","last_fullsized":"2024-09-21T09:58:31.386514+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8439,"log_dups_size":0,"ondisk_log_size":8439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384172056,"num_objects":5863,"num_object_clones":0,"num_object_copies":17589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5863,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8439,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.653224+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T09:58:25.653224+0000","last_peered":"2024-09-21T09:58:25.653224+0000","last_clean":"2024-09-21T09:58:25.653224+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T09:58:25.653224+0000","last_undegraded":"2024-09-21T09:58:25.653224+0000","last_fullsized":"2024-09-21T09:58:25.653224+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1392,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.677075+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:58:29.677075+0000","last_peered":"2024-09-21T09:58:29.677075+0000","last_clean":"2024-09-21T09:58:29.677075+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:58:29.677075+0000","last_undegraded":"2024-09-21T09:58:29.677075+0000","last_fullsized":"2024-09-21T09:58:29.677075+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1114'8220","reported_seq":11045,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.371602+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T09:58:31.371602+0000","last_peered":"2024-09-21T09:58:31.371602+0000","last_clean":"2024-09-21T09:58:31.371602+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T09:58:31.371602+0000","last_undegraded":"2024-09-21T09:58:31.371602+0000","last_fullsized":"2024-09-21T09:58:31.371602+0000","mapping_epoch":1113,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8220,"log_dups_size":0,"ondisk_log_size":8220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379322368,"num_objects":5788,"num_object_clones":0,"num_object_copies":17364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8220,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1371,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673777+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:58:28.673777+0000","last_peered":"2024-09-21T09:58:28.673777+0000","last_clean":"2024-09-21T09:58:28.673777+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:58:28.673777+0000","last_undegraded":"2024-09-21T09:58:28.673777+0000","last_fullsized":"2024-09-21T09:58:28.673777+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1071,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673453+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:58:28.673453+0000","last_peered":"2024-09-21T09:58:28.673453+0000","last_clean":"2024-09-21T09:58:28.673453+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:58:28.673453+0000","last_undegraded":"2024-09-21T09:58:28.673453+0000","last_fullsized":"2024-09-21T09:58:28.673453+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.650736+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T09:58:25.650736+0000","last_peered":"2024-09-21T09:58:25.650736+0000","last_clean":"2024-09-21T09:58:25.650736+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T09:58:25.650736+0000","last_undegraded":"2024-09-21T09:58:25.650736+0000","last_fullsized":"2024-09-21T09:58:25.650736+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1029,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678932+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:58:28.678932+0000","last_peered":"2024-09-21T09:58:28.678932+0000","last_clean":"2024-09-21T09:58:28.678932+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:58:28.678932+0000","last_undegraded":"2024-09-21T09:58:28.678932+0000","last_fullsized":"2024-09-21T09:58:28.678932+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1114'8434","reported_seq":11469,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.993264+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T09:58:31.993264+0000","last_peered":"2024-09-21T09:58:31.993264+0000","last_clean":"2024-09-21T09:58:31.993264+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T09:58:31.993264+0000","last_undegraded":"2024-09-21T09:58:31.993264+0000","last_fullsized":"2024-09-21T09:58:31.993264+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8434,"log_dups_size":0,"ondisk_log_size":8434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375128064,"num_objects":5724,"num_object_clones":0,"num_object_copies":17172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8434,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1069,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673631+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:58:28.673631+0000","last_peered":"2024-09-21T09:58:28.673631+0000","last_clean":"2024-09-21T09:58:28.673631+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:58:28.673631+0000","last_undegraded":"2024-09-21T09:58:28.673631+0000","last_fullsized":"2024-09-21T09:58:28.673631+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1417,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692386+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T09:58:28.692386+0000","last_peered":"2024-09-21T09:58:28.692386+0000","last_clean":"2024-09-21T09:58:28.692386+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T09:58:28.692386+0000","last_undegraded":"2024-09-21T09:58:28.692386+0000","last_fullsized":"2024-09-21T09:58:28.692386+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1415,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692446+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:58:28.692446+0000","last_peered":"2024-09-21T09:58:28.692446+0000","last_clean":"2024-09-21T09:58:28.692446+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:58:28.692446+0000","last_undegraded":"2024-09-21T09:58:28.692446+0000","last_fullsized":"2024-09-21T09:58:28.692446+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1114'8311","reported_seq":8354,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:31.331131+0000","last_change":"2024-09-21T09:56:46.337646+0000","last_active":"2024-09-21T09:58:31.331131+0000","last_peered":"2024-09-21T09:58:31.331131+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:56:46.335218+0000","last_became_peered":"2024-09-21T09:56:46.335218+0000","last_unstale":"2024-09-21T09:58:31.331131+0000","last_undegraded":"2024-09-21T09:56:46.260383+0000","last_fullsized":"2024-09-21T09:56:46.244582+0000","mapping_epoch":1093,"log_start":"1089'200","ondisk_log_start":"1089'200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8111,"log_dups_size":0,"ondisk_log_size":8111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:03.383491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382533632,"num_objects":5837,"num_object_clones":0,"num_object_copies":17511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1404,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8311,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":167},{"shards":"6,7","objects":4433}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1398,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.681929+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T09:58:29.681929+0000","last_peered":"2024-09-21T09:58:29.681929+0000","last_clean":"2024-09-21T09:58:29.681929+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T09:58:29.681929+0000","last_undegraded":"2024-09-21T09:58:29.681929+0000","last_fullsized":"2024-09-21T09:58:29.681929+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1379,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.675884+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:58:29.675884+0000","last_peered":"2024-09-21T09:58:29.675884+0000","last_clean":"2024-09-21T09:58:29.675884+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:58:29.675884+0000","last_undegraded":"2024-09-21T09:58:29.675884+0000","last_fullsized":"2024-09-21T09:58:29.675884+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1387,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676947+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:58:29.676947+0000","last_peered":"2024-09-21T09:58:29.676947+0000","last_clean":"2024-09-21T09:58:29.676947+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:58:29.676947+0000","last_undegraded":"2024-09-21T09:58:29.676947+0000","last_fullsized":"2024-09-21T09:58:29.676947+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1112'8252","reported_seq":11210,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.260939+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T09:58:28.260939+0000","last_peered":"2024-09-21T09:58:28.260939+0000","last_clean":"2024-09-21T09:58:28.260939+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T09:58:28.260939+0000","last_undegraded":"2024-09-21T09:58:28.260939+0000","last_fullsized":"2024-09-21T09:58:28.260939+0000","mapping_epoch":1096,"log_start":"1089'900","ondisk_log_start":"1089'900","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7352,"log_dups_size":0,"ondisk_log_size":7352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390070272,"num_objects":5952,"num_object_clones":0,"num_object_copies":17856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8252,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1072,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673355+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:58:28.673355+0000","last_peered":"2024-09-21T09:58:28.673355+0000","last_clean":"2024-09-21T09:58:28.673355+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:58:28.673355+0000","last_undegraded":"2024-09-21T09:58:28.673355+0000","last_fullsized":"2024-09-21T09:58:28.673355+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1063,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679170+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:58:28.679170+0000","last_peered":"2024-09-21T09:58:28.679170+0000","last_clean":"2024-09-21T09:58:28.679170+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:58:28.679170+0000","last_undegraded":"2024-09-21T09:58:28.679170+0000","last_fullsized":"2024-09-21T09:58:28.679170+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1404,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676440+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:58:29.676440+0000","last_peered":"2024-09-21T09:58:29.676440+0000","last_clean":"2024-09-21T09:58:29.676440+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:58:29.676440+0000","last_undegraded":"2024-09-21T09:58:29.676440+0000","last_fullsized":"2024-09-21T09:58:29.676440+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1113'8282","reported_seq":8349,"reported_epoch":1113,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:28.867991+0000","last_change":"2024-09-21T09:56:46.343261+0000","last_active":"2024-09-21T09:58:28.867991+0000","last_peered":"2024-09-21T09:58:28.867991+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T09:58:28.867991+0000","last_undegraded":"2024-09-21T09:56:46.247241+0000","last_fullsized":"2024-09-21T09:56:46.239078+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8282,"log_dups_size":0,"ondisk_log_size":8282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385744896,"num_objects":5886,"num_object_clones":0,"num_object_copies":17658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8282,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":4425},{"shards":"0,2,5","objects":263}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1090,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673958+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:58:28.673958+0000","last_peered":"2024-09-21T09:58:28.673958+0000","last_clean":"2024-09-21T09:58:28.673958+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:58:28.673958+0000","last_undegraded":"2024-09-21T09:58:28.673958+0000","last_fullsized":"2024-09-21T09:58:28.673958+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1401,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676534+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:58:29.676534+0000","last_peered":"2024-09-21T09:58:29.676534+0000","last_clean":"2024-09-21T09:58:29.676534+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:58:29.676534+0000","last_undegraded":"2024-09-21T09:58:29.676534+0000","last_fullsized":"2024-09-21T09:58:29.676534+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":56,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692129+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T09:58:28.692129+0000","last_peered":"2024-09-21T09:58:28.692129+0000","last_clean":"2024-09-21T09:58:28.692129+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T09:58:28.692129+0000","last_undegraded":"2024-09-21T09:58:28.692129+0000","last_fullsized":"2024-09-21T09:58:28.692129+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1114'8645","reported_seq":8713,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:32.004262+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T09:58:32.004262+0000","last_peered":"2024-09-21T09:58:32.004262+0000","last_clean":"2024-09-21T09:58:32.004262+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T09:58:32.004262+0000","last_undegraded":"2024-09-21T09:58:32.004262+0000","last_fullsized":"2024-09-21T09:58:32.004262+0000","mapping_epoch":1086,"log_start":"1089'100","ondisk_log_start":"1089'100","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8545,"log_dups_size":0,"ondisk_log_size":8545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388694016,"num_objects":5931,"num_object_clones":0,"num_object_copies":17793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8645,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1378,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676733+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:58:29.676733+0000","last_peered":"2024-09-21T09:58:29.676733+0000","last_clean":"2024-09-21T09:58:29.676733+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:58:29.676733+0000","last_undegraded":"2024-09-21T09:58:29.676733+0000","last_fullsized":"2024-09-21T09:58:29.676733+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1054,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674949+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:58:28.674949+0000","last_peered":"2024-09-21T09:58:28.674949+0000","last_clean":"2024-09-21T09:58:28.674949+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:58:28.674949+0000","last_undegraded":"2024-09-21T09:58:28.674949+0000","last_fullsized":"2024-09-21T09:58:28.674949+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1393,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692460+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:58:28.692460+0000","last_peered":"2024-09-21T09:58:28.692460+0000","last_clean":"2024-09-21T09:58:28.692460+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:58:28.692460+0000","last_undegraded":"2024-09-21T09:58:28.692460+0000","last_fullsized":"2024-09-21T09:58:28.692460+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1114'8407","reported_seq":8453,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:31.360036+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T09:58:31.360036+0000","last_peered":"2024-09-21T09:58:31.360036+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T09:58:31.360036+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1089'500","ondisk_log_start":"1089'500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388562944,"num_objects":5929,"num_object_clones":0,"num_object_copies":17787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8407,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":4517},{"shards":"2,4,6","objects":173}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":54,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.658663+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T09:58:25.658663+0000","last_peered":"2024-09-21T09:58:25.658663+0000","last_clean":"2024-09-21T09:58:25.658663+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T09:58:25.658663+0000","last_undegraded":"2024-09-21T09:58:25.658663+0000","last_fullsized":"2024-09-21T09:58:25.658663+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1379,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676734+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:58:29.676734+0000","last_peered":"2024-09-21T09:58:29.676734+0000","last_clean":"2024-09-21T09:58:29.676734+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:58:29.676734+0000","last_undegraded":"2024-09-21T09:58:29.676734+0000","last_fullsized":"2024-09-21T09:58:29.676734+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.653146+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T09:58:25.653146+0000","last_peered":"2024-09-21T09:58:25.653146+0000","last_clean":"2024-09-21T09:58:25.653146+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T09:58:25.653146+0000","last_undegraded":"2024-09-21T09:58:25.653146+0000","last_fullsized":"2024-09-21T09:58:25.653146+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1114'8444","reported_seq":8482,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:31.347494+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T09:58:31.347494+0000","last_peered":"2024-09-21T09:58:31.347494+0000","last_clean":"2024-09-21T09:58:31.347494+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T09:58:31.347494+0000","last_undegraded":"2024-09-21T09:58:31.347494+0000","last_fullsized":"2024-09-21T09:58:31.347494+0000","mapping_epoch":1086,"log_start":"1089'300","ondisk_log_start":"1089'300","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382205952,"num_objects":5832,"num_object_clones":0,"num_object_copies":17496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8444,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1407,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.693080+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:58:28.693080+0000","last_peered":"2024-09-21T09:58:28.693080+0000","last_clean":"2024-09-21T09:58:28.693080+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:58:28.693080+0000","last_undegraded":"2024-09-21T09:58:28.693080+0000","last_fullsized":"2024-09-21T09:58:28.693080+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1057,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678869+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:58:28.678869+0000","last_peered":"2024-09-21T09:58:28.678869+0000","last_clean":"2024-09-21T09:58:28.678869+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:58:28.678869+0000","last_undegraded":"2024-09-21T09:58:28.678869+0000","last_fullsized":"2024-09-21T09:58:28.678869+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1047,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678896+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:58:28.678896+0000","last_peered":"2024-09-21T09:58:28.678896+0000","last_clean":"2024-09-21T09:58:28.678896+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:58:28.678896+0000","last_undegraded":"2024-09-21T09:58:28.678896+0000","last_fullsized":"2024-09-21T09:58:28.678896+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1061,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679148+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:58:28.679148+0000","last_peered":"2024-09-21T09:58:28.679148+0000","last_clean":"2024-09-21T09:58:28.679148+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:58:28.679148+0000","last_undegraded":"2024-09-21T09:58:28.679148+0000","last_fullsized":"2024-09-21T09:58:28.679148+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1070,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679276+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:58:28.679276+0000","last_peered":"2024-09-21T09:58:28.679276+0000","last_clean":"2024-09-21T09:58:28.679276+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:58:28.679276+0000","last_undegraded":"2024-09-21T09:58:28.679276+0000","last_fullsized":"2024-09-21T09:58:28.679276+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1378,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676134+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:58:29.676134+0000","last_peered":"2024-09-21T09:58:29.676134+0000","last_clean":"2024-09-21T09:58:29.676134+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:58:29.676134+0000","last_undegraded":"2024-09-21T09:58:29.676134+0000","last_fullsized":"2024-09-21T09:58:29.676134+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1073,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674753+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T09:58:28.674753+0000","last_peered":"2024-09-21T09:58:28.674753+0000","last_clean":"2024-09-21T09:58:28.674753+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T09:58:28.674753+0000","last_undegraded":"2024-09-21T09:58:28.674753+0000","last_fullsized":"2024-09-21T09:58:28.674753+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1056,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673577+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:58:28.673577+0000","last_peered":"2024-09-21T09:58:28.673577+0000","last_clean":"2024-09-21T09:58:28.673577+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:58:28.673577+0000","last_undegraded":"2024-09-21T09:58:28.673577+0000","last_fullsized":"2024-09-21T09:58:28.673577+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.647881+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T09:58:25.647881+0000","last_peered":"2024-09-21T09:58:25.647881+0000","last_clean":"2024-09-21T09:58:25.647881+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T09:58:25.647881+0000","last_undegraded":"2024-09-21T09:58:25.647881+0000","last_fullsized":"2024-09-21T09:58:25.647881+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1415,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692757+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T09:58:28.692757+0000","last_peered":"2024-09-21T09:58:28.692757+0000","last_clean":"2024-09-21T09:58:28.692757+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T09:58:28.692757+0000","last_undegraded":"2024-09-21T09:58:28.692757+0000","last_fullsized":"2024-09-21T09:58:28.692757+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.653305+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T09:58:25.653305+0000","last_peered":"2024-09-21T09:58:25.653305+0000","last_clean":"2024-09-21T09:58:25.653305+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T09:58:25.653305+0000","last_undegraded":"2024-09-21T09:58:25.653305+0000","last_fullsized":"2024-09-21T09:58:25.653305+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1053,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678791+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:58:28.678791+0000","last_peered":"2024-09-21T09:58:28.678791+0000","last_clean":"2024-09-21T09:58:28.678791+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:58:28.678791+0000","last_undegraded":"2024-09-21T09:58:28.678791+0000","last_fullsized":"2024-09-21T09:58:28.678791+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.653053+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T09:58:25.653053+0000","last_peered":"2024-09-21T09:58:25.653053+0000","last_clean":"2024-09-21T09:58:25.653053+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T09:58:25.653053+0000","last_undegraded":"2024-09-21T09:58:25.653053+0000","last_fullsized":"2024-09-21T09:58:25.653053+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1418,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692946+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T09:58:28.692946+0000","last_peered":"2024-09-21T09:58:28.692946+0000","last_clean":"2024-09-21T09:58:28.692946+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T09:58:28.692946+0000","last_undegraded":"2024-09-21T09:58:28.692946+0000","last_fullsized":"2024-09-21T09:58:28.692946+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1391,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676013+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:58:29.676013+0000","last_peered":"2024-09-21T09:58:29.676013+0000","last_clean":"2024-09-21T09:58:29.676013+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:58:29.676013+0000","last_undegraded":"2024-09-21T09:58:29.676013+0000","last_fullsized":"2024-09-21T09:58:29.676013+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.653278+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T09:58:25.653278+0000","last_peered":"2024-09-21T09:58:25.653278+0000","last_clean":"2024-09-21T09:58:25.653278+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T09:58:25.653278+0000","last_undegraded":"2024-09-21T09:58:25.653278+0000","last_fullsized":"2024-09-21T09:58:25.653278+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1077,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.681211+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:58:28.681211+0000","last_peered":"2024-09-21T09:58:28.681211+0000","last_clean":"2024-09-21T09:58:28.681211+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:58:28.681211+0000","last_undegraded":"2024-09-21T09:58:28.681211+0000","last_fullsized":"2024-09-21T09:58:28.681211+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1374,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.682026+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T09:58:29.682026+0000","last_peered":"2024-09-21T09:58:29.682026+0000","last_clean":"2024-09-21T09:58:29.682026+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T09:58:29.682026+0000","last_undegraded":"2024-09-21T09:58:29.682026+0000","last_fullsized":"2024-09-21T09:58:29.682026+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1394,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692545+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:58:28.692545+0000","last_peered":"2024-09-21T09:58:28.692545+0000","last_clean":"2024-09-21T09:58:28.692545+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:58:28.692545+0000","last_undegraded":"2024-09-21T09:58:28.692545+0000","last_fullsized":"2024-09-21T09:58:28.692545+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1093,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679073+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T09:58:28.679073+0000","last_peered":"2024-09-21T09:58:28.679073+0000","last_clean":"2024-09-21T09:58:28.679073+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T09:58:28.679073+0000","last_undegraded":"2024-09-21T09:58:28.679073+0000","last_fullsized":"2024-09-21T09:58:28.679073+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1057,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674473+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:58:28.674473+0000","last_peered":"2024-09-21T09:58:28.674473+0000","last_clean":"2024-09-21T09:58:28.674473+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:58:28.674473+0000","last_undegraded":"2024-09-21T09:58:28.674473+0000","last_fullsized":"2024-09-21T09:58:28.674473+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1093,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.681072+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T09:58:28.681072+0000","last_peered":"2024-09-21T09:58:28.681072+0000","last_clean":"2024-09-21T09:58:28.681072+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T09:58:28.681072+0000","last_undegraded":"2024-09-21T09:58:28.681072+0000","last_fullsized":"2024-09-21T09:58:28.681072+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1042,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673893+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:58:28.673893+0000","last_peered":"2024-09-21T09:58:28.673893+0000","last_clean":"2024-09-21T09:58:28.673893+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:58:28.673893+0000","last_undegraded":"2024-09-21T09:58:28.673893+0000","last_fullsized":"2024-09-21T09:58:28.673893+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.648072+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T09:58:25.648072+0000","last_peered":"2024-09-21T09:58:25.648072+0000","last_clean":"2024-09-21T09:58:25.648072+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T09:58:25.648072+0000","last_undegraded":"2024-09-21T09:58:25.648072+0000","last_fullsized":"2024-09-21T09:58:25.648072+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":50,"reported_epoch":1112,"state":"active+clean","last_fresh":"2024-09-21T09:58:25.650887+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T09:58:25.650887+0000","last_peered":"2024-09-21T09:58:25.650887+0000","last_clean":"2024-09-21T09:58:25.650887+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T09:58:25.650887+0000","last_undegraded":"2024-09-21T09:58:25.650887+0000","last_fullsized":"2024-09-21T09:58:25.650887+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1069,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679374+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:58:28.679374+0000","last_peered":"2024-09-21T09:58:28.679374+0000","last_clean":"2024-09-21T09:58:28.679374+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:58:28.679374+0000","last_undegraded":"2024-09-21T09:58:28.679374+0000","last_fullsized":"2024-09-21T09:58:28.679374+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1410,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.692838+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:58:28.692838+0000","last_peered":"2024-09-21T09:58:28.692838+0000","last_clean":"2024-09-21T09:58:28.692838+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:58:28.692838+0000","last_undegraded":"2024-09-21T09:58:28.692838+0000","last_fullsized":"2024-09-21T09:58:28.692838+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":6155337752,"num_objects":93924,"num_object_clones":0,"num_object_copies":281772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6904,"num_objects_misplaced":81,"num_objects_unfound":0,"num_objects_dirty":93924,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133884,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7361,"num_bytes_recovered":482410496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19400241152,"data_stored":19400228936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130384,"ondisk_log_size":130384,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234272,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2488172,"kb_used_data":2362720,"kb_used_omap":1,"kb_used_meta":125438,"kb_avail":91256980,"statfs":{"total":95995035648,"available":93447147520,"internally_reserved":0,"allocated":2419425280,"data_stored":2417396429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":128448971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495909,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2716772,"kb_used_data":2573336,"kb_used_omap":1,"kb_used_meta":143422,"kb_avail":91028380,"statfs":{"total":95995035648,"available":93213061120,"internally_reserved":0,"allocated":2635096064,"data_stored":2633075278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":146864583},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757547,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2126560,"kb_used_data":1969168,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":91618592,"statfs":{"total":95995035648,"available":93817438208,"internally_reserved":0,"allocated":2016428032,"data_stored":2014410744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019183,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2015208,"kb_used_data":1893848,"kb_used_omap":1,"kb_used_meta":121342,"kb_avail":91729944,"statfs":{"total":95995035648,"available":93931462656,"internally_reserved":0,"allocated":1939300352,"data_stored":1937279079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":124254662},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888479,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3954292,"kb_used_data":3733356,"kb_used_omap":34,"kb_used_meta":220765,"kb_avail":89790860,"statfs":{"total":95995035648,"available":91945840640,"internally_reserved":0,"allocated":3822956544,"data_stored":3820958628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":226064371},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967718,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3297160,"kb_used_data":3152636,"kb_used_omap":41,"kb_used_meta":144470,"kb_avail":90447992,"statfs":{"total":95995035648,"available":92618743808,"internally_reserved":0,"allocated":3228299264,"data_stored":3226293950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":147937695},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046957,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1960016,"kb_used_data":1873800,"kb_used_omap":41,"kb_used_meta":86166,"kb_avail":91785136,"statfs":{"total":95995035648,"available":93987979264,"internally_reserved":0,"allocated":1918771200,"data_stored":1916761464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":88234921},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126195,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1627904,"kb_used_data":1532664,"kb_used_omap":72,"kb_used_meta":95095,"kb_avail":92117248,"statfs":{"total":95995035648,"available":94328061952,"internally_reserved":0,"allocated":1569447936,"data_stored":1567438814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":97377323},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1554518016,"data_stored":1554513944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1900154880,"data_stored":1900150808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3214082048,"data_stored":3214082048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3810197504,"data_stored":3810197504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1916338176,"data_stored":1916338176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1994592256,"data_stored":1994588184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2610757632,"data_stored":2610757632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2399600640,"data_stored":2399600640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:58:33.894 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:58:33.927 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:33.950 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:34.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:34.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:33 smithi191 ceph-mon[102206]: pgmap v1234: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 954 op/s; 6904/281850 objects degraded (2.450%); 81/281850 objects misplaced (0.029%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:58:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:33 smithi191 ceph-mon[102206]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:33 smithi076 ceph-mon[106482]: pgmap v1234: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 954 op/s; 6904/281850 objects degraded (2.450%); 81/281850 objects misplaced (0.029%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:58:34.114 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:33 smithi076 ceph-mon[106482]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:34.114 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:33 smithi076 ceph-mon[110963]: pgmap v1234: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 954 op/s; 6904/281850 objects degraded (2.450%); 81/281850 objects misplaced (0.029%); 5.0 MiB/s, 80 objects/s recovering 2024-09-21T09:58:34.114 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:33 smithi076 ceph-mon[110963]: from='client.44705 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:34.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:58:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.152 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:34.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:34.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:34.304 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:34.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:34.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:34.438 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:34.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:34.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:34.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:34.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:34.750 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:34.852 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 6904/281850 objects degraded (2.450%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 6904/281850 objects degraded (2.450%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:34.877 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:34.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:34.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:35.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:35.072 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 6904/281850 objects degraded (2.450%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:35.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:35.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:35.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:35.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:35.300 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:35.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:35.369 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:35.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:35.421 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.479 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:35.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:35.491 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:35.491 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:35.581 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:35.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:35.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:35.722 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:35.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:35.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:35.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:35.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:35.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:36.010 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:35 smithi076 ceph-mon[106482]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 6904/279120 objects degraded (2.473%); 81/279120 objects misplaced (0.029%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T09:58:36.010 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:35 smithi076 ceph-mon[110963]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 6904/279120 objects degraded (2.473%); 81/279120 objects misplaced (0.029%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T09:58:36.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.021 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:36.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:35 smithi191 ceph-mon[102206]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 6904/279120 objects degraded (2.473%); 81/279120 objects misplaced (0.029%); 2.4 MiB/s, 38 objects/s recovering 2024-09-21T09:58:36.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:36.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:36.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:36.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T09:58:36.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:36.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:36.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:36.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:36.559 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:36.566 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:36.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:36.668 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:36.709 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:36.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:36.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:36.835 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:36.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:36.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:36.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:36.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:37.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:37.128 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:37.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:37.257 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:37.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:37.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:37.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:37.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:37.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:37.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:37.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:37.619 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:37.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:37.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.795 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:37.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:37.831 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:37 smithi191 ceph-mon[102206]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 6799/271728 objects degraded (2.502%); 3.0 MiB/s, 61 objects/s recovering 2024-09-21T09:58:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:37 smithi076 ceph-mon[106482]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 6799/271728 objects degraded (2.502%); 3.0 MiB/s, 61 objects/s recovering 2024-09-21T09:58:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:37 smithi076 ceph-mon[110963]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 6799/271728 objects degraded (2.502%); 3.0 MiB/s, 61 objects/s recovering 2024-09-21T09:58:37.896 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:37.932 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:37.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:38.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:38.066 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:38.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:38.204 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:38.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:38.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:38.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:38.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:38.468 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:38.502 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:38.502 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:38.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:38.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:38.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:38.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:38.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:38.898 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:38.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:39.000 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:39.029 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:39.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:39.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:39.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:39.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:39.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T09:58:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.287 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:39.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:39.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:39.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.434 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:39.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:39.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:39.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:39.601 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:39.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:39.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:39.741 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T09:58:39.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:39.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:58:39.820 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:39.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:39.861 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:58:39.861 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:39.902 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:58:39.902 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:39.909 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:58:39.911 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:58:39.924 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:39.950 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:58:39.950 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:39.966 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:39.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:39.989 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:58:39.989 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:39.989 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:39 smithi076 ceph-mon[106482]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 6799/268944 objects degraded (2.528%); 2.5 MiB/s, 50 objects/s recovering 2024-09-21T09:58:39.990 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 6799/268944 objects degraded (2.528%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:39.990 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2887875209' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:58:39.990 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:39 smithi076 ceph-mon[110963]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 6799/268944 objects degraded (2.528%); 2.5 MiB/s, 50 objects/s recovering 2024-09-21T09:58:39.990 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 6799/268944 objects degraded (2.528%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:39.990 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2887875209' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:58:40.021 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:58:40.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:40.054 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:58:40.055 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:40.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:40.093 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:58:40.093 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:40.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:39 smithi191 ceph-mon[102206]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 6799/268944 objects degraded (2.528%); 2.5 MiB/s, 50 objects/s recovering 2024-09-21T09:58:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 6799/268944 objects degraded (2.528%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2887875209' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:58:40.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:40.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:40.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:40.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:40.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:40.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:40.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:40.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.639 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:40.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:40.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:40.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:40.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:40.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:40.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:40.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2887875209' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[110963]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3219140917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[110963]: from='client.34809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:40.953 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:40.954 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2887875209' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:58:40.954 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[106482]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-21T09:58:40.954 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3219140917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:40.954 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[106482]: from='client.34809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:40.954 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:41.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:40 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:58:41.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2887875209' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:58:41.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:40 smithi191 ceph-mon[102206]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-21T09:58:41.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3219140917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:41.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:40 smithi191 ceph-mon[102206]: from='client.34809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:41.010 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:41.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:41.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:41.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:41.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:41.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:41.337 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:41.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:41.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:41.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:41.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:41.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:41.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:41.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:41.793 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:41 smithi191 ceph-mon[102206]: pgmap v1239: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6632/266745 objects degraded (2.486%); 1.3 MiB/s, 47 objects/s recovering 2024-09-21T09:58:41.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:41.922 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:41.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:41.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:42.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:41 smithi076 ceph-mon[110963]: pgmap v1239: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6632/266745 objects degraded (2.486%); 1.3 MiB/s, 47 objects/s recovering 2024-09-21T09:58:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:41 smithi076 ceph-mon[106482]: pgmap v1239: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 6632/266745 objects degraded (2.486%); 1.3 MiB/s, 47 objects/s recovering 2024-09-21T09:58:42.148 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:42.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:42.241 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:58:42.241 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T09:58:42.241 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T09:58:42.241 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T09:58:42.241 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:58:42.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:42.369 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1238,"stamp":"2024-09-21T09:58:39.738307+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5842343933,"num_objects":89165,"num_object_clones":0,"num_object_copies":267495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6799,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89165,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":138948,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7560,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133277,"ondisk_log_size":133277,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19363524,"kb_used_data":18202756,"kb_used_omap":195,"kb_used_meta":1160444,"kb_avail":730597692,"statfs":{"total":767960285184,"available":748132036608,"internally_reserved":0,"allocated":18639622144,"data_stored":18623492655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1188294788},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-489160704,"num_objects":-7464,"num_object_clones":0,"num_object_copies":-22392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-364,"num_objects_misplaced":-127,"num_objects_unfound":0,"num_objects_dirty":-7464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7464,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":491,"num_bytes_recovered":25296896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.063538"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.680180+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T09:58:29.680180+0000","last_peered":"2024-09-21T09:58:29.680180+0000","last_clean":"2024-09-21T09:58:29.680180+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T09:58:29.680180+0000","last_undegraded":"2024-09-21T09:58:29.680180+0000","last_fullsized":"2024-09-21T09:58:29.680180+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1042,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.676664+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:58:28.676664+0000","last_peered":"2024-09-21T09:58:28.676664+0000","last_clean":"2024-09-21T09:58:28.676664+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:58:28.676664+0000","last_undegraded":"2024-09-21T09:58:28.676664+0000","last_fullsized":"2024-09-21T09:58:28.676664+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1073,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673823+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:58:28.673823+0000","last_peered":"2024-09-21T09:58:28.673823+0000","last_clean":"2024-09-21T09:58:28.673823+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:58:28.673823+0000","last_undegraded":"2024-09-21T09:58:28.673823+0000","last_fullsized":"2024-09-21T09:58:28.673823+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1419,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.741615+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T09:58:29.741615+0000","last_peered":"2024-09-21T09:58:29.741615+0000","last_clean":"2024-09-21T09:58:29.741615+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T09:58:29.741615+0000","last_undegraded":"2024-09-21T09:58:29.741615+0000","last_fullsized":"2024-09-21T09:58:29.741615+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1418,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.742603+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T09:58:29.742603+0000","last_peered":"2024-09-21T09:58:29.742603+0000","last_clean":"2024-09-21T09:58:29.742603+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T09:58:29.742603+0000","last_undegraded":"2024-09-21T09:58:29.742603+0000","last_fullsized":"2024-09-21T09:58:29.742603+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1025,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678549+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:58:28.678549+0000","last_peered":"2024-09-21T09:58:28.678549+0000","last_clean":"2024-09-21T09:58:28.678549+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:58:28.678549+0000","last_undegraded":"2024-09-21T09:58:28.678549+0000","last_fullsized":"2024-09-21T09:58:28.678549+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.680107+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T09:58:29.680107+0000","last_peered":"2024-09-21T09:58:29.680107+0000","last_clean":"2024-09-21T09:58:29.680107+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T09:58:29.680107+0000","last_undegraded":"2024-09-21T09:58:29.680107+0000","last_fullsized":"2024-09-21T09:58:29.680107+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1052,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678361+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:58:28.678361+0000","last_peered":"2024-09-21T09:58:28.678361+0000","last_clean":"2024-09-21T09:58:28.678361+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:58:28.678361+0000","last_undegraded":"2024-09-21T09:58:28.678361+0000","last_fullsized":"2024-09-21T09:58:28.678361+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1114'8568","reported_seq":8606,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.398453+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T09:58:36.398453+0000","last_peered":"2024-09-21T09:58:36.398453+0000","last_clean":"2024-09-21T09:58:36.398453+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T09:58:36.398453+0000","last_undegraded":"2024-09-21T09:58:36.398453+0000","last_fullsized":"2024-09-21T09:58:36.398453+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8568,"log_dups_size":0,"ondisk_log_size":8568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362807296,"num_objects":5536,"num_object_clones":0,"num_object_copies":16608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8568,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1041,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674151+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:58:28.674151+0000","last_peered":"2024-09-21T09:58:28.674151+0000","last_clean":"2024-09-21T09:58:28.674151+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:58:28.674151+0000","last_undegraded":"2024-09-21T09:58:28.674151+0000","last_fullsized":"2024-09-21T09:58:28.674151+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1041,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673098+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:58:28.673098+0000","last_peered":"2024-09-21T09:58:28.673098+0000","last_clean":"2024-09-21T09:58:28.673098+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:58:28.673098+0000","last_undegraded":"2024-09-21T09:58:28.673098+0000","last_fullsized":"2024-09-21T09:58:28.673098+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1114'8682","reported_seq":11623,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:38.306665+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T09:58:38.306665+0000","last_peered":"2024-09-21T09:58:38.306665+0000","last_clean":"2024-09-21T09:58:38.306665+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T09:58:38.306665+0000","last_undegraded":"2024-09-21T09:58:38.306665+0000","last_fullsized":"2024-09-21T09:58:38.306665+0000","mapping_epoch":1108,"log_start":"1091'1300","ondisk_log_start":"1091'1300","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7382,"log_dups_size":0,"ondisk_log_size":7382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356122624,"num_objects":5434,"num_object_clones":0,"num_object_copies":16302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8682,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1042,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674521+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:58:28.674521+0000","last_peered":"2024-09-21T09:58:28.674521+0000","last_clean":"2024-09-21T09:58:28.674521+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:58:28.674521+0000","last_undegraded":"2024-09-21T09:58:28.674521+0000","last_fullsized":"2024-09-21T09:58:28.674521+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1095,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.675842+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T09:58:29.675842+0000","last_peered":"2024-09-21T09:58:29.675842+0000","last_clean":"2024-09-21T09:58:29.675842+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T09:58:29.675842+0000","last_undegraded":"2024-09-21T09:58:29.675842+0000","last_fullsized":"2024-09-21T09:58:29.675842+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1114'8702","reported_seq":11624,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.345920+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T09:58:36.345920+0000","last_peered":"2024-09-21T09:58:36.345920+0000","last_clean":"2024-09-21T09:58:36.345920+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T09:58:36.345920+0000","last_undegraded":"2024-09-21T09:58:36.345920+0000","last_fullsized":"2024-09-21T09:58:36.345920+0000","mapping_epoch":1100,"log_start":"1089'500","ondisk_log_start":"1089'500","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8202,"log_dups_size":0,"ondisk_log_size":8202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372899840,"num_objects":5690,"num_object_clones":0,"num_object_copies":17070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8702,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1026,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674875+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:58:28.674875+0000","last_peered":"2024-09-21T09:58:28.674875+0000","last_clean":"2024-09-21T09:58:28.674875+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:58:28.674875+0000","last_undegraded":"2024-09-21T09:58:28.674875+0000","last_fullsized":"2024-09-21T09:58:28.674875+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1044,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.743365+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:58:29.743365+0000","last_peered":"2024-09-21T09:58:29.743365+0000","last_clean":"2024-09-21T09:58:29.743365+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:58:29.743365+0000","last_undegraded":"2024-09-21T09:58:29.743365+0000","last_fullsized":"2024-09-21T09:58:29.743365+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1114'8785","reported_seq":8853,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:38.930453+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T09:58:38.930453+0000","last_peered":"2024-09-21T09:58:38.930453+0000","last_clean":"2024-09-21T09:58:38.930453+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T09:58:38.930453+0000","last_undegraded":"2024-09-21T09:58:38.930453+0000","last_fullsized":"2024-09-21T09:58:38.930453+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8785,"log_dups_size":0,"ondisk_log_size":8785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362741760,"num_objects":5535,"num_object_clones":0,"num_object_copies":16605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8785,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1074,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674607+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:58:28.674607+0000","last_peered":"2024-09-21T09:58:28.674607+0000","last_clean":"2024-09-21T09:58:28.674607+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:58:28.674607+0000","last_undegraded":"2024-09-21T09:58:28.674607+0000","last_fullsized":"2024-09-21T09:58:28.674607+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1407,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.741716+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:58:29.741716+0000","last_peered":"2024-09-21T09:58:29.741716+0000","last_clean":"2024-09-21T09:58:29.741716+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:58:29.741716+0000","last_undegraded":"2024-09-21T09:58:29.741716+0000","last_fullsized":"2024-09-21T09:58:29.741716+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1114'8634","reported_seq":8671,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.365887+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T09:58:36.365887+0000","last_peered":"2024-09-21T09:58:36.365887+0000","last_clean":"2024-09-21T09:58:36.365887+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T09:58:36.365887+0000","last_undegraded":"2024-09-21T09:58:36.365887+0000","last_fullsized":"2024-09-21T09:58:36.365887+0000","mapping_epoch":1086,"log_start":"1089'700","ondisk_log_start":"1089'700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7934,"log_dups_size":0,"ondisk_log_size":7934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362545152,"num_objects":5532,"num_object_clones":0,"num_object_copies":16596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8634,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":58,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679716+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T09:58:29.679716+0000","last_peered":"2024-09-21T09:58:29.679716+0000","last_clean":"2024-09-21T09:58:29.679716+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T09:58:29.679716+0000","last_undegraded":"2024-09-21T09:58:29.679716+0000","last_fullsized":"2024-09-21T09:58:29.679716+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1079,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679179+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T09:58:28.679179+0000","last_peered":"2024-09-21T09:58:28.679179+0000","last_clean":"2024-09-21T09:58:28.679179+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T09:58:28.679179+0000","last_undegraded":"2024-09-21T09:58:28.679179+0000","last_fullsized":"2024-09-21T09:58:28.679179+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1114'8605","reported_seq":9404,"reported_epoch":1114,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:35.020089+0000","last_change":"2024-09-21T09:58:28.149252+0000","last_active":"2024-09-21T09:58:35.020089+0000","last_peered":"2024-09-21T09:58:35.020089+0000","last_clean":"2024-09-21T09:56:44.221905+0000","last_became_active":"2024-09-21T09:56:46.358647+0000","last_became_peered":"2024-09-21T09:56:46.358647+0000","last_unstale":"2024-09-21T09:58:35.020089+0000","last_undegraded":"2024-09-21T09:56:46.247338+0000","last_fullsized":"2024-09-21T09:56:46.236810+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8605,"log_dups_size":0,"ondisk_log_size":8605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377159680,"num_objects":5755,"num_object_clones":0,"num_object_copies":17265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1046,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8605,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":364,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":4709}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1045,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678672+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:58:28.678672+0000","last_peered":"2024-09-21T09:58:28.678672+0000","last_clean":"2024-09-21T09:58:28.678672+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:58:28.678672+0000","last_undegraded":"2024-09-21T09:58:28.678672+0000","last_fullsized":"2024-09-21T09:58:28.678672+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1076,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679464+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:58:28.679464+0000","last_peered":"2024-09-21T09:58:28.679464+0000","last_clean":"2024-09-21T09:58:28.679464+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:58:28.679464+0000","last_undegraded":"2024-09-21T09:58:28.679464+0000","last_fullsized":"2024-09-21T09:58:28.679464+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1114'8651","reported_seq":8693,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:36.368358+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T09:58:36.368358+0000","last_peered":"2024-09-21T09:58:36.368358+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T09:58:36.368358+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8651,"log_dups_size":0,"ondisk_log_size":8651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361955328,"num_objects":5523,"num_object_clones":0,"num_object_copies":16569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8651,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":4047}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1063,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678966+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:58:28.678966+0000","last_peered":"2024-09-21T09:58:28.678966+0000","last_clean":"2024-09-21T09:58:28.678966+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:58:28.678966+0000","last_undegraded":"2024-09-21T09:58:28.678966+0000","last_fullsized":"2024-09-21T09:58:28.678966+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1070,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679115+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T09:58:28.679115+0000","last_peered":"2024-09-21T09:58:28.679115+0000","last_clean":"2024-09-21T09:58:28.679115+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T09:58:28.679115+0000","last_undegraded":"2024-09-21T09:58:28.679115+0000","last_fullsized":"2024-09-21T09:58:28.679115+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1114'8691","reported_seq":8729,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.401068+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T09:58:36.401068+0000","last_peered":"2024-09-21T09:58:36.401068+0000","last_clean":"2024-09-21T09:58:36.401068+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T09:58:36.401068+0000","last_undegraded":"2024-09-21T09:58:36.401068+0000","last_fullsized":"2024-09-21T09:58:36.401068+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8691,"log_dups_size":0,"ondisk_log_size":8691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367656984,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8691,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679598+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T09:58:29.679598+0000","last_peered":"2024-09-21T09:58:29.679598+0000","last_clean":"2024-09-21T09:58:29.679598+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T09:58:29.679598+0000","last_undegraded":"2024-09-21T09:58:29.679598+0000","last_fullsized":"2024-09-21T09:58:29.679598+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1393,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.677108+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:58:29.677108+0000","last_peered":"2024-09-21T09:58:29.677108+0000","last_clean":"2024-09-21T09:58:29.677108+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:58:29.677108+0000","last_undegraded":"2024-09-21T09:58:29.677108+0000","last_fullsized":"2024-09-21T09:58:29.677108+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1114'8474","reported_seq":11298,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.362725+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T09:58:36.362725+0000","last_peered":"2024-09-21T09:58:36.362725+0000","last_clean":"2024-09-21T09:58:36.362725+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T09:58:36.362725+0000","last_undegraded":"2024-09-21T09:58:36.362725+0000","last_fullsized":"2024-09-21T09:58:36.362725+0000","mapping_epoch":1113,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8474,"log_dups_size":0,"ondisk_log_size":8474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362676224,"num_objects":5534,"num_object_clones":0,"num_object_copies":16602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8474,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1371,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673777+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:58:28.673777+0000","last_peered":"2024-09-21T09:58:28.673777+0000","last_clean":"2024-09-21T09:58:28.673777+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:58:28.673777+0000","last_undegraded":"2024-09-21T09:58:28.673777+0000","last_fullsized":"2024-09-21T09:58:28.673777+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1071,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673453+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:58:28.673453+0000","last_peered":"2024-09-21T09:58:28.673453+0000","last_clean":"2024-09-21T09:58:28.673453+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:58:28.673453+0000","last_undegraded":"2024-09-21T09:58:28.673453+0000","last_fullsized":"2024-09-21T09:58:28.673453+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679880+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T09:58:29.679880+0000","last_peered":"2024-09-21T09:58:29.679880+0000","last_clean":"2024-09-21T09:58:29.679880+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T09:58:29.679880+0000","last_undegraded":"2024-09-21T09:58:29.679880+0000","last_fullsized":"2024-09-21T09:58:29.679880+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1029,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678932+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:58:28.678932+0000","last_peered":"2024-09-21T09:58:28.678932+0000","last_clean":"2024-09-21T09:58:28.678932+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:58:28.678932+0000","last_undegraded":"2024-09-21T09:58:28.678932+0000","last_fullsized":"2024-09-21T09:58:28.678932+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1114'8659","reported_seq":11695,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.985330+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T09:58:36.985330+0000","last_peered":"2024-09-21T09:58:36.985330+0000","last_clean":"2024-09-21T09:58:36.985330+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T09:58:36.985330+0000","last_undegraded":"2024-09-21T09:58:36.985330+0000","last_fullsized":"2024-09-21T09:58:36.985330+0000","mapping_epoch":1105,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8659,"log_dups_size":0,"ondisk_log_size":8659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360382464,"num_objects":5499,"num_object_clones":0,"num_object_copies":16497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8659,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1069,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673631+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:58:28.673631+0000","last_peered":"2024-09-21T09:58:28.673631+0000","last_clean":"2024-09-21T09:58:28.673631+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:58:28.673631+0000","last_undegraded":"2024-09-21T09:58:28.673631+0000","last_fullsized":"2024-09-21T09:58:28.673631+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1419,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.741195+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T09:58:29.741195+0000","last_peered":"2024-09-21T09:58:29.741195+0000","last_clean":"2024-09-21T09:58:29.741195+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T09:58:29.741195+0000","last_undegraded":"2024-09-21T09:58:29.741195+0000","last_fullsized":"2024-09-21T09:58:29.741195+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1417,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.742473+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:58:29.742473+0000","last_peered":"2024-09-21T09:58:29.742473+0000","last_clean":"2024-09-21T09:58:29.742473+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:58:29.742473+0000","last_undegraded":"2024-09-21T09:58:29.742473+0000","last_fullsized":"2024-09-21T09:58:29.742473+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1114'8550","reported_seq":8593,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:36.323684+0000","last_change":"2024-09-21T09:56:46.337646+0000","last_active":"2024-09-21T09:58:36.323684+0000","last_peered":"2024-09-21T09:58:36.323684+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:56:46.335218+0000","last_became_peered":"2024-09-21T09:56:46.335218+0000","last_unstale":"2024-09-21T09:58:36.323684+0000","last_undegraded":"2024-09-21T09:56:46.260383+0000","last_fullsized":"2024-09-21T09:56:46.244582+0000","mapping_epoch":1093,"log_start":"1089'400","ondisk_log_start":"1089'400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8150,"log_dups_size":0,"ondisk_log_size":8150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:03.383491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366870528,"num_objects":5598,"num_object_clones":0,"num_object_copies":16794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1404,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8550,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":4194}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1398,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.681929+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T09:58:29.681929+0000","last_peered":"2024-09-21T09:58:29.681929+0000","last_clean":"2024-09-21T09:58:29.681929+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T09:58:29.681929+0000","last_undegraded":"2024-09-21T09:58:29.681929+0000","last_fullsized":"2024-09-21T09:58:29.681929+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1379,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.675884+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:58:29.675884+0000","last_peered":"2024-09-21T09:58:29.675884+0000","last_clean":"2024-09-21T09:58:29.675884+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:58:29.675884+0000","last_undegraded":"2024-09-21T09:58:29.675884+0000","last_fullsized":"2024-09-21T09:58:29.675884+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1387,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676947+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:58:29.676947+0000","last_peered":"2024-09-21T09:58:29.676947+0000","last_clean":"2024-09-21T09:58:29.676947+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:58:29.676947+0000","last_undegraded":"2024-09-21T09:58:29.676947+0000","last_fullsized":"2024-09-21T09:58:29.676947+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1114'8705","reported_seq":11667,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:38.296117+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T09:58:38.296117+0000","last_peered":"2024-09-21T09:58:38.296117+0000","last_clean":"2024-09-21T09:58:38.296117+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T09:58:38.296117+0000","last_undegraded":"2024-09-21T09:58:38.296117+0000","last_fullsized":"2024-09-21T09:58:38.296117+0000","mapping_epoch":1096,"log_start":"1091'1300","ondisk_log_start":"1091'1300","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7405,"log_dups_size":0,"ondisk_log_size":7405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360382464,"num_objects":5499,"num_object_clones":0,"num_object_copies":16497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8705,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1072,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673355+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:58:28.673355+0000","last_peered":"2024-09-21T09:58:28.673355+0000","last_clean":"2024-09-21T09:58:28.673355+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:58:28.673355+0000","last_undegraded":"2024-09-21T09:58:28.673355+0000","last_fullsized":"2024-09-21T09:58:28.673355+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1063,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679170+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:58:28.679170+0000","last_peered":"2024-09-21T09:58:28.679170+0000","last_clean":"2024-09-21T09:58:28.679170+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:58:28.679170+0000","last_undegraded":"2024-09-21T09:58:28.679170+0000","last_fullsized":"2024-09-21T09:58:28.679170+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1404,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676440+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:58:29.676440+0000","last_peered":"2024-09-21T09:58:29.676440+0000","last_clean":"2024-09-21T09:58:29.676440+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:58:29.676440+0000","last_undegraded":"2024-09-21T09:58:29.676440+0000","last_fullsized":"2024-09-21T09:58:29.676440+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1114'8746","reported_seq":8815,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:38.933997+0000","last_change":"2024-09-21T09:56:46.343261+0000","last_active":"2024-09-21T09:58:38.933997+0000","last_peered":"2024-09-21T09:58:38.933997+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T09:58:38.933997+0000","last_undegraded":"2024-09-21T09:56:46.247241+0000","last_fullsized":"2024-09-21T09:56:46.239078+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8746,"log_dups_size":0,"ondisk_log_size":8746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355336192,"num_objects":5422,"num_object_clones":0,"num_object_copies":16266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8746,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":3961}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1090,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673958+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:58:28.673958+0000","last_peered":"2024-09-21T09:58:28.673958+0000","last_clean":"2024-09-21T09:58:28.673958+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:58:28.673958+0000","last_undegraded":"2024-09-21T09:58:28.673958+0000","last_fullsized":"2024-09-21T09:58:28.673958+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1401,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676534+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:58:29.676534+0000","last_peered":"2024-09-21T09:58:29.676534+0000","last_clean":"2024-09-21T09:58:29.676534+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:58:29.676534+0000","last_undegraded":"2024-09-21T09:58:29.676534+0000","last_fullsized":"2024-09-21T09:58:29.676534+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.741012+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T09:58:29.741012+0000","last_peered":"2024-09-21T09:58:29.741012+0000","last_clean":"2024-09-21T09:58:29.741012+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T09:58:29.741012+0000","last_undegraded":"2024-09-21T09:58:29.741012+0000","last_fullsized":"2024-09-21T09:58:29.741012+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1114'8870","reported_seq":8938,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.922939+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T09:58:36.922939+0000","last_peered":"2024-09-21T09:58:36.922939+0000","last_clean":"2024-09-21T09:58:36.922939+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T09:58:36.922939+0000","last_undegraded":"2024-09-21T09:58:36.922939+0000","last_fullsized":"2024-09-21T09:58:36.922939+0000","mapping_epoch":1086,"log_start":"1089'200","ondisk_log_start":"1089'200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8670,"log_dups_size":0,"ondisk_log_size":8670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373948416,"num_objects":5706,"num_object_clones":0,"num_object_copies":17118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8870,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1378,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676733+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:58:29.676733+0000","last_peered":"2024-09-21T09:58:29.676733+0000","last_clean":"2024-09-21T09:58:29.676733+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:58:29.676733+0000","last_undegraded":"2024-09-21T09:58:29.676733+0000","last_fullsized":"2024-09-21T09:58:29.676733+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1054,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674949+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:58:28.674949+0000","last_peered":"2024-09-21T09:58:28.674949+0000","last_clean":"2024-09-21T09:58:28.674949+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:58:28.674949+0000","last_undegraded":"2024-09-21T09:58:28.674949+0000","last_fullsized":"2024-09-21T09:58:28.674949+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1395,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.741807+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:58:29.741807+0000","last_peered":"2024-09-21T09:58:29.741807+0000","last_clean":"2024-09-21T09:58:29.741807+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:58:29.741807+0000","last_undegraded":"2024-09-21T09:58:29.741807+0000","last_fullsized":"2024-09-21T09:58:29.741807+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1114'8653","reported_seq":8699,"reported_epoch":1114,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:36.338615+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T09:58:36.338615+0000","last_peered":"2024-09-21T09:58:36.338615+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T09:58:36.338615+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1089'700","ondisk_log_start":"1089'700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372441088,"num_objects":5683,"num_object_clones":0,"num_object_copies":17049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8653,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":4271}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":58,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.680186+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T09:58:29.680186+0000","last_peered":"2024-09-21T09:58:29.680186+0000","last_clean":"2024-09-21T09:58:29.680186+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T09:58:29.680186+0000","last_undegraded":"2024-09-21T09:58:29.680186+0000","last_fullsized":"2024-09-21T09:58:29.680186+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1379,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676734+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:58:29.676734+0000","last_peered":"2024-09-21T09:58:29.676734+0000","last_clean":"2024-09-21T09:58:29.676734+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:58:29.676734+0000","last_undegraded":"2024-09-21T09:58:29.676734+0000","last_fullsized":"2024-09-21T09:58:29.676734+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679505+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T09:58:29.679505+0000","last_peered":"2024-09-21T09:58:29.679505+0000","last_clean":"2024-09-21T09:58:29.679505+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T09:58:29.679505+0000","last_undegraded":"2024-09-21T09:58:29.679505+0000","last_fullsized":"2024-09-21T09:58:29.679505+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1114'8694","reported_seq":8732,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:36.333362+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T09:58:36.333362+0000","last_peered":"2024-09-21T09:58:36.333362+0000","last_clean":"2024-09-21T09:58:36.333362+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T09:58:36.333362+0000","last_undegraded":"2024-09-21T09:58:36.333362+0000","last_fullsized":"2024-09-21T09:58:36.333362+0000","mapping_epoch":1086,"log_start":"1089'500","ondisk_log_start":"1089'500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8194,"log_dups_size":0,"ondisk_log_size":8194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365821952,"num_objects":5582,"num_object_clones":0,"num_object_copies":16746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8694,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1409,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.742669+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:58:29.742669+0000","last_peered":"2024-09-21T09:58:29.742669+0000","last_clean":"2024-09-21T09:58:29.742669+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:58:29.742669+0000","last_undegraded":"2024-09-21T09:58:29.742669+0000","last_fullsized":"2024-09-21T09:58:29.742669+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1057,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678869+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:58:28.678869+0000","last_peered":"2024-09-21T09:58:28.678869+0000","last_clean":"2024-09-21T09:58:28.678869+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:58:28.678869+0000","last_undegraded":"2024-09-21T09:58:28.678869+0000","last_fullsized":"2024-09-21T09:58:28.678869+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1047,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678896+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:58:28.678896+0000","last_peered":"2024-09-21T09:58:28.678896+0000","last_clean":"2024-09-21T09:58:28.678896+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:58:28.678896+0000","last_undegraded":"2024-09-21T09:58:28.678896+0000","last_fullsized":"2024-09-21T09:58:28.678896+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1061,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679148+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:58:28.679148+0000","last_peered":"2024-09-21T09:58:28.679148+0000","last_clean":"2024-09-21T09:58:28.679148+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:58:28.679148+0000","last_undegraded":"2024-09-21T09:58:28.679148+0000","last_fullsized":"2024-09-21T09:58:28.679148+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1070,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679276+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:58:28.679276+0000","last_peered":"2024-09-21T09:58:28.679276+0000","last_clean":"2024-09-21T09:58:28.679276+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:58:28.679276+0000","last_undegraded":"2024-09-21T09:58:28.679276+0000","last_fullsized":"2024-09-21T09:58:28.679276+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1378,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676134+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:58:29.676134+0000","last_peered":"2024-09-21T09:58:29.676134+0000","last_clean":"2024-09-21T09:58:29.676134+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:58:29.676134+0000","last_undegraded":"2024-09-21T09:58:29.676134+0000","last_fullsized":"2024-09-21T09:58:29.676134+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1073,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674753+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T09:58:28.674753+0000","last_peered":"2024-09-21T09:58:28.674753+0000","last_clean":"2024-09-21T09:58:28.674753+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T09:58:28.674753+0000","last_undegraded":"2024-09-21T09:58:28.674753+0000","last_fullsized":"2024-09-21T09:58:28.674753+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1056,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673577+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:58:28.673577+0000","last_peered":"2024-09-21T09:58:28.673577+0000","last_clean":"2024-09-21T09:58:28.673577+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:58:28.673577+0000","last_undegraded":"2024-09-21T09:58:28.673577+0000","last_fullsized":"2024-09-21T09:58:28.673577+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679437+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T09:58:29.679437+0000","last_peered":"2024-09-21T09:58:29.679437+0000","last_clean":"2024-09-21T09:58:29.679437+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T09:58:29.679437+0000","last_undegraded":"2024-09-21T09:58:29.679437+0000","last_fullsized":"2024-09-21T09:58:29.679437+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1417,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.742292+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T09:58:29.742292+0000","last_peered":"2024-09-21T09:58:29.742292+0000","last_clean":"2024-09-21T09:58:29.742292+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T09:58:29.742292+0000","last_undegraded":"2024-09-21T09:58:29.742292+0000","last_fullsized":"2024-09-21T09:58:29.742292+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679698+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T09:58:29.679698+0000","last_peered":"2024-09-21T09:58:29.679698+0000","last_clean":"2024-09-21T09:58:29.679698+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T09:58:29.679698+0000","last_undegraded":"2024-09-21T09:58:29.679698+0000","last_fullsized":"2024-09-21T09:58:29.679698+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1053,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.678791+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:58:28.678791+0000","last_peered":"2024-09-21T09:58:28.678791+0000","last_clean":"2024-09-21T09:58:28.678791+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:58:28.678791+0000","last_undegraded":"2024-09-21T09:58:28.678791+0000","last_fullsized":"2024-09-21T09:58:28.678791+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679391+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T09:58:29.679391+0000","last_peered":"2024-09-21T09:58:29.679391+0000","last_clean":"2024-09-21T09:58:29.679391+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T09:58:29.679391+0000","last_undegraded":"2024-09-21T09:58:29.679391+0000","last_fullsized":"2024-09-21T09:58:29.679391+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1420,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.742538+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T09:58:29.742538+0000","last_peered":"2024-09-21T09:58:29.742538+0000","last_clean":"2024-09-21T09:58:29.742538+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T09:58:29.742538+0000","last_undegraded":"2024-09-21T09:58:29.742538+0000","last_fullsized":"2024-09-21T09:58:29.742538+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1391,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.676013+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:58:29.676013+0000","last_peered":"2024-09-21T09:58:29.676013+0000","last_clean":"2024-09-21T09:58:29.676013+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:58:29.676013+0000","last_undegraded":"2024-09-21T09:58:29.676013+0000","last_fullsized":"2024-09-21T09:58:29.676013+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.680069+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T09:58:29.680069+0000","last_peered":"2024-09-21T09:58:29.680069+0000","last_clean":"2024-09-21T09:58:29.680069+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T09:58:29.680069+0000","last_undegraded":"2024-09-21T09:58:29.680069+0000","last_fullsized":"2024-09-21T09:58:29.680069+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1077,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.681211+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:58:28.681211+0000","last_peered":"2024-09-21T09:58:28.681211+0000","last_clean":"2024-09-21T09:58:28.681211+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:58:28.681211+0000","last_undegraded":"2024-09-21T09:58:28.681211+0000","last_fullsized":"2024-09-21T09:58:28.681211+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1374,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.682026+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T09:58:29.682026+0000","last_peered":"2024-09-21T09:58:29.682026+0000","last_clean":"2024-09-21T09:58:29.682026+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T09:58:29.682026+0000","last_undegraded":"2024-09-21T09:58:29.682026+0000","last_fullsized":"2024-09-21T09:58:29.682026+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1396,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.741890+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:58:29.741890+0000","last_peered":"2024-09-21T09:58:29.741890+0000","last_clean":"2024-09-21T09:58:29.741890+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:58:29.741890+0000","last_undegraded":"2024-09-21T09:58:29.741890+0000","last_fullsized":"2024-09-21T09:58:29.741890+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1093,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.679073+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T09:58:28.679073+0000","last_peered":"2024-09-21T09:58:28.679073+0000","last_clean":"2024-09-21T09:58:28.679073+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T09:58:28.679073+0000","last_undegraded":"2024-09-21T09:58:28.679073+0000","last_fullsized":"2024-09-21T09:58:28.679073+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1057,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.674473+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:58:28.674473+0000","last_peered":"2024-09-21T09:58:28.674473+0000","last_clean":"2024-09-21T09:58:28.674473+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:58:28.674473+0000","last_undegraded":"2024-09-21T09:58:28.674473+0000","last_fullsized":"2024-09-21T09:58:28.674473+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1093,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.681072+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T09:58:28.681072+0000","last_peered":"2024-09-21T09:58:28.681072+0000","last_clean":"2024-09-21T09:58:28.681072+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T09:58:28.681072+0000","last_undegraded":"2024-09-21T09:58:28.681072+0000","last_fullsized":"2024-09-21T09:58:28.681072+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1042,"reported_epoch":1113,"state":"active+clean","last_fresh":"2024-09-21T09:58:28.673893+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:58:28.673893+0000","last_peered":"2024-09-21T09:58:28.673893+0000","last_clean":"2024-09-21T09:58:28.673893+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:58:28.673893+0000","last_undegraded":"2024-09-21T09:58:28.673893+0000","last_fullsized":"2024-09-21T09:58:28.673893+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.679580+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T09:58:29.679580+0000","last_peered":"2024-09-21T09:58:29.679580+0000","last_clean":"2024-09-21T09:58:29.679580+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T09:58:29.679580+0000","last_undegraded":"2024-09-21T09:58:29.679580+0000","last_fullsized":"2024-09-21T09:58:29.679580+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":54,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.680241+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T09:58:29.680241+0000","last_peered":"2024-09-21T09:58:29.680241+0000","last_clean":"2024-09-21T09:58:29.680241+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T09:58:29.680241+0000","last_undegraded":"2024-09-21T09:58:29.680241+0000","last_fullsized":"2024-09-21T09:58:29.680241+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1070,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.675740+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:58:29.675740+0000","last_peered":"2024-09-21T09:58:29.675740+0000","last_clean":"2024-09-21T09:58:29.675740+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:58:29.675740+0000","last_undegraded":"2024-09-21T09:58:29.675740+0000","last_fullsized":"2024-09-21T09:58:29.675740+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1412,"reported_epoch":1114,"state":"active+clean","last_fresh":"2024-09-21T09:58:29.742383+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:58:29.742383+0000","last_peered":"2024-09-21T09:58:29.742383+0000","last_clean":"2024-09-21T09:58:29.742383+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:58:29.742383+0000","last_undegraded":"2024-09-21T09:58:29.742383+0000","last_fullsized":"2024-09-21T09:58:29.742383+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5841747992,"num_objects":89139,"num_object_clones":0,"num_object_copies":267417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6799,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89139,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138669,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7547,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18475069440,"data_stored":18475057224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133069,"ondisk_log_size":133069,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234273,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2394280,"kb_used_data":2264736,"kb_used_omap":1,"kb_used_meta":129534,"kb_avail":91350872,"statfs":{"total":95995035648,"available":93543292928,"internally_reserved":0,"allocated":2319089664,"data_stored":2317060813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":132643275},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495910,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2617888,"kb_used_data":2452312,"kb_used_omap":1,"kb_used_meta":165566,"kb_avail":91127264,"statfs":{"total":95995035648,"available":93314318336,"internally_reserved":0,"allocated":2511167488,"data_stored":2509146702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":169540039},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757548,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2050072,"kb_used_data":1888592,"kb_used_omap":1,"kb_used_meta":161470,"kb_avail":91695080,"statfs":{"total":95995035648,"available":93895761920,"internally_reserved":0,"allocated":1933918208,"data_stored":1931900920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165345734},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019184,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1958304,"kb_used_data":1814808,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":91786848,"statfs":{"total":95995035648,"available":93989732352,"internally_reserved":0,"allocated":1858363392,"data_stored":1856342119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888481,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3756492,"kb_used_data":3535684,"kb_used_omap":34,"kb_used_meta":220765,"kb_avail":89988660,"statfs":{"total":95995035648,"available":92148387840,"internally_reserved":0,"allocated":3620540416,"data_stored":3618534823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":226064371},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967719,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3188816,"kb_used_data":3040072,"kb_used_omap":41,"kb_used_meta":148630,"kb_avail":90556336,"statfs":{"total":95995035648,"available":92729688064,"internally_reserved":0,"allocated":3113033728,"data_stored":3111024565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":152197535},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046958,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1902032,"kb_used_data":1793608,"kb_used_omap":41,"kb_used_meta":108374,"kb_avail":91843120,"statfs":{"total":95995035648,"available":94047354880,"internally_reserved":0,"allocated":1836654592,"data_stored":1834644856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":110975913},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126197,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1495640,"kb_used_data":1412944,"kb_used_omap":72,"kb_used_meta":82615,"kb_avail":92249512,"statfs":{"total":95995035648,"available":94463500288,"internally_reserved":0,"allocated":1446854656,"data_stored":1444837857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":84597803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1430065152,"data_stored":1430061080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1821118464,"data_stored":1821114392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3079995392,"data_stored":3079995392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3601858560,"data_stored":3601858560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1836777472,"data_stored":1836777472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1913196544,"data_stored":1913192472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2490040320,"data_stored":2490040320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2302017536,"data_stored":2302017536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:58:42.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T09:58:42.425 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:42.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:42.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.625 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:42.726 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:42.762 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:42.864 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:42.890 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:42.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:42.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T09:58:42.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:42.992 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:58:43.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:43.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:43.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:43.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:43.278 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:43.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:43.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:43.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:58:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:58:43] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.51.0" 2024-09-21T09:58:43.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:43.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:43.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:43.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:43.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:43.747 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:43.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:43 smithi076 ceph-mon[110963]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 6632/258324 objects degraded (2.567%); 518 KiB/s, 35 objects/s recovering 2024-09-21T09:58:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:43 smithi076 ceph-mon[106482]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 6632/258324 objects degraded (2.567%); 518 KiB/s, 35 objects/s recovering 2024-09-21T09:58:43.888 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:43.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:43.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:44.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:43 smithi191 ceph-mon[102206]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 6632/258324 objects degraded (2.567%); 518 KiB/s, 35 objects/s recovering 2024-09-21T09:58:44.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:44.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:44.263 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:44.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:44.384 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:44.400 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:44.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:44.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:44.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:44.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:44.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:44.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:44.832 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 6632/258324 objects degraded (2.567%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:44.832 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 6632/258324 objects degraded (2.567%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:44.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 6632/258324 objects degraded (2.567%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:44.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:44.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:44.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:45.006 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:45.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:45.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:45.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:45.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80506 6.c 6:31936050:::benchmark_data_smithi076_154645_object80505:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.750702+0000", 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 75.448223458000001, 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.103213288, 2024-09-21T09:58:45.271 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80506 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750702+0000", 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750702+0000", 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750701+0000", 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750789+0000", 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8430999999999994e-05 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750801+0000", 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2156e-05 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750808+0000", 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3350000000000001e-06 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.750921+0000", 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112685 2024-09-21T09:58:45.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.751061+0000", 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140116 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.751208+0000", 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014668299999999999 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754076+0000", 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0028681660000000001 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754583+0000", 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00050727400000000005 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.853164+0000", 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.098580650000000006 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.853237+0000", 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2668000000000003e-05 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.853915+0000", 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067824699999999999 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:105877 6.c 6:30ad38cf:::benchmark_data_smithi076_154645_object105876:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:56.943963+0000", 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 48.254962253999999, 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.109530211, 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 105877 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.278 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.943963+0000", 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.943963+0000", 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.943962+0000", 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:58:45.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.944032+0000", 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9690000000000005e-05 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.944051+0000", 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9627999999999999e-05 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.944060+0000", 2024-09-21T09:58:45.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9199999999999993e-06 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.953893+0000", 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098332149999999993 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.954055+0000", 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016187800000000001 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.954226+0000", 2024-09-21T09:58:45.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017078800000000001 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.963254+0000", 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0090277619999999999 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.973862+0000", 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010607917 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:57.052712+0000", 2024-09-21T09:58:45.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.078850502000000003 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:57.052747+0000", 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5172000000000001e-05 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:57.053493+0000", 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00074569800000000004 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.283 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:129707 6.7 6:e94b811a:::benchmark_data_smithi076_154645_object15801:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:25.023398+0000", 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 20.175527200000001, 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.103899045, 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 129707 2024-09-21T09:58:45.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023398+0000", 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023398+0000", 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023398+0000", 2024-09-21T09:58:45.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023399+0000", 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7710000000000001e-06 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023403+0000", 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6610000000000002e-06 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023406+0000", 2024-09-21T09:58:45.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.774e-06 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.027492+0000", 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040864849999999999 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.027620+0000", 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012725 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.027741+0000", 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121235 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.084535+0000", 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.056794598000000002 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.084893+0000", 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00035744699999999999 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.126980+0000", 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042087490999999998 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.127049+0000", 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8497999999999994e-05 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.127297+0000", 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00024809300000000001 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:129708 6.7 6:e753a80e:::benchmark_data_smithi076_154645_object15802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1111)", 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:25.023437+0000", 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 20.175488409, 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.110392618, 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 129708 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023437+0000", 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023437+0000", 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023436+0000", 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023439+0000", 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.514e-06 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023445+0000", 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.331e-06 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.023449+0000", 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1949999999999996e-06 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.080277+0000", 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.056827895000000003 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.080421+0000", 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014367199999999999 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.080598+0000", 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000176992 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.084899+0000", 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043011949999999998 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.115208+0000", 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030308512999999999 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.133534+0000", 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018326123 2024-09-21T09:58:45.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.133586+0000", 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2266999999999999e-05 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:25.133829+0000", 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000243339 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132669 6.7 6:ee3236d4:::benchmark_data_smithi076_154645_object18763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.701663+0000", 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.497261796, 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0345916079999999, 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132669 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701663+0000", 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701663+0000", 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701662+0000", 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701668+0000", 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5389999999999998e-06 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701685+0000", 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6798000000000001e-05 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701693+0000", 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2679999999999994e-06 2024-09-21T09:58:45.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701770+0000", 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7224000000000005e-05 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701782+0000", 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2174000000000001e-05 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723044+0000", 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0212620189999999 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723136+0000", 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1699000000000004e-05 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.300 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723211+0000", 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5343e-05 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729587+0000", 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063752499999999998 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732288+0000", 2024-09-21T09:58:45.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027012820000000002 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736194+0000", 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003905974 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736229+0000", 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5385000000000001e-05 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736255+0000", 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5789e-05 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132674 6.7 6:e6149bab:::benchmark_data_smithi076_154645_object18768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.709017+0000", 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.489907855999999, 2024-09-21T09:58:45.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.027360359, 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132674 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709017+0000", 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709017+0000", 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709016+0000", 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709023+0000", 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1690000000000003e-06 2024-09-21T09:58:45.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709040+0000", 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7295999999999998e-05 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709048+0000", 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0309999999999998e-06 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.306 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709157+0000", 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109302 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709170+0000", 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3006999999999999e-05 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726637+0000", 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.017466998 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726724+0000", 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6982000000000002e-05 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726801+0000", 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6508999999999995e-05 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736264+0000", 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094625979999999991 2024-09-21T09:58:45.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736288+0000", 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4314000000000001e-05 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736328+0000", 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0627000000000003e-05 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736358+0000", 2024-09-21T09:58:45.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9179999999999998e-05 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736378+0000", 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9995000000000001e-05 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132678 6.7 6:e83a6585:::benchmark_data_smithi076_154645_object18772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.716074+0000", 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.482851234999998, 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0204681069999999, 2024-09-21T09:58:45.310 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132678 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716074+0000", 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716074+0000", 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716072+0000", 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716080+0000", 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2679999999999997e-06 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.312 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716092+0000", 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2102e-05 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716100+0000", 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9829999999999996e-06 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716143+0000", 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2979999999999998e-05 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716151+0000", 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9750000000000007e-06 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729620+0000", 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.013469213 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729708+0000", 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7615999999999996e-05 2024-09-21T09:58:45.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729777+0000", 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9509000000000001e-05 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736385+0000", 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006608198 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736425+0000", 2024-09-21T09:58:45.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0061999999999997e-05 2024-09-21T09:58:45.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736481+0000", 2024-09-21T09:58:45.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5494000000000001e-05 2024-09-21T09:58:45.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736512+0000", 2024-09-21T09:58:45.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1439000000000002e-05 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736542+0000", 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9791e-05 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132696 6.7 6:e3a68bfc:::benchmark_data_smithi076_154645_object18790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.735329+0000", 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.463595683000001, 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.001302906, 2024-09-21T09:58:45.318 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132696 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735329+0000", 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735329+0000", 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735328+0000", 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735335+0000", 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4699999999999999e-06 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735349+0000", 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4493e-05 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735357+0000", 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5259999999999996e-06 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735402+0000", 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5572999999999999e-05 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735420+0000", 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7569999999999999e-05 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732376+0000", 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99695626999999998 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732509+0000", 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013274200000000001 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732636+0000", 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000127346 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736458+0000", 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038217440000000002 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736462+0000", 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5299999999999998e-06 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736583+0000", 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120709 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736612+0000", 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9308000000000001e-05 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736632+0000", 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.001e-05 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.324 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132704 6.7 6:ee4531e9:::benchmark_data_smithi076_154645_object18798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.744678+0000", 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.454246967, 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.004240655, 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132704 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.325 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744678+0000", 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744678+0000", 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744676+0000", 2024-09-21T09:58:45.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744685+0000", 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5040000000000002e-06 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744702+0000", 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7399000000000001e-05 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744711+0000", 2024-09-21T09:58:45.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4200000000000007e-06 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744768+0000", 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6725999999999999e-05 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744776+0000", 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5329999999999994e-06 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736669+0000", 2024-09-21T09:58:45.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99189255799999998 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736774+0000", 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105721 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736877+0000", 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010238699999999999 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748781+0000", 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011904068 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748790+0000", 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.679e-06 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748855+0000", 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4892000000000001e-05 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748889+0000", 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3206e-05 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748919+0000", 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0226000000000002e-05 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132719 6.7 6:ecb28ace:::benchmark_data_smithi076_154645_object18813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.758432+0000", 2024-09-21T09:58:45.331 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.440493101000001, 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99870091400000005, 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132719 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758432+0000", 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758432+0000", 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758430+0000", 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758438+0000", 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7360000000000001e-06 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758453+0000", 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4759999999999999e-05 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758460+0000", 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6350000000000006e-06 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758522+0000", 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2273000000000005e-05 2024-09-21T09:58:45.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758533+0000", 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0237e-05 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741029+0000", 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98249609900000001 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741177+0000", 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014844599999999999 2024-09-21T09:58:45.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741300+0000", 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122748 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748825+0000", 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075247990000000004 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748931+0000", 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010597799999999999 2024-09-21T09:58:45.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757064+0000", 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008132841 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757105+0000", 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1198000000000001e-05 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757133+0000", 2024-09-21T09:58:45.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8116999999999999e-05 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132729 6.7 6:ec0c304a:::benchmark_data_smithi076_154645_object18823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.776958+0000", 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.421967303999999, 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98031006499999995, 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.338 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132729 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776958+0000", 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776958+0000", 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776957+0000", 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776962+0000", 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4190000000000001e-06 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776974+0000", 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2303999999999999e-05 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776982+0000", 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.486e-06 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.777032+0000", 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9852999999999999e-05 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.777039+0000", 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4900000000000003e-06 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745236+0000", 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96819714999999995 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745349+0000", 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112273 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745455+0000", 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106005 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757033+0000", 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011578564 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757190+0000", 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156661 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757223+0000", 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3413e-05 2024-09-21T09:58:45.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757249+0000", 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5522000000000001e-05 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757268+0000", 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9046999999999999e-05 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.344 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132747 6.7 6:ef3db1a5:::benchmark_data_smithi076_154645_object18841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.807098+0000", 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.391827068000001, 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95029417199999999, 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132747 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807098+0000", 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807098+0000", 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807097+0000", 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807101+0000", 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4649999999999999e-06 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807107+0000", 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.207e-06 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807111+0000", 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7620000000000001e-06 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807144+0000", 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2768999999999997e-05 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807150+0000", 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9540000000000001e-06 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748957+0000", 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94180735199999999 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.749051+0000", 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3607e-05 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.749151+0000", 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100497 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757182+0000", 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080311150000000001 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757197+0000", 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4562000000000001e-05 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757311+0000", 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011408000000000001 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757358+0000", 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7392999999999998e-05 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757392+0000", 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4044000000000001e-05 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132752 6.7 6:e393e829:::benchmark_data_smithi076_154645_object18846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.808248+0000", 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.390677230000001, 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96938841399999998, 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.351 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132752 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808248+0000", 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808248+0000", 2024-09-21T09:58:45.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808247+0000", 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808251+0000", 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6559999999999998e-06 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808259+0000", 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8590000000000005e-06 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808263+0000", 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4039999999999997e-06 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808297+0000", 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3346000000000002e-05 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808302+0000", 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.964e-06 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753045+0000", 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94474228999999998 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753148+0000", 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103346 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753251+0000", 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010297 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757404+0000", 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041525709999999999 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757440+0000", 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6028999999999997e-05 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777567+0000", 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020127794000000001 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777606+0000", 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8336e-05 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777636+0000", 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0482000000000001e-05 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.357 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132759 6.7 6:ec6776a2:::benchmark_data_smithi076_154645_object18853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.822499+0000", 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.376426228, 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95524379100000001, 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132759 2024-09-21T09:58:45.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822499+0000", 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822499+0000", 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822498+0000", 2024-09-21T09:58:45.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822504+0000", 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2910000000000004e-06 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822515+0000", 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1122e-05 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822521+0000", 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8850000000000004e-06 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822576+0000", 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5012000000000001e-05 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822586+0000", 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0198e-05 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757477+0000", 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93489060499999999 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757603+0000", 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000125699 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757727+0000", 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012464799999999999 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.769569+0000", 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011841539 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777648+0000", 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080794749999999992 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777682+0000", 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3782000000000003e-05 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777710+0000", 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8453e-05 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777743+0000", 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2187000000000002e-05 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132762 6.7 6:e3ba75f7:::benchmark_data_smithi076_154645_object18856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.826045+0000", 2024-09-21T09:58:45.364 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.372879849, 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97741979800000001, 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132762 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826049+0000", 2024-09-21T09:58:45.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7199999999999997e-06 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826058+0000", 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8810000000000002e-06 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826063+0000", 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5870000000000003e-06 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826098+0000", 2024-09-21T09:58:45.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5222999999999998e-05 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826104+0000", 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1140000000000002e-06 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761541+0000", 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93543742500000004 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761637+0000", 2024-09-21T09:58:45.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5283000000000002e-05 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761731+0000", 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3795999999999994e-05 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777779+0000", 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016048020999999999 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777784+0000", 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1710000000000001e-06 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803357+0000", 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025573056 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803440+0000", 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3496000000000002e-05 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803465+0000", 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4794999999999999e-05 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132777 6.7 6:e0ee62be:::benchmark_data_smithi076_154645_object18871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.868119+0000", 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.330806351, 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94012129, 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.371 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132777 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868119+0000", 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868119+0000", 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868117+0000", 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868125+0000", 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4200000000000001e-06 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868139+0000", 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4076999999999999e-05 2024-09-21T09:58:45.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868146+0000", 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7950000000000002e-06 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868192+0000", 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.693e-05 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868201+0000", 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6580000000000008e-06 2024-09-21T09:58:45.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765747+0000", 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89754568400000001 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765878+0000", 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013122700000000001 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765999+0000", 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120541 2024-09-21T09:58:45.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777788+0000", 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011788997000000001 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777826+0000", 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8155999999999999e-05 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808156+0000", 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030329964000000001 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808203+0000", 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7398000000000003e-05 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808240+0000", 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6976e-05 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.377 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132793 6.7 6:ee55f5b7:::benchmark_data_smithi076_154645_object18887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.895577+0000", 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.303348541999998, 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91277691299999997, 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132793 2024-09-21T09:58:45.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895577+0000", 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895577+0000", 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895575+0000", 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895581+0000", 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6899999999999997e-06 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895597+0000", 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5475999999999999e-05 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895604+0000", 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1509999999999998e-06 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895661+0000", 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7431e-05 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.895674+0000", 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3372e-05 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.769596+0000", 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87392107600000002 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.769704+0000", 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108557 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.769820+0000", 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115734 2024-09-21T09:58:45.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777791+0000", 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079714520000000004 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.788087+0000", 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010295281 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808275+0000", 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020188544999999999 2024-09-21T09:58:45.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808317+0000", 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1721000000000003e-05 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808353+0000", 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6667e-05 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.384 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132818 6.7 6:eb8c91cf:::benchmark_data_smithi076_154645_object18912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.949946+0000", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.248979143, 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.864644038, 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132818 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.949946+0000", 2024-09-21T09:58:45.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.949946+0000", 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.949945+0000", 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.949951+0000", 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.832e-06 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.949961+0000", 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2930000000000006e-06 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.949966+0000", 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1170000000000003e-06 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.950015+0000", 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9268000000000003e-05 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.950024+0000", 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5050000000000007e-06 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.773652+0000", 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.82362855999999995 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.773754+0000", 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010156499999999999 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.773855+0000", 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100765 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777795+0000", 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039404749999999997 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.794982+0000", 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017186561 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814510+0000", 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019528620999999999 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814552+0000", 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1452999999999998e-05 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814590+0000", 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8334000000000003e-05 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132844 6.7 6:ed26285c:::benchmark_data_smithi076_154645_object18938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:29.014446+0000", 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.184479107000001, 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80025672000000003, 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.391 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132844 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014446+0000", 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014446+0000", 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014444+0000", 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014452+0000", 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3100000000000001e-06 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014468+0000", 2024-09-21T09:58:45.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5500000000000001e-05 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014475+0000", 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.708e-06 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014538+0000", 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3058999999999994e-05 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.014549+0000", 2024-09-21T09:58:45.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1894e-05 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777846+0000", 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76329647 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777918+0000", 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2214999999999999e-05 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777993+0000", 2024-09-21T09:58:45.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5014999999999999e-05 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.788094+0000", 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010100367000000001 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.795041+0000", 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069472040000000002 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814632+0000", 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019591478999999998 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814672+0000", 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0231999999999997e-05 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814703+0000", 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0297000000000001e-05 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132852 6.7 6:ebb48829:::benchmark_data_smithi076_154645_object18946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:29.064634+0000", 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 16.134291156, 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.750207022, 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132852 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:58:45.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064634+0000", 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064634+0000", 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064632+0000", 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064642+0000", 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8260000000000005e-06 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064663+0000", 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0936000000000001e-05 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064672+0000", 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7289999999999995e-06 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064725+0000", 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.308e-05 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.064735+0000", 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4099999999999997e-06 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.784006+0000", 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.71927104399999997 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.784103+0000", 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7132000000000004e-05 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.784211+0000", 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010824599999999999 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.794993+0000", 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01078135 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.795083+0000", 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0488999999999994e-05 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814756+0000", 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.019673099999999999 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814804+0000", 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7613e-05 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:58:45.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.814841+0000", 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7194999999999999e-05 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:58:45.404 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:58:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:45.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:45.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:45.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:45.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:45.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:45.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:45.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:45 smithi191 ceph-mon[102206]: pgmap v1241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 6632/255423 objects degraded (2.596%); 518 KiB/s, 35 objects/s recovering 2024-09-21T09:58:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:45 smithi076 ceph-mon[106482]: pgmap v1241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 6632/255423 objects degraded (2.596%); 518 KiB/s, 35 objects/s recovering 2024-09-21T09:58:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:45 smithi076 ceph-mon[110963]: pgmap v1241: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 6632/255423 objects degraded (2.596%); 518 KiB/s, 35 objects/s recovering 2024-09-21T09:58:45.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:45.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:45.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:45.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:46.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:46.093 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:46.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:46.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:46.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:46.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T09:58:46.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:58:46.329 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:46.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:46.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:46.478 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:46.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:46.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:46.564 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:46.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:46.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:46.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:46.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:46.790 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:46.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:46.892 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:46.932 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:46.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:46.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:47.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:47.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:47.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:47.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:47.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:47.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:47.381 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:47.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:47.533 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:47.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:47.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:47.665 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:47.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:47.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:47.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:47.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:47.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:47.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:48.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:48.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:47 smithi076 ceph-mon[106482]: pgmap v1242: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6437/248118 objects degraded (2.594%); 0 B/s, 36 objects/s recovering 2024-09-21T09:58:48.047 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:47 smithi076 ceph-mon[110963]: pgmap v1242: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6437/248118 objects degraded (2.594%); 0 B/s, 36 objects/s recovering 2024-09-21T09:58:48.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:48.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:48.087 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:47 smithi191 ceph-mon[102206]: pgmap v1242: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6437/248118 objects degraded (2.594%); 0 B/s, 36 objects/s recovering 2024-09-21T09:58:48.175 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:48.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:48.252 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:58:48.252 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:58:48.252 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:58:48.252 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:58:48.253 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:58:48.254 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:48.254 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:48.276 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:58:48.276 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:48.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:48.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:48.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:48.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:48.596 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:48.621 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:48.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:48.726 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:48.726 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.754 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:48.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:48.854 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:48.886 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:48.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:48.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:49.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:49.063 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3878818922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3878818922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3878818922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:58:49.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:49.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:49.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:49.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:49.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:49.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.423 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:49.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:49.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T09:58:49.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:49.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:49.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:49.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:49.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:49.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:49.772 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:49.808 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:49.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:49.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:49.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:49.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:49 smithi191 ceph-mon[102206]: pgmap v1243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6437/245130 objects degraded (2.626%); 0 B/s, 36 objects/s recovering 2024-09-21T09:58:50.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 6437/245130 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:50.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.102 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:50.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:49 smithi076 ceph-mon[106482]: pgmap v1243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6437/245130 objects degraded (2.626%); 0 B/s, 36 objects/s recovering 2024-09-21T09:58:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 6437/245130 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:50.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:49 smithi076 ceph-mon[110963]: pgmap v1243: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6437/245130 objects degraded (2.626%); 0 B/s, 36 objects/s recovering 2024-09-21T09:58:50.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 6437/245130 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:50.159 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:58:50.160 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 6437/245130 objects degraded (2.626%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":833,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1115,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":8,"osd_in_since":1726912604,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":84},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":81710,"data_bytes":5353773053,"bytes_used":18233331712,"bytes_avail":749726953472,"bytes_total":767960285184,"degraded_objects":6437,"degraded_total":245130,"degraded_ratio":0.026259535756537347,"recovering_objects_per_sec":36,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":362,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":793},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":24,"modified":"2024-09-21T09:58:42.469069+0000","services":{"mon":{"daemons":{"summary":"","c":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-09-21T09:58:50.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:50.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:50.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:50.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:50.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:50.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:50.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:50.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:50.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:50.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:50.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:50.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:50.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:50.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:51.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:51.102 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4249440250' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:58:51.109 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4249440250' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:58:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4249440250' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:58:51.210 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:51.215 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:51.222 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T09:58:51.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:58:51.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:51.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:51.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:51.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:51.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:51.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:51.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:51.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:51.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:51.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:51.850 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:51 smithi076 ceph-mon[106482]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 6253/242805 objects degraded (2.575%); 0 B/s, 50 objects/s recovering 2024-09-21T09:58:51.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:51 smithi076 ceph-mon[110963]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 6253/242805 objects degraded (2.575%); 0 B/s, 50 objects/s recovering 2024-09-21T09:58:51.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:51.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:51.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:51.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:52.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:52.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:52.038 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:52.038 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:52.038 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:51 smithi191 ceph-mon[102206]: pgmap v1244: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 6253/242805 objects degraded (2.575%); 0 B/s, 50 objects/s recovering 2024-09-21T09:58:52.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:52.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:52.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:52.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:52.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:52.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:52.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:52.525 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:58:52.529 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:52.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:52.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:52.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:52.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:52.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:52.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:52.919 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:52.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:53.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:53.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:53.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:53.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:53.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:53.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:53.318 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:53.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:53.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:53.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:53.448 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:53.477 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:58:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:58:53] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.51.0" 2024-09-21T09:58:53.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:53.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:53.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:53.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:53.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:53.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:53.943 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:53 smithi191 ceph-mon[102206]: pgmap v1245: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 6253/234603 objects degraded (2.665%); 0 B/s, 31 objects/s recovering 2024-09-21T09:58:53.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:53.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T09:58:53.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:53.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:54.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:53 smithi076 ceph-mon[106482]: pgmap v1245: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 6253/234603 objects degraded (2.665%); 0 B/s, 31 objects/s recovering 2024-09-21T09:58:54.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:53 smithi076 ceph-mon[110963]: pgmap v1245: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 6253/234603 objects degraded (2.665%); 0 B/s, 31 objects/s recovering 2024-09-21T09:58:54.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:54.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:54.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:54.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.276 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:54.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:54.311 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:54.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:54.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:54.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:54.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:54.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:54.674 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:54.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:54.814 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:54.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:54.824 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T09:58:54.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:54.947 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:54.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:54.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:55.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:55.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.091 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:55.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 6253/234603 objects degraded (2.665%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3156497361' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:55.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:54 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 6253/234603 objects degraded (2.665%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3156497361' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:54 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 6253/234603 objects degraded (2.665%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:58:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3156497361' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:55.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:54 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:58:55.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:55.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:55.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:55.322 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:55.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:55.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:55.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:55.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:55.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:55.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:55.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:55.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 6253/231513 objects degraded (2.701%); 0 B/s, 31 objects/s recovering 2024-09-21T09:58:55.804 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:55.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.824 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:55.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:55.864 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:55.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:55.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:55.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:55.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:56.066 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:56.066 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-21T09:58:56.066 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:56.066 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:56.066 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:56.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 6253/231513 objects degraded (2.701%); 0 B/s, 31 objects/s recovering 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: pgmap v1246: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 6253/231513 objects degraded (2.701%); 0 B/s, 31 objects/s recovering 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:58:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:56.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:56.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:56.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:56.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:56.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:56.363 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T09:58:56.363 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:58:56.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:56.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:56.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:56.543 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:56.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:56.681 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:56.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:56.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:56.808 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:56.909 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:56.910 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:58:56.919 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:58:56.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:56.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:57.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:57.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:57.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:58:57.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:58:57.314 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:58:57.338 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:57.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:58:57.440 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:57.491 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:57.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:57.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:57.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:57.753 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:57.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:57.855 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:57 smithi076 ceph-mon[106482]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 6053/224217 objects degraded (2.700%); 0 B/s, 38 objects/s recovering 2024-09-21T09:58:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:57 smithi076 ceph-mon[106482]: from='client.44738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:57 smithi076 ceph-mon[110963]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 6053/224217 objects degraded (2.700%); 0 B/s, 38 objects/s recovering 2024-09-21T09:58:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:57 smithi076 ceph-mon[110963]: from='client.44738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:57.890 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:57.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:57.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:58:57.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:58:57.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T09:58:57.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:58.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:58.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:57 smithi191 ceph-mon[102206]: pgmap v1248: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 6053/224217 objects degraded (2.700%); 0 B/s, 38 objects/s recovering 2024-09-21T09:58:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:57 smithi191 ceph-mon[102206]: from='client.44738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:58:58.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:58.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:58.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:58.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:58.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:58.440 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:58.456 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1248,"stamp":"2024-09-21T09:58:56.477339+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4896921597,"num_objects":74739,"num_object_clones":0,"num_object_copies":224217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6053,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74739,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":153374,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8306,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139403,"ondisk_log_size":139403,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16503392,"kb_used_data":15314452,"kb_used_omap":195,"kb_used_meta":1188412,"kb_avail":733457824,"statfs":{"total":767960285184,"available":751060811776,"internally_reserved":0,"allocated":15681998848,"data_stored":15665823075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1216934020},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-522125312,"num_objects":-7967,"num_object_clones":0,"num_object_copies":-23901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-384,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7967,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":384,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007235"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752683+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T09:58:39.752683+0000","last_peered":"2024-09-21T09:58:39.752683+0000","last_clean":"2024-09-21T09:58:39.752683+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T09:58:39.752683+0000","last_undegraded":"2024-09-21T09:58:39.752683+0000","last_fullsized":"2024-09-21T09:58:39.752683+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1044,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752996+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:58:39.752996+0000","last_peered":"2024-09-21T09:58:39.752996+0000","last_clean":"2024-09-21T09:58:39.752996+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:58:39.752996+0000","last_undegraded":"2024-09-21T09:58:39.752996+0000","last_fullsized":"2024-09-21T09:58:39.752996+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1075,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740637+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:58:39.740637+0000","last_peered":"2024-09-21T09:58:39.740637+0000","last_clean":"2024-09-21T09:58:39.740637+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:58:39.740637+0000","last_undegraded":"2024-09-21T09:58:39.740637+0000","last_fullsized":"2024-09-21T09:58:39.740637+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1421,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.759455+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T09:58:39.759455+0000","last_peered":"2024-09-21T09:58:39.759455+0000","last_clean":"2024-09-21T09:58:39.759455+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T09:58:39.759455+0000","last_undegraded":"2024-09-21T09:58:39.759455+0000","last_fullsized":"2024-09-21T09:58:39.759455+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1421,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.755445+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T09:58:39.755445+0000","last_peered":"2024-09-21T09:58:39.755445+0000","last_clean":"2024-09-21T09:58:39.755445+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T09:58:39.755445+0000","last_undegraded":"2024-09-21T09:58:39.755445+0000","last_fullsized":"2024-09-21T09:58:39.755445+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1027,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753391+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:58:39.753391+0000","last_peered":"2024-09-21T09:58:39.753391+0000","last_clean":"2024-09-21T09:58:39.753391+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:58:39.753391+0000","last_undegraded":"2024-09-21T09:58:39.753391+0000","last_fullsized":"2024-09-21T09:58:39.753391+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753647+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T09:58:39.753647+0000","last_peered":"2024-09-21T09:58:39.753647+0000","last_clean":"2024-09-21T09:58:39.753647+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T09:58:39.753647+0000","last_undegraded":"2024-09-21T09:58:39.753647+0000","last_fullsized":"2024-09-21T09:58:39.753647+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1054,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751120+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:58:39.751120+0000","last_peered":"2024-09-21T09:58:39.751120+0000","last_clean":"2024-09-21T09:58:39.751120+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:58:39.751120+0000","last_undegraded":"2024-09-21T09:58:39.751120+0000","last_fullsized":"2024-09-21T09:58:39.751120+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1116'9550","reported_seq":9590,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:56.427258+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T09:58:56.427258+0000","last_peered":"2024-09-21T09:58:56.427258+0000","last_clean":"2024-09-21T09:58:56.427258+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T09:58:56.427258+0000","last_undegraded":"2024-09-21T09:58:56.427258+0000","last_fullsized":"2024-09-21T09:58:56.427258+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9550,"log_dups_size":0,"ondisk_log_size":9550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298450944,"num_objects":4554,"num_object_clones":0,"num_object_copies":13662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9550,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1043,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753064+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:58:39.753064+0000","last_peered":"2024-09-21T09:58:39.753064+0000","last_clean":"2024-09-21T09:58:39.753064+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:58:39.753064+0000","last_undegraded":"2024-09-21T09:58:39.753064+0000","last_fullsized":"2024-09-21T09:58:39.753064+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1043,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740279+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:58:39.740279+0000","last_peered":"2024-09-21T09:58:39.740279+0000","last_clean":"2024-09-21T09:58:39.740279+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:58:39.740279+0000","last_undegraded":"2024-09-21T09:58:39.740279+0000","last_fullsized":"2024-09-21T09:58:39.740279+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1115'9419","reported_seq":12362,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:53.293091+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T09:58:53.293091+0000","last_peered":"2024-09-21T09:58:53.293091+0000","last_clean":"2024-09-21T09:58:53.293091+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T09:58:53.293091+0000","last_undegraded":"2024-09-21T09:58:53.293091+0000","last_fullsized":"2024-09-21T09:58:53.293091+0000","mapping_epoch":1108,"log_start":"1095'2100","ondisk_log_start":"1095'2100","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7319,"log_dups_size":0,"ondisk_log_size":7319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307822592,"num_objects":4697,"num_object_clones":0,"num_object_copies":14091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9419,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1044,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754635+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:58:39.754635+0000","last_peered":"2024-09-21T09:58:39.754635+0000","last_clean":"2024-09-21T09:58:39.754635+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:58:39.754635+0000","last_undegraded":"2024-09-21T09:58:39.754635+0000","last_fullsized":"2024-09-21T09:58:39.754635+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1097,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739620+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T09:58:39.739620+0000","last_peered":"2024-09-21T09:58:39.739620+0000","last_clean":"2024-09-21T09:58:39.739620+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T09:58:39.739620+0000","last_undegraded":"2024-09-21T09:58:39.739620+0000","last_fullsized":"2024-09-21T09:58:39.739620+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1116'9740","reported_seq":12664,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:56.349491+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T09:58:56.349491+0000","last_peered":"2024-09-21T09:58:56.349491+0000","last_clean":"2024-09-21T09:58:56.349491+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T09:58:56.349491+0000","last_undegraded":"2024-09-21T09:58:56.349491+0000","last_fullsized":"2024-09-21T09:58:56.349491+0000","mapping_epoch":1100,"log_start":"1094'1600","ondisk_log_start":"1094'1600","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304873472,"num_objects":4652,"num_object_clones":0,"num_object_copies":13956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9740,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1028,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753321+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:58:39.753321+0000","last_peered":"2024-09-21T09:58:39.753321+0000","last_clean":"2024-09-21T09:58:39.753321+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:58:39.753321+0000","last_undegraded":"2024-09-21T09:58:39.753321+0000","last_fullsized":"2024-09-21T09:58:39.753321+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1046,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754065+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:58:39.754065+0000","last_peered":"2024-09-21T09:58:39.754065+0000","last_clean":"2024-09-21T09:58:39.754065+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:58:39.754065+0000","last_undegraded":"2024-09-21T09:58:39.754065+0000","last_fullsized":"2024-09-21T09:58:39.754065+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1115'9576","reported_seq":9646,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:53.979697+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T09:58:53.979697+0000","last_peered":"2024-09-21T09:58:53.979697+0000","last_clean":"2024-09-21T09:58:53.979697+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T09:58:53.979697+0000","last_undegraded":"2024-09-21T09:58:53.979697+0000","last_fullsized":"2024-09-21T09:58:53.979697+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9576,"log_dups_size":0,"ondisk_log_size":9576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310902784,"num_objects":4744,"num_object_clones":0,"num_object_copies":14232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1076,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740929+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:58:39.740929+0000","last_peered":"2024-09-21T09:58:39.740929+0000","last_clean":"2024-09-21T09:58:39.740929+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:58:39.740929+0000","last_undegraded":"2024-09-21T09:58:39.740929+0000","last_fullsized":"2024-09-21T09:58:39.740929+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1409,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754307+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:58:39.754307+0000","last_peered":"2024-09-21T09:58:39.754307+0000","last_clean":"2024-09-21T09:58:39.754307+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:58:39.754307+0000","last_undegraded":"2024-09-21T09:58:39.754307+0000","last_fullsized":"2024-09-21T09:58:39.754307+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1116'9611","reported_seq":9651,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:56.371214+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T09:58:56.371214+0000","last_peered":"2024-09-21T09:58:56.371214+0000","last_clean":"2024-09-21T09:58:56.371214+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T09:58:56.371214+0000","last_undegraded":"2024-09-21T09:58:56.371214+0000","last_fullsized":"2024-09-21T09:58:56.371214+0000","mapping_epoch":1086,"log_start":"1094'1700","ondisk_log_start":"1094'1700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7911,"log_dups_size":0,"ondisk_log_size":7911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9611,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":60,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.748621+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T09:58:39.748621+0000","last_peered":"2024-09-21T09:58:39.748621+0000","last_clean":"2024-09-21T09:58:39.748621+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T09:58:39.748621+0000","last_undegraded":"2024-09-21T09:58:39.748621+0000","last_fullsized":"2024-09-21T09:58:39.748621+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1081,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739874+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T09:58:39.739874+0000","last_peered":"2024-09-21T09:58:39.739874+0000","last_clean":"2024-09-21T09:58:39.739874+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T09:58:39.739874+0000","last_undegraded":"2024-09-21T09:58:39.739874+0000","last_fullsized":"2024-09-21T09:58:39.739874+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1116'9636","reported_seq":11931,"reported_epoch":1116,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:55.083062+0000","last_change":"2024-09-21T09:58:28.149252+0000","last_active":"2024-09-21T09:58:55.083062+0000","last_peered":"2024-09-21T09:58:55.083062+0000","last_clean":"2024-09-21T09:56:44.221905+0000","last_became_active":"2024-09-21T09:56:46.358647+0000","last_became_peered":"2024-09-21T09:56:46.358647+0000","last_unstale":"2024-09-21T09:58:55.083062+0000","last_undegraded":"2024-09-21T09:56:46.247338+0000","last_fullsized":"2024-09-21T09:56:46.236810+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9636,"log_dups_size":0,"ondisk_log_size":9636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309592064,"num_objects":4724,"num_object_clones":0,"num_object_copies":14172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9636,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1110,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":4424}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1047,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754208+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:58:39.754208+0000","last_peered":"2024-09-21T09:58:39.754208+0000","last_clean":"2024-09-21T09:58:39.754208+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:58:39.754208+0000","last_undegraded":"2024-09-21T09:58:39.754208+0000","last_fullsized":"2024-09-21T09:58:39.754208+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1078,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740224+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:58:39.740224+0000","last_peered":"2024-09-21T09:58:39.740224+0000","last_clean":"2024-09-21T09:58:39.740224+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:58:39.740224+0000","last_undegraded":"2024-09-21T09:58:39.740224+0000","last_fullsized":"2024-09-21T09:58:39.740224+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1116'9591","reported_seq":9635,"reported_epoch":1116,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:56.393535+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T09:58:56.393535+0000","last_peered":"2024-09-21T09:58:56.393535+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T09:58:56.393535+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9591,"log_dups_size":0,"ondisk_log_size":9591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9591,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":3107}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1065,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739779+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:58:39.739779+0000","last_peered":"2024-09-21T09:58:39.739779+0000","last_clean":"2024-09-21T09:58:39.739779+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:58:39.739779+0000","last_undegraded":"2024-09-21T09:58:39.739779+0000","last_fullsized":"2024-09-21T09:58:39.739779+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1072,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740274+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T09:58:39.740274+0000","last_peered":"2024-09-21T09:58:39.740274+0000","last_clean":"2024-09-21T09:58:39.740274+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T09:58:39.740274+0000","last_undegraded":"2024-09-21T09:58:39.740274+0000","last_fullsized":"2024-09-21T09:58:39.740274+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1116'9695","reported_seq":9736,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:56.367750+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T09:58:56.367750+0000","last_peered":"2024-09-21T09:58:56.367750+0000","last_clean":"2024-09-21T09:58:56.367750+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T09:58:56.367750+0000","last_undegraded":"2024-09-21T09:58:56.367750+0000","last_fullsized":"2024-09-21T09:58:56.367750+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9695,"log_dups_size":0,"ondisk_log_size":9695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301858840,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9695,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753803+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T09:58:39.753803+0000","last_peered":"2024-09-21T09:58:39.753803+0000","last_clean":"2024-09-21T09:58:39.753803+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T09:58:39.753803+0000","last_undegraded":"2024-09-21T09:58:39.753803+0000","last_fullsized":"2024-09-21T09:58:39.753803+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1395,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.750877+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:58:39.750877+0000","last_peered":"2024-09-21T09:58:39.750877+0000","last_clean":"2024-09-21T09:58:39.750877+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:58:39.750877+0000","last_undegraded":"2024-09-21T09:58:39.750877+0000","last_fullsized":"2024-09-21T09:58:39.750877+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1116'9449","reported_seq":12277,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:56.386571+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T09:58:56.386571+0000","last_peered":"2024-09-21T09:58:56.386571+0000","last_clean":"2024-09-21T09:58:56.386571+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T09:58:56.386571+0000","last_undegraded":"2024-09-21T09:58:56.386571+0000","last_fullsized":"2024-09-21T09:58:56.386571+0000","mapping_epoch":1113,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9449,"log_dups_size":0,"ondisk_log_size":9449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9449,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1373,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752620+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:58:39.752620+0000","last_peered":"2024-09-21T09:58:39.752620+0000","last_clean":"2024-09-21T09:58:39.752620+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:58:39.752620+0000","last_undegraded":"2024-09-21T09:58:39.752620+0000","last_fullsized":"2024-09-21T09:58:39.752620+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1073,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740469+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:58:39.740469+0000","last_peered":"2024-09-21T09:58:39.740469+0000","last_clean":"2024-09-21T09:58:39.740469+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:58:39.740469+0000","last_undegraded":"2024-09-21T09:58:39.740469+0000","last_fullsized":"2024-09-21T09:58:39.740469+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":57,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.747196+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T09:58:39.747196+0000","last_peered":"2024-09-21T09:58:39.747196+0000","last_clean":"2024-09-21T09:58:39.747196+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T09:58:39.747196+0000","last_undegraded":"2024-09-21T09:58:39.747196+0000","last_fullsized":"2024-09-21T09:58:39.747196+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1031,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751151+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:58:39.751151+0000","last_peered":"2024-09-21T09:58:39.751151+0000","last_clean":"2024-09-21T09:58:39.751151+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:58:39.751151+0000","last_undegraded":"2024-09-21T09:58:39.751151+0000","last_fullsized":"2024-09-21T09:58:39.751151+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1115'9387","reported_seq":12425,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:51.983891+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T09:58:51.983891+0000","last_peered":"2024-09-21T09:58:51.983891+0000","last_clean":"2024-09-21T09:58:51.983891+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T09:58:51.983891+0000","last_undegraded":"2024-09-21T09:58:51.983891+0000","last_fullsized":"2024-09-21T09:58:51.983891+0000","mapping_epoch":1105,"log_start":"1089'800","ondisk_log_start":"1089'800","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8587,"log_dups_size":0,"ondisk_log_size":8587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312672256,"num_objects":4771,"num_object_clones":0,"num_object_copies":14313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9387,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1071,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752383+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:58:39.752383+0000","last_peered":"2024-09-21T09:58:39.752383+0000","last_clean":"2024-09-21T09:58:39.752383+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:58:39.752383+0000","last_undegraded":"2024-09-21T09:58:39.752383+0000","last_fullsized":"2024-09-21T09:58:39.752383+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1421,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754550+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T09:58:39.754550+0000","last_peered":"2024-09-21T09:58:39.754550+0000","last_clean":"2024-09-21T09:58:39.754550+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T09:58:39.754550+0000","last_undegraded":"2024-09-21T09:58:39.754550+0000","last_fullsized":"2024-09-21T09:58:39.754550+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1419,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754255+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:58:39.754255+0000","last_peered":"2024-09-21T09:58:39.754255+0000","last_clean":"2024-09-21T09:58:39.754255+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:58:39.754255+0000","last_undegraded":"2024-09-21T09:58:39.754255+0000","last_fullsized":"2024-09-21T09:58:39.754255+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1116'9547","reported_seq":9592,"reported_epoch":1116,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:56.340986+0000","last_change":"2024-09-21T09:56:46.337646+0000","last_active":"2024-09-21T09:58:56.340986+0000","last_peered":"2024-09-21T09:58:56.340986+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:56:46.335218+0000","last_became_peered":"2024-09-21T09:56:46.335218+0000","last_unstale":"2024-09-21T09:58:56.340986+0000","last_undegraded":"2024-09-21T09:56:46.260383+0000","last_fullsized":"2024-09-21T09:56:46.244582+0000","mapping_epoch":1093,"log_start":"1091'1400","ondisk_log_start":"1091'1400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8147,"log_dups_size":0,"ondisk_log_size":8147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:03.383491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301531136,"num_objects":4601,"num_object_clones":0,"num_object_copies":13803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1404,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9547,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":3197}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1402,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.825926+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T09:58:54.825926+0000","last_peered":"2024-09-21T09:58:54.825926+0000","last_clean":"2024-09-21T09:58:54.825926+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T09:58:54.825926+0000","last_undegraded":"2024-09-21T09:58:54.825926+0000","last_fullsized":"2024-09-21T09:58:54.825926+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1381,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.750349+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:58:39.750349+0000","last_peered":"2024-09-21T09:58:39.750349+0000","last_clean":"2024-09-21T09:58:39.750349+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:58:39.750349+0000","last_undegraded":"2024-09-21T09:58:39.750349+0000","last_fullsized":"2024-09-21T09:58:39.750349+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1389,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.750761+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:58:39.750761+0000","last_peered":"2024-09-21T09:58:39.750761+0000","last_clean":"2024-09-21T09:58:39.750761+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:58:39.750761+0000","last_undegraded":"2024-09-21T09:58:39.750761+0000","last_fullsized":"2024-09-21T09:58:39.750761+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1115'9444","reported_seq":12408,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:53.322221+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T09:58:53.322221+0000","last_peered":"2024-09-21T09:58:53.322221+0000","last_clean":"2024-09-21T09:58:53.322221+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T09:58:53.322221+0000","last_undegraded":"2024-09-21T09:58:53.322221+0000","last_fullsized":"2024-09-21T09:58:53.322221+0000","mapping_epoch":1096,"log_start":"1095'2100","ondisk_log_start":"1095'2100","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7344,"log_dups_size":0,"ondisk_log_size":7344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311951360,"num_objects":4760,"num_object_clones":0,"num_object_copies":14280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9444,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1074,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740627+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:58:39.740627+0000","last_peered":"2024-09-21T09:58:39.740627+0000","last_clean":"2024-09-21T09:58:39.740627+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:58:39.740627+0000","last_undegraded":"2024-09-21T09:58:39.740627+0000","last_fullsized":"2024-09-21T09:58:39.740627+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1065,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739983+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:58:39.739983+0000","last_peered":"2024-09-21T09:58:39.739983+0000","last_clean":"2024-09-21T09:58:39.739983+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:58:39.739983+0000","last_undegraded":"2024-09-21T09:58:39.739983+0000","last_fullsized":"2024-09-21T09:58:39.739983+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1406,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752165+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:58:39.752165+0000","last_peered":"2024-09-21T09:58:39.752165+0000","last_clean":"2024-09-21T09:58:39.752165+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:58:39.752165+0000","last_undegraded":"2024-09-21T09:58:39.752165+0000","last_fullsized":"2024-09-21T09:58:39.752165+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1115'9506","reported_seq":9578,"reported_epoch":1115,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:53.955629+0000","last_change":"2024-09-21T09:56:46.343261+0000","last_active":"2024-09-21T09:58:53.955629+0000","last_peered":"2024-09-21T09:58:53.955629+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T09:58:53.955629+0000","last_undegraded":"2024-09-21T09:56:46.247241+0000","last_fullsized":"2024-09-21T09:56:46.239078+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9506,"log_dups_size":0,"ondisk_log_size":9506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305528832,"num_objects":4662,"num_object_clones":0,"num_object_copies":13986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9506,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":3201}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1092,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740737+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:58:39.740737+0000","last_peered":"2024-09-21T09:58:39.740737+0000","last_clean":"2024-09-21T09:58:39.740737+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:58:39.740737+0000","last_undegraded":"2024-09-21T09:58:39.740737+0000","last_fullsized":"2024-09-21T09:58:39.740737+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1403,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752244+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:58:39.752244+0000","last_peered":"2024-09-21T09:58:39.752244+0000","last_clean":"2024-09-21T09:58:39.752244+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:58:39.752244+0000","last_undegraded":"2024-09-21T09:58:39.752244+0000","last_fullsized":"2024-09-21T09:58:39.752244+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.757071+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T09:58:39.757071+0000","last_peered":"2024-09-21T09:58:39.757071+0000","last_clean":"2024-09-21T09:58:39.757071+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T09:58:39.757071+0000","last_undegraded":"2024-09-21T09:58:39.757071+0000","last_fullsized":"2024-09-21T09:58:39.757071+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1115'9627","reported_seq":9696,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:51.989281+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T09:58:51.989281+0000","last_peered":"2024-09-21T09:58:51.989281+0000","last_clean":"2024-09-21T09:58:51.989281+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T09:58:51.989281+0000","last_undegraded":"2024-09-21T09:58:51.989281+0000","last_fullsized":"2024-09-21T09:58:51.989281+0000","mapping_epoch":1086,"log_start":"1090'1000","ondisk_log_start":"1090'1000","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324337664,"num_objects":4949,"num_object_clones":0,"num_object_copies":14847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9627,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1380,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752423+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:58:39.752423+0000","last_peered":"2024-09-21T09:58:39.752423+0000","last_clean":"2024-09-21T09:58:39.752423+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:58:39.752423+0000","last_undegraded":"2024-09-21T09:58:39.752423+0000","last_fullsized":"2024-09-21T09:58:39.752423+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1056,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753347+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:58:39.753347+0000","last_peered":"2024-09-21T09:58:39.753347+0000","last_clean":"2024-09-21T09:58:39.753347+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:58:39.753347+0000","last_undegraded":"2024-09-21T09:58:39.753347+0000","last_fullsized":"2024-09-21T09:58:39.753347+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1397,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754400+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:58:39.754400+0000","last_peered":"2024-09-21T09:58:39.754400+0000","last_clean":"2024-09-21T09:58:39.754400+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:58:39.754400+0000","last_undegraded":"2024-09-21T09:58:39.754400+0000","last_fullsized":"2024-09-21T09:58:39.754400+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1116'9609","reported_seq":9657,"reported_epoch":1116,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:58:56.336347+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T09:58:56.336347+0000","last_peered":"2024-09-21T09:58:56.336347+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T09:58:56.336347+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1094'1700","ondisk_log_start":"1094'1700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7909,"log_dups_size":0,"ondisk_log_size":7909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309788672,"num_objects":4727,"num_object_clones":0,"num_object_copies":14181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9609,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":3315}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":60,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753743+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T09:58:39.753743+0000","last_peered":"2024-09-21T09:58:39.753743+0000","last_clean":"2024-09-21T09:58:39.753743+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T09:58:39.753743+0000","last_undegraded":"2024-09-21T09:58:39.753743+0000","last_fullsized":"2024-09-21T09:58:39.753743+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1381,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754415+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:58:39.754415+0000","last_peered":"2024-09-21T09:58:39.754415+0000","last_clean":"2024-09-21T09:58:39.754415+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:58:39.754415+0000","last_undegraded":"2024-09-21T09:58:39.754415+0000","last_fullsized":"2024-09-21T09:58:39.754415+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753778+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T09:58:39.753778+0000","last_peered":"2024-09-21T09:58:39.753778+0000","last_clean":"2024-09-21T09:58:39.753778+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T09:58:39.753778+0000","last_undegraded":"2024-09-21T09:58:39.753778+0000","last_fullsized":"2024-09-21T09:58:39.753778+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1116'9708","reported_seq":9746,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:56.376035+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T09:58:56.376035+0000","last_peered":"2024-09-21T09:58:56.376035+0000","last_clean":"2024-09-21T09:58:56.376035+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T09:58:56.376035+0000","last_undegraded":"2024-09-21T09:58:56.376035+0000","last_fullsized":"2024-09-21T09:58:56.376035+0000","mapping_epoch":1086,"log_start":"1094'1500","ondisk_log_start":"1094'1500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8208,"log_dups_size":0,"ondisk_log_size":8208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299368448,"num_objects":4568,"num_object_clones":0,"num_object_copies":13704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9708,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1411,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.755493+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:58:39.755493+0000","last_peered":"2024-09-21T09:58:39.755493+0000","last_clean":"2024-09-21T09:58:39.755493+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:58:39.755493+0000","last_undegraded":"2024-09-21T09:58:39.755493+0000","last_fullsized":"2024-09-21T09:58:39.755493+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1059,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751458+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:58:39.751458+0000","last_peered":"2024-09-21T09:58:39.751458+0000","last_clean":"2024-09-21T09:58:39.751458+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:58:39.751458+0000","last_undegraded":"2024-09-21T09:58:39.751458+0000","last_fullsized":"2024-09-21T09:58:39.751458+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1049,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739722+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:58:39.739722+0000","last_peered":"2024-09-21T09:58:39.739722+0000","last_clean":"2024-09-21T09:58:39.739722+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:58:39.739722+0000","last_undegraded":"2024-09-21T09:58:39.739722+0000","last_fullsized":"2024-09-21T09:58:39.739722+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1063,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739919+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:58:39.739919+0000","last_peered":"2024-09-21T09:58:39.739919+0000","last_clean":"2024-09-21T09:58:39.739919+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:58:39.739919+0000","last_undegraded":"2024-09-21T09:58:39.739919+0000","last_fullsized":"2024-09-21T09:58:39.739919+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1072,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740020+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:58:39.740020+0000","last_peered":"2024-09-21T09:58:39.740020+0000","last_clean":"2024-09-21T09:58:39.740020+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:58:39.740020+0000","last_undegraded":"2024-09-21T09:58:39.740020+0000","last_fullsized":"2024-09-21T09:58:39.740020+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1380,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.750572+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:58:39.750572+0000","last_peered":"2024-09-21T09:58:39.750572+0000","last_clean":"2024-09-21T09:58:39.750572+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:58:39.750572+0000","last_undegraded":"2024-09-21T09:58:39.750572+0000","last_fullsized":"2024-09-21T09:58:39.750572+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1075,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753203+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T09:58:39.753203+0000","last_peered":"2024-09-21T09:58:39.753203+0000","last_clean":"2024-09-21T09:58:39.753203+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T09:58:39.753203+0000","last_undegraded":"2024-09-21T09:58:39.753203+0000","last_fullsized":"2024-09-21T09:58:39.753203+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1058,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740542+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:58:39.740542+0000","last_peered":"2024-09-21T09:58:39.740542+0000","last_clean":"2024-09-21T09:58:39.740542+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:58:39.740542+0000","last_undegraded":"2024-09-21T09:58:39.740542+0000","last_fullsized":"2024-09-21T09:58:39.740542+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.746929+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T09:58:39.746929+0000","last_peered":"2024-09-21T09:58:39.746929+0000","last_clean":"2024-09-21T09:58:39.746929+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T09:58:39.746929+0000","last_undegraded":"2024-09-21T09:58:39.746929+0000","last_fullsized":"2024-09-21T09:58:39.746929+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1420,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.755160+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T09:58:39.755160+0000","last_peered":"2024-09-21T09:58:39.755160+0000","last_clean":"2024-09-21T09:58:39.755160+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T09:58:39.755160+0000","last_undegraded":"2024-09-21T09:58:39.755160+0000","last_fullsized":"2024-09-21T09:58:39.755160+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753823+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T09:58:39.753823+0000","last_peered":"2024-09-21T09:58:39.753823+0000","last_clean":"2024-09-21T09:58:39.753823+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T09:58:39.753823+0000","last_undegraded":"2024-09-21T09:58:39.753823+0000","last_fullsized":"2024-09-21T09:58:39.753823+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1055,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751426+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:58:39.751426+0000","last_peered":"2024-09-21T09:58:39.751426+0000","last_clean":"2024-09-21T09:58:39.751426+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:58:39.751426+0000","last_undegraded":"2024-09-21T09:58:39.751426+0000","last_fullsized":"2024-09-21T09:58:39.751426+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753648+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T09:58:39.753648+0000","last_peered":"2024-09-21T09:58:39.753648+0000","last_clean":"2024-09-21T09:58:39.753648+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T09:58:39.753648+0000","last_undegraded":"2024-09-21T09:58:39.753648+0000","last_fullsized":"2024-09-21T09:58:39.753648+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1422,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.755325+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T09:58:39.755325+0000","last_peered":"2024-09-21T09:58:39.755325+0000","last_clean":"2024-09-21T09:58:39.755325+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T09:58:39.755325+0000","last_undegraded":"2024-09-21T09:58:39.755325+0000","last_fullsized":"2024-09-21T09:58:39.755325+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1393,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752777+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:58:39.752777+0000","last_peered":"2024-09-21T09:58:39.752777+0000","last_clean":"2024-09-21T09:58:39.752777+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:58:39.752777+0000","last_undegraded":"2024-09-21T09:58:39.752777+0000","last_fullsized":"2024-09-21T09:58:39.752777+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.756496+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T09:58:39.756496+0000","last_peered":"2024-09-21T09:58:39.756496+0000","last_clean":"2024-09-21T09:58:39.756496+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T09:58:39.756496+0000","last_undegraded":"2024-09-21T09:58:39.756496+0000","last_fullsized":"2024-09-21T09:58:39.756496+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1079,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739877+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:58:39.739877+0000","last_peered":"2024-09-21T09:58:39.739877+0000","last_clean":"2024-09-21T09:58:39.739877+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:58:39.739877+0000","last_undegraded":"2024-09-21T09:58:39.739877+0000","last_fullsized":"2024-09-21T09:58:39.739877+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1378,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.824784+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T09:58:54.824784+0000","last_peered":"2024-09-21T09:58:54.824784+0000","last_clean":"2024-09-21T09:58:54.824784+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T09:58:54.824784+0000","last_undegraded":"2024-09-21T09:58:54.824784+0000","last_fullsized":"2024-09-21T09:58:54.824784+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1398,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754481+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:58:39.754481+0000","last_peered":"2024-09-21T09:58:39.754481+0000","last_clean":"2024-09-21T09:58:39.754481+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:58:39.754481+0000","last_undegraded":"2024-09-21T09:58:39.754481+0000","last_fullsized":"2024-09-21T09:58:39.754481+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1095,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739895+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T09:58:39.739895+0000","last_peered":"2024-09-21T09:58:39.739895+0000","last_clean":"2024-09-21T09:58:39.739895+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T09:58:39.739895+0000","last_undegraded":"2024-09-21T09:58:39.739895+0000","last_fullsized":"2024-09-21T09:58:39.739895+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1059,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740795+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:58:39.740795+0000","last_peered":"2024-09-21T09:58:39.740795+0000","last_clean":"2024-09-21T09:58:39.740795+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:58:39.740795+0000","last_undegraded":"2024-09-21T09:58:39.740795+0000","last_fullsized":"2024-09-21T09:58:39.740795+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1095,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739739+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T09:58:39.739739+0000","last_peered":"2024-09-21T09:58:39.739739+0000","last_clean":"2024-09-21T09:58:39.739739+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T09:58:39.739739+0000","last_undegraded":"2024-09-21T09:58:39.739739+0000","last_fullsized":"2024-09-21T09:58:39.739739+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1044,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740770+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:58:39.740770+0000","last_peered":"2024-09-21T09:58:39.740770+0000","last_clean":"2024-09-21T09:58:39.740770+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:58:39.740770+0000","last_undegraded":"2024-09-21T09:58:39.740770+0000","last_fullsized":"2024-09-21T09:58:39.740770+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":56,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.747067+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T09:58:39.747067+0000","last_peered":"2024-09-21T09:58:39.747067+0000","last_clean":"2024-09-21T09:58:39.747067+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T09:58:39.747067+0000","last_undegraded":"2024-09-21T09:58:39.747067+0000","last_fullsized":"2024-09-21T09:58:39.747067+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":57,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.747970+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T09:58:39.747970+0000","last_peered":"2024-09-21T09:58:39.747970+0000","last_clean":"2024-09-21T09:58:39.747970+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T09:58:39.747970+0000","last_undegraded":"2024-09-21T09:58:39.747970+0000","last_fullsized":"2024-09-21T09:58:39.747970+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1072,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739553+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:58:39.739553+0000","last_peered":"2024-09-21T09:58:39.739553+0000","last_clean":"2024-09-21T09:58:39.739553+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:58:39.739553+0000","last_undegraded":"2024-09-21T09:58:39.739553+0000","last_fullsized":"2024-09-21T09:58:39.739553+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1414,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.755210+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:58:39.755210+0000","last_peered":"2024-09-21T09:58:39.755210+0000","last_clean":"2024-09-21T09:58:39.755210+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:58:39.755210+0000","last_undegraded":"2024-09-21T09:58:39.755210+0000","last_fullsized":"2024-09-21T09:58:39.755210+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4896325656,"num_objects":74713,"num_object_clones":0,"num_object_copies":224139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6053,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74713,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153095,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8293,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15534600192,"data_stored":15534587976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139195,"ondisk_log_size":139195,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234277,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2016644,"kb_used_data":1877240,"kb_used_omap":1,"kb_used_meta":139390,"kb_avail":91728508,"statfs":{"total":95995035648,"available":93929992192,"internally_reserved":0,"allocated":1922293760,"data_stored":1920257195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":142735819},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495914,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2168892,"kb_used_data":2015600,"kb_used_omap":1,"kb_used_meta":153278,"kb_avail":91576260,"statfs":{"total":95995035648,"available":93774090240,"internally_reserved":0,"allocated":2063974400,"data_stored":2061945900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":156957127},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757552,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1730800,"kb_used_data":1565160,"kb_used_omap":1,"kb_used_meta":165566,"kb_avail":92014352,"statfs":{"total":95995035648,"available":94222696448,"internally_reserved":0,"allocated":1602723840,"data_stored":1600698838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":169540038},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019188,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1647292,"kb_used_data":1499696,"kb_used_omap":1,"kb_used_meta":147582,"kb_avail":92097860,"statfs":{"total":95995035648,"available":94308208640,"internally_reserved":0,"allocated":1535688704,"data_stored":1533659717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":151124422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888484,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3297816,"kb_used_data":3062992,"kb_used_omap":34,"kb_used_meta":234653,"kb_avail":90447336,"statfs":{"total":95995035648,"available":92618072064,"internally_reserved":0,"allocated":3136503808,"data_stored":3134494358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":240285683},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967723,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664732,"kb_used_data":2520148,"kb_used_omap":41,"kb_used_meta":144470,"kb_avail":91080420,"statfs":{"total":95995035648,"available":93266350080,"internally_reserved":0,"allocated":2580631552,"data_stored":2578618532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":147937695},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046961,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1666012,"kb_used_data":1557588,"kb_used_omap":41,"kb_used_meta":108374,"kb_avail":92079140,"statfs":{"total":95995035648,"available":94289039360,"internally_reserved":0,"allocated":1594970112,"data_stored":1592956519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":110975913},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126200,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1311204,"kb_used_data":1216028,"kb_used_omap":72,"kb_used_meta":95095,"kb_avail":92433948,"statfs":{"total":95995035648,"available":94652362752,"internally_reserved":0,"allocated":1245212672,"data_stored":1243192016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":97377323},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1232080896,"data_stored":1232076824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1578176512,"data_stored":1578172440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2552954880,"data_stored":2552954880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3113943040,"data_stored":3113943040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1515126784,"data_stored":1515126784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1594101760,"data_stored":1594097688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2042757120,"data_stored":2042757120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1905459200,"data_stored":1905459200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:58:58.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:58.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:58.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:58.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:58.750 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:58:58.761 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:58:58.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:58.865 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:58.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:58.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:58.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:58:59.008 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:58:59.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:59.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.152 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:59.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:59.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:58:59.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:58:59.362 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:58:59.363 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:58:59.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:59.402 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:58:59.403 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:58:59.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:59.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:59.443 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:58:59.443 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:58:59.479 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:58:59.480 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:58:59.512 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:58:59.512 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:58:59.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:58:59.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:58:59.548 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:58:59.548 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:58:59.577 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:58:59.577 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:58:59.610 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:58:59.610 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:58:59.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:58:59.665 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:58:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:58:59.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:58:59.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:59.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:58:59.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:58:59.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:58:59.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:58:59.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:00.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:00.081 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:00.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:00.098 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:59 smithi191 ceph-mon[102206]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6053/221316 objects degraded (2.735%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:00.099 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 6053/221316 objects degraded (2.735%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:00.099 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:58:59 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/473446383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:59 smithi076 ceph-mon[106482]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6053/221316 objects degraded (2.735%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 6053/221316 objects degraded (2.735%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:58:59 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/473446383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:59 smithi076 ceph-mon[110963]: pgmap v1249: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 6053/221316 objects degraded (2.735%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 6053/221316 objects degraded (2.735%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:58:59 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/473446383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:00.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:00.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:00.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:00.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:00.356 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:00.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:00.503 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:00.604 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:00.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:00.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:00.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:00.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:00.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:00.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:00.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:01.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:01.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:01.148 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:01.177 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:01.279 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:01.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:01.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:01.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:01.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:59:01.538 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:01.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:01.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:01.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:01.738 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:01.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:01.814 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:158303 6.5 6:a04cc71f:::benchmark_data_smithi076_154645_object44397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1116)", 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:01.805059+0000", 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0071799070000000001, 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0072020449999999998, 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 158303 2024-09-21T09:59:01.815 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805059+0000", 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805059+0000", 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805057+0000", 2024-09-21T09:59:01.816 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805063+0000", 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4020000000000001e-06 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805076+0000", 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3341e-05 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.817 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805083+0000", 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2570000000000001e-06 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805126+0000", 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3103999999999999e-05 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805269+0000", 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014243099999999999 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.818 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.805420+0000", 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015105399999999999 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.811443+0000", 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006022838 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:01.811507+0000", 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4363999999999996e-05 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:01.819 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:01.820 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:01.820 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2024-09-21T09:59:01.820 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 1 2024-09-21T09:59:01.820 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:59:01.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:01.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:01.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:01.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:01.990 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:01 smithi076 ceph-mon[106482]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 5863/219033 objects degraded (2.677%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:01.990 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:01 smithi076 ceph-mon[110963]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 5863/219033 objects degraded (2.677%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:02.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:02.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:01 smithi191 ceph-mon[102206]: pgmap v1250: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 5863/219033 objects degraded (2.677%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:02.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:02.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:02.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:02.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:02.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:02.423 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:02.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:02.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:59:02.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:02.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:02.673 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:02.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:02.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:02.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:02.856 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T09:59:02.886 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:02.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:02.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:02.988 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:03.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:03.098 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/121448346' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:03.098 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:02 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:03.098 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/121448346' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:03.098 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:02 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/121448346' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:02 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T09:59:03.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:03.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:03.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:03.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:03.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:59:03] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-09-21T09:59:03.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:03.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:03.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:03.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:03.532 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:03.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:03.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:03.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T09:59:03.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:03.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:03.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:03.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:03.908 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:03.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:04.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:04.027 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:03 smithi191 ceph-mon[102206]: pgmap v1251: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 5863/211089 objects degraded (2.778%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:04.027 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:03 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:04.027 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:03 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:59:04.027 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:03 smithi191 ceph-mon[102206]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-21T09:59:04.043 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:04.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[106482]: pgmap v1251: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 5863/211089 objects degraded (2.778%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:59:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[106482]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-21T09:59:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[110963]: pgmap v1251: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 5863/211089 objects degraded (2.778%); 0 B/s, 38 objects/s recovering 2024-09-21T09:59:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T09:59:04.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:03 smithi076 ceph-mon[110963]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-21T09:59:04.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:04.176 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:04.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:04.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:04.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:04.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.460 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:04.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:04.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:04.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:04.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:04.605 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:04.699 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.750 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:04.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:04.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:04.874 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:04.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:04.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:05.010 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:04 smithi076 ceph-mon[106482]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-21T09:59:05.010 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 5863/211089 objects degraded (2.778%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:05.010 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:04 smithi076 ceph-mon[110963]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-21T09:59:05.011 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 5863/211089 objects degraded (2.778%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:05.034 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:05.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:04 smithi191 ceph-mon[102206]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-21T09:59:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 5863/211089 objects degraded (2.778%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:05.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:05.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.171 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:05.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:05.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.305 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:05.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:05.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:05.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:05.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:05.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:05.570 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:05.673 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:05.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:05.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:05.775 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:59:05.776 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:59:05.832 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:05.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:05.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:05.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:06.010 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:06.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:05 smithi191 ceph-mon[102206]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 5863/208509 objects degraded (2.812%); 0 B/s, 23 objects/s recovering 2024-09-21T09:59:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:05 smithi076 ceph-mon[106482]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 5863/208509 objects degraded (2.812%); 0 B/s, 23 objects/s recovering 2024-09-21T09:59:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:05 smithi076 ceph-mon[110963]: pgmap v1254: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 5863/208509 objects degraded (2.812%); 0 B/s, 23 objects/s recovering 2024-09-21T09:59:06.135 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:06.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:06.237 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.276 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:06.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:06.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:06.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:06.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:06.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:06.530 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:06.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:06.632 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:06.674 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:06.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:06.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:06.912 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1254,"stamp":"2024-09-21T09:59:04.487482+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4553775101,"num_objects":69503,"num_object_clones":0,"num_object_copies":208509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5863,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69503,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":158610,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8496,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141339,"ondisk_log_size":141339,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15422496,"kb_used_data":14270940,"kb_used_omap":195,"kb_used_meta":1151228,"kb_avail":734538720,"statfs":{"total":767960285184,"available":752167649280,"internally_reserved":0,"allocated":14613442560,"data_stored":14597243645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1178857604},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,5,4,2,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-343146496,"num_objects":-5236,"num_object_clones":0,"num_object_copies":-15708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-190,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5236,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":190,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.010142"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.866404+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T09:59:02.866404+0000","last_peered":"2024-09-21T09:59:02.866404+0000","last_clean":"2024-09-21T09:59:02.866404+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T09:59:02.866404+0000","last_undegraded":"2024-09-21T09:59:02.866404+0000","last_fullsized":"2024-09-21T09:59:02.866404+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1044,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752996+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:58:39.752996+0000","last_peered":"2024-09-21T09:58:39.752996+0000","last_clean":"2024-09-21T09:58:39.752996+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:58:39.752996+0000","last_undegraded":"2024-09-21T09:58:39.752996+0000","last_fullsized":"2024-09-21T09:58:39.752996+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1075,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740637+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:58:39.740637+0000","last_peered":"2024-09-21T09:58:39.740637+0000","last_clean":"2024-09-21T09:58:39.740637+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:58:39.740637+0000","last_undegraded":"2024-09-21T09:58:39.740637+0000","last_fullsized":"2024-09-21T09:58:39.740637+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1427,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.872883+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T09:59:03.872883+0000","last_peered":"2024-09-21T09:59:03.872883+0000","last_clean":"2024-09-21T09:59:03.872883+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T09:59:03.872883+0000","last_undegraded":"2024-09-21T09:59:03.872883+0000","last_fullsized":"2024-09-21T09:59:03.872883+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1427,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873378+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T09:59:03.873378+0000","last_peered":"2024-09-21T09:59:03.873378+0000","last_clean":"2024-09-21T09:59:03.873378+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T09:59:03.873378+0000","last_undegraded":"2024-09-21T09:59:03.873378+0000","last_fullsized":"2024-09-21T09:59:03.873378+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1027,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753391+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:58:39.753391+0000","last_peered":"2024-09-21T09:58:39.753391+0000","last_clean":"2024-09-21T09:58:39.753391+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:58:39.753391+0000","last_undegraded":"2024-09-21T09:58:39.753391+0000","last_fullsized":"2024-09-21T09:58:39.753391+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.850936+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T09:59:02.850936+0000","last_peered":"2024-09-21T09:59:02.850936+0000","last_clean":"2024-09-21T09:59:02.850936+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T09:59:02.850936+0000","last_undegraded":"2024-09-21T09:59:02.850936+0000","last_fullsized":"2024-09-21T09:59:02.850936+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1054,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751120+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:58:39.751120+0000","last_peered":"2024-09-21T09:58:39.751120+0000","last_clean":"2024-09-21T09:58:39.751120+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:58:39.751120+0000","last_undegraded":"2024-09-21T09:58:39.751120+0000","last_fullsized":"2024-09-21T09:58:39.751120+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1116'9780","reported_seq":9820,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.428648+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T09:59:01.428648+0000","last_peered":"2024-09-21T09:59:01.428648+0000","last_clean":"2024-09-21T09:59:01.428648+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T09:59:01.428648+0000","last_undegraded":"2024-09-21T09:59:01.428648+0000","last_fullsized":"2024-09-21T09:59:01.428648+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9780,"log_dups_size":0,"ondisk_log_size":9780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283377664,"num_objects":4324,"num_object_clones":0,"num_object_copies":12972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9780,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1043,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753064+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:58:39.753064+0000","last_peered":"2024-09-21T09:58:39.753064+0000","last_clean":"2024-09-21T09:58:39.753064+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:58:39.753064+0000","last_undegraded":"2024-09-21T09:58:39.753064+0000","last_fullsized":"2024-09-21T09:58:39.753064+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1043,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740279+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:58:39.740279+0000","last_peered":"2024-09-21T09:58:39.740279+0000","last_clean":"2024-09-21T09:58:39.740279+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:58:39.740279+0000","last_undegraded":"2024-09-21T09:58:39.740279+0000","last_fullsized":"2024-09-21T09:58:39.740279+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1117'9883","reported_seq":12830,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.358264+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T09:59:03.358264+0000","last_peered":"2024-09-21T09:59:03.358264+0000","last_clean":"2024-09-21T09:59:03.358264+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T09:59:03.358264+0000","last_undegraded":"2024-09-21T09:59:03.358264+0000","last_fullsized":"2024-09-21T09:59:03.358264+0000","mapping_epoch":1108,"log_start":"1095'2500","ondisk_log_start":"1095'2500","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277413888,"num_objects":4233,"num_object_clones":0,"num_object_copies":12699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9883,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1044,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754635+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:58:39.754635+0000","last_peered":"2024-09-21T09:58:39.754635+0000","last_clean":"2024-09-21T09:58:39.754635+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:58:39.754635+0000","last_undegraded":"2024-09-21T09:58:39.754635+0000","last_fullsized":"2024-09-21T09:58:39.754635+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1097,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739620+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T09:58:39.739620+0000","last_peered":"2024-09-21T09:58:39.739620+0000","last_clean":"2024-09-21T09:58:39.739620+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T09:58:39.739620+0000","last_undegraded":"2024-09-21T09:58:39.739620+0000","last_fullsized":"2024-09-21T09:58:39.739620+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1116'9961","reported_seq":12885,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.352214+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T09:59:01.352214+0000","last_peered":"2024-09-21T09:59:01.352214+0000","last_clean":"2024-09-21T09:59:01.352214+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T09:59:01.352214+0000","last_undegraded":"2024-09-21T09:59:01.352214+0000","last_fullsized":"2024-09-21T09:59:01.352214+0000","mapping_epoch":1100,"log_start":"1094'1800","ondisk_log_start":"1094'1800","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8161,"log_dups_size":0,"ondisk_log_size":8161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9961,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1028,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753321+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:58:39.753321+0000","last_peered":"2024-09-21T09:58:39.753321+0000","last_clean":"2024-09-21T09:58:39.753321+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:58:39.753321+0000","last_undegraded":"2024-09-21T09:58:39.753321+0000","last_fullsized":"2024-09-21T09:58:39.753321+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1046,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754065+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:58:39.754065+0000","last_peered":"2024-09-21T09:58:39.754065+0000","last_clean":"2024-09-21T09:58:39.754065+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:58:39.754065+0000","last_undegraded":"2024-09-21T09:58:39.754065+0000","last_fullsized":"2024-09-21T09:58:39.754065+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1118'10036","reported_seq":10112,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.975179+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T09:59:03.975179+0000","last_peered":"2024-09-21T09:59:03.975179+0000","last_clean":"2024-09-21T09:59:03.975179+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T09:59:03.975179+0000","last_undegraded":"2024-09-21T09:59:03.975179+0000","last_fullsized":"2024-09-21T09:59:03.975179+0000","mapping_epoch":1086,"log_start":"1089'300","ondisk_log_start":"1089'300","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9736,"log_dups_size":0,"ondisk_log_size":9736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280756224,"num_objects":4284,"num_object_clones":0,"num_object_copies":12852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10036,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1076,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740929+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:58:39.740929+0000","last_peered":"2024-09-21T09:58:39.740929+0000","last_clean":"2024-09-21T09:58:39.740929+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:58:39.740929+0000","last_undegraded":"2024-09-21T09:58:39.740929+0000","last_fullsized":"2024-09-21T09:58:39.740929+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1415,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.872831+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:59:03.872831+0000","last_peered":"2024-09-21T09:59:03.872831+0000","last_clean":"2024-09-21T09:59:03.872831+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:59:03.872831+0000","last_undegraded":"2024-09-21T09:59:03.872831+0000","last_fullsized":"2024-09-21T09:59:03.872831+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1116'9864","reported_seq":9904,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.375164+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T09:59:01.375164+0000","last_peered":"2024-09-21T09:59:01.375164+0000","last_clean":"2024-09-21T09:59:01.375164+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T09:59:01.375164+0000","last_undegraded":"2024-09-21T09:59:01.375164+0000","last_fullsized":"2024-09-21T09:59:01.375164+0000","mapping_epoch":1086,"log_start":"1094'1900","ondisk_log_start":"1094'1900","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7964,"log_dups_size":0,"ondisk_log_size":7964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281935872,"num_objects":4302,"num_object_clones":0,"num_object_copies":12906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9864,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":65,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.849949+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T09:59:02.849949+0000","last_peered":"2024-09-21T09:59:02.849949+0000","last_clean":"2024-09-21T09:59:02.849949+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T09:59:02.849949+0000","last_undegraded":"2024-09-21T09:59:02.849949+0000","last_fullsized":"2024-09-21T09:59:02.849949+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1081,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739874+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T09:58:39.739874+0000","last_peered":"2024-09-21T09:58:39.739874+0000","last_clean":"2024-09-21T09:58:39.739874+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T09:58:39.739874+0000","last_undegraded":"2024-09-21T09:58:39.739874+0000","last_fullsized":"2024-09-21T09:58:39.739874+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1116'9901","reported_seq":12577,"reported_epoch":1116,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:00.085545+0000","last_change":"2024-09-21T09:58:28.149252+0000","last_active":"2024-09-21T09:59:00.085545+0000","last_peered":"2024-09-21T09:59:00.085545+0000","last_clean":"2024-09-21T09:56:44.221905+0000","last_became_active":"2024-09-21T09:56:46.358647+0000","last_became_peered":"2024-09-21T09:56:46.358647+0000","last_unstale":"2024-09-21T09:59:00.085545+0000","last_undegraded":"2024-09-21T09:56:46.247338+0000","last_fullsized":"2024-09-21T09:56:46.236810+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9901,"log_dups_size":0,"ondisk_log_size":9901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292225024,"num_objects":4459,"num_object_clones":0,"num_object_copies":13377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":110,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9901,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1300,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":4349}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1047,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.754208+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:58:39.754208+0000","last_peered":"2024-09-21T09:58:39.754208+0000","last_clean":"2024-09-21T09:58:39.754208+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:58:39.754208+0000","last_undegraded":"2024-09-21T09:58:39.754208+0000","last_fullsized":"2024-09-21T09:58:39.754208+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1078,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740224+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:58:39.740224+0000","last_peered":"2024-09-21T09:58:39.740224+0000","last_clean":"2024-09-21T09:58:39.740224+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:58:39.740224+0000","last_undegraded":"2024-09-21T09:58:39.740224+0000","last_fullsized":"2024-09-21T09:58:39.740224+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1116'9831","reported_seq":9874,"reported_epoch":1116,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:01.405128+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T09:59:01.405128+0000","last_peered":"2024-09-21T09:59:01.405128+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T09:59:01.405128+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9831,"log_dups_size":0,"ondisk_log_size":9831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284622848,"num_objects":4343,"num_object_clones":0,"num_object_copies":13029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9831,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":2867}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1065,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739779+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:58:39.739779+0000","last_peered":"2024-09-21T09:58:39.739779+0000","last_clean":"2024-09-21T09:58:39.739779+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:58:39.739779+0000","last_undegraded":"2024-09-21T09:58:39.739779+0000","last_fullsized":"2024-09-21T09:58:39.739779+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1072,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740274+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T09:58:39.740274+0000","last_peered":"2024-09-21T09:58:39.740274+0000","last_clean":"2024-09-21T09:58:39.740274+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T09:58:39.740274+0000","last_undegraded":"2024-09-21T09:58:39.740274+0000","last_fullsized":"2024-09-21T09:58:39.740274+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1116'9942","reported_seq":9983,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.434304+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T09:59:01.434304+0000","last_peered":"2024-09-21T09:59:01.434304+0000","last_clean":"2024-09-21T09:59:01.434304+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T09:59:01.434304+0000","last_undegraded":"2024-09-21T09:59:01.434304+0000","last_fullsized":"2024-09-21T09:59:01.434304+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9942,"log_dups_size":0,"ondisk_log_size":9942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285671448,"num_objects":4360,"num_object_clones":0,"num_object_copies":13080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4360,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9942,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":61,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.872698+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T09:59:02.872698+0000","last_peered":"2024-09-21T09:59:02.872698+0000","last_clean":"2024-09-21T09:59:02.872698+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T09:59:02.872698+0000","last_undegraded":"2024-09-21T09:59:02.872698+0000","last_fullsized":"2024-09-21T09:59:02.872698+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1397,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.819433+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:58:54.819433+0000","last_peered":"2024-09-21T09:58:54.819433+0000","last_clean":"2024-09-21T09:58:54.819433+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:58:54.819433+0000","last_undegraded":"2024-09-21T09:58:54.819433+0000","last_fullsized":"2024-09-21T09:58:54.819433+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1116'9685","reported_seq":12513,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.363984+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T09:59:01.363984+0000","last_peered":"2024-09-21T09:59:01.363984+0000","last_clean":"2024-09-21T09:59:01.363984+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T09:59:01.363984+0000","last_undegraded":"2024-09-21T09:59:01.363984+0000","last_fullsized":"2024-09-21T09:59:01.363984+0000","mapping_epoch":1113,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9685,"log_dups_size":0,"ondisk_log_size":9685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283312128,"num_objects":4323,"num_object_clones":0,"num_object_copies":12969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9685,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1374,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.835588+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:58:54.835588+0000","last_peered":"2024-09-21T09:58:54.835588+0000","last_clean":"2024-09-21T09:58:54.835588+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:58:54.835588+0000","last_undegraded":"2024-09-21T09:58:54.835588+0000","last_fullsized":"2024-09-21T09:58:54.835588+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1073,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740469+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:58:39.740469+0000","last_peered":"2024-09-21T09:58:39.740469+0000","last_clean":"2024-09-21T09:58:39.740469+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:58:39.740469+0000","last_undegraded":"2024-09-21T09:58:39.740469+0000","last_fullsized":"2024-09-21T09:58:39.740469+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":61,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.872952+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T09:59:02.872952+0000","last_peered":"2024-09-21T09:59:02.872952+0000","last_clean":"2024-09-21T09:59:02.872952+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T09:59:02.872952+0000","last_undegraded":"2024-09-21T09:59:02.872952+0000","last_fullsized":"2024-09-21T09:59:02.872952+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1031,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751151+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:58:39.751151+0000","last_peered":"2024-09-21T09:58:39.751151+0000","last_clean":"2024-09-21T09:58:39.751151+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:58:39.751151+0000","last_undegraded":"2024-09-21T09:58:39.751151+0000","last_fullsized":"2024-09-21T09:58:39.751151+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1116'9867","reported_seq":12907,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.971194+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T09:59:01.971194+0000","last_peered":"2024-09-21T09:59:01.971194+0000","last_clean":"2024-09-21T09:59:01.971194+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T09:59:01.971194+0000","last_undegraded":"2024-09-21T09:59:01.971194+0000","last_fullsized":"2024-09-21T09:59:01.971194+0000","mapping_epoch":1105,"log_start":"1091'1200","ondisk_log_start":"1091'1200","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281214976,"num_objects":4291,"num_object_clones":0,"num_object_copies":12873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9867,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1071,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.752383+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:58:39.752383+0000","last_peered":"2024-09-21T09:58:39.752383+0000","last_clean":"2024-09-21T09:58:39.752383+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:58:39.752383+0000","last_undegraded":"2024-09-21T09:58:39.752383+0000","last_fullsized":"2024-09-21T09:58:39.752383+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1427,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873595+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T09:59:03.873595+0000","last_peered":"2024-09-21T09:59:03.873595+0000","last_clean":"2024-09-21T09:59:03.873595+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T09:59:03.873595+0000","last_undegraded":"2024-09-21T09:59:03.873595+0000","last_fullsized":"2024-09-21T09:59:03.873595+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1425,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873230+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:59:03.873230+0000","last_peered":"2024-09-21T09:59:03.873230+0000","last_clean":"2024-09-21T09:59:03.873230+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:59:03.873230+0000","last_undegraded":"2024-09-21T09:59:03.873230+0000","last_fullsized":"2024-09-21T09:59:03.873230+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1116'9783","reported_seq":9828,"reported_epoch":1116,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:01.316203+0000","last_change":"2024-09-21T09:56:46.337646+0000","last_active":"2024-09-21T09:59:01.316203+0000","last_peered":"2024-09-21T09:59:01.316203+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:56:46.335218+0000","last_became_peered":"2024-09-21T09:56:46.335218+0000","last_unstale":"2024-09-21T09:59:01.316203+0000","last_undegraded":"2024-09-21T09:56:46.260383+0000","last_fullsized":"2024-09-21T09:56:46.244582+0000","mapping_epoch":1093,"log_start":"1094'1600","ondisk_log_start":"1094'1600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8183,"log_dups_size":0,"ondisk_log_size":8183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:03.383491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286064640,"num_objects":4365,"num_object_clones":0,"num_object_copies":13095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1404,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9783,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":2961}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1402,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.825926+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T09:58:54.825926+0000","last_peered":"2024-09-21T09:58:54.825926+0000","last_clean":"2024-09-21T09:58:54.825926+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T09:58:54.825926+0000","last_undegraded":"2024-09-21T09:58:54.825926+0000","last_fullsized":"2024-09-21T09:58:54.825926+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1383,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.819380+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:58:54.819380+0000","last_peered":"2024-09-21T09:58:54.819380+0000","last_clean":"2024-09-21T09:58:54.819380+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:58:54.819380+0000","last_undegraded":"2024-09-21T09:58:54.819380+0000","last_fullsized":"2024-09-21T09:58:54.819380+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1391,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.819288+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:58:54.819288+0000","last_peered":"2024-09-21T09:58:54.819288+0000","last_clean":"2024-09-21T09:58:54.819288+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:58:54.819288+0000","last_undegraded":"2024-09-21T09:58:54.819288+0000","last_fullsized":"2024-09-21T09:58:54.819288+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1117'9917","reported_seq":12885,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.167822+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T09:59:03.167822+0000","last_peered":"2024-09-21T09:59:03.167822+0000","last_clean":"2024-09-21T09:59:03.167822+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T09:59:03.167822+0000","last_undegraded":"2024-09-21T09:59:03.167822+0000","last_fullsized":"2024-09-21T09:59:03.167822+0000","mapping_epoch":1096,"log_start":"1095'2500","ondisk_log_start":"1095'2500","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280952832,"num_objects":4287,"num_object_clones":0,"num_object_copies":12861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9917,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1074,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740627+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:58:39.740627+0000","last_peered":"2024-09-21T09:58:39.740627+0000","last_clean":"2024-09-21T09:58:39.740627+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:58:39.740627+0000","last_undegraded":"2024-09-21T09:58:39.740627+0000","last_fullsized":"2024-09-21T09:58:39.740627+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1065,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739983+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:58:39.739983+0000","last_peered":"2024-09-21T09:58:39.739983+0000","last_clean":"2024-09-21T09:58:39.739983+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:58:39.739983+0000","last_undegraded":"2024-09-21T09:58:39.739983+0000","last_fullsized":"2024-09-21T09:58:39.739983+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1408,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.827730+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:58:54.827730+0000","last_peered":"2024-09-21T09:58:54.827730+0000","last_clean":"2024-09-21T09:58:54.827730+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:58:54.827730+0000","last_undegraded":"2024-09-21T09:58:54.827730+0000","last_fullsized":"2024-09-21T09:58:54.827730+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1118'9947","reported_seq":10024,"reported_epoch":1118,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:03.984267+0000","last_change":"2024-09-21T09:56:46.343261+0000","last_active":"2024-09-21T09:59:03.984267+0000","last_peered":"2024-09-21T09:59:03.984267+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T09:59:03.984267+0000","last_undegraded":"2024-09-21T09:56:46.247241+0000","last_fullsized":"2024-09-21T09:56:46.239078+0000","mapping_epoch":1093,"log_start":"1089'200","ondisk_log_start":"1089'200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9747,"log_dups_size":0,"ondisk_log_size":9747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276627456,"num_objects":4221,"num_object_clones":0,"num_object_copies":12663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9947,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":2760}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1092,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740737+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:58:39.740737+0000","last_peered":"2024-09-21T09:58:39.740737+0000","last_clean":"2024-09-21T09:58:39.740737+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:58:39.740737+0000","last_undegraded":"2024-09-21T09:58:39.740737+0000","last_fullsized":"2024-09-21T09:58:39.740737+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1405,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.827803+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:58:54.827803+0000","last_peered":"2024-09-21T09:58:54.827803+0000","last_clean":"2024-09-21T09:58:54.827803+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:58:54.827803+0000","last_undegraded":"2024-09-21T09:58:54.827803+0000","last_fullsized":"2024-09-21T09:58:54.827803+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":67,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.872704+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T09:59:03.872704+0000","last_peered":"2024-09-21T09:59:03.872704+0000","last_clean":"2024-09-21T09:59:03.872704+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T09:59:03.872704+0000","last_undegraded":"2024-09-21T09:59:03.872704+0000","last_fullsized":"2024-09-21T09:59:03.872704+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1116'10131","reported_seq":10203,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.982027+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T09:59:01.982027+0000","last_peered":"2024-09-21T09:59:01.982027+0000","last_clean":"2024-09-21T09:59:01.982027+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T09:59:01.982027+0000","last_undegraded":"2024-09-21T09:59:01.982027+0000","last_fullsized":"2024-09-21T09:59:01.982027+0000","mapping_epoch":1086,"log_start":"1094'1500","ondisk_log_start":"1094'1500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8631,"log_dups_size":0,"ondisk_log_size":8631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291307520,"num_objects":4445,"num_object_clones":0,"num_object_copies":13335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10131,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1382,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.827970+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:58:54.827970+0000","last_peered":"2024-09-21T09:58:54.827970+0000","last_clean":"2024-09-21T09:58:54.827970+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:58:54.827970+0000","last_undegraded":"2024-09-21T09:58:54.827970+0000","last_fullsized":"2024-09-21T09:58:54.827970+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1056,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753347+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:58:39.753347+0000","last_peered":"2024-09-21T09:58:39.753347+0000","last_clean":"2024-09-21T09:58:39.753347+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:58:39.753347+0000","last_undegraded":"2024-09-21T09:58:39.753347+0000","last_fullsized":"2024-09-21T09:58:39.753347+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1403,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.872926+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:59:03.872926+0000","last_peered":"2024-09-21T09:59:03.872926+0000","last_clean":"2024-09-21T09:59:03.872926+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:59:03.872926+0000","last_undegraded":"2024-09-21T09:59:03.872926+0000","last_fullsized":"2024-09-21T09:59:03.872926+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1116'9859","reported_seq":9907,"reported_epoch":1116,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:01.364237+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T09:59:01.364237+0000","last_peered":"2024-09-21T09:59:01.364237+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T09:59:01.364237+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1095'1900","ondisk_log_start":"1095'1900","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293404672,"num_objects":4477,"num_object_clones":0,"num_object_copies":13431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":3065}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":65,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.851013+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T09:59:02.851013+0000","last_peered":"2024-09-21T09:59:02.851013+0000","last_clean":"2024-09-21T09:59:02.851013+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T09:59:02.851013+0000","last_undegraded":"2024-09-21T09:59:02.851013+0000","last_fullsized":"2024-09-21T09:59:02.851013+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1383,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.820117+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:58:54.820117+0000","last_peered":"2024-09-21T09:58:54.820117+0000","last_clean":"2024-09-21T09:58:54.820117+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:58:54.820117+0000","last_undegraded":"2024-09-21T09:58:54.820117+0000","last_fullsized":"2024-09-21T09:58:54.820117+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":61,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.872141+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T09:59:02.872141+0000","last_peered":"2024-09-21T09:59:02.872141+0000","last_clean":"2024-09-21T09:59:02.872141+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T09:59:02.872141+0000","last_undegraded":"2024-09-21T09:59:02.872141+0000","last_fullsized":"2024-09-21T09:59:02.872141+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1116'9944","reported_seq":9984,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:59:01.358913+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T09:59:01.358913+0000","last_peered":"2024-09-21T09:59:01.358913+0000","last_clean":"2024-09-21T09:59:01.358913+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T09:59:01.358913+0000","last_undegraded":"2024-09-21T09:59:01.358913+0000","last_fullsized":"2024-09-21T09:59:01.358913+0000","mapping_epoch":1086,"log_start":"1094'1800","ondisk_log_start":"1094'1800","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283901952,"num_objects":4332,"num_object_clones":0,"num_object_copies":12996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9944,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1417,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873448+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:59:03.873448+0000","last_peered":"2024-09-21T09:59:03.873448+0000","last_clean":"2024-09-21T09:59:03.873448+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:59:03.873448+0000","last_undegraded":"2024-09-21T09:59:03.873448+0000","last_fullsized":"2024-09-21T09:59:03.873448+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1059,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751458+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:58:39.751458+0000","last_peered":"2024-09-21T09:58:39.751458+0000","last_clean":"2024-09-21T09:58:39.751458+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:58:39.751458+0000","last_undegraded":"2024-09-21T09:58:39.751458+0000","last_fullsized":"2024-09-21T09:58:39.751458+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1049,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739722+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:58:39.739722+0000","last_peered":"2024-09-21T09:58:39.739722+0000","last_clean":"2024-09-21T09:58:39.739722+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:58:39.739722+0000","last_undegraded":"2024-09-21T09:58:39.739722+0000","last_fullsized":"2024-09-21T09:58:39.739722+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1063,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739919+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:58:39.739919+0000","last_peered":"2024-09-21T09:58:39.739919+0000","last_clean":"2024-09-21T09:58:39.739919+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:58:39.739919+0000","last_undegraded":"2024-09-21T09:58:39.739919+0000","last_fullsized":"2024-09-21T09:58:39.739919+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1072,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740020+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:58:39.740020+0000","last_peered":"2024-09-21T09:58:39.740020+0000","last_clean":"2024-09-21T09:58:39.740020+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:58:39.740020+0000","last_undegraded":"2024-09-21T09:58:39.740020+0000","last_fullsized":"2024-09-21T09:58:39.740020+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1383,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.820905+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:58:54.820905+0000","last_peered":"2024-09-21T09:58:54.820905+0000","last_clean":"2024-09-21T09:58:54.820905+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:58:54.820905+0000","last_undegraded":"2024-09-21T09:58:54.820905+0000","last_fullsized":"2024-09-21T09:58:54.820905+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1075,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.753203+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T09:58:39.753203+0000","last_peered":"2024-09-21T09:58:39.753203+0000","last_clean":"2024-09-21T09:58:39.753203+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T09:58:39.753203+0000","last_undegraded":"2024-09-21T09:58:39.753203+0000","last_fullsized":"2024-09-21T09:58:39.753203+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1058,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740542+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:58:39.740542+0000","last_peered":"2024-09-21T09:58:39.740542+0000","last_clean":"2024-09-21T09:58:39.740542+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:58:39.740542+0000","last_undegraded":"2024-09-21T09:58:39.740542+0000","last_fullsized":"2024-09-21T09:58:39.740542+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.850126+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T09:59:02.850126+0000","last_peered":"2024-09-21T09:59:02.850126+0000","last_clean":"2024-09-21T09:59:02.850126+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T09:59:02.850126+0000","last_undegraded":"2024-09-21T09:59:02.850126+0000","last_fullsized":"2024-09-21T09:59:02.850126+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1426,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873694+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T09:59:03.873694+0000","last_peered":"2024-09-21T09:59:03.873694+0000","last_clean":"2024-09-21T09:59:03.873694+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T09:59:03.873694+0000","last_undegraded":"2024-09-21T09:59:03.873694+0000","last_fullsized":"2024-09-21T09:59:03.873694+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.872852+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T09:59:02.872852+0000","last_peered":"2024-09-21T09:59:02.872852+0000","last_clean":"2024-09-21T09:59:02.872852+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T09:59:02.872852+0000","last_undegraded":"2024-09-21T09:59:02.872852+0000","last_fullsized":"2024-09-21T09:59:02.872852+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1055,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.751426+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:58:39.751426+0000","last_peered":"2024-09-21T09:58:39.751426+0000","last_clean":"2024-09-21T09:58:39.751426+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:58:39.751426+0000","last_undegraded":"2024-09-21T09:58:39.751426+0000","last_fullsized":"2024-09-21T09:58:39.751426+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.870335+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T09:59:02.870335+0000","last_peered":"2024-09-21T09:59:02.870335+0000","last_clean":"2024-09-21T09:59:02.870335+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T09:59:02.870335+0000","last_undegraded":"2024-09-21T09:59:02.870335+0000","last_fullsized":"2024-09-21T09:59:02.870335+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1429,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873339+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T09:59:03.873339+0000","last_peered":"2024-09-21T09:59:03.873339+0000","last_clean":"2024-09-21T09:59:03.873339+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T09:59:03.873339+0000","last_undegraded":"2024-09-21T09:59:03.873339+0000","last_fullsized":"2024-09-21T09:59:03.873339+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1395,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.821175+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:58:54.821175+0000","last_peered":"2024-09-21T09:58:54.821175+0000","last_clean":"2024-09-21T09:58:54.821175+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:58:54.821175+0000","last_undegraded":"2024-09-21T09:58:54.821175+0000","last_fullsized":"2024-09-21T09:58:54.821175+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.867398+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T09:59:02.867398+0000","last_peered":"2024-09-21T09:59:02.867398+0000","last_clean":"2024-09-21T09:59:02.867398+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T09:59:02.867398+0000","last_undegraded":"2024-09-21T09:59:02.867398+0000","last_fullsized":"2024-09-21T09:59:02.867398+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1079,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739877+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:58:39.739877+0000","last_peered":"2024-09-21T09:58:39.739877+0000","last_clean":"2024-09-21T09:58:39.739877+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:58:39.739877+0000","last_undegraded":"2024-09-21T09:58:39.739877+0000","last_fullsized":"2024-09-21T09:58:39.739877+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1378,"reported_epoch":1116,"state":"active+clean","last_fresh":"2024-09-21T09:58:54.824784+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T09:58:54.824784+0000","last_peered":"2024-09-21T09:58:54.824784+0000","last_clean":"2024-09-21T09:58:54.824784+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T09:58:54.824784+0000","last_undegraded":"2024-09-21T09:58:54.824784+0000","last_fullsized":"2024-09-21T09:58:54.824784+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1405,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873013+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:59:03.873013+0000","last_peered":"2024-09-21T09:59:03.873013+0000","last_clean":"2024-09-21T09:59:03.873013+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:59:03.873013+0000","last_undegraded":"2024-09-21T09:59:03.873013+0000","last_fullsized":"2024-09-21T09:59:03.873013+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1095,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739895+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T09:58:39.739895+0000","last_peered":"2024-09-21T09:58:39.739895+0000","last_clean":"2024-09-21T09:58:39.739895+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T09:58:39.739895+0000","last_undegraded":"2024-09-21T09:58:39.739895+0000","last_fullsized":"2024-09-21T09:58:39.739895+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1059,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740795+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:58:39.740795+0000","last_peered":"2024-09-21T09:58:39.740795+0000","last_clean":"2024-09-21T09:58:39.740795+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:58:39.740795+0000","last_undegraded":"2024-09-21T09:58:39.740795+0000","last_fullsized":"2024-09-21T09:58:39.740795+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1095,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739739+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T09:58:39.739739+0000","last_peered":"2024-09-21T09:58:39.739739+0000","last_clean":"2024-09-21T09:58:39.739739+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T09:58:39.739739+0000","last_undegraded":"2024-09-21T09:58:39.739739+0000","last_fullsized":"2024-09-21T09:58:39.739739+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1044,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.740770+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:58:39.740770+0000","last_peered":"2024-09-21T09:58:39.740770+0000","last_clean":"2024-09-21T09:58:39.740770+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:58:39.740770+0000","last_undegraded":"2024-09-21T09:58:39.740770+0000","last_fullsized":"2024-09-21T09:58:39.740770+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":60,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.850249+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T09:59:02.850249+0000","last_peered":"2024-09-21T09:59:02.850249+0000","last_clean":"2024-09-21T09:59:02.850249+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T09:59:02.850249+0000","last_undegraded":"2024-09-21T09:59:02.850249+0000","last_fullsized":"2024-09-21T09:59:02.850249+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":61,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.849947+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T09:59:02.849947+0000","last_peered":"2024-09-21T09:59:02.849947+0000","last_clean":"2024-09-21T09:59:02.849947+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T09:59:02.849947+0000","last_undegraded":"2024-09-21T09:59:02.849947+0000","last_fullsized":"2024-09-21T09:59:02.849947+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1072,"reported_epoch":1115,"state":"active+clean","last_fresh":"2024-09-21T09:58:39.739553+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:58:39.739553+0000","last_peered":"2024-09-21T09:58:39.739553+0000","last_clean":"2024-09-21T09:58:39.739553+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:58:39.739553+0000","last_undegraded":"2024-09-21T09:58:39.739553+0000","last_fullsized":"2024-09-21T09:58:39.739553+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1420,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.873778+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:59:03.873778+0000","last_peered":"2024-09-21T09:59:03.873778+0000","last_clean":"2024-09-21T09:59:03.873778+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:59:03.873778+0000","last_undegraded":"2024-09-21T09:59:03.873778+0000","last_fullsized":"2024-09-21T09:59:03.873778+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4553179160,"num_objects":69477,"num_object_clones":0,"num_object_copies":208431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5863,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":158331,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8483,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14478487552,"data_stored":14478475336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141131,"ondisk_log_size":141131,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234278,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1928960,"kb_used_data":1785464,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":91816192,"statfs":{"total":95995035648,"available":94019780608,"internally_reserved":0,"allocated":1828315136,"data_stored":1826278571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":146930123},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495915,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2084280,"kb_used_data":1926896,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":91660872,"statfs":{"total":95995035648,"available":93860732928,"internally_reserved":0,"allocated":1973141504,"data_stored":1971113004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":161151431},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757553,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1641776,"kb_used_data":1488488,"kb_used_omap":1,"kb_used_meta":153278,"kb_avail":92103376,"statfs":{"total":95995035648,"available":94313857024,"internally_reserved":0,"allocated":1524211712,"data_stored":1522186710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":156957126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019189,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1555128,"kb_used_data":1419824,"kb_used_omap":1,"kb_used_meta":135294,"kb_avail":92190024,"statfs":{"total":95995035648,"available":94402584576,"internally_reserved":0,"allocated":1453899776,"data_stored":1451870789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":138541510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888486,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2993460,"kb_used_data":2754536,"kb_used_omap":34,"kb_used_meta":238813,"kb_avail":90751692,"statfs":{"total":95995035648,"available":92929732608,"internally_reserved":0,"allocated":2820644864,"data_stored":2818627700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":244545523},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967724,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2533032,"kb_used_data":2396832,"kb_used_omap":41,"kb_used_meta":136150,"kb_avail":91212120,"statfs":{"total":95995035648,"available":93401210880,"internally_reserved":0,"allocated":2454355968,"data_stored":2452339091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":139418015},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046963,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1500776,"kb_used_data":1400672,"kb_used_omap":41,"kb_used_meta":100054,"kb_avail":92244376,"statfs":{"total":95995035648,"available":94458241024,"internally_reserved":0,"allocated":1434288128,"data_stored":1432270678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":102456233},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126202,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185084,"kb_used_data":1098228,"kb_used_omap":72,"kb_used_meta":86775,"kb_avail":92560068,"statfs":{"total":95995035648,"available":94781509632,"internally_reserved":0,"allocated":1124585472,"data_stored":1122557102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":88857643},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1115820032,"data_stored":1115815960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1416695808,"data_stored":1416691736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2425815040,"data_stored":2425815040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2806972416,"data_stored":2806972416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1431896064,"data_stored":1431896064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1515327488,"data_stored":1515323416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1953824768,"data_stored":1953824768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1812135936,"data_stored":1812135936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:59:06.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T09:59:06.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:06.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:06.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:06.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:07.026 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:06 smithi191 ceph-mon[102206]: from='client.54493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:07.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:07.081 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:07.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:07.183 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:06 smithi076 ceph-mon[106482]: from='client.54493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:07.183 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:06 smithi076 ceph-mon[110963]: from='client.54493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:07.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:07.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:07.302 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:07.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:07.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:07.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:07.521 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:07.534 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:59:07.534 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T09:59:07.534 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T09:59:07.534 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T09:59:07.534 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:59:07.622 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:07.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:07.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:07.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:07.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:07.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:07.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:07.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:07.953 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:07 smithi191 ceph-mon[102206]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5484/202731 objects degraded (2.705%); 0 B/s, 71 objects/s recovering 2024-09-21T09:59:08.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:08.055 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:08.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:08.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:08.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T09:59:08.236 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:07 smithi076 ceph-mon[106482]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5484/202731 objects degraded (2.705%); 0 B/s, 71 objects/s recovering 2024-09-21T09:59:08.237 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:07 smithi076 ceph-mon[110963]: pgmap v1255: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5484/202731 objects degraded (2.705%); 0 B/s, 71 objects/s recovering 2024-09-21T09:59:08.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:08.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:08.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:08.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:08.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:08.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:08.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:08.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:08.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:08.723 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:08.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:08.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T09:59:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:08.867 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:08.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:08.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:09.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:09.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:09.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:09.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:09.228 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:59:09.249 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:09.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:09.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:09.410 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:09.410 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:09.411 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:09 smithi191 ceph-mon[102206]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5484/200271 objects degraded (2.738%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:09.411 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 5484/202731 objects degraded (2.705%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:09.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.473 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:09.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:09.574 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:09.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:09.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:09.619 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:09.619 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:09.619 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[106482]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5484/200271 objects degraded (2.738%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:09.619 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 5484/202731 objects degraded (2.705%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:09.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:09.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:09.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[110963]: pgmap v1256: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5484/200271 objects degraded (2.738%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:09.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 5484/202731 objects degraded (2.705%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:09.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:09.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:09.855 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:09.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:09.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:10.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:10.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:10.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:10.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:10.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:10.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:10.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:10.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:10.455 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:10.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:10.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:10.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:10.716 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:10.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:10.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:10.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:10.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:10.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:11.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.137 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:11.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:11.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:11.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7899 6.3 6:ceef0145:::benchmark_data_smithi076_154645_object7898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.012600+0000", 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.25079548299999, 2024-09-21T09:59:11.269 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.099283558999999993, 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7899 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012600+0000", 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.270 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012600+0000", 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012598+0000", 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012728+0000", 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129985 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.271 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012740+0000", 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1921e-05 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012747+0000", 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2139999999999999e-06 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012949+0000", 2024-09-21T09:59:11.272 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020183399999999999 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013057+0000", 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010762100000000001 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013186+0000", 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129228 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014926+0000", 2024-09-21T09:59:11.273 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017401669999999999 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.016435+0000", 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001509334 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111798+0000", 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.095362858999999994 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.274 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111845+0000", 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6743000000000003e-05 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111883+0000", 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8164999999999998e-05 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7900 6.8 6:1d7bd920:::benchmark_data_smithi076_154645_object7899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.012886+0000", 2024-09-21T09:59:11.275 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.25050915099999, 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.098657053999999994, 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7900 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012886+0000", 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.276 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012886+0000", 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012885+0000", 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012930+0000", 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4450999999999998e-05 2024-09-21T09:59:11.277 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012976+0000", 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6285999999999997e-05 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012983+0000", 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0609999999999998e-06 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013049+0000", 2024-09-21T09:59:11.278 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5939999999999995e-05 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013180+0000", 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013155 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013329+0000", 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014864199999999999 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014454+0000", 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011247480000000001 2024-09-21T09:59:11.279 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015925+0000", 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014708740000000001 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111469+0000", 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.095544542999999996 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111509+0000", 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9894000000000001e-05 2024-09-21T09:59:11.280 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111543+0000", 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3785000000000003e-05 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7903 6.b 6:d080dfec:::benchmark_data_smithi076_154645_object7902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.012992+0000", 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.25040289899999, 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.095366643000000001, 2024-09-21T09:59:11.281 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7903 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012992+0000", 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.282 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012992+0000", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.012991+0000", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013026+0000", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4164e-05 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.283 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013033+0000", 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4490000000000002e-06 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013039+0000", 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4580000000000001e-06 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013128+0000", 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9732000000000001e-05 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.284 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013245+0000", 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011660800000000001 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,6", 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013373+0000", 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012845599999999999 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015131+0000", 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017574979999999999 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.285 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.016331+0000", 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001200303 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.108287+0000", 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.091956357000000002 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.108330+0000", 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2342999999999998e-05 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.286 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.108359+0000", 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8958999999999998e-05 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7906 6.5 6:aeb082ad:::benchmark_data_smithi076_154645_object7905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.013050+0000", 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.250345399, 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.098175714999999997, 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.287 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7906 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013050+0000", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013050+0000", 2024-09-21T09:59:11.288 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013049+0000", 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013081+0000", 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1723000000000001e-05 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013088+0000", 2024-09-21T09:59:11.289 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8700000000000003e-06 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013095+0000", 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0949999999999998e-06 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013797+0000", 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070188300000000004 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013930+0000", 2024-09-21T09:59:11.290 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000133533 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014074+0000", 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143342 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015062+0000", 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00098805799999999999 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015880+0000", 2024-09-21T09:59:11.291 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000818087 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111142+0000", 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.095262272999999995 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111189+0000", 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6578e-05 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111225+0000", 2024-09-21T09:59:11.292 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6733000000000002e-05 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7907 6.8 6:1edb872f:::benchmark_data_smithi076_154645_object7906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.013771+0000", 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.24962386499999, 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.098129791999999993, 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.293 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7907 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013771+0000", 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013771+0000", 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.294 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.295 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013770+0000", 2024-09-21T09:59:11.295 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:59:11.295 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.295 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013810+0000", 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9539000000000002e-05 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013820+0000", 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0180000000000001e-05 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013827+0000", 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7719999999999997e-06 2024-09-21T09:59:11.296 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013865+0000", 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7868999999999999e-05 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013997+0000", 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000131827 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014117+0000", 2024-09-21T09:59:11.297 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012020599999999999 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015052+0000", 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093497199999999995 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015951+0000", 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00089971400000000003 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.298 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111636+0000", 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.095684488999999998 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111871+0000", 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00023482099999999999 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111901+0000", 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0171e-05 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.299 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7908 6.8 6:18f299fc:::benchmark_data_smithi076_154645_object7907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.013836+0000", 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.249558699, 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.099270424999999995, 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7908 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.300 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013836+0000", 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013836+0000", 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013836+0000", 2024-09-21T09:59:11.301 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013864+0000", 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7957999999999999e-05 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013871+0000", 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7049999999999999e-06 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.302 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.013877+0000", 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.614e-06 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014476+0000", 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000599246 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014581+0000", 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010519200000000001 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.303 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014708+0000", 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012680999999999999 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015264+0000", 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00055621000000000002 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.016152+0000", 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00088719500000000002 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.304 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.113028+0000", 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.096876484999999998 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.113073+0000", 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4498e-05 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.113107+0000", 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4159999999999998e-05 2024-09-21T09:59:11.305 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:7909 6.3 6:c42df8d1:::benchmark_data_smithi076_154645_object7908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1089)", 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:39.014028+0000", 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 152.24936656, 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.097969174000000006, 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.306 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 7909 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014028+0000", 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014028+0000", 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014028+0000", 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014146+0000", 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118547 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014157+0000", 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0134e-05 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.308 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014163+0000", 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.251e-06 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014254+0000", 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1527000000000006e-05 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014367+0000", 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112946 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.309 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.014503+0000", 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013614899999999999 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.015262+0000", 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00075870899999999999 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.016506+0000", 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012442760000000001 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.310 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111922+0000", 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.095415847999999998 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111971+0000", 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8350000000000003e-05 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:39.111998+0000", 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6988e-05 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.311 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80507 6.5 6:a3ef16e8:::benchmark_data_smithi076_154645_object80506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.753204+0000", 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 101.510191432, 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.081050328000000005, 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80507 2024-09-21T09:59:11.312 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753204+0000", 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753204+0000", 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.313 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753202+0000", 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753258+0000", 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6038000000000001e-05 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753271+0000", 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2384e-05 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.314 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753277+0000", 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8050000000000001e-06 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.755530+0000", 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022525570000000001 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.755688+0000", 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000158343 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T09:59:11.315 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.755847+0000", 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015884199999999999 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.759804+0000", 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039567810000000004 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.760004+0000", 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000199886 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.316 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.833395+0000", 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.073391214999999996 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.833438+0000", 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2725999999999997e-05 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.834254+0000", 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00081627200000000005 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.317 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80517 6.3 6:c4181fc8:::benchmark_data_smithi076_154645_object80516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.762328+0000", 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 101.501067095, 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.074006481999999998, 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80517 2024-09-21T09:59:11.318 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762328+0000", 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762328+0000", 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762327+0000", 2024-09-21T09:59:11.319 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762372+0000", 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5374000000000003e-05 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762385+0000", 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2583e-05 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762392+0000", 2024-09-21T09:59:11.320 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3429999999999998e-06 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762439+0000", 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6990000000000002e-05 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762535+0000", 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6015999999999997e-05 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.762629+0000", 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4370000000000006e-05 2024-09-21T09:59:11.321 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.764749+0000", 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002119681 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.764947+0000", 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019797700000000001 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.836267+0000", 2024-09-21T09:59:11.322 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071320016999999999 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.836311+0000", 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3850999999999997e-05 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.836334+0000", 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3512000000000001e-05 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.323 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:81314 6.5 6:a5a18d3c:::benchmark_data_smithi076_154645_object81313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:30.593086+0000", 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 100.670308822, 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071296096000000003, 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 81314 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.324 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.593086+0000", 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.593086+0000", 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.593086+0000", 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.325 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.593119+0000", 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3046999999999997e-05 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.593128+0000", 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2769999999999994e-06 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.593134+0000", 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2330000000000003e-06 2024-09-21T09:59:11.326 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.601772+0000", 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086373460000000006 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.601914+0000", 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000142934 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.602066+0000", 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000152008 2024-09-21T09:59:11.327 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.610546+0000", 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0084793779999999992 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.613563+0000", 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030168 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.663369+0000", 2024-09-21T09:59:11.328 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.049806034999999999 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.663446+0000", 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7179000000000001e-05 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:30.664382+0000", 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093646500000000004 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.329 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34674.0:105156 6.6 e1107/1096 6:607d88ce:::benchmark_data_smithi076_154645_object105155:head v 1107'6563, mlcod=1107'6560)", 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:56.130898+0000", 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 75.132496899000003, 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.074977557, 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.130898+0000", 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.330 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.130898+0000", 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.130897+0000", 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.131021+0000", 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012345100000000001 2024-09-21T09:59:11.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.131087+0000", 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6186000000000006e-05 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.131142+0000", 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5216999999999997e-05 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.131211+0000", 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8969999999999999e-05 2024-09-21T09:59:11.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.131228+0000", 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7051999999999999e-05 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.205704+0000", 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.074475909000000007 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.205876+0000", 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017145300000000001 2024-09-21T09:59:11.333 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:105166 6.3 6:c58b1508:::benchmark_data_smithi076_154645_object105165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:56.134719+0000", 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 75.128675642000005, 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.072882848, 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.334 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 105166 2024-09-21T09:59:11.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.335 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.134719+0000", 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.134719+0000", 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.134718+0000", 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.134829+0000", 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011081700000000001 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.198838+0000", 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.064008850000000006 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.198848+0000", 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0409e-05 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.198978+0000", 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012939099999999999 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.199129+0000", 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015182300000000001 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.199281+0000", 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015126500000000001 2024-09-21T09:59:11.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.207443+0000", 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008161939 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.207449+0000", 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4389999999999997e-06 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.207477+0000", 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7551000000000001e-05 2024-09-21T09:59:11.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.207564+0000", 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7548999999999998e-05 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.207602+0000", 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8067e-05 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:130957 6.3 6:cf1ac98f:::benchmark_data_smithi076_154645_object17051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1112)", 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:26.593702+0000", 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 44.669692550000001, 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.112349855, 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 130957 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593702+0000", 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593702+0000", 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593701+0000", 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593707+0000", 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2090000000000002e-06 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593719+0000", 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1409999999999999e-05 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593725+0000", 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6749999999999996e-06 2024-09-21T09:59:11.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593771+0000", 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5563999999999999e-05 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593871+0000", 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100487 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.593973+0000", 2024-09-21T09:59:11.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101945 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.601864+0000", 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078904779999999994 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.603328+0000", 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014642069999999999 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.705991+0000", 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.10266298 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.706025+0000", 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3819999999999998e-05 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.706052+0000", 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7412000000000001e-05 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.346 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:130965 6.3 6:ccc6b65b:::benchmark_data_smithi076_154645_object17059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1112)", 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:26.597609+0000", 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 44.665785610999997, 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.108533989, 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 130965 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597609+0000", 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597609+0000", 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597608+0000", 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597615+0000", 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6259999999999997e-06 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597628+0000", 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3739e-05 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597637+0000", 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8440000000000004e-06 2024-09-21T09:59:11.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597994+0000", 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00035678000000000001 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.598112+0000", 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000118107 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:11.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.598236+0000", 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012360600000000001 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.601973+0000", 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037373599999999999 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.603461+0000", 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001487749 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.706089+0000", 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.10262837599999999 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.706122+0000", 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2373000000000003e-05 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.706143+0000", 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1880999999999999e-05 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.352 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:130977 6.8 6:1ef63bc6:::benchmark_data_smithi076_154645_object17071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1112)", 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:26.614176+0000", 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 44.649219037000002, 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.102361703, 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 130977 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.353 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614176+0000", 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614176+0000", 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614175+0000", 2024-09-21T09:59:11.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614180+0000", 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0610000000000003e-06 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614192+0000", 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1545999999999999e-05 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614197+0000", 2024-09-21T09:59:11.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4380000000000003e-06 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614231+0000", 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3581000000000003e-05 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614306+0000", 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5403999999999998e-05 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.614372+0000", 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5883999999999995e-05 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.617517+0000", 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031448299999999999 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.618600+0000", 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001082843 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.716427+0000", 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.097827128999999999 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.716499+0000", 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.216e-05 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.716538+0000", 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8858e-05 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.358 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:159232 6.8 6:1aca8329:::benchmark_data_smithi076_154645_object45326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:03.169594+0000", 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 8.0938014559999996, 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.145834931, 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 159232 2024-09-21T09:59:11.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169594+0000", 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169594+0000", 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169591+0000", 2024-09-21T09:59:11.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169599+0000", 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8599999999999993e-06 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169612+0000", 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2659e-05 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169619+0000", 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1119999999999998e-06 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169653+0000", 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4334e-05 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169725+0000", 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1693000000000006e-05 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.169791+0000", 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5500000000000006e-05 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315259+0000", 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14546883399999999 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315317+0000", 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7364000000000002e-05 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315328+0000", 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0957e-05 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315374+0000", 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5967000000000002e-05 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315429+0000", 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4728999999999997e-05 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.364 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGRecoveryDeleteReply(6.5 e1117,1093 6:abd891ac:::benchmark_data_smithi076_154645_object424:head,1109'7104)", 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:03.172183+0000", 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 8.0912120380000001, 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14315351700000001, 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172183+0000", 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172183+0000", 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172182+0000", 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172188+0000", 2024-09-21T09:59:11.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6279999999999999e-06 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172201+0000", 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2612000000000001e-05 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172207+0000", 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9909999999999999e-06 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.172251+0000", 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3809e-05 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315337+0000", 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14308570700000001 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34674.0:159232 6.8 e1117/1100 ondisk, result = 0)", 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:03.173254+0000", 2024-09-21T09:59:11.368 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 8.0901410939999998, 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.142012571, 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173254+0000", 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173254+0000", 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173253+0000", 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173260+0000", 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9689999999999997e-06 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315156+0000", 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14189529100000001 2024-09-21T09:59:11.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315169+0000", 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3448000000000001e-05 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315242+0000", 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2947999999999993e-05 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315252+0000", 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0311e-05 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315267+0000", 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4236000000000001e-05 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:159234 6.8 6:1a590547:::benchmark_data_smithi076_154645_object45328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:03.173346+0000", 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 8.090049166, 2024-09-21T09:59:11.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.149370372, 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 159234 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173346+0000", 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173346+0000", 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173345+0000", 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.173349+0000", 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7919999999999999e-06 2024-09-21T09:59:11.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315170+0000", 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14182098500000001 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315183+0000", 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.295e-05 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315467+0000", 2024-09-21T09:59:11.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028378399999999998 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315586+0000", 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011896 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3", 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315698+0000", 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112143 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.322575+0000", 2024-09-21T09:59:11.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068772099999999999 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.322621+0000", 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6007000000000002e-05 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.322631+0000", 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0396e-05 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T09:59:11.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.322676+0000", 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4453999999999998e-05 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.322716+0000", 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0494000000000002e-05 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.34674.0:159232 6.8 e1117/1100 ondisk, result = 0)", 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:03.175349+0000", 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 8.088046018, 2024-09-21T09:59:11.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.140083866, 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.175349+0000", 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.175349+0000", 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.175349+0000", 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.175352+0000", 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6030000000000001e-06 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315184+0000", 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.13983190200000001 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315194+0000", 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0434999999999999e-05 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315302+0000", 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107977 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315311+0000", 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4679999999999992e-06 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T09:59:11.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:03.315433+0000", 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121991 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T09:59:11.382 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T09:59:11.382 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:11.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:11.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:11.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:11.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:11.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:11.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:11.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:11.849 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:11.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:11 smithi191 ceph-mon[102206]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 5484/198246 objects degraded (2.766%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:11 smithi076 ceph-mon[106482]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 5484/198246 objects degraded (2.766%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:11 smithi076 ceph-mon[110963]: pgmap v1257: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 5484/198246 objects degraded (2.766%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:11.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:11.984 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:11.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:12.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:12.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.123 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:12.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:12.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T09:59:12.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:12.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:12.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:12.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:12.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:12.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:12.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:12.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:12.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:12.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:12.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:12.871 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:12.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:12.957 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:59:12.957 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:12.972 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:13.004 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:59:13.005 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:13.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:13.046 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:59:13.046 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:13.077 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:59:13.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:13.108 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:59:13.108 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:13.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:13.142 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:59:13.143 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:13.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:13.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:13.173 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:59:13.174 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:13.206 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:59:13.207 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:13.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:13.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:13.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:13.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:13.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:13.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:13.559 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:59:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:59:13] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.51.0" 2024-09-21T09:59:13.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:13.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:13.688 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:13.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.819 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:13.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:13 smithi191 ceph-mon[102206]: pgmap v1258: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5244/190380 objects degraded (2.754%); 0 B/s, 64 objects/s recovering 2024-09-21T09:59:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3137022910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:13 smithi076 ceph-mon[106482]: pgmap v1258: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5244/190380 objects degraded (2.754%); 0 B/s, 64 objects/s recovering 2024-09-21T09:59:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3137022910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:13 smithi076 ceph-mon[110963]: pgmap v1258: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5244/190380 objects degraded (2.754%); 0 B/s, 64 objects/s recovering 2024-09-21T09:59:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3137022910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:13.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:13.965 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:13.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:14.066 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:14.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:14.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:14.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:14.345 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:14.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:14.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:14.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:14.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:14.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:14.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.644 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:14.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:14.745 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:14.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 5244/190380 objects degraded (2.754%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 5244/190380 objects degraded (2.754%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 5244/190380 objects degraded (2.754%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:14.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:14.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:14.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:15.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:15.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:15.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:15.134 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:15.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:15.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:15.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:15.400 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:15.400 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:15.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:15.479 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:15.581 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.659 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:15.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:15.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:15.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:15.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:15.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:15.933 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:59:15.933 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5244/187422 objects degraded (2.798%), 4 pgs degraded, 4 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":858,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1118,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":8,"osd_in_since":1726912604,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":62474,"data_bytes":4093122557,"bytes_used":14446366720,"bytes_avail":753513918464,"bytes_total":767960285184,"degraded_objects":5244,"degraded_total":187422,"degraded_ratio":0.027979639530044498,"recovering_objects_per_sec":58,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":619,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":697},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":25,"modified":"2024-09-21T09:58:56.477511+0000","services":{}},"progress_events":{}} 2024-09-21T09:59:15.960 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:15.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:16.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:16.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:16.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[106482]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 5244/187422 objects degraded (2.798%); 0 B/s, 58 objects/s recovering 2024-09-21T09:59:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[110963]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 5244/187422 objects degraded (2.798%); 0 B/s, 58 objects/s recovering 2024-09-21T09:59:16.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:16.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:16.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:16.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:16.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:59:16.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:16.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:16.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:15 smithi191 ceph-mon[102206]: pgmap v1259: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 5244/187422 objects degraded (2.798%); 0 B/s, 58 objects/s recovering 2024-09-21T09:59:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:16.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.502 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:16.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:16.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.646 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:16.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:16.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:16.769 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:16.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:16.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:16.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:16.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:16.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:17.026 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:17.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:17.164 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3468839411' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:59:17.164 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3468839411' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:59:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:17.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:17.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:17.300 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:17.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:17.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3468839411' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:59:17.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:17.429 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:17.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:17.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:17.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:17.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:17.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:17.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:17.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:17.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:17.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:17.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:17.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:18.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:18.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:17 smithi076 ceph-mon[110963]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 4990/180420 objects degraded (2.766%); 0 B/s, 72 objects/s recovering 2024-09-21T09:59:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:17 smithi076 ceph-mon[106482]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 4990/180420 objects degraded (2.766%); 0 B/s, 72 objects/s recovering 2024-09-21T09:59:18.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:18.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:18.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:18.356 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:17 smithi191 ceph-mon[102206]: pgmap v1260: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 4990/180420 objects degraded (2.766%); 0 B/s, 72 objects/s recovering 2024-09-21T09:59:18.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:18.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:18.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:18.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:18.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:18.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:18.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:18.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:18.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:18.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:18.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:19.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:19.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:19.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:19.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:19.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:19.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:19.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:19.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:19.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:19.527 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:19.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.572 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:19.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:19.673 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:19.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:19.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:19.866 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:19.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:19.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:19.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:20.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:20.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:19 smithi076 ceph-mon[106482]: pgmap v1261: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 4990/177414 objects degraded (2.813%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 4990/177414 objects degraded (2.813%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:19 smithi076 ceph-mon[110963]: pgmap v1261: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 4990/177414 objects degraded (2.813%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 4990/177414 objects degraded (2.813%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.139 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:20.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:20.219 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T09:59:20.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:59:20.240 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:20.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:19 smithi191 ceph-mon[102206]: pgmap v1261: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 4990/177414 objects degraded (2.813%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 4990/177414 objects degraded (2.813%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:20.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:20.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:20.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:20.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:20.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:20.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:20.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:20.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:20.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:20.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:20.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:21.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:21.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:21.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:21.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:21.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:21.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:21.316 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:21.367 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:21.367 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:21.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:21.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:21.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:21.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:21.622 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:21.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:21.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:21.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:21.851 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:21.912 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T09:59:21.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:21.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:21.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:21.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:22.059 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:21 smithi076 ceph-mon[106482]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4990/175200 objects degraded (2.848%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:22.059 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2050653365' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:22.060 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:22.060 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:21 smithi076 ceph-mon[110963]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4990/175200 objects degraded (2.848%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:22.060 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2050653365' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:22.060 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:22.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:22.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:22.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:22.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:22.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:22.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:22.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:21 smithi191 ceph-mon[102206]: pgmap v1262: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 4990/175200 objects degraded (2.848%); 0 B/s, 41 objects/s recovering 2024-09-21T09:59:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2050653365' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:22.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:22.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:22.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:22.512 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:22.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:22.643 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:22.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T09:59:22.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:22.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:22.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:22.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:22.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:23.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:23.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:22 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:22 smithi076 ceph-mon[106482]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-21T09:59:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:22 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:23.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:23.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:22 smithi076 ceph-mon[110963]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-21T09:59:23.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:23.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:23.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:23.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:23.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:22 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:22 smithi191 ceph-mon[102206]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-21T09:59:23.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:23.458 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:23.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:23.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:23.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:23.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:59:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:59:23] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.51.0" 2024-09-21T09:59:23.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:23.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:23.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:23.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:23.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:23.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:23.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:24.005 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:24.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:24.133 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:24.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:24.264 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:24.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:23 smithi191 ceph-mon[102206]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4728/167268 objects degraded (2.827%); 0 B/s, 51 objects/s recovering 2024-09-21T09:59:24.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:23 smithi076 ceph-mon[106482]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4728/167268 objects degraded (2.827%); 0 B/s, 51 objects/s recovering 2024-09-21T09:59:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:23 smithi076 ceph-mon[110963]: pgmap v1264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4728/167268 objects degraded (2.827%); 0 B/s, 51 objects/s recovering 2024-09-21T09:59:24.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:24.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:24.523 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:24.625 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.664 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:24.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:24.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:24.791 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:24.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:24.892 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:24.935 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:24.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:25.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:25.077 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:25.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:25.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 4728/167268 objects degraded (2.827%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:25.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 4728/167268 objects degraded (2.827%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:25.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:25.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:25.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:25.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:25.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:25.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 4728/167268 objects degraded (2.827%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:25.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:25.461 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:25.562 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:25.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:25.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:25.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:25.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:25.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:25.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:25.849 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:25.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:25.973 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T09:59:25.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:25.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:25.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:26.052 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:26.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:26.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:25 smithi076 ceph-mon[106482]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4728/164328 objects degraded (2.877%); 0 B/s, 51 objects/s recovering 2024-09-21T09:59:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:25 smithi076 ceph-mon[110963]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4728/164328 objects degraded (2.877%); 0 B/s, 51 objects/s recovering 2024-09-21T09:59:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:26.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:26.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:26.362 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:25 smithi191 ceph-mon[102206]: pgmap v1265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 4728/164328 objects degraded (2.877%); 0 B/s, 51 objects/s recovering 2024-09-21T09:59:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:26.388 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:59:26.389 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:26.390 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:26.423 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:59:26.423 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:26.457 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:59:26.457 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:26.460 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:59:26.467 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:59:26.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:26.502 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:59:26.502 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:26.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:26.542 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:59:26.543 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:26.580 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:59:26.580 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:26.617 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:59:26.617 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:26.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:26.648 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:59:26.648 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:26.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:26.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:26.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:26.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:26.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:26.916 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:26.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:27.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:27.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:27.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3332797067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:27.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3332797067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:27.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:27.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:27.313 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:27.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:27.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:27.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3332797067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:27.453 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:27.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:27.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:27.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:27.737 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:27.768 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:27.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:27.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:27.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:27.995 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:27.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:28.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:28.132 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:28.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:28.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:28.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:28.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:28.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:28.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:27 smithi191 ceph-mon[102206]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:27 smithi191 ceph-mon[102206]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4437/156588 objects degraded (2.834%); 0 B/s, 55 objects/s recovering 2024-09-21T09:59:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:27 smithi076 ceph-mon[106482]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:27 smithi076 ceph-mon[106482]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4437/156588 objects degraded (2.834%); 0 B/s, 55 objects/s recovering 2024-09-21T09:59:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:27 smithi076 ceph-mon[110963]: from='client.34863 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:27 smithi076 ceph-mon[110963]: pgmap v1266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4437/156588 objects degraded (2.834%); 0 B/s, 55 objects/s recovering 2024-09-21T09:59:28.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:28.410 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:28.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:28.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:28.544 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1265,"stamp":"2024-09-21T09:59:24.499641+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3588626429,"num_objects":54776,"num_object_clones":0,"num_object_copies":164328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4728,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54776,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":173337,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9631,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142166,"ondisk_log_size":142166,"up":267,"acting":263,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12557920,"kb_used_data":11361884,"kb_used_omap":195,"kb_used_meta":1195388,"kb_avail":737403296,"statfs":{"total":767960285184,"available":755100975104,"internally_reserved":0,"allocated":11634569216,"data_stored":11618308363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1224077444},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504496128,"num_objects":-7698,"num_object_clones":0,"num_object_copies":-23094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7698,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":516,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006298"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.930762+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T09:59:21.930762+0000","last_peered":"2024-09-21T09:59:21.930762+0000","last_clean":"2024-09-21T09:59:21.930762+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T09:59:21.930762+0000","last_undegraded":"2024-09-21T09:59:21.930762+0000","last_fullsized":"2024-09-21T09:59:21.930762+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1046,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.847563+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:59:02.847563+0000","last_peered":"2024-09-21T09:59:02.847563+0000","last_clean":"2024-09-21T09:59:02.847563+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:59:02.847563+0000","last_undegraded":"2024-09-21T09:59:02.847563+0000","last_fullsized":"2024-09-21T09:59:02.847563+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1077,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.877065+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:59:02.877065+0000","last_peered":"2024-09-21T09:59:02.877065+0000","last_clean":"2024-09-21T09:59:02.877065+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:59:02.877065+0000","last_undegraded":"2024-09-21T09:59:02.877065+0000","last_fullsized":"2024-09-21T09:59:02.877065+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1429,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.921022+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T09:59:21.921022+0000","last_peered":"2024-09-21T09:59:21.921022+0000","last_clean":"2024-09-21T09:59:21.921022+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T09:59:21.921022+0000","last_undegraded":"2024-09-21T09:59:21.921022+0000","last_fullsized":"2024-09-21T09:59:21.921022+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1429,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.918423+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T09:59:21.918423+0000","last_peered":"2024-09-21T09:59:21.918423+0000","last_clean":"2024-09-21T09:59:21.918423+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T09:59:21.918423+0000","last_undegraded":"2024-09-21T09:59:21.918423+0000","last_fullsized":"2024-09-21T09:59:21.918423+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1029,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.847312+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:59:02.847312+0000","last_peered":"2024-09-21T09:59:02.847312+0000","last_clean":"2024-09-21T09:59:02.847312+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:59:02.847312+0000","last_undegraded":"2024-09-21T09:59:02.847312+0000","last_fullsized":"2024-09-21T09:59:02.847312+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.929884+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T09:59:21.929884+0000","last_peered":"2024-09-21T09:59:21.929884+0000","last_clean":"2024-09-21T09:59:21.929884+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T09:59:21.929884+0000","last_undegraded":"2024-09-21T09:59:21.929884+0000","last_fullsized":"2024-09-21T09:59:21.929884+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1056,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.849628+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:59:02.849628+0000","last_peered":"2024-09-21T09:59:02.849628+0000","last_clean":"2024-09-21T09:59:02.849628+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:59:02.849628+0000","last_undegraded":"2024-09-21T09:59:02.849628+0000","last_fullsized":"2024-09-21T09:59:02.849628+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1118'10634","reported_seq":10675,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.460486+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T09:59:21.460486+0000","last_peered":"2024-09-21T09:59:21.460486+0000","last_clean":"2024-09-21T09:59:21.460486+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T09:59:21.460486+0000","last_undegraded":"2024-09-21T09:59:21.460486+0000","last_fullsized":"2024-09-21T09:59:21.460486+0000","mapping_epoch":1086,"log_start":"1089'600","ondisk_log_start":"1089'600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10034,"log_dups_size":0,"ondisk_log_size":10034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10634,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1046,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841708+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:59:03.841708+0000","last_peered":"2024-09-21T09:59:03.841708+0000","last_clean":"2024-09-21T09:59:03.841708+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:59:03.841708+0000","last_undegraded":"2024-09-21T09:59:03.841708+0000","last_fullsized":"2024-09-21T09:59:03.841708+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1045,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.876896+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:59:02.876896+0000","last_peered":"2024-09-21T09:59:02.876896+0000","last_clean":"2024-09-21T09:59:02.876896+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:59:02.876896+0000","last_undegraded":"2024-09-21T09:59:02.876896+0000","last_fullsized":"2024-09-21T09:59:02.876896+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1119'10850","reported_seq":13800,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:23.443041+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T09:59:23.443041+0000","last_peered":"2024-09-21T09:59:23.443041+0000","last_clean":"2024-09-21T09:59:23.443041+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T09:59:23.443041+0000","last_undegraded":"2024-09-21T09:59:23.443041+0000","last_fullsized":"2024-09-21T09:59:23.443041+0000","mapping_epoch":1108,"log_start":"1098'3500","ondisk_log_start":"1098'3500","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214040576,"num_objects":3266,"num_object_clones":0,"num_object_copies":9798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10850,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1047,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841874+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:59:03.841874+0000","last_peered":"2024-09-21T09:59:03.841874+0000","last_clean":"2024-09-21T09:59:03.841874+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:59:03.841874+0000","last_undegraded":"2024-09-21T09:59:03.841874+0000","last_fullsized":"2024-09-21T09:59:03.841874+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1099,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840150+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T09:59:02.840150+0000","last_peered":"2024-09-21T09:59:02.840150+0000","last_clean":"2024-09-21T09:59:02.840150+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T09:59:02.840150+0000","last_undegraded":"2024-09-21T09:59:02.840150+0000","last_fullsized":"2024-09-21T09:59:02.840150+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1118'10885","reported_seq":13811,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.402463+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T09:59:21.402463+0000","last_peered":"2024-09-21T09:59:21.402463+0000","last_clean":"2024-09-21T09:59:21.402463+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T09:59:21.402463+0000","last_undegraded":"2024-09-21T09:59:21.402463+0000","last_fullsized":"2024-09-21T09:59:21.402463+0000","mapping_epoch":1100,"log_start":"1095'2700","ondisk_log_start":"1095'2700","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8185,"log_dups_size":0,"ondisk_log_size":8185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229834752,"num_objects":3507,"num_object_clones":0,"num_object_copies":10521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10885,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1031,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841225+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:59:03.841225+0000","last_peered":"2024-09-21T09:59:03.841225+0000","last_clean":"2024-09-21T09:59:03.841225+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:59:03.841225+0000","last_undegraded":"2024-09-21T09:59:03.841225+0000","last_fullsized":"2024-09-21T09:59:03.841225+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1048,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.849780+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:59:02.849780+0000","last_peered":"2024-09-21T09:59:02.849780+0000","last_clean":"2024-09-21T09:59:02.849780+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:59:02.849780+0000","last_undegraded":"2024-09-21T09:59:02.849780+0000","last_fullsized":"2024-09-21T09:59:02.849780+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1119'10971","reported_seq":11049,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:23.981756+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T09:59:23.981756+0000","last_peered":"2024-09-21T09:59:23.981756+0000","last_clean":"2024-09-21T09:59:23.981756+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T09:59:23.981756+0000","last_undegraded":"2024-09-21T09:59:23.981756+0000","last_fullsized":"2024-09-21T09:59:23.981756+0000","mapping_epoch":1086,"log_start":"1091'1200","ondisk_log_start":"1091'1200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9771,"log_dups_size":0,"ondisk_log_size":9771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219480064,"num_objects":3349,"num_object_clones":0,"num_object_copies":10047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10971,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1079,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841299+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:59:03.841299+0000","last_peered":"2024-09-21T09:59:03.841299+0000","last_clean":"2024-09-21T09:59:03.841299+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:59:03.841299+0000","last_undegraded":"2024-09-21T09:59:03.841299+0000","last_fullsized":"2024-09-21T09:59:03.841299+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1417,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920025+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:59:21.920025+0000","last_peered":"2024-09-21T09:59:21.920025+0000","last_clean":"2024-09-21T09:59:21.920025+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:59:21.920025+0000","last_undegraded":"2024-09-21T09:59:21.920025+0000","last_fullsized":"2024-09-21T09:59:21.920025+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1118'10777","reported_seq":10819,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.395356+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T09:59:21.395356+0000","last_peered":"2024-09-21T09:59:21.395356+0000","last_clean":"2024-09-21T09:59:21.395356+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T09:59:21.395356+0000","last_undegraded":"2024-09-21T09:59:21.395356+0000","last_fullsized":"2024-09-21T09:59:21.395356+0000","mapping_epoch":1086,"log_start":"1095'2800","ondisk_log_start":"1095'2800","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7977,"log_dups_size":0,"ondisk_log_size":7977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222101504,"num_objects":3389,"num_object_clones":0,"num_object_copies":10167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10777,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":69,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.919861+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T09:59:21.919861+0000","last_peered":"2024-09-21T09:59:21.919861+0000","last_clean":"2024-09-21T09:59:21.919861+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T09:59:21.919861+0000","last_undegraded":"2024-09-21T09:59:21.919861+0000","last_fullsized":"2024-09-21T09:59:21.919861+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1084,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.843783+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T09:59:03.843783+0000","last_peered":"2024-09-21T09:59:03.843783+0000","last_clean":"2024-09-21T09:59:03.843783+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T09:59:03.843783+0000","last_undegraded":"2024-09-21T09:59:03.843783+0000","last_fullsized":"2024-09-21T09:59:03.843783+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1118'10839","reported_seq":13747,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:20.113611+0000","last_change":"2024-09-21T09:59:03.867059+0000","last_active":"2024-09-21T09:59:20.113611+0000","last_peered":"2024-09-21T09:59:20.113611+0000","last_clean":"2024-09-21T09:59:20.113611+0000","last_became_active":"2024-09-21T09:59:03.844371+0000","last_became_peered":"2024-09-21T09:59:03.844371+0000","last_unstale":"2024-09-21T09:59:20.113611+0000","last_undegraded":"2024-09-21T09:59:20.113611+0000","last_fullsized":"2024-09-21T09:59:20.113611+0000","mapping_epoch":1117,"log_start":"1089'800","ondisk_log_start":"1089'800","created":1086,"last_epoch_clean":1118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:34:30.990234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230752256,"num_objects":3521,"num_object_clones":0,"num_object_copies":10563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10839,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1049,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.849907+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:59:02.849907+0000","last_peered":"2024-09-21T09:59:02.849907+0000","last_clean":"2024-09-21T09:59:02.849907+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:59:02.849907+0000","last_undegraded":"2024-09-21T09:59:02.849907+0000","last_fullsized":"2024-09-21T09:59:02.849907+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1081,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.843764+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:59:03.843764+0000","last_peered":"2024-09-21T09:59:03.843764+0000","last_clean":"2024-09-21T09:59:03.843764+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:59:03.843764+0000","last_undegraded":"2024-09-21T09:59:03.843764+0000","last_fullsized":"2024-09-21T09:59:03.843764+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1118'10742","reported_seq":10789,"reported_epoch":1118,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:21.423133+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T09:59:21.423133+0000","last_peered":"2024-09-21T09:59:21.423133+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T09:59:21.423133+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"1089'700","ondisk_log_start":"1089'700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10042,"log_dups_size":0,"ondisk_log_size":10042,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224919552,"num_objects":3432,"num_object_clones":0,"num_object_copies":10296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10742,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":1956}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1067,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840247+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:59:02.840247+0000","last_peered":"2024-09-21T09:59:02.840247+0000","last_clean":"2024-09-21T09:59:02.840247+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:59:02.840247+0000","last_undegraded":"2024-09-21T09:59:02.840247+0000","last_fullsized":"2024-09-21T09:59:02.840247+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1075,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.843588+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T09:59:03.843588+0000","last_peered":"2024-09-21T09:59:03.843588+0000","last_clean":"2024-09-21T09:59:03.843588+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T09:59:03.843588+0000","last_undegraded":"2024-09-21T09:59:03.843588+0000","last_fullsized":"2024-09-21T09:59:03.843588+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1118'10823","reported_seq":10866,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.455851+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T09:59:21.455851+0000","last_peered":"2024-09-21T09:59:21.455851+0000","last_clean":"2024-09-21T09:59:21.455851+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T09:59:21.455851+0000","last_undegraded":"2024-09-21T09:59:21.455851+0000","last_fullsized":"2024-09-21T09:59:21.455851+0000","mapping_epoch":1086,"log_start":"1089'800","ondisk_log_start":"1089'800","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10023,"log_dups_size":0,"ondisk_log_size":10023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227934232,"num_objects":3479,"num_object_clones":0,"num_object_copies":10437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3479,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10823,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920228+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T09:59:21.920228+0000","last_peered":"2024-09-21T09:59:21.920228+0000","last_clean":"2024-09-21T09:59:21.920228+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T09:59:21.920228+0000","last_undegraded":"2024-09-21T09:59:21.920228+0000","last_fullsized":"2024-09-21T09:59:21.920228+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1403,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.907825+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:59:21.907825+0000","last_peered":"2024-09-21T09:59:21.907825+0000","last_clean":"2024-09-21T09:59:21.907825+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:59:21.907825+0000","last_undegraded":"2024-09-21T09:59:21.907825+0000","last_fullsized":"2024-09-21T09:59:21.907825+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1118'10565","reported_seq":13395,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.442324+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T09:59:21.442324+0000","last_peered":"2024-09-21T09:59:21.442324+0000","last_clean":"2024-09-21T09:59:21.442324+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T09:59:21.442324+0000","last_undegraded":"2024-09-21T09:59:21.442324+0000","last_fullsized":"2024-09-21T09:59:21.442324+0000","mapping_epoch":1113,"log_start":"1089'500","ondisk_log_start":"1089'500","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10065,"log_dups_size":0,"ondisk_log_size":10065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225640448,"num_objects":3443,"num_object_clones":0,"num_object_copies":10329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10565,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1376,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.851118+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:59:02.851118+0000","last_peered":"2024-09-21T09:59:02.851118+0000","last_clean":"2024-09-21T09:59:02.851118+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:59:02.851118+0000","last_undegraded":"2024-09-21T09:59:02.851118+0000","last_fullsized":"2024-09-21T09:59:02.851118+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1075,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.876991+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:59:02.876991+0000","last_peered":"2024-09-21T09:59:02.876991+0000","last_clean":"2024-09-21T09:59:02.876991+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:59:02.876991+0000","last_undegraded":"2024-09-21T09:59:02.876991+0000","last_fullsized":"2024-09-21T09:59:02.876991+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920405+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T09:59:21.920405+0000","last_peered":"2024-09-21T09:59:21.920405+0000","last_clean":"2024-09-21T09:59:21.920405+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T09:59:21.920405+0000","last_undegraded":"2024-09-21T09:59:21.920405+0000","last_fullsized":"2024-09-21T09:59:21.920405+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1033,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.852202+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:59:02.852202+0000","last_peered":"2024-09-21T09:59:02.852202+0000","last_clean":"2024-09-21T09:59:02.852202+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:59:02.852202+0000","last_undegraded":"2024-09-21T09:59:02.852202+0000","last_fullsized":"2024-09-21T09:59:02.852202+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1119'10780","reported_seq":13826,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.957791+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T09:59:21.957791+0000","last_peered":"2024-09-21T09:59:21.957791+0000","last_clean":"2024-09-21T09:59:21.957791+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T09:59:21.957791+0000","last_undegraded":"2024-09-21T09:59:21.957791+0000","last_fullsized":"2024-09-21T09:59:21.957791+0000","mapping_epoch":1105,"log_start":"1095'2200","ondisk_log_start":"1095'2200","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8580,"log_dups_size":0,"ondisk_log_size":8580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221380608,"num_objects":3378,"num_object_clones":0,"num_object_copies":10134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10780,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1073,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.851011+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:59:02.851011+0000","last_peered":"2024-09-21T09:59:02.851011+0000","last_clean":"2024-09-21T09:59:02.851011+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:59:02.851011+0000","last_undegraded":"2024-09-21T09:59:02.851011+0000","last_fullsized":"2024-09-21T09:59:02.851011+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1429,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920407+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T09:59:21.920407+0000","last_peered":"2024-09-21T09:59:21.920407+0000","last_clean":"2024-09-21T09:59:21.920407+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T09:59:21.920407+0000","last_undegraded":"2024-09-21T09:59:21.920407+0000","last_fullsized":"2024-09-21T09:59:21.920407+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1427,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.918590+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:59:21.918590+0000","last_peered":"2024-09-21T09:59:21.918590+0000","last_clean":"2024-09-21T09:59:21.918590+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:59:21.918590+0000","last_undegraded":"2024-09-21T09:59:21.918590+0000","last_fullsized":"2024-09-21T09:59:21.918590+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1118'10682","reported_seq":12780,"reported_epoch":1118,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:21.395588+0000","last_change":"2024-09-21T09:59:02.617400+0000","last_active":"2024-09-21T09:59:21.395588+0000","last_peered":"2024-09-21T09:59:21.395588+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:56:46.335218+0000","last_became_peered":"2024-09-21T09:56:46.335218+0000","last_unstale":"2024-09-21T09:59:21.395588+0000","last_undegraded":"2024-09-21T09:56:46.260383+0000","last_fullsized":"2024-09-21T09:56:46.244582+0000","mapping_epoch":1093,"log_start":"1095'2500","ondisk_log_start":"1095'2500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:38:03.383491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227147776,"num_objects":3466,"num_object_clones":0,"num_object_copies":10398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":379,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10682,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1025,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"6,7","objects":3087}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1406,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841308+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T09:59:03.841308+0000","last_peered":"2024-09-21T09:59:03.841308+0000","last_clean":"2024-09-21T09:59:03.841308+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T09:59:03.841308+0000","last_undegraded":"2024-09-21T09:59:03.841308+0000","last_fullsized":"2024-09-21T09:59:03.841308+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1389,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.907340+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:59:21.907340+0000","last_peered":"2024-09-21T09:59:21.907340+0000","last_clean":"2024-09-21T09:59:21.907340+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:59:21.907340+0000","last_undegraded":"2024-09-21T09:59:21.907340+0000","last_fullsized":"2024-09-21T09:59:21.907340+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1397,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.907698+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:59:21.907698+0000","last_peered":"2024-09-21T09:59:21.907698+0000","last_clean":"2024-09-21T09:59:21.907698+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:59:21.907698+0000","last_undegraded":"2024-09-21T09:59:21.907698+0000","last_fullsized":"2024-09-21T09:59:21.907698+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1119'10891","reported_seq":13864,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:23.450557+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T09:59:23.450557+0000","last_peered":"2024-09-21T09:59:23.450557+0000","last_clean":"2024-09-21T09:59:23.450557+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T09:59:23.450557+0000","last_undegraded":"2024-09-21T09:59:23.450557+0000","last_fullsized":"2024-09-21T09:59:23.450557+0000","mapping_epoch":1096,"log_start":"1098'3500","ondisk_log_start":"1098'3500","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7391,"log_dups_size":0,"ondisk_log_size":7391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217120768,"num_objects":3313,"num_object_clones":0,"num_object_copies":9939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10891,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1076,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.876983+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:59:02.876983+0000","last_peered":"2024-09-21T09:59:02.876983+0000","last_clean":"2024-09-21T09:59:02.876983+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:59:02.876983+0000","last_undegraded":"2024-09-21T09:59:02.876983+0000","last_fullsized":"2024-09-21T09:59:02.876983+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1067,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840456+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:59:02.840456+0000","last_peered":"2024-09-21T09:59:02.840456+0000","last_clean":"2024-09-21T09:59:02.840456+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:59:02.840456+0000","last_undegraded":"2024-09-21T09:59:02.840456+0000","last_fullsized":"2024-09-21T09:59:02.840456+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1414,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.908809+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:59:21.908809+0000","last_peered":"2024-09-21T09:59:21.908809+0000","last_clean":"2024-09-21T09:59:21.908809+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:59:21.908809+0000","last_undegraded":"2024-09-21T09:59:21.908809+0000","last_fullsized":"2024-09-21T09:59:21.908809+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1119'10897","reported_seq":10977,"reported_epoch":1119,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:23.998854+0000","last_change":"2024-09-21T09:56:46.343261+0000","last_active":"2024-09-21T09:59:23.998854+0000","last_peered":"2024-09-21T09:59:23.998854+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T09:59:23.998854+0000","last_undegraded":"2024-09-21T09:56:46.247241+0000","last_fullsized":"2024-09-21T09:56:46.239078+0000","mapping_epoch":1093,"log_start":"1091'1200","ondisk_log_start":"1091'1200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9697,"log_dups_size":0,"ondisk_log_size":9697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214368256,"num_objects":3271,"num_object_clones":0,"num_object_copies":9813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10897,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":1810}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1094,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.877139+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:59:02.877139+0000","last_peered":"2024-09-21T09:59:02.877139+0000","last_clean":"2024-09-21T09:59:02.877139+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:59:02.877139+0000","last_undegraded":"2024-09-21T09:59:02.877139+0000","last_fullsized":"2024-09-21T09:59:02.877139+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1411,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.908779+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:59:21.908779+0000","last_peered":"2024-09-21T09:59:21.908779+0000","last_clean":"2024-09-21T09:59:21.908779+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:59:21.908779+0000","last_undegraded":"2024-09-21T09:59:21.908779+0000","last_fullsized":"2024-09-21T09:59:21.908779+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920849+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T09:59:21.920849+0000","last_peered":"2024-09-21T09:59:21.920849+0000","last_clean":"2024-09-21T09:59:21.920849+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T09:59:21.920849+0000","last_undegraded":"2024-09-21T09:59:21.920849+0000","last_fullsized":"2024-09-21T09:59:21.920849+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1119'11096","reported_seq":11174,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:22.023796+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T09:59:22.023796+0000","last_peered":"2024-09-21T09:59:22.023796+0000","last_clean":"2024-09-21T09:59:22.023796+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T09:59:22.023796+0000","last_undegraded":"2024-09-21T09:59:22.023796+0000","last_fullsized":"2024-09-21T09:59:22.023796+0000","mapping_epoch":1086,"log_start":"1095'2500","ondisk_log_start":"1095'2500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8596,"log_dups_size":0,"ondisk_log_size":8596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228065280,"num_objects":3480,"num_object_clones":0,"num_object_copies":10440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11096,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1388,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.909139+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:59:21.909139+0000","last_peered":"2024-09-21T09:59:21.909139+0000","last_clean":"2024-09-21T09:59:21.909139+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:59:21.909139+0000","last_undegraded":"2024-09-21T09:59:21.909139+0000","last_fullsized":"2024-09-21T09:59:21.909139+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1058,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.847881+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:59:02.847881+0000","last_peered":"2024-09-21T09:59:02.847881+0000","last_clean":"2024-09-21T09:59:02.847881+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:59:02.847881+0000","last_undegraded":"2024-09-21T09:59:02.847881+0000","last_fullsized":"2024-09-21T09:59:02.847881+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1405,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920119+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:59:21.920119+0000","last_peered":"2024-09-21T09:59:21.920119+0000","last_clean":"2024-09-21T09:59:21.920119+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:59:21.920119+0000","last_undegraded":"2024-09-21T09:59:21.920119+0000","last_fullsized":"2024-09-21T09:59:21.920119+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1118'10802","reported_seq":10852,"reported_epoch":1118,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:21.329678+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T09:59:21.329678+0000","last_peered":"2024-09-21T09:59:21.329678+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T09:59:21.329678+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1096'2900","ondisk_log_start":"1096'2900","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10802,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":2122}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":69,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.929984+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T09:59:21.929984+0000","last_peered":"2024-09-21T09:59:21.929984+0000","last_clean":"2024-09-21T09:59:21.929984+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T09:59:21.929984+0000","last_undegraded":"2024-09-21T09:59:21.929984+0000","last_fullsized":"2024-09-21T09:59:21.929984+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1389,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.909098+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:59:21.909098+0000","last_peered":"2024-09-21T09:59:21.909098+0000","last_clean":"2024-09-21T09:59:21.909098+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:59:21.909098+0000","last_undegraded":"2024-09-21T09:59:21.909098+0000","last_fullsized":"2024-09-21T09:59:21.909098+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.919590+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T09:59:21.919590+0000","last_peered":"2024-09-21T09:59:21.919590+0000","last_clean":"2024-09-21T09:59:21.919590+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T09:59:21.919590+0000","last_undegraded":"2024-09-21T09:59:21.919590+0000","last_fullsized":"2024-09-21T09:59:21.919590+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1118'10824","reported_seq":10866,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.293098+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T09:59:21.293098+0000","last_peered":"2024-09-21T09:59:21.293098+0000","last_clean":"2024-09-21T09:59:21.293098+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T09:59:21.293098+0000","last_undegraded":"2024-09-21T09:59:21.293098+0000","last_fullsized":"2024-09-21T09:59:21.293098+0000","mapping_epoch":1086,"log_start":"1095'2700","ondisk_log_start":"1095'2700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8124,"log_dups_size":0,"ondisk_log_size":8124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226230272,"num_objects":3452,"num_object_clones":0,"num_object_copies":10356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10824,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1419,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.918496+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:59:21.918496+0000","last_peered":"2024-09-21T09:59:21.918496+0000","last_clean":"2024-09-21T09:59:21.918496+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:59:21.918496+0000","last_undegraded":"2024-09-21T09:59:21.918496+0000","last_fullsized":"2024-09-21T09:59:21.918496+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1062,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.881176+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:59:03.881176+0000","last_peered":"2024-09-21T09:59:03.881176+0000","last_clean":"2024-09-21T09:59:03.881176+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:59:03.881176+0000","last_undegraded":"2024-09-21T09:59:03.881176+0000","last_fullsized":"2024-09-21T09:59:03.881176+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1051,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840214+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:59:02.840214+0000","last_peered":"2024-09-21T09:59:02.840214+0000","last_clean":"2024-09-21T09:59:02.840214+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:59:02.840214+0000","last_undegraded":"2024-09-21T09:59:02.840214+0000","last_fullsized":"2024-09-21T09:59:02.840214+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1065,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840356+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:59:02.840356+0000","last_peered":"2024-09-21T09:59:02.840356+0000","last_clean":"2024-09-21T09:59:02.840356+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:59:02.840356+0000","last_undegraded":"2024-09-21T09:59:02.840356+0000","last_fullsized":"2024-09-21T09:59:02.840356+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1074,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840470+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:59:02.840470+0000","last_peered":"2024-09-21T09:59:02.840470+0000","last_clean":"2024-09-21T09:59:02.840470+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:59:02.840470+0000","last_undegraded":"2024-09-21T09:59:02.840470+0000","last_fullsized":"2024-09-21T09:59:02.840470+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1389,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.907555+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:59:21.907555+0000","last_peered":"2024-09-21T09:59:21.907555+0000","last_clean":"2024-09-21T09:59:21.907555+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:59:21.907555+0000","last_undegraded":"2024-09-21T09:59:21.907555+0000","last_fullsized":"2024-09-21T09:59:21.907555+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1077,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.847708+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T09:59:02.847708+0000","last_peered":"2024-09-21T09:59:02.847708+0000","last_clean":"2024-09-21T09:59:02.847708+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T09:59:02.847708+0000","last_undegraded":"2024-09-21T09:59:02.847708+0000","last_fullsized":"2024-09-21T09:59:02.847708+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1060,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.877143+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:59:02.877143+0000","last_peered":"2024-09-21T09:59:02.877143+0000","last_clean":"2024-09-21T09:59:02.877143+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:59:02.877143+0000","last_undegraded":"2024-09-21T09:59:02.877143+0000","last_fullsized":"2024-09-21T09:59:02.877143+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.917891+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T09:59:21.917891+0000","last_peered":"2024-09-21T09:59:21.917891+0000","last_clean":"2024-09-21T09:59:21.917891+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T09:59:21.917891+0000","last_undegraded":"2024-09-21T09:59:21.917891+0000","last_fullsized":"2024-09-21T09:59:21.917891+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1428,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920646+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T09:59:21.920646+0000","last_peered":"2024-09-21T09:59:21.920646+0000","last_clean":"2024-09-21T09:59:21.920646+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T09:59:21.920646+0000","last_undegraded":"2024-09-21T09:59:21.920646+0000","last_fullsized":"2024-09-21T09:59:21.920646+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.919774+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T09:59:21.919774+0000","last_peered":"2024-09-21T09:59:21.919774+0000","last_clean":"2024-09-21T09:59:21.919774+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T09:59:21.919774+0000","last_undegraded":"2024-09-21T09:59:21.919774+0000","last_fullsized":"2024-09-21T09:59:21.919774+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1057,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.850174+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:59:02.850174+0000","last_peered":"2024-09-21T09:59:02.850174+0000","last_clean":"2024-09-21T09:59:02.850174+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:59:02.850174+0000","last_undegraded":"2024-09-21T09:59:02.850174+0000","last_fullsized":"2024-09-21T09:59:02.850174+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.917898+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T09:59:21.917898+0000","last_peered":"2024-09-21T09:59:21.917898+0000","last_clean":"2024-09-21T09:59:21.917898+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T09:59:21.917898+0000","last_undegraded":"2024-09-21T09:59:21.917898+0000","last_fullsized":"2024-09-21T09:59:21.917898+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1431,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.918332+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T09:59:21.918332+0000","last_peered":"2024-09-21T09:59:21.918332+0000","last_clean":"2024-09-21T09:59:21.918332+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T09:59:21.918332+0000","last_undegraded":"2024-09-21T09:59:21.918332+0000","last_fullsized":"2024-09-21T09:59:21.918332+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1402,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.907587+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:59:21.907587+0000","last_peered":"2024-09-21T09:59:21.907587+0000","last_clean":"2024-09-21T09:59:21.907587+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:59:21.907587+0000","last_undegraded":"2024-09-21T09:59:21.907587+0000","last_fullsized":"2024-09-21T09:59:21.907587+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.926836+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T09:59:21.926836+0000","last_peered":"2024-09-21T09:59:21.926836+0000","last_clean":"2024-09-21T09:59:21.926836+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T09:59:21.926836+0000","last_undegraded":"2024-09-21T09:59:21.926836+0000","last_fullsized":"2024-09-21T09:59:21.926836+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1082,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.843280+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:59:03.843280+0000","last_peered":"2024-09-21T09:59:03.843280+0000","last_clean":"2024-09-21T09:59:03.843280+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:59:03.843280+0000","last_undegraded":"2024-09-21T09:59:03.843280+0000","last_fullsized":"2024-09-21T09:59:03.843280+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1382,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841380+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T09:59:03.841380+0000","last_peered":"2024-09-21T09:59:03.841380+0000","last_clean":"2024-09-21T09:59:03.841380+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T09:59:03.841380+0000","last_undegraded":"2024-09-21T09:59:03.841380+0000","last_fullsized":"2024-09-21T09:59:03.841380+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1407,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920203+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:59:21.920203+0000","last_peered":"2024-09-21T09:59:21.920203+0000","last_clean":"2024-09-21T09:59:21.920203+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:59:21.920203+0000","last_undegraded":"2024-09-21T09:59:21.920203+0000","last_fullsized":"2024-09-21T09:59:21.920203+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1098,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.843874+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T09:59:03.843874+0000","last_peered":"2024-09-21T09:59:03.843874+0000","last_clean":"2024-09-21T09:59:03.843874+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T09:59:03.843874+0000","last_undegraded":"2024-09-21T09:59:03.843874+0000","last_fullsized":"2024-09-21T09:59:03.843874+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1062,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841167+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:59:03.841167+0000","last_peered":"2024-09-21T09:59:03.841167+0000","last_clean":"2024-09-21T09:59:03.841167+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:59:03.841167+0000","last_undegraded":"2024-09-21T09:59:03.841167+0000","last_fullsized":"2024-09-21T09:59:03.841167+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1098,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.843132+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T09:59:03.843132+0000","last_peered":"2024-09-21T09:59:03.843132+0000","last_clean":"2024-09-21T09:59:03.843132+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T09:59:03.843132+0000","last_undegraded":"2024-09-21T09:59:03.843132+0000","last_fullsized":"2024-09-21T09:59:03.843132+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1047,"reported_epoch":1118,"state":"active+clean","last_fresh":"2024-09-21T09:59:03.841930+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:59:03.841930+0000","last_peered":"2024-09-21T09:59:03.841930+0000","last_clean":"2024-09-21T09:59:03.841930+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:59:03.841930+0000","last_undegraded":"2024-09-21T09:59:03.841930+0000","last_fullsized":"2024-09-21T09:59:03.841930+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.918044+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T09:59:21.918044+0000","last_peered":"2024-09-21T09:59:21.918044+0000","last_clean":"2024-09-21T09:59:21.918044+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T09:59:21.918044+0000","last_undegraded":"2024-09-21T09:59:21.918044+0000","last_fullsized":"2024-09-21T09:59:21.918044+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":65,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.929364+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T09:59:21.929364+0000","last_peered":"2024-09-21T09:59:21.929364+0000","last_clean":"2024-09-21T09:59:21.929364+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T09:59:21.929364+0000","last_undegraded":"2024-09-21T09:59:21.929364+0000","last_fullsized":"2024-09-21T09:59:21.929364+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1074,"reported_epoch":1117,"state":"active+clean","last_fresh":"2024-09-21T09:59:02.840073+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:59:02.840073+0000","last_peered":"2024-09-21T09:59:02.840073+0000","last_clean":"2024-09-21T09:59:02.840073+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:59:02.840073+0000","last_undegraded":"2024-09-21T09:59:02.840073+0000","last_fullsized":"2024-09-21T09:59:02.840073+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1422,"reported_epoch":1119,"state":"active+clean","last_fresh":"2024-09-21T09:59:21.920724+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:59:21.920724+0000","last_peered":"2024-09-21T09:59:21.920724+0000","last_clean":"2024-09-21T09:59:21.920724+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:59:21.920724+0000","last_undegraded":"2024-09-21T09:59:21.920724+0000","last_fullsized":"2024-09-21T09:59:21.920724+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3588030488,"num_objects":54750,"num_object_clones":0,"num_object_copies":164250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4728,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54750,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173058,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9618,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11501187072,"data_stored":11501174856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141958,"ondisk_log_size":141958,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234282,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1579544,"kb_used_data":1427856,"kb_used_omap":1,"kb_used_meta":151678,"kb_avail":92165608,"statfs":{"total":95995035648,"available":94377582592,"internally_reserved":0,"allocated":1462124544,"data_stored":1460080244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":155318731},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495919,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1723280,"kb_used_data":1561736,"kb_used_omap":1,"kb_used_meta":161470,"kb_avail":92021872,"statfs":{"total":95995035648,"available":94230396928,"internally_reserved":0,"allocated":1599217664,"data_stored":1597181429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":165345735},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757557,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357192,"kb_used_data":1203904,"kb_used_omap":1,"kb_used_meta":153278,"kb_avail":92387960,"statfs":{"total":95995035648,"available":94605271040,"internally_reserved":0,"allocated":1232797696,"data_stored":1230764959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":156957126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019193,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259984,"kb_used_data":1120584,"kb_used_omap":1,"kb_used_meta":139390,"kb_avail":92485168,"statfs":{"total":95995035648,"available":94704812032,"internally_reserved":0,"allocated":1147478016,"data_stored":1145441294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":142735814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888490,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2385288,"kb_used_data":2158784,"kb_used_omap":34,"kb_used_meta":226333,"kb_avail":91359864,"statfs":{"total":95995035648,"available":93552500736,"internally_reserved":0,"allocated":2210594816,"data_stored":2208569888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":231766003},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967728,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2084352,"kb_used_data":1921592,"kb_used_omap":41,"kb_used_meta":162518,"kb_avail":91660800,"statfs":{"total":95995035648,"available":93860659200,"internally_reserved":0,"allocated":1967710208,"data_stored":1965685596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":166418847},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046967,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1206784,"kb_used_data":1106680,"kb_used_omap":41,"kb_used_meta":100054,"kb_avail":92538368,"statfs":{"total":95995035648,"available":94759288832,"internally_reserved":0,"allocated":1133240320,"data_stored":1131215135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":102456233},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126206,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":961496,"kb_used_data":860748,"kb_used_omap":72,"kb_used_meta":100663,"kb_avail":92783656,"statfs":{"total":95995035648,"available":95010463744,"internally_reserved":0,"allocated":881405952,"data_stored":879369818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":103078955},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":870584320,"data_stored":870580248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1114705920,"data_stored":1114701848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1937440768,"data_stored":1937440768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2186936320,"data_stored":2186936320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1130758144,"data_stored":1130758144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1224347648,"data_stored":1224343576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1584267264,"data_stored":1584267264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1452146688,"data_stored":1452146688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:59:28.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:28.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:28.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:28.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:28.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:28.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:28.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:28.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:29.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:29.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:29.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:29.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:29.154 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:29.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:29.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:29.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:29.330 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:28 smithi076 ceph-mon[106482]: osdmap e1120: 8 total, 8 up, 8 in 2024-09-21T09:59:29.330 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:28 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 4437/156588 objects degraded (2.834%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:29.331 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:28 smithi076 ceph-mon[110963]: osdmap e1120: 8 total, 8 up, 8 in 2024-09-21T09:59:29.331 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:28 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 4437/156588 objects degraded (2.834%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:29.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:28 smithi191 ceph-mon[102206]: osdmap e1120: 8 total, 8 up, 8 in 2024-09-21T09:59:29.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:28 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 4437/156588 objects degraded (2.834%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:29.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:29.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:29.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:29.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:29.620 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:59:29.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.706 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:29.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:29.807 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:29.878 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:29.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:29.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:29.902 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:59:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T09:59:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T09:59:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T09:59:29.902 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:59:29.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:30.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:30.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:29 smithi076 ceph-mon[106482]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 4437/153765 objects degraded (2.886%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:29 smithi076 ceph-mon[106482]: osdmap e1121: 8 total, 8 up, 8 in 2024-09-21T09:59:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:29 smithi076 ceph-mon[110963]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 4437/153765 objects degraded (2.886%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:29 smithi076 ceph-mon[110963]: osdmap e1121: 8 total, 8 up, 8 in 2024-09-21T09:59:30.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:30.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:30.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:30.299 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:30.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:29 smithi191 ceph-mon[102206]: pgmap v1268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 4437/153765 objects degraded (2.886%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:29 smithi191 ceph-mon[102206]: osdmap e1121: 8 total, 8 up, 8 in 2024-09-21T09:59:30.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:30.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:30.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.583 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:30.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:30.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:30.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:30.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:30.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:30.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:30.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:30.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.000 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:31.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:31.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:31.125 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T09:59:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:31.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:31.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:31.258 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:31.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:31.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.389 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:31.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:31.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:31.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:31.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:59:31.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:31.698 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:31.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:31.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:31.783 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:31.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:31.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:31.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:31.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:32.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:32.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:31 smithi076 ceph-mon[106482]: pgmap v1270: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 4262/151926 objects degraded (2.805%); 0 B/s, 58 objects/s recovering 2024-09-21T09:59:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:31 smithi076 ceph-mon[110963]: pgmap v1270: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 4262/151926 objects degraded (2.805%); 0 B/s, 58 objects/s recovering 2024-09-21T09:59:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:32.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:32.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.270 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:32.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:31 smithi191 ceph-mon[102206]: pgmap v1270: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 4262/151926 objects degraded (2.805%); 0 B/s, 58 objects/s recovering 2024-09-21T09:59:32.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:32.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:32.508 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:32.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:32.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:32.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:32.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:32.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:32.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:32.970 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:32.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:32.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:33.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:33.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:33.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:33.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:33.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:33.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:33.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:33.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:33.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:33.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:33.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:33.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:59:33] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-09-21T09:59:33.635 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:33.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:33.737 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:33.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:33.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:33.908 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:33.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:33.966 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T09:59:34.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.042 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:34.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:34.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[106482]: pgmap v1271: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 4174/145299 objects degraded (2.873%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3008454473' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 4174/145299 objects degraded (2.873%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[110963]: pgmap v1271: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 4174/145299 objects degraded (2.873%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3008454473' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:34.074 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:33 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 4174/145299 objects degraded (2.873%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:34.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:34.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:34.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:34.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:34.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:33 smithi191 ceph-mon[102206]: pgmap v1271: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 4174/145299 objects degraded (2.873%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:34.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:33 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3008454473' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:34.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:33 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T09:59:34.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:33 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 4174/145299 objects degraded (2.873%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:34.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:34.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:34.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:34.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:34.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:34.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:34.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:34.893 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:34.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:34.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:34.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:35.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:35.067 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:35.119 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:35.165 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:35.188 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[106482]: osdmap e1122: 8 total, 8 up, 8 in 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/435180730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[110963]: osdmap e1122: 8 total, 8 up, 8 in 2024-09-21T09:59:35.189 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:34 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/435180730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:35.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:35.300 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:35.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:34 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T09:59:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:34 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T09:59:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:34 smithi191 ceph-mon[102206]: osdmap e1122: 8 total, 8 up, 8 in 2024-09-21T09:59:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:34 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/435180730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:35.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:35.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.433 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:35.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:35.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:35.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:35.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:35.653 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:59:35.653 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:59:35.653 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:59:35.653 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:59:35.653 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:35.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:35.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:35.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:35.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:35.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:35.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:35.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:35.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:35.980 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:35.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:35.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:36.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:36.128 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:36.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.260 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:36.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:36.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:36.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:35 smithi191 ceph-mon[102206]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 3985/142701 objects degraded (2.793%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:35 smithi076 ceph-mon[106482]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 3985/142701 objects degraded (2.793%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:35 smithi076 ceph-mon[110963]: pgmap v1273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 3985/142701 objects degraded (2.793%); 0 B/s, 69 objects/s recovering 2024-09-21T09:59:36.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:36.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:36.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:36.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:36.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:36.639 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:36.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:36.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:36.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:36.871 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:36.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:36.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:36.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:37.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:37.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:37.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:37.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:37.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.175 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:37.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:37.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:37.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:37.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:37.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.434 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:37.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:37.535 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:37.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T09:59:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:37.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:37.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:37.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:37.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:37.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:37.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:37.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:37.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:38.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:38.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:38.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:38.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:38.249 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:38.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:38.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:38 smithi191 ceph-mon[102206]: pgmap v1274: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 3985/135879 objects degraded (2.933%); 0 B/s, 56 objects/s recovering 2024-09-21T09:59:38.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:38 smithi076 ceph-mon[106482]: pgmap v1274: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 3985/135879 objects degraded (2.933%); 0 B/s, 56 objects/s recovering 2024-09-21T09:59:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:38 smithi076 ceph-mon[110963]: pgmap v1274: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 3985/135879 objects degraded (2.933%); 0 B/s, 56 objects/s recovering 2024-09-21T09:59:38.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:38.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:38.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:38.525 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:38.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:38.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:38.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:38.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:38.818 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:38.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:38.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:38.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:38.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:39.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:39.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:39.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:39.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:39.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:39.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:39.355 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3985/135879 objects degraded (2.933%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:39.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3985/135879 objects degraded (2.933%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:39.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3985/135879 objects degraded (2.933%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:39.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:39.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:39.399 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T09:59:39.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:39.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:39.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:39.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:39.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:39.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:39.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:39.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:39.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:39.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:39.936 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:39.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:39.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:40.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:40.066 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:40.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:40.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:40.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:40.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:40.254 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:40 smithi076 ceph-mon[106482]: pgmap v1275: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 3985/134388 objects degraded (2.965%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:40.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:40 smithi076 ceph-mon[110963]: pgmap v1275: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 3985/134388 objects degraded (2.965%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:40.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:40.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:40.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:40 smithi191 ceph-mon[102206]: pgmap v1275: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 3985/134388 objects degraded (2.965%); 0 B/s, 47 objects/s recovering 2024-09-21T09:59:40.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:40.458 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:40.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:40.560 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:40.580 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T09:59:40.580 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T09:59:40.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:40.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:40.699 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:40.742 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:40.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:40.874 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:40.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:40.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:40.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.006 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:41.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:41.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:41.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:41.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:41.250 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:41.271 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:41.271 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:41.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:41.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:41.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T09:59:41.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:41.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.414 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:41.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:41.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:41.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:41.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:41.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:41.818 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:41.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:41.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:41.995 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:41.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:42.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:42.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:42.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:42.286 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:42 smithi076 ceph-mon[106482]: pgmap v1276: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 3813/130905 objects degraded (2.913%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:42.286 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:42 smithi076 ceph-mon[110963]: pgmap v1276: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 3813/130905 objects degraded (2.913%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:42.300 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:42.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:42 smithi191 ceph-mon[102206]: pgmap v1276: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 3813/130905 objects degraded (2.913%); 0 B/s, 44 objects/s recovering 2024-09-21T09:59:42.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:42.423 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:42.423 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:42.435 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:42.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.582 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:42.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:42.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:42.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:42.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:42.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:42.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:42.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:42.967 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:42.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:43.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:43.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:43.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:43.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:43.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:43.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:43.377 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:43.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:43.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:43.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:43.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:43.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:59:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:59:43] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-09-21T09:59:43.640 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:43.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:43.769 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:43.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:43.870 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:43.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:43.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:44.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:44.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:44.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:44.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:44.183 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:44.200 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:59:44.202 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T09:59:44.209 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:44.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:44.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:44.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:44.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:44 smithi191 ceph-mon[102206]: pgmap v1277: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3813/122532 objects degraded (3.112%); 0 B/s, 36 objects/s recovering 2024-09-21T09:59:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3813/122532 objects degraded (3.112%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:44 smithi076 ceph-mon[106482]: pgmap v1277: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3813/122532 objects degraded (3.112%); 0 B/s, 36 objects/s recovering 2024-09-21T09:59:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3813/122532 objects degraded (3.112%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:44 smithi076 ceph-mon[110963]: pgmap v1277: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3813/122532 objects degraded (3.112%); 0 B/s, 36 objects/s recovering 2024-09-21T09:59:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3813/122532 objects degraded (3.112%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:44.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:44.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:44.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:44.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:44.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:44.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:44.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:44.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:44.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:44.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:44.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:45.016 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:45.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:45.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:45.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:45.272 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:45 smithi076 ceph-mon[106482]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:45.273 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:45 smithi076 ceph-mon[110963]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:45.276 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:45.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:45 smithi191 ceph-mon[102206]: from='client.44780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:59:45.378 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:45.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:45.526 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:45.567 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:45.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:45.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:45.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:45.799 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:45.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:45.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:45.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:45.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:46.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:46.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:46.187 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:46.223 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:46.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:46.328 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:46 smithi076 ceph-mon[106482]: pgmap v1278: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 3645/119544 objects degraded (3.049%); 0 B/s, 50 objects/s recovering 2024-09-21T09:59:46.328 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:46 smithi076 ceph-mon[110963]: pgmap v1278: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 3645/119544 objects degraded (3.049%); 0 B/s, 50 objects/s recovering 2024-09-21T09:59:46.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:46.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:46 smithi191 ceph-mon[102206]: pgmap v1278: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 3645/119544 objects degraded (3.049%); 0 B/s, 50 objects/s recovering 2024-09-21T09:59:46.461 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:46.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:46.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:46.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:46.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:46.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.775 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:46.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:46.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:46.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:46.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:47.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:47.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:47.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:47.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:47.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:47.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:47.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:47.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:47.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:47.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:47.574 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:47.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:47.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:47.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:47.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:47.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:47.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:47.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:47.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:47.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:47.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:48.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:48.123 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:48.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.266 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:48.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:48 smithi191 ceph-mon[102206]: pgmap v1279: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 3645/112206 objects degraded (3.248%); 0 B/s, 28 objects/s recovering 2024-09-21T09:59:48.367 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:48 smithi076 ceph-mon[106482]: pgmap v1279: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 3645/112206 objects degraded (3.248%); 0 B/s, 28 objects/s recovering 2024-09-21T09:59:48.373 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T09:59:48.373 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T09:59:48.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:48 smithi076 ceph-mon[110963]: pgmap v1279: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 3645/112206 objects degraded (3.248%); 0 B/s, 28 objects/s recovering 2024-09-21T09:59:48.413 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:48.417 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T09:59:48.417 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T09:59:48.459 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T09:59:48.460 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T09:59:48.495 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T09:59:48.495 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T09:59:48.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:48.530 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T09:59:48.530 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T09:59:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.553 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:48.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:48.566 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T09:59:48.566 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T09:59:48.597 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T09:59:48.597 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T09:59:48.631 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T09:59:48.631 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T09:59:48.655 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:48.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:48.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:48.842 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:48.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:48.984 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:48.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:49.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:49.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:49.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:49.249 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:49.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:49 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4181539390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3645/112206 objects degraded (3.248%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:49 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4181539390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3645/112206 objects degraded (3.248%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:49 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4181539390' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T09:59:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3645/112206 objects degraded (3.248%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:49.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:49.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.537 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:49.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:49.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:49.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:49.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.815 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:49.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:49.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:49.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:49.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:50.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:50.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:50.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:50 smithi076 ceph-mon[106482]: pgmap v1280: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 3645/110754 objects degraded (3.291%); 0 B/s, 28 objects/s recovering 2024-09-21T09:59:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:50 smithi076 ceph-mon[110963]: pgmap v1280: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 3645/110754 objects degraded (3.291%); 0 B/s, 28 objects/s recovering 2024-09-21T09:59:50.170 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1277,"stamp":"2024-09-21T09:59:42.506898+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2675578877,"num_objects":40844,"num_object_clones":0,"num_object_copies":122532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40844,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":187269,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10547,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142298,"ondisk_log_size":142298,"up":267,"acting":264,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9883076,"kb_used_data":8660676,"kb_used_omap":195,"kb_used_meta":1221884,"kb_avail":740078140,"statfs":{"total":767960285184,"available":757840015360,"internally_reserved":0,"allocated":8868532224,"data_stored":8852154391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1251209348},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-497352704,"num_objects":-7589,"num_object_clones":0,"num_object_copies":-22767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-361,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7589,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":361,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004192"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.982450+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T09:59:33.982450+0000","last_peered":"2024-09-21T09:59:33.982450+0000","last_clean":"2024-09-21T09:59:33.982450+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T09:59:33.982450+0000","last_undegraded":"2024-09-21T09:59:33.982450+0000","last_fullsized":"2024-09-21T09:59:33.982450+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1050,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.955311+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T09:59:28.955311+0000","last_peered":"2024-09-21T09:59:28.955311+0000","last_clean":"2024-09-21T09:59:28.955311+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T09:59:28.955311+0000","last_undegraded":"2024-09-21T09:59:28.955311+0000","last_fullsized":"2024-09-21T09:59:28.955311+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1081,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080737+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:59:29.080737+0000","last_peered":"2024-09-21T09:59:29.080737+0000","last_clean":"2024-09-21T09:59:29.080737+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:59:29.080737+0000","last_undegraded":"2024-09-21T09:59:29.080737+0000","last_fullsized":"2024-09-21T09:59:29.080737+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1435,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.971237+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T09:59:33.971237+0000","last_peered":"2024-09-21T09:59:33.971237+0000","last_clean":"2024-09-21T09:59:33.971237+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T09:59:33.971237+0000","last_undegraded":"2024-09-21T09:59:33.971237+0000","last_fullsized":"2024-09-21T09:59:33.971237+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1435,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.974397+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T09:59:33.974397+0000","last_peered":"2024-09-21T09:59:33.974397+0000","last_clean":"2024-09-21T09:59:33.974397+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T09:59:33.974397+0000","last_undegraded":"2024-09-21T09:59:33.974397+0000","last_fullsized":"2024-09-21T09:59:33.974397+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1033,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.955135+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T09:59:28.955135+0000","last_peered":"2024-09-21T09:59:28.955135+0000","last_clean":"2024-09-21T09:59:28.955135+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T09:59:28.955135+0000","last_undegraded":"2024-09-21T09:59:28.955135+0000","last_fullsized":"2024-09-21T09:59:28.955135+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.974905+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T09:59:33.974905+0000","last_peered":"2024-09-21T09:59:33.974905+0000","last_clean":"2024-09-21T09:59:33.974905+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T09:59:33.974905+0000","last_undegraded":"2024-09-21T09:59:33.974905+0000","last_fullsized":"2024-09-21T09:59:33.974905+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1060,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953446+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T09:59:28.953446+0000","last_peered":"2024-09-21T09:59:28.953446+0000","last_clean":"2024-09-21T09:59:28.953446+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T09:59:28.953446+0000","last_undegraded":"2024-09-21T09:59:28.953446+0000","last_fullsized":"2024-09-21T09:59:28.953446+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1122'11601","reported_seq":11647,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.449960+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T09:59:41.449960+0000","last_peered":"2024-09-21T09:59:41.449960+0000","last_clean":"2024-09-21T09:59:41.449960+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T09:59:41.449960+0000","last_undegraded":"2024-09-21T09:59:41.449960+0000","last_fullsized":"2024-09-21T09:59:41.449960+0000","mapping_epoch":1086,"log_start":"1094'1600","ondisk_log_start":"1094'1600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10001,"log_dups_size":0,"ondisk_log_size":10001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164036608,"num_objects":2503,"num_object_clones":0,"num_object_copies":7509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11601,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1050,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.955471+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T09:59:28.955471+0000","last_peered":"2024-09-21T09:59:28.955471+0000","last_clean":"2024-09-21T09:59:28.955471+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T09:59:28.955471+0000","last_undegraded":"2024-09-21T09:59:28.955471+0000","last_fullsized":"2024-09-21T09:59:28.955471+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1049,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080339+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:59:29.080339+0000","last_peered":"2024-09-21T09:59:29.080339+0000","last_clean":"2024-09-21T09:59:29.080339+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:59:29.080339+0000","last_undegraded":"2024-09-21T09:59:29.080339+0000","last_fullsized":"2024-09-21T09:59:29.080339+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1122'11511","reported_seq":14468,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:38.479156+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T09:59:38.479156+0000","last_peered":"2024-09-21T09:59:38.479156+0000","last_clean":"2024-09-21T09:59:38.479156+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T09:59:38.479156+0000","last_undegraded":"2024-09-21T09:59:38.479156+0000","last_fullsized":"2024-09-21T09:59:38.479156+0000","mapping_epoch":1108,"log_start":"1099'4100","ondisk_log_start":"1099'4100","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7411,"log_dups_size":0,"ondisk_log_size":7411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170721280,"num_objects":2605,"num_object_clones":0,"num_object_copies":7815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11511,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1051,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.955463+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T09:59:28.955463+0000","last_peered":"2024-09-21T09:59:28.955463+0000","last_clean":"2024-09-21T09:59:28.955463+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T09:59:28.955463+0000","last_undegraded":"2024-09-21T09:59:28.955463+0000","last_fullsized":"2024-09-21T09:59:28.955463+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1103,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953906+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T09:59:28.953906+0000","last_peered":"2024-09-21T09:59:28.953906+0000","last_clean":"2024-09-21T09:59:28.953906+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T09:59:28.953906+0000","last_undegraded":"2024-09-21T09:59:28.953906+0000","last_fullsized":"2024-09-21T09:59:28.953906+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1122'11863","reported_seq":14793,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.338523+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T09:59:41.338523+0000","last_peered":"2024-09-21T09:59:41.338523+0000","last_clean":"2024-09-21T09:59:41.338523+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T09:59:41.338523+0000","last_undegraded":"2024-09-21T09:59:41.338523+0000","last_fullsized":"2024-09-21T09:59:41.338523+0000","mapping_epoch":1100,"log_start":"1098'3700","ondisk_log_start":"1098'3700","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165740544,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11863,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1035,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.958933+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T09:59:28.958933+0000","last_peered":"2024-09-21T09:59:28.958933+0000","last_clean":"2024-09-21T09:59:28.958933+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T09:59:28.958933+0000","last_undegraded":"2024-09-21T09:59:28.958933+0000","last_fullsized":"2024-09-21T09:59:28.958933+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1052,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953617+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T09:59:28.953617+0000","last_peered":"2024-09-21T09:59:28.953617+0000","last_clean":"2024-09-21T09:59:28.953617+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T09:59:28.953617+0000","last_undegraded":"2024-09-21T09:59:28.953617+0000","last_fullsized":"2024-09-21T09:59:28.953617+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1122'11642","reported_seq":11724,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:38.947632+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T09:59:38.947632+0000","last_peered":"2024-09-21T09:59:38.947632+0000","last_clean":"2024-09-21T09:59:38.947632+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T09:59:38.947632+0000","last_undegraded":"2024-09-21T09:59:38.947632+0000","last_fullsized":"2024-09-21T09:59:38.947632+0000","mapping_epoch":1086,"log_start":"1095'1900","ondisk_log_start":"1095'1900","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9742,"log_dups_size":0,"ondisk_log_size":9742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505408,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11642,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1083,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080670+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:59:29.080670+0000","last_peered":"2024-09-21T09:59:29.080670+0000","last_clean":"2024-09-21T09:59:29.080670+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:59:29.080670+0000","last_undegraded":"2024-09-21T09:59:29.080670+0000","last_fullsized":"2024-09-21T09:59:29.080670+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1423,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.970859+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T09:59:33.970859+0000","last_peered":"2024-09-21T09:59:33.970859+0000","last_clean":"2024-09-21T09:59:33.970859+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T09:59:33.970859+0000","last_undegraded":"2024-09-21T09:59:33.970859+0000","last_fullsized":"2024-09-21T09:59:33.970859+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1122'11679","reported_seq":11725,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.404599+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T09:59:41.404599+0000","last_peered":"2024-09-21T09:59:41.404599+0000","last_clean":"2024-09-21T09:59:41.404599+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T09:59:41.404599+0000","last_undegraded":"2024-09-21T09:59:41.404599+0000","last_fullsized":"2024-09-21T09:59:41.404599+0000","mapping_epoch":1086,"log_start":"1098'3700","ondisk_log_start":"1098'3700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7979,"log_dups_size":0,"ondisk_log_size":7979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162988032,"num_objects":2487,"num_object_clones":0,"num_object_copies":7461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11679,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":75,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.972785+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T09:59:33.972785+0000","last_peered":"2024-09-21T09:59:33.972785+0000","last_clean":"2024-09-21T09:59:33.972785+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T09:59:33.972785+0000","last_undegraded":"2024-09-21T09:59:33.972785+0000","last_fullsized":"2024-09-21T09:59:33.972785+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1088,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953555+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T09:59:28.953555+0000","last_peered":"2024-09-21T09:59:28.953555+0000","last_clean":"2024-09-21T09:59:28.953555+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T09:59:28.953555+0000","last_undegraded":"2024-09-21T09:59:28.953555+0000","last_fullsized":"2024-09-21T09:59:28.953555+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1122'11725","reported_seq":14641,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:40.148887+0000","last_change":"2024-09-21T09:59:03.867059+0000","last_active":"2024-09-21T09:59:40.148887+0000","last_peered":"2024-09-21T09:59:40.148887+0000","last_clean":"2024-09-21T09:59:40.148887+0000","last_became_active":"2024-09-21T09:59:03.844371+0000","last_became_peered":"2024-09-21T09:59:03.844371+0000","last_unstale":"2024-09-21T09:59:40.148887+0000","last_undegraded":"2024-09-21T09:59:40.148887+0000","last_fullsized":"2024-09-21T09:59:40.148887+0000","mapping_epoch":1117,"log_start":"1094'1700","ondisk_log_start":"1094'1700","created":1086,"last_epoch_clean":1118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10025,"log_dups_size":0,"ondisk_log_size":10025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:34:30.990234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172687360,"num_objects":2635,"num_object_clones":0,"num_object_copies":7905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11725,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1053,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953914+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T09:59:28.953914+0000","last_peered":"2024-09-21T09:59:28.953914+0000","last_clean":"2024-09-21T09:59:28.953914+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T09:59:28.953914+0000","last_undegraded":"2024-09-21T09:59:28.953914+0000","last_fullsized":"2024-09-21T09:59:28.953914+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1085,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954227+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T09:59:28.954227+0000","last_peered":"2024-09-21T09:59:28.954227+0000","last_clean":"2024-09-21T09:59:28.954227+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T09:59:28.954227+0000","last_undegraded":"2024-09-21T09:59:28.954227+0000","last_fullsized":"2024-09-21T09:59:28.954227+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1122'11648","reported_seq":11698,"reported_epoch":1122,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:41.429277+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T09:59:41.429277+0000","last_peered":"2024-09-21T09:59:41.429277+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T09:59:41.429277+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"1094'1600","ondisk_log_start":"1094'1600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10048,"log_dups_size":0,"ondisk_log_size":10048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165543936,"num_objects":2526,"num_object_clones":0,"num_object_copies":7578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11648,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":1050}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1071,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954321+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T09:59:28.954321+0000","last_peered":"2024-09-21T09:59:28.954321+0000","last_clean":"2024-09-21T09:59:28.954321+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T09:59:28.954321+0000","last_undegraded":"2024-09-21T09:59:28.954321+0000","last_fullsized":"2024-09-21T09:59:28.954321+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1079,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954004+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T09:59:28.954004+0000","last_peered":"2024-09-21T09:59:28.954004+0000","last_clean":"2024-09-21T09:59:28.954004+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T09:59:28.954004+0000","last_undegraded":"2024-09-21T09:59:28.954004+0000","last_fullsized":"2024-09-21T09:59:28.954004+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1122'11771","reported_seq":11817,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.490860+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T09:59:41.490860+0000","last_peered":"2024-09-21T09:59:41.490860+0000","last_clean":"2024-09-21T09:59:41.490860+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T09:59:41.490860+0000","last_undegraded":"2024-09-21T09:59:41.490860+0000","last_fullsized":"2024-09-21T09:59:41.490860+0000","mapping_epoch":1086,"log_start":"1094'1700","ondisk_log_start":"1094'1700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10071,"log_dups_size":0,"ondisk_log_size":10071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806104,"num_objects":2531,"num_object_clones":0,"num_object_copies":7593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2531,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11771,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.977258+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T09:59:33.977258+0000","last_peered":"2024-09-21T09:59:33.977258+0000","last_clean":"2024-09-21T09:59:33.977258+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T09:59:33.977258+0000","last_undegraded":"2024-09-21T09:59:33.977258+0000","last_fullsized":"2024-09-21T09:59:33.977258+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1409,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961558+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:59:33.961558+0000","last_peered":"2024-09-21T09:59:33.961558+0000","last_clean":"2024-09-21T09:59:33.961558+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:59:33.961558+0000","last_undegraded":"2024-09-21T09:59:33.961558+0000","last_fullsized":"2024-09-21T09:59:33.961558+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1122'11519","reported_seq":14353,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.487170+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T09:59:41.487170+0000","last_peered":"2024-09-21T09:59:41.487170+0000","last_clean":"2024-09-21T09:59:41.487170+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T09:59:41.487170+0000","last_undegraded":"2024-09-21T09:59:41.487170+0000","last_fullsized":"2024-09-21T09:59:41.487170+0000","mapping_epoch":1113,"log_start":"1094'1500","ondisk_log_start":"1094'1500","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10019,"log_dups_size":0,"ondisk_log_size":10019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11519,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1380,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.955168+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T09:59:28.955168+0000","last_peered":"2024-09-21T09:59:28.955168+0000","last_clean":"2024-09-21T09:59:28.955168+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T09:59:28.955168+0000","last_undegraded":"2024-09-21T09:59:28.955168+0000","last_fullsized":"2024-09-21T09:59:28.955168+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1079,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080485+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:59:29.080485+0000","last_peered":"2024-09-21T09:59:29.080485+0000","last_clean":"2024-09-21T09:59:29.080485+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:59:29.080485+0000","last_undegraded":"2024-09-21T09:59:29.080485+0000","last_fullsized":"2024-09-21T09:59:29.080485+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.967597+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T09:59:33.967597+0000","last_peered":"2024-09-21T09:59:33.967597+0000","last_clean":"2024-09-21T09:59:33.967597+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T09:59:33.967597+0000","last_undegraded":"2024-09-21T09:59:33.967597+0000","last_fullsized":"2024-09-21T09:59:33.967597+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1037,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953798+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T09:59:28.953798+0000","last_peered":"2024-09-21T09:59:28.953798+0000","last_clean":"2024-09-21T09:59:28.953798+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T09:59:28.953798+0000","last_undegraded":"2024-09-21T09:59:28.953798+0000","last_fullsized":"2024-09-21T09:59:28.953798+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1122'11681","reported_seq":14733,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:42.041508+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T09:59:42.041508+0000","last_peered":"2024-09-21T09:59:42.041508+0000","last_clean":"2024-09-21T09:59:42.041508+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T09:59:42.041508+0000","last_undegraded":"2024-09-21T09:59:42.041508+0000","last_fullsized":"2024-09-21T09:59:42.041508+0000","mapping_epoch":1105,"log_start":"1097'3100","ondisk_log_start":"1097'3100","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8581,"log_dups_size":0,"ondisk_log_size":8581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162332672,"num_objects":2477,"num_object_clones":0,"num_object_copies":7431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11681,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1078,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:31.406895+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T09:59:31.406895+0000","last_peered":"2024-09-21T09:59:31.406895+0000","last_clean":"2024-09-21T09:59:31.406895+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T09:59:31.406895+0000","last_undegraded":"2024-09-21T09:59:31.406895+0000","last_fullsized":"2024-09-21T09:59:31.406895+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1436,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.970598+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T09:59:33.970598+0000","last_peered":"2024-09-21T09:59:33.970598+0000","last_clean":"2024-09-21T09:59:33.970598+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T09:59:33.970598+0000","last_undegraded":"2024-09-21T09:59:33.970598+0000","last_fullsized":"2024-09-21T09:59:33.970598+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1433,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.974240+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T09:59:33.974240+0000","last_peered":"2024-09-21T09:59:33.974240+0000","last_clean":"2024-09-21T09:59:33.974240+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T09:59:33.974240+0000","last_undegraded":"2024-09-21T09:59:33.974240+0000","last_fullsized":"2024-09-21T09:59:33.974240+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1122'11648","reported_seq":14517,"reported_epoch":1122,"state":"active+recovery_wait","last_fresh":"2024-09-21T09:59:41.385848+0000","last_change":"2024-09-21T09:59:29.091758+0000","last_active":"2024-09-21T09:59:41.385848+0000","last_peered":"2024-09-21T09:59:41.385848+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:59:29.047032+0000","last_became_peered":"2024-09-21T09:59:29.047032+0000","last_unstale":"2024-09-21T09:59:41.385848+0000","last_undegraded":"2024-09-21T09:59:41.385848+0000","last_fullsized":"2024-09-21T09:59:41.385848+0000","mapping_epoch":1120,"log_start":"1098'3500","ondisk_log_start":"1098'3500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:33.224146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11648,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1405,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":["7","2","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":2500}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1414,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.972827+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T09:59:33.972827+0000","last_peered":"2024-09-21T09:59:33.972827+0000","last_clean":"2024-09-21T09:59:33.972827+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T09:59:33.972827+0000","last_undegraded":"2024-09-21T09:59:33.972827+0000","last_fullsized":"2024-09-21T09:59:33.972827+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1396,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961144+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:59:33.961144+0000","last_peered":"2024-09-21T09:59:33.961144+0000","last_clean":"2024-09-21T09:59:33.961144+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:59:33.961144+0000","last_undegraded":"2024-09-21T09:59:33.961144+0000","last_fullsized":"2024-09-21T09:59:33.961144+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1403,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961411+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:59:33.961411+0000","last_peered":"2024-09-21T09:59:33.961411+0000","last_clean":"2024-09-21T09:59:33.961411+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:59:33.961411+0000","last_undegraded":"2024-09-21T09:59:33.961411+0000","last_fullsized":"2024-09-21T09:59:33.961411+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1122'11575","reported_seq":14554,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:38.518664+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T09:59:38.518664+0000","last_peered":"2024-09-21T09:59:38.518664+0000","last_clean":"2024-09-21T09:59:38.518664+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T09:59:38.518664+0000","last_undegraded":"2024-09-21T09:59:38.518664+0000","last_fullsized":"2024-09-21T09:59:38.518664+0000","mapping_epoch":1096,"log_start":"1099'4200","ondisk_log_start":"1099'4200","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172294144,"num_objects":2629,"num_object_clones":0,"num_object_copies":7887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11575,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1080,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080444+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:59:29.080444+0000","last_peered":"2024-09-21T09:59:29.080444+0000","last_clean":"2024-09-21T09:59:29.080444+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:59:29.080444+0000","last_undegraded":"2024-09-21T09:59:29.080444+0000","last_fullsized":"2024-09-21T09:59:29.080444+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1072,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:31.413275+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T09:59:31.413275+0000","last_peered":"2024-09-21T09:59:31.413275+0000","last_clean":"2024-09-21T09:59:31.413275+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T09:59:31.413275+0000","last_undegraded":"2024-09-21T09:59:31.413275+0000","last_fullsized":"2024-09-21T09:59:31.413275+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1420,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961397+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:59:33.961397+0000","last_peered":"2024-09-21T09:59:33.961397+0000","last_clean":"2024-09-21T09:59:33.961397+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:59:33.961397+0000","last_undegraded":"2024-09-21T09:59:33.961397+0000","last_fullsized":"2024-09-21T09:59:33.961397+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1122'11572","reported_seq":12731,"reported_epoch":1122,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:38.993966+0000","last_change":"2024-09-21T09:59:27.940280+0000","last_active":"2024-09-21T09:59:38.993966+0000","last_peered":"2024-09-21T09:59:38.993966+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T09:59:38.993966+0000","last_undegraded":"2024-09-21T09:56:46.247241+0000","last_fullsized":"2024-09-21T09:56:46.239078+0000","mapping_epoch":1093,"log_start":"1094'1800","ondisk_log_start":"1094'1800","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9772,"log_dups_size":0,"ondisk_log_size":9772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170131456,"num_objects":2596,"num_object_clones":0,"num_object_copies":7788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11572,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":536,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":1671}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1099,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962850+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:59:33.962850+0000","last_peered":"2024-09-21T09:59:33.962850+0000","last_clean":"2024-09-21T09:59:33.962850+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:59:33.962850+0000","last_undegraded":"2024-09-21T09:59:33.962850+0000","last_fullsized":"2024-09-21T09:59:33.962850+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1417,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961483+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:59:33.961483+0000","last_peered":"2024-09-21T09:59:33.961483+0000","last_clean":"2024-09-21T09:59:33.961483+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:59:33.961483+0000","last_undegraded":"2024-09-21T09:59:33.961483+0000","last_fullsized":"2024-09-21T09:59:33.961483+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":75,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.970637+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T09:59:33.970637+0000","last_peered":"2024-09-21T09:59:33.970637+0000","last_clean":"2024-09-21T09:59:33.970637+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T09:59:33.970637+0000","last_undegraded":"2024-09-21T09:59:33.970637+0000","last_fullsized":"2024-09-21T09:59:33.970637+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1122'12049","reported_seq":12133,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.965538+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T09:59:41.965538+0000","last_peered":"2024-09-21T09:59:41.965538+0000","last_clean":"2024-09-21T09:59:41.965538+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T09:59:41.965538+0000","last_undegraded":"2024-09-21T09:59:41.965538+0000","last_fullsized":"2024-09-21T09:59:41.965538+0000","mapping_epoch":1086,"log_start":"1097'3400","ondisk_log_start":"1097'3400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8649,"log_dups_size":0,"ondisk_log_size":8649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165609472,"num_objects":2527,"num_object_clones":0,"num_object_copies":7581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12049,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1394,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961635+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:59:33.961635+0000","last_peered":"2024-09-21T09:59:33.961635+0000","last_clean":"2024-09-21T09:59:33.961635+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:59:33.961635+0000","last_undegraded":"2024-09-21T09:59:33.961635+0000","last_fullsized":"2024-09-21T09:59:33.961635+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1062,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.958927+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T09:59:28.958927+0000","last_peered":"2024-09-21T09:59:28.958927+0000","last_clean":"2024-09-21T09:59:28.958927+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T09:59:28.958927+0000","last_undegraded":"2024-09-21T09:59:28.958927+0000","last_fullsized":"2024-09-21T09:59:28.958927+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1412,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.970960+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T09:59:33.970960+0000","last_peered":"2024-09-21T09:59:33.970960+0000","last_clean":"2024-09-21T09:59:33.970960+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T09:59:33.970960+0000","last_undegraded":"2024-09-21T09:59:33.970960+0000","last_fullsized":"2024-09-21T09:59:33.970960+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1122'11729","reported_seq":11783,"reported_epoch":1122,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T09:59:41.394039+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T09:59:41.394039+0000","last_peered":"2024-09-21T09:59:41.394039+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T09:59:41.394039+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1098'3800","ondisk_log_start":"1098'3800","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7929,"log_dups_size":0,"ondisk_log_size":7929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11729,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":1195}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":75,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.974999+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T09:59:33.974999+0000","last_peered":"2024-09-21T09:59:33.974999+0000","last_clean":"2024-09-21T09:59:33.974999+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T09:59:33.974999+0000","last_undegraded":"2024-09-21T09:59:33.974999+0000","last_fullsized":"2024-09-21T09:59:33.974999+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1395,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961789+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:59:33.961789+0000","last_peered":"2024-09-21T09:59:33.961789+0000","last_clean":"2024-09-21T09:59:33.961789+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:59:33.961789+0000","last_undegraded":"2024-09-21T09:59:33.961789+0000","last_fullsized":"2024-09-21T09:59:33.961789+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.977090+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T09:59:33.977090+0000","last_peered":"2024-09-21T09:59:33.977090+0000","last_clean":"2024-09-21T09:59:33.977090+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T09:59:33.977090+0000","last_undegraded":"2024-09-21T09:59:33.977090+0000","last_fullsized":"2024-09-21T09:59:33.977090+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1122'11777","reported_seq":11823,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.343589+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T09:59:41.343589+0000","last_peered":"2024-09-21T09:59:41.343589+0000","last_clean":"2024-09-21T09:59:41.343589+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T09:59:41.343589+0000","last_undegraded":"2024-09-21T09:59:41.343589+0000","last_fullsized":"2024-09-21T09:59:41.343589+0000","mapping_epoch":1086,"log_start":"1098'3600","ondisk_log_start":"1098'3600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8177,"log_dups_size":0,"ondisk_log_size":8177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163774464,"num_objects":2499,"num_object_clones":0,"num_object_copies":7497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11777,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1425,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.974472+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T09:59:33.974472+0000","last_peered":"2024-09-21T09:59:33.974472+0000","last_clean":"2024-09-21T09:59:33.974472+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T09:59:33.974472+0000","last_undegraded":"2024-09-21T09:59:33.974472+0000","last_fullsized":"2024-09-21T09:59:33.974472+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1066,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953809+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T09:59:28.953809+0000","last_peered":"2024-09-21T09:59:28.953809+0000","last_clean":"2024-09-21T09:59:28.953809+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T09:59:28.953809+0000","last_undegraded":"2024-09-21T09:59:28.953809+0000","last_fullsized":"2024-09-21T09:59:28.953809+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1056,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:31.413199+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T09:59:31.413199+0000","last_peered":"2024-09-21T09:59:31.413199+0000","last_clean":"2024-09-21T09:59:31.413199+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T09:59:31.413199+0000","last_undegraded":"2024-09-21T09:59:31.413199+0000","last_fullsized":"2024-09-21T09:59:31.413199+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1069,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954392+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T09:59:28.954392+0000","last_peered":"2024-09-21T09:59:28.954392+0000","last_clean":"2024-09-21T09:59:28.954392+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T09:59:28.954392+0000","last_undegraded":"2024-09-21T09:59:28.954392+0000","last_fullsized":"2024-09-21T09:59:28.954392+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1078,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953715+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T09:59:28.953715+0000","last_peered":"2024-09-21T09:59:28.953715+0000","last_clean":"2024-09-21T09:59:28.953715+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T09:59:28.953715+0000","last_undegraded":"2024-09-21T09:59:28.953715+0000","last_fullsized":"2024-09-21T09:59:28.953715+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1395,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.964097+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:59:33.964097+0000","last_peered":"2024-09-21T09:59:33.964097+0000","last_clean":"2024-09-21T09:59:33.964097+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:59:33.964097+0000","last_undegraded":"2024-09-21T09:59:33.964097+0000","last_fullsized":"2024-09-21T09:59:33.964097+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1081,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.958860+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T09:59:28.958860+0000","last_peered":"2024-09-21T09:59:28.958860+0000","last_clean":"2024-09-21T09:59:28.958860+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T09:59:28.958860+0000","last_undegraded":"2024-09-21T09:59:28.958860+0000","last_fullsized":"2024-09-21T09:59:28.958860+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1064,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080534+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:59:29.080534+0000","last_peered":"2024-09-21T09:59:29.080534+0000","last_clean":"2024-09-21T09:59:29.080534+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:59:29.080534+0000","last_undegraded":"2024-09-21T09:59:29.080534+0000","last_fullsized":"2024-09-21T09:59:29.080534+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.967131+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T09:59:33.967131+0000","last_peered":"2024-09-21T09:59:33.967131+0000","last_clean":"2024-09-21T09:59:33.967131+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T09:59:33.967131+0000","last_undegraded":"2024-09-21T09:59:33.967131+0000","last_fullsized":"2024-09-21T09:59:33.967131+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1434,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.971053+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T09:59:33.971053+0000","last_peered":"2024-09-21T09:59:33.971053+0000","last_clean":"2024-09-21T09:59:33.971053+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T09:59:33.971053+0000","last_undegraded":"2024-09-21T09:59:33.971053+0000","last_fullsized":"2024-09-21T09:59:33.971053+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.977254+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T09:59:33.977254+0000","last_peered":"2024-09-21T09:59:33.977254+0000","last_clean":"2024-09-21T09:59:33.977254+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T09:59:33.977254+0000","last_undegraded":"2024-09-21T09:59:33.977254+0000","last_fullsized":"2024-09-21T09:59:33.977254+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1062,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.964506+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T09:59:33.964506+0000","last_peered":"2024-09-21T09:59:33.964506+0000","last_clean":"2024-09-21T09:59:33.964506+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T09:59:33.964506+0000","last_undegraded":"2024-09-21T09:59:33.964506+0000","last_fullsized":"2024-09-21T09:59:33.964506+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.977036+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T09:59:33.977036+0000","last_peered":"2024-09-21T09:59:33.977036+0000","last_clean":"2024-09-21T09:59:33.977036+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T09:59:33.977036+0000","last_undegraded":"2024-09-21T09:59:33.977036+0000","last_fullsized":"2024-09-21T09:59:33.977036+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1437,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.974325+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T09:59:33.974325+0000","last_peered":"2024-09-21T09:59:33.974325+0000","last_clean":"2024-09-21T09:59:33.974325+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T09:59:33.974325+0000","last_undegraded":"2024-09-21T09:59:33.974325+0000","last_fullsized":"2024-09-21T09:59:33.974325+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1408,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.961156+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:59:33.961156+0000","last_peered":"2024-09-21T09:59:33.961156+0000","last_clean":"2024-09-21T09:59:33.961156+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:59:33.961156+0000","last_undegraded":"2024-09-21T09:59:33.961156+0000","last_fullsized":"2024-09-21T09:59:33.961156+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.977750+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T09:59:33.977750+0000","last_peered":"2024-09-21T09:59:33.977750+0000","last_clean":"2024-09-21T09:59:33.977750+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T09:59:33.977750+0000","last_undegraded":"2024-09-21T09:59:33.977750+0000","last_fullsized":"2024-09-21T09:59:33.977750+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1086,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954282+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T09:59:28.954282+0000","last_peered":"2024-09-21T09:59:28.954282+0000","last_clean":"2024-09-21T09:59:28.954282+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T09:59:28.954282+0000","last_undegraded":"2024-09-21T09:59:28.954282+0000","last_fullsized":"2024-09-21T09:59:28.954282+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1390,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.966393+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T09:59:33.966393+0000","last_peered":"2024-09-21T09:59:33.966393+0000","last_clean":"2024-09-21T09:59:33.966393+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T09:59:33.966393+0000","last_undegraded":"2024-09-21T09:59:33.966393+0000","last_fullsized":"2024-09-21T09:59:33.966393+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1413,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.971044+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T09:59:33.971044+0000","last_peered":"2024-09-21T09:59:33.971044+0000","last_clean":"2024-09-21T09:59:33.971044+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T09:59:33.971044+0000","last_undegraded":"2024-09-21T09:59:33.971044+0000","last_fullsized":"2024-09-21T09:59:33.971044+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1102,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954388+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T09:59:28.954388+0000","last_peered":"2024-09-21T09:59:28.954388+0000","last_clean":"2024-09-21T09:59:28.954388+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T09:59:28.954388+0000","last_undegraded":"2024-09-21T09:59:28.954388+0000","last_fullsized":"2024-09-21T09:59:28.954388+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1066,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080536+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:59:29.080536+0000","last_peered":"2024-09-21T09:59:29.080536+0000","last_clean":"2024-09-21T09:59:29.080536+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:59:29.080536+0000","last_undegraded":"2024-09-21T09:59:29.080536+0000","last_fullsized":"2024-09-21T09:59:29.080536+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1102,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.954149+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T09:59:28.954149+0000","last_peered":"2024-09-21T09:59:28.954149+0000","last_clean":"2024-09-21T09:59:28.954149+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T09:59:28.954149+0000","last_undegraded":"2024-09-21T09:59:28.954149+0000","last_fullsized":"2024-09-21T09:59:28.954149+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1051,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:29.080777+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:59:29.080777+0000","last_peered":"2024-09-21T09:59:29.080777+0000","last_clean":"2024-09-21T09:59:29.080777+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:59:29.080777+0000","last_undegraded":"2024-09-21T09:59:29.080777+0000","last_fullsized":"2024-09-21T09:59:29.080777+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.967261+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T09:59:33.967261+0000","last_peered":"2024-09-21T09:59:33.967261+0000","last_clean":"2024-09-21T09:59:33.967261+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T09:59:33.967261+0000","last_undegraded":"2024-09-21T09:59:33.967261+0000","last_fullsized":"2024-09-21T09:59:33.967261+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":71,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.967377+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T09:59:33.967377+0000","last_peered":"2024-09-21T09:59:33.967377+0000","last_clean":"2024-09-21T09:59:33.967377+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T09:59:33.967377+0000","last_undegraded":"2024-09-21T09:59:33.967377+0000","last_fullsized":"2024-09-21T09:59:33.967377+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1078,"reported_epoch":1121,"state":"active+clean","last_fresh":"2024-09-21T09:59:28.953630+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T09:59:28.953630+0000","last_peered":"2024-09-21T09:59:28.953630+0000","last_clean":"2024-09-21T09:59:28.953630+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T09:59:28.953630+0000","last_undegraded":"2024-09-21T09:59:28.953630+0000","last_fullsized":"2024-09-21T09:59:28.953630+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1429,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.971127+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T09:59:33.971127+0000","last_peered":"2024-09-21T09:59:33.971127+0000","last_clean":"2024-09-21T09:59:33.971127+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T09:59:33.971127+0000","last_undegraded":"2024-09-21T09:59:33.971127+0000","last_fullsized":"2024-09-21T09:59:33.971127+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2674982936,"num_objects":40818,"num_object_clones":0,"num_object_copies":122454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40818,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186990,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10534,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8726523904,"data_stored":8726511688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142090,"ondisk_log_size":142090,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234286,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1211976,"kb_used_data":1072576,"kb_used_omap":1,"kb_used_meta":139390,"kb_avail":92533176,"statfs":{"total":95995035648,"available":94753972224,"internally_reserved":0,"allocated":1098317824,"data_stored":1096257930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":142735819},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495923,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1344000,"kb_used_data":1182520,"kb_used_omap":1,"kb_used_meta":161470,"kb_avail":92401152,"statfs":{"total":95995035648,"available":94618779648,"internally_reserved":0,"allocated":1210900480,"data_stored":1208848651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":165345735},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757561,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1062072,"kb_used_data":904688,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":92683080,"statfs":{"total":95995035648,"available":94907473920,"internally_reserved":0,"allocated":926400512,"data_stored":924352181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019197,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":965440,"kb_used_data":826040,"kb_used_omap":1,"kb_used_meta":139390,"kb_avail":92779712,"statfs":{"total":95995035648,"available":95006425088,"internally_reserved":0,"allocated":845864960,"data_stored":843812644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":142735814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888493,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1982636,"kb_used_data":1729764,"kb_used_omap":34,"kb_used_meta":252701,"kb_avail":91762516,"statfs":{"total":95995035648,"available":93964816384,"internally_reserved":0,"allocated":1771278336,"data_stored":1769241700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":258766835},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967732,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1600304,"kb_used_data":1441768,"kb_used_omap":41,"kb_used_meta":158358,"kb_avail":92144848,"statfs":{"total":95995035648,"available":94356324352,"internally_reserved":0,"allocated":1476370432,"data_stored":1474330226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":162159007},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046971,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":916400,"kb_used_data":816296,"kb_used_omap":41,"kb_used_meta":100054,"kb_avail":92828752,"statfs":{"total":95995035648,"available":95056642048,"internally_reserved":0,"allocated":835887104,"data_stored":833846325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":102456233},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126209,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":800248,"kb_used_data":687024,"kb_used_omap":72,"kb_used_meta":113143,"kb_avail":92944904,"statfs":{"total":95995035648,"available":95175581696,"internally_reserved":0,"allocated":703512576,"data_stored":701464734,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":115858475},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":692326400,"data_stored":692322328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":812781568,"data_stored":812777496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1452474368,"data_stored":1452474368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1744633856,"data_stored":1744633856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":829095936,"data_stored":829095936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":918425600,"data_stored":918421528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1194590208,"data_stored":1194590208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1082195968,"data_stored":1082195968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:59:50.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T09:59:50.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:50.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:50.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:50.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:50.351 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:50 smithi191 ceph-mon[102206]: pgmap v1280: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 3645/110754 objects degraded (3.291%); 0 B/s, 28 objects/s recovering 2024-09-21T09:59:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:50.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:50.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:50.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:50.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:50.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:50.758 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:50.798 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:50.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:50.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:50.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:50.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:51.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:194980 6.d e1122/1086 6:bcff4587:::benchmark_data_smithi076_154645_object81074:head v 1122'12228, mlcod=1122'12226)", 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:51.023746+0000", 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0014777359999999999, 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015053379999999999, 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.029 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023746+0000", 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023746+0000", 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023747+0000", 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.285e-06 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.030 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023756+0000", 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9129999999999992e-06 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023774+0000", 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.855e-05 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023782+0000", 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0209999999999999e-06 2024-09-21T09:59:51.031 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023855+0000", 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3028000000000006e-05 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023875+0000", 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9412999999999999e-05 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.032 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:194979 6.1 6:8c2334dc:::benchmark_data_smithi076_154645_object81073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1122)", 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:51.023791+0000", 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0014327929999999999, 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001515321, 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 194979 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T09:59:51.033 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023791+0000", 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023791+0000", 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023792+0000", 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7770000000000001e-06 2024-09-21T09:59:51.034 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023798+0000", 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1279999999999996e-06 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023812+0000", 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.327e-05 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023820+0000", 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2190000000000004e-06 2024-09-21T09:59:51.035 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.023937+0000", 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116539 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.024340+0000", 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00040325700000000002 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:51.024495+0000", 2024-09-21T09:59:51.036 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000155396 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 2 2024-09-21T09:59:51.037 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:59:51.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:51.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:51.086 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T09:59:51.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:51.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:51.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:51.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:51.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2321128259' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:51.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2321128259' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:51.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2321128259' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T09:59:51.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:51.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:51.564 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:51.597 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:51.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:51.699 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:51.730 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:51.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:51.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:51.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:51.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:51.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:51.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:52.097 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:52.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:52.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:52.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:52.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:52 smithi191 ceph-mon[102206]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3436/107091 objects degraded (3.208%); 0 B/s, 45 objects/s recovering 2024-09-21T09:59:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:52 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:59:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2321128259' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:52 smithi191 ceph-mon[102206]: osdmap e1123: 8 total, 8 up, 8 in 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[106482]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3436/107091 objects degraded (3.208%); 0 B/s, 45 objects/s recovering 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2321128259' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[106482]: osdmap e1123: 8 total, 8 up, 8 in 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[110963]: pgmap v1281: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 3436/107091 objects degraded (3.208%); 0 B/s, 45 objects/s recovering 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2321128259' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T09:59:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:52 smithi076 ceph-mon[110963]: osdmap e1123: 8 total, 8 up, 8 in 2024-09-21T09:59:52.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:52.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:52.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:52.435 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:52.471 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T09:59:52.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:52.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:52.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:52.663 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:52.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:52.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:52.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:52.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:52.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:52.938 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:52.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:53.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:53.064 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:53.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:53.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:53.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:53.323 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:53.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:53.425 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:53.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T09:59:53.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:53.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:53.498 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 09:59:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:09:59:53] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-09-21T09:59:53.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:53.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:53.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:53.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:53.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:53.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:53.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:53.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:54.034 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:54.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:54.112 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:54.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:54.186 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:54.288 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:54.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:54 smithi191 ceph-mon[102206]: pgmap v1283: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3436/99048 objects degraded (3.469%); 0 B/s, 37 objects/s recovering 2024-09-21T09:59:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3436/99048 objects degraded (3.469%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:54 smithi076 ceph-mon[106482]: pgmap v1283: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3436/99048 objects degraded (3.469%); 0 B/s, 37 objects/s recovering 2024-09-21T09:59:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3436/99048 objects degraded (3.469%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:54 smithi076 ceph-mon[110963]: pgmap v1283: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3436/99048 objects degraded (3.469%); 0 B/s, 37 objects/s recovering 2024-09-21T09:59:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3436/99048 objects degraded (3.469%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:54.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:54.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:54.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:54.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:54.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:54.724 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:54.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:54.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:54.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:54.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T09:59:54.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:55.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:55.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:55.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:55.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:55.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.345 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:55.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:55.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:55.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:55.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:55.597 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:55.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:55.723 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:55.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:55.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:55.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:55.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:56.011 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:56.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:56.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:56.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:56 smithi191 ceph-mon[102206]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3240/96222 objects degraded (3.367%); 0 B/s, 40 objects/s recovering 2024-09-21T09:59:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:56.372 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:56 smithi076 ceph-mon[106482]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3240/96222 objects degraded (3.367%); 0 B/s, 40 objects/s recovering 2024-09-21T09:59:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:56 smithi076 ceph-mon[110963]: pgmap v1284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 3240/96222 objects degraded (3.367%); 0 B/s, 40 objects/s recovering 2024-09-21T09:59:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T09:59:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:56.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:56.497 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:56.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:56.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:56.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:56.739 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T09:59:56.739 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 3240/96222 objects degraded (3.367%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":899,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1123,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":8,"osd_in_since":1726912604,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":32074,"data_bytes":2100828157,"bytes_used":8335589376,"bytes_avail":759624695808,"bytes_total":767960285184,"degraded_objects":3240,"degraded_total":96222,"degraded_ratio":0.033672133191993518,"recovering_objects_per_sec":40,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":405,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":777},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":25,"modified":"2024-09-21T09:58:56.477511+0000","services":{}},"progress_events":{}} 2024-09-21T09:59:56.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:56.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:56.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:56.957 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:56.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:57.058 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:57.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:57.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:57.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:57.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:57.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:57 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3924831936' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:59:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:57 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3924831936' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:59:57.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:57 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3924831936' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T09:59:57.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:57.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:57.514 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:57.552 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T09:59:57.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:59:57.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:57.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:57.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:57.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:57.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:57.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:57.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:58.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:58.101 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:58.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:58.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:58.266 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:58.346 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:58 smithi076 ceph-mon[106482]: pgmap v1285: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 3240/88698 objects degraded (3.653%); 0 B/s, 40 objects/s recovering 2024-09-21T09:59:58.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:58 smithi076 ceph-mon[110963]: pgmap v1285: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 3240/88698 objects degraded (3.653%); 0 B/s, 40 objects/s recovering 2024-09-21T09:59:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:58 smithi191 ceph-mon[102206]: pgmap v1285: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 3240/88698 objects degraded (3.653%); 0 B/s, 40 objects/s recovering 2024-09-21T09:59:58.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T09:59:58.395 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T09:59:58.395 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T09:59:58.395 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T09:59:58.395 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T09:59:58.395 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T09:59:58.396 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:58.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T09:59:58.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T09:59:58.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T09:59:58.640 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:59:58.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T09:59:58.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T09:59:58.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T09:59:58.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:58.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T09:59:58.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:58.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:58.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:59.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T09:59:59.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T09:59:59.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:59.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:59.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:59.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T09:59:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 09:59:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3240/88698 objects degraded (3.653%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 09:59:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3240/88698 objects degraded (3.653%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:59.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:59.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:59.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T09:59:59.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 09:59:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3240/88698 objects degraded (3.653%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T09:59:59.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T09:59:59.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T09:59:59.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T09:59:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T09:59:59.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T09:59:59.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T09:59:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:59:59.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:00.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.042 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:00.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:00.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:00.179 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:00.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:00.303 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[106482]: pgmap v1286: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3240/87279 objects degraded (3.712%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[106482]: Health detail: HEALTH_WARN Degraded data redundancy: 3240/87279 objects degraded (3.712%), 3 pgs degraded, 3 pgs undersized 2024-09-21T10:00:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[106482]: [WRN] PG_DEGRADED: Degraded data redundancy: 3240/87279 objects degraded (3.712%), 3 pgs degraded, 3 pgs undersized 2024-09-21T10:00:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[106482]: pg 6.0 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [0,2] 2024-09-21T10:00:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[106482]: pg 6.2 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[106482]: pg 6.c is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[110963]: pgmap v1286: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3240/87279 objects degraded (3.712%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[110963]: Health detail: HEALTH_WARN Degraded data redundancy: 3240/87279 objects degraded (3.712%), 3 pgs degraded, 3 pgs undersized 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[110963]: [WRN] PG_DEGRADED: Degraded data redundancy: 3240/87279 objects degraded (3.712%), 3 pgs degraded, 3 pgs undersized 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[110963]: pg 6.0 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [0,2] 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[110963]: pg 6.2 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2024-09-21T10:00:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:00 smithi076 ceph-mon[110963]: pg 6.c is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2024-09-21T10:00:00.404 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:00.440 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:00.542 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:00.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:00.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:00.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:00 smithi191 ceph-mon[102206]: pgmap v1286: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 3240/87279 objects degraded (3.712%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:00 smithi191 ceph-mon[102206]: Health detail: HEALTH_WARN Degraded data redundancy: 3240/87279 objects degraded (3.712%), 3 pgs degraded, 3 pgs undersized 2024-09-21T10:00:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:00 smithi191 ceph-mon[102206]: [WRN] PG_DEGRADED: Degraded data redundancy: 3240/87279 objects degraded (3.712%), 3 pgs degraded, 3 pgs undersized 2024-09-21T10:00:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:00 smithi191 ceph-mon[102206]: pg 6.0 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [0,2] 2024-09-21T10:00:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:00 smithi191 ceph-mon[102206]: pg 6.2 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2024-09-21T10:00:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:00 smithi191 ceph-mon[102206]: pg 6.c is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,6] 2024-09-21T10:00:00.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:00.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:00.812 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:00.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:00.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:00.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:00.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:01.085 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:01.109 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:01.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:01.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:01.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:01.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:01.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:01.477 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:01.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:01.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:01.663 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:01.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:01.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:01.792 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:01.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:01.922 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:01.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:02.024 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:02.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:02.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:02.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:02.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:02.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:02.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:02.297 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:02.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:02.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:02.338 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:00:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:02 smithi076 ceph-mon[106482]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 3039/83715 objects degraded (3.630%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:02.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:02 smithi076 ceph-mon[110963]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 3039/83715 objects degraded (3.630%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:02.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:02.448 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:02.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:02.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:02.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:02.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:02 smithi191 ceph-mon[102206]: pgmap v1287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 3039/83715 objects degraded (3.630%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:02.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:02.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:02.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:02.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:02.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:02.869 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:02.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:02.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:03.000 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:03.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:03.102 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:03.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:03.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:03.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:03.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:03.360 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:00:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:00:03] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.51.0" 2024-09-21T10:00:03.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:03.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:03.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:03.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:03.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:03.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:03.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:03.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:03.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:03.717 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:03.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:03.805 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:03.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:03.928 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:03.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:03.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:04.030 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:04.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:04.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:04.174 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:04.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:04.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:04.306 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[106482]: pgmap v1288: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 3039/75363 objects degraded (4.032%); 0 B/s, 34 objects/s recovering 2024-09-21T10:00:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[106482]: osdmap e1124: 8 total, 8 up, 8 in 2024-09-21T10:00:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2388307597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3039/75363 objects degraded (4.032%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[110963]: pgmap v1288: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 3039/75363 objects degraded (4.032%); 0 B/s, 34 objects/s recovering 2024-09-21T10:00:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[110963]: osdmap e1124: 8 total, 8 up, 8 in 2024-09-21T10:00:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2388307597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3039/75363 objects degraded (4.032%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:04.408 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:04.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:04.544 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:04.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:04 smithi191 ceph-mon[102206]: pgmap v1288: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 3039/75363 objects degraded (4.032%); 0 B/s, 34 objects/s recovering 2024-09-21T10:00:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:04 smithi191 ceph-mon[102206]: osdmap e1124: 8 total, 8 up, 8 in 2024-09-21T10:00:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2388307597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3039/75363 objects degraded (4.032%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:04.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:04.730 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:04.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:04.832 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:04.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:04.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:04.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:05.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:05.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:05.147 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:05.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:05.165 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:00:05.250 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:05.288 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:05.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:05.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:05.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:05.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:05.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:05 smithi191 ceph-mon[102206]: osdmap e1125: 8 total, 8 up, 8 in 2024-09-21T10:00:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:05 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/653797432' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:05 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:05.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:05 smithi076 ceph-mon[106482]: osdmap e1125: 8 total, 8 up, 8 in 2024-09-21T10:00:05.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:05 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/653797432' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:05.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:05 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:05 smithi076 ceph-mon[110963]: osdmap e1125: 8 total, 8 up, 8 in 2024-09-21T10:00:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:05 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/653797432' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:05.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:05 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:00:05.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:05.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:05.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:05.813 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:05.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:05.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:05.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:06.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:06.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:06.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:06.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:06.227 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[106482]: pgmap v1291: 89 pgs: 1 peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 2888/72705 objects degraded (3.972%); 0 B/s, 43 objects/s recovering 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[106482]: osdmap e1126: 8 total, 8 up, 8 in 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[110963]: pgmap v1291: 89 pgs: 1 peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 2888/72705 objects degraded (3.972%); 0 B/s, 43 objects/s recovering 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:00:06.304 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:06 smithi076 ceph-mon[110963]: osdmap e1126: 8 total, 8 up, 8 in 2024-09-21T10:00:06.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:06.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:06.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.492 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:06.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:06.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:00:06.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:06 smithi191 ceph-mon[102206]: pgmap v1291: 89 pgs: 1 peering, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 2888/72705 objects degraded (3.972%); 0 B/s, 43 objects/s recovering 2024-09-21T10:00:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:06 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:06 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:00:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:06 smithi191 ceph-mon[102206]: osdmap e1126: 8 total, 8 up, 8 in 2024-09-21T10:00:06.617 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:06.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:06.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:06.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:06.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:06.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:06.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:06.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:07.012 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:07.017 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.067 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:07.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:07.103 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:07.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:07.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:07.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:07.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:07.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:07.380 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:00:07.384 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:00:07.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:07.459 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:07.560 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:07.594 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:07.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:07.696 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:07.733 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:07.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:07.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.866 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:07.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:07.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:07.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:07.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:07.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:08.097 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:08.128 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:08.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:08.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:08.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:08.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:08.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:08.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:08.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:08.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:08.567 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:08 smithi076 ceph-mon[106482]: pgmap v1293: 89 pgs: 1 active+recovery_wait, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 2888/68973 objects degraded (4.187%); 0 B/s, 25 objects/s recovering 2024-09-21T10:00:08.567 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:08 smithi076 ceph-mon[110963]: pgmap v1293: 89 pgs: 1 active+recovery_wait, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 2888/68973 objects degraded (4.187%); 0 B/s, 25 objects/s recovering 2024-09-21T10:00:08.606 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1293,"stamp":"2024-09-21T10:00:06.518057+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1505564669,"num_objects":22991,"num_object_clones":0,"num_object_copies":68973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22991,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":205122,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11472,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142451,"ondisk_log_size":142451,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6350788,"kb_used_data":5039988,"kb_used_omap":195,"kb_used_meta":1310524,"kb_avail":743610428,"statfs":{"total":767960285184,"available":761457078272,"internally_reserved":0,"allocated":5160947712,"data_stored":5144491408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1341976708},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,5,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-322043904,"num_objects":-4914,"num_object_clones":0,"num_object_copies":-14742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-151,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4914,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.004477"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.156210+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T10:00:03.156210+0000","last_peered":"2024-09-21T10:00:03.156210+0000","last_clean":"2024-09-21T10:00:03.156210+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T10:00:03.156210+0000","last_undegraded":"2024-09-21T10:00:03.156210+0000","last_fullsized":"2024-09-21T10:00:03.156210+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1055,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.164496+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T10:00:05.164496+0000","last_peered":"2024-09-21T10:00:05.164496+0000","last_clean":"2024-09-21T10:00:05.164496+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T10:00:05.164496+0000","last_undegraded":"2024-09-21T10:00:05.164496+0000","last_fullsized":"2024-09-21T10:00:05.164496+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1082,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962762+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T09:59:33.962762+0000","last_peered":"2024-09-21T09:59:33.962762+0000","last_clean":"2024-09-21T09:59:33.962762+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T09:59:33.962762+0000","last_undegraded":"2024-09-21T09:59:33.962762+0000","last_fullsized":"2024-09-21T09:59:33.962762+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1439,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.153843+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T10:00:03.153843+0000","last_peered":"2024-09-21T10:00:03.153843+0000","last_clean":"2024-09-21T10:00:03.153843+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T10:00:03.153843+0000","last_undegraded":"2024-09-21T10:00:03.153843+0000","last_fullsized":"2024-09-21T10:00:03.153843+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1439,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154276+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T10:00:03.154276+0000","last_peered":"2024-09-21T10:00:03.154276+0000","last_clean":"2024-09-21T10:00:03.154276+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T10:00:03.154276+0000","last_undegraded":"2024-09-21T10:00:03.154276+0000","last_fullsized":"2024-09-21T10:00:03.154276+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1038,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.141626+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T10:00:04.141626+0000","last_peered":"2024-09-21T10:00:04.141626+0000","last_clean":"2024-09-21T10:00:04.141626+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T10:00:04.141626+0000","last_undegraded":"2024-09-21T10:00:04.141626+0000","last_fullsized":"2024-09-21T10:00:04.141626+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.158552+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T10:00:03.158552+0000","last_peered":"2024-09-21T10:00:03.158552+0000","last_clean":"2024-09-21T10:00:03.158552+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T10:00:03.158552+0000","last_undegraded":"2024-09-21T10:00:03.158552+0000","last_fullsized":"2024-09-21T10:00:03.158552+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1065,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212176+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T10:00:04.212176+0000","last_peered":"2024-09-21T10:00:04.212176+0000","last_clean":"2024-09-21T10:00:04.212176+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T10:00:04.212176+0000","last_undegraded":"2024-09-21T10:00:04.212176+0000","last_fullsized":"2024-09-21T10:00:04.212176+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1123'12617","reported_seq":12662,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T10:00:01.521513+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T10:00:01.521513+0000","last_peered":"2024-09-21T10:00:01.521513+0000","last_clean":"2024-09-21T10:00:01.521513+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T10:00:01.521513+0000","last_undegraded":"2024-09-21T10:00:01.521513+0000","last_fullsized":"2024-09-21T10:00:01.521513+0000","mapping_epoch":1086,"log_start":"1095'2600","ondisk_log_start":"1095'2600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97452032,"num_objects":1487,"num_object_clones":0,"num_object_copies":4461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12617,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1054,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142052+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T10:00:04.142052+0000","last_peered":"2024-09-21T10:00:04.142052+0000","last_clean":"2024-09-21T10:00:04.142052+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T10:00:04.142052+0000","last_undegraded":"2024-09-21T10:00:04.142052+0000","last_fullsized":"2024-09-21T10:00:04.142052+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1050,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962396+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T09:59:33.962396+0000","last_peered":"2024-09-21T09:59:33.962396+0000","last_clean":"2024-09-21T09:59:33.962396+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T09:59:33.962396+0000","last_undegraded":"2024-09-21T09:59:33.962396+0000","last_fullsized":"2024-09-21T09:59:33.962396+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1124'12686","reported_seq":15647,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.516846+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T10:00:03.516846+0000","last_peered":"2024-09-21T10:00:03.516846+0000","last_clean":"2024-09-21T10:00:03.516846+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T10:00:03.516846+0000","last_undegraded":"2024-09-21T10:00:03.516846+0000","last_fullsized":"2024-09-21T10:00:03.516846+0000","mapping_epoch":1108,"log_start":"1103'5300","ondisk_log_start":"1103'5300","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7386,"log_dups_size":0,"ondisk_log_size":7386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93716480,"num_objects":1430,"num_object_clones":0,"num_object_copies":4290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12686,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1055,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142056+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T10:00:04.142056+0000","last_peered":"2024-09-21T10:00:04.142056+0000","last_clean":"2024-09-21T10:00:04.142056+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T10:00:04.142056+0000","last_undegraded":"2024-09-21T10:00:04.142056+0000","last_fullsized":"2024-09-21T10:00:04.142056+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1107,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205320+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T10:00:04.205320+0000","last_peered":"2024-09-21T10:00:04.205320+0000","last_clean":"2024-09-21T10:00:04.205320+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T10:00:04.205320+0000","last_undegraded":"2024-09-21T10:00:04.205320+0000","last_fullsized":"2024-09-21T10:00:04.205320+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1126'13075","reported_seq":16009,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:06.390166+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T10:00:06.390166+0000","last_peered":"2024-09-21T10:00:06.390166+0000","last_clean":"2024-09-21T10:00:06.390166+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T10:00:06.390166+0000","last_undegraded":"2024-09-21T10:00:06.390166+0000","last_fullsized":"2024-09-21T10:00:06.390166+0000","mapping_epoch":1100,"log_start":"1101'4900","ondisk_log_start":"1101'4900","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8175,"log_dups_size":0,"ondisk_log_size":8175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86310912,"num_objects":1317,"num_object_clones":0,"num_object_copies":3951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13075,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1039,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142247+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T10:00:04.142247+0000","last_peered":"2024-09-21T10:00:04.142247+0000","last_clean":"2024-09-21T10:00:04.142247+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T10:00:04.142247+0000","last_undegraded":"2024-09-21T10:00:04.142247+0000","last_fullsized":"2024-09-21T10:00:04.142247+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1056,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212236+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T10:00:04.212236+0000","last_peered":"2024-09-21T10:00:04.212236+0000","last_clean":"2024-09-21T10:00:04.212236+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T10:00:04.212236+0000","last_undegraded":"2024-09-21T10:00:04.212236+0000","last_fullsized":"2024-09-21T10:00:04.212236+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1124'12862","reported_seq":12950,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.415587+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T10:00:03.415587+0000","last_peered":"2024-09-21T10:00:03.415587+0000","last_clean":"2024-09-21T10:00:03.415587+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T10:00:03.415587+0000","last_undegraded":"2024-09-21T10:00:03.415587+0000","last_fullsized":"2024-09-21T10:00:03.415587+0000","mapping_epoch":1086,"log_start":"1097'3100","ondisk_log_start":"1097'3100","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9762,"log_dups_size":0,"ondisk_log_size":9762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95551488,"num_objects":1458,"num_object_clones":0,"num_object_copies":4374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12862,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1084,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962744+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T09:59:33.962744+0000","last_peered":"2024-09-21T09:59:33.962744+0000","last_clean":"2024-09-21T09:59:33.962744+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T09:59:33.962744+0000","last_undegraded":"2024-09-21T09:59:33.962744+0000","last_fullsized":"2024-09-21T09:59:33.962744+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1427,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.153601+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T10:00:03.153601+0000","last_peered":"2024-09-21T10:00:03.153601+0000","last_clean":"2024-09-21T10:00:03.153601+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T10:00:03.153601+0000","last_undegraded":"2024-09-21T10:00:03.153601+0000","last_fullsized":"2024-09-21T10:00:03.153601+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1126'12820","reported_seq":12870,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:06.429567+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T10:00:06.429567+0000","last_peered":"2024-09-21T10:00:06.429567+0000","last_clean":"2024-09-21T10:00:06.429567+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T10:00:06.429567+0000","last_undegraded":"2024-09-21T10:00:06.429567+0000","last_fullsized":"2024-09-21T10:00:06.429567+0000","mapping_epoch":1086,"log_start":"1101'4900","ondisk_log_start":"1101'4900","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7920,"log_dups_size":0,"ondisk_log_size":7920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12820,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":79,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.156995+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T10:00:03.156995+0000","last_peered":"2024-09-21T10:00:03.156995+0000","last_clean":"2024-09-21T10:00:03.156995+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T10:00:03.156995+0000","last_undegraded":"2024-09-21T10:00:03.156995+0000","last_fullsized":"2024-09-21T10:00:03.156995+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1092,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205424+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T10:00:04.205424+0000","last_peered":"2024-09-21T10:00:04.205424+0000","last_clean":"2024-09-21T10:00:04.205424+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T10:00:04.205424+0000","last_undegraded":"2024-09-21T10:00:04.205424+0000","last_fullsized":"2024-09-21T10:00:04.205424+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1126'12884","reported_seq":15808,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.231412+0000","last_change":"2024-09-21T09:59:03.867059+0000","last_active":"2024-09-21T10:00:05.231412+0000","last_peered":"2024-09-21T10:00:05.231412+0000","last_clean":"2024-09-21T10:00:05.231412+0000","last_became_active":"2024-09-21T09:59:03.844371+0000","last_became_peered":"2024-09-21T09:59:03.844371+0000","last_unstale":"2024-09-21T10:00:05.231412+0000","last_undegraded":"2024-09-21T10:00:05.231412+0000","last_fullsized":"2024-09-21T10:00:05.231412+0000","mapping_epoch":1117,"log_start":"1095'2800","ondisk_log_start":"1095'2800","created":1086,"last_epoch_clean":1118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:34:30.990234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96731136,"num_objects":1476,"num_object_clones":0,"num_object_copies":4428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12884,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1057,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212311+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T10:00:04.212311+0000","last_peered":"2024-09-21T10:00:04.212311+0000","last_clean":"2024-09-21T10:00:04.212311+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T10:00:04.212311+0000","last_undegraded":"2024-09-21T10:00:04.212311+0000","last_fullsized":"2024-09-21T10:00:04.212311+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1089,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205773+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T10:00:04.205773+0000","last_peered":"2024-09-21T10:00:04.205773+0000","last_clean":"2024-09-21T10:00:04.205773+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T10:00:04.205773+0000","last_undegraded":"2024-09-21T10:00:04.205773+0000","last_fullsized":"2024-09-21T10:00:04.205773+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1123'12662","reported_seq":12713,"reported_epoch":1123,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:00:01.488575+0000","last_change":"2024-09-21T09:56:46.373388+0000","last_active":"2024-09-21T10:00:01.488575+0000","last_peered":"2024-09-21T10:00:01.488575+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T10:00:01.488575+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"1095'2600","ondisk_log_start":"1095'2600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99090432,"num_objects":1512,"num_object_clones":0,"num_object_copies":4536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1476,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12662,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":36}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1075,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205420+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T10:00:04.205420+0000","last_peered":"2024-09-21T10:00:04.205420+0000","last_clean":"2024-09-21T10:00:04.205420+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T10:00:04.205420+0000","last_undegraded":"2024-09-21T10:00:04.205420+0000","last_fullsized":"2024-09-21T10:00:04.205420+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1083,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205935+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T10:00:04.205935+0000","last_peered":"2024-09-21T10:00:04.205935+0000","last_clean":"2024-09-21T10:00:04.205935+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T10:00:04.205935+0000","last_undegraded":"2024-09-21T10:00:04.205935+0000","last_fullsized":"2024-09-21T10:00:04.205935+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1123'12736","reported_seq":12784,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T10:00:01.502357+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T10:00:01.502357+0000","last_peered":"2024-09-21T10:00:01.502357+0000","last_clean":"2024-09-21T10:00:01.502357+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T10:00:01.502357+0000","last_undegraded":"2024-09-21T10:00:01.502357+0000","last_fullsized":"2024-09-21T10:00:01.502357+0000","mapping_epoch":1086,"log_start":"1095'2700","ondisk_log_start":"1095'2700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102563864,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12736,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.155290+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T10:00:03.155290+0000","last_peered":"2024-09-21T10:00:03.155290+0000","last_clean":"2024-09-21T10:00:03.155290+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T10:00:03.155290+0000","last_undegraded":"2024-09-21T10:00:03.155290+0000","last_fullsized":"2024-09-21T10:00:03.155290+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1411,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.094759+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T09:59:51.094759+0000","last_peered":"2024-09-21T09:59:51.094759+0000","last_clean":"2024-09-21T09:59:51.094759+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T09:59:51.094759+0000","last_undegraded":"2024-09-21T09:59:51.094759+0000","last_fullsized":"2024-09-21T09:59:51.094759+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1123'12458","reported_seq":15293,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T10:00:01.531255+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T10:00:01.531255+0000","last_peered":"2024-09-21T10:00:01.531255+0000","last_clean":"2024-09-21T10:00:01.531255+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T10:00:01.531255+0000","last_undegraded":"2024-09-21T10:00:01.531255+0000","last_fullsized":"2024-09-21T10:00:01.531255+0000","mapping_epoch":1113,"log_start":"1095'2400","ondisk_log_start":"1095'2400","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101580800,"num_objects":1550,"num_object_clones":0,"num_object_copies":4650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12458,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1384,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.141621+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T10:00:04.141621+0000","last_peered":"2024-09-21T10:00:04.141621+0000","last_clean":"2024-09-21T10:00:04.141621+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T10:00:04.141621+0000","last_undegraded":"2024-09-21T10:00:04.141621+0000","last_fullsized":"2024-09-21T10:00:04.141621+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1080,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962546+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T09:59:33.962546+0000","last_peered":"2024-09-21T09:59:33.962546+0000","last_clean":"2024-09-21T09:59:33.962546+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T09:59:33.962546+0000","last_undegraded":"2024-09-21T09:59:33.962546+0000","last_fullsized":"2024-09-21T09:59:33.962546+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.153016+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T10:00:03.153016+0000","last_peered":"2024-09-21T10:00:03.153016+0000","last_clean":"2024-09-21T10:00:03.153016+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T10:00:03.153016+0000","last_undegraded":"2024-09-21T10:00:03.153016+0000","last_fullsized":"2024-09-21T10:00:03.153016+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1042,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212296+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T10:00:04.212296+0000","last_peered":"2024-09-21T10:00:04.212296+0000","last_clean":"2024-09-21T10:00:04.212296+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T10:00:04.212296+0000","last_undegraded":"2024-09-21T10:00:04.212296+0000","last_fullsized":"2024-09-21T10:00:04.212296+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1123'12669","reported_seq":15721,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T10:00:02.129551+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T10:00:02.129551+0000","last_peered":"2024-09-21T10:00:02.129551+0000","last_clean":"2024-09-21T10:00:02.129551+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T10:00:02.129551+0000","last_undegraded":"2024-09-21T10:00:02.129551+0000","last_fullsized":"2024-09-21T10:00:02.129551+0000","mapping_epoch":1105,"log_start":"1099'4000","ondisk_log_start":"1099'4000","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8669,"log_dups_size":0,"ondisk_log_size":8669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97583104,"num_objects":1489,"num_object_clones":0,"num_object_copies":4467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12669,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1082,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.141538+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T10:00:04.141538+0000","last_peered":"2024-09-21T10:00:04.141538+0000","last_clean":"2024-09-21T10:00:04.141538+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T10:00:04.141538+0000","last_undegraded":"2024-09-21T10:00:04.141538+0000","last_fullsized":"2024-09-21T10:00:04.141538+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1440,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.156456+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T10:00:03.156456+0000","last_peered":"2024-09-21T10:00:03.156456+0000","last_clean":"2024-09-21T10:00:03.156456+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T10:00:03.156456+0000","last_undegraded":"2024-09-21T10:00:03.156456+0000","last_fullsized":"2024-09-21T10:00:03.156456+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1438,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154094+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T10:00:03.154094+0000","last_peered":"2024-09-21T10:00:03.154094+0000","last_clean":"2024-09-21T10:00:03.154094+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T10:00:03.154094+0000","last_undegraded":"2024-09-21T10:00:03.154094+0000","last_fullsized":"2024-09-21T10:00:03.154094+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1126'12855","reported_seq":15728,"reported_epoch":1126,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:00:06.382610+0000","last_change":"2024-09-21T09:59:29.091758+0000","last_active":"2024-09-21T10:00:06.382610+0000","last_peered":"2024-09-21T10:00:06.382610+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:59:29.047032+0000","last_became_peered":"2024-09-21T09:59:29.047032+0000","last_unstale":"2024-09-21T10:00:06.382610+0000","last_undegraded":"2024-09-21T10:00:06.382610+0000","last_fullsized":"2024-09-21T10:00:06.382610+0000","mapping_epoch":1120,"log_start":"1101'4700","ondisk_log_start":"1101'4700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8155,"log_dups_size":0,"ondisk_log_size":8155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:33.224146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84738048,"num_objects":1293,"num_object_clones":0,"num_object_copies":3879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12855,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1405,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":["7","2","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":1293}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1422,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.173190+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T10:00:05.173190+0000","last_peered":"2024-09-21T10:00:05.173190+0000","last_clean":"2024-09-21T10:00:05.173190+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T10:00:05.173190+0000","last_undegraded":"2024-09-21T10:00:05.173190+0000","last_fullsized":"2024-09-21T10:00:05.173190+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1398,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.094223+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T09:59:51.094223+0000","last_peered":"2024-09-21T09:59:51.094223+0000","last_clean":"2024-09-21T09:59:51.094223+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T09:59:51.094223+0000","last_undegraded":"2024-09-21T09:59:51.094223+0000","last_fullsized":"2024-09-21T09:59:51.094223+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1405,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.094645+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T09:59:51.094645+0000","last_peered":"2024-09-21T09:59:51.094645+0000","last_clean":"2024-09-21T09:59:51.094645+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T09:59:51.094645+0000","last_undegraded":"2024-09-21T09:59:51.094645+0000","last_fullsized":"2024-09-21T09:59:51.094645+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1124'12799","reported_seq":15782,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.470586+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T10:00:03.470586+0000","last_peered":"2024-09-21T10:00:03.470586+0000","last_clean":"2024-09-21T10:00:03.470586+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T10:00:03.470586+0000","last_undegraded":"2024-09-21T10:00:03.470586+0000","last_fullsized":"2024-09-21T10:00:03.470586+0000","mapping_epoch":1096,"log_start":"1103'5400","ondisk_log_start":"1103'5400","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7399,"log_dups_size":0,"ondisk_log_size":7399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92078080,"num_objects":1405,"num_object_clones":0,"num_object_copies":4215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12799,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1081,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962585+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T09:59:33.962585+0000","last_peered":"2024-09-21T09:59:33.962585+0000","last_clean":"2024-09-21T09:59:33.962585+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T09:59:33.962585+0000","last_undegraded":"2024-09-21T09:59:33.962585+0000","last_fullsized":"2024-09-21T09:59:33.962585+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1076,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205721+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T10:00:04.205721+0000","last_peered":"2024-09-21T10:00:04.205721+0000","last_clean":"2024-09-21T10:00:04.205721+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T10:00:04.205721+0000","last_undegraded":"2024-09-21T10:00:04.205721+0000","last_fullsized":"2024-09-21T10:00:04.205721+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1423,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.101683+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T09:59:51.101683+0000","last_peered":"2024-09-21T09:59:51.101683+0000","last_clean":"2024-09-21T09:59:51.101683+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T09:59:51.101683+0000","last_undegraded":"2024-09-21T09:59:51.101683+0000","last_fullsized":"2024-09-21T09:59:51.101683+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1123'12735","reported_seq":15747,"reported_epoch":1124,"state":"peering","last_fresh":"2024-09-21T10:00:03.171081+0000","last_change":"2024-09-21T10:00:03.171081+0000","last_active":"2024-09-21T10:00:03.171081+0000","last_peered":"2024-09-21T10:00:03.085759+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T09:56:46.340094+0000","last_became_peered":"2024-09-21T09:56:46.340094+0000","last_unstale":"2024-09-21T10:00:03.171081+0000","last_undegraded":"2024-09-21T10:00:03.171081+0000","last_fullsized":"2024-09-21T10:00:03.171081+0000","mapping_epoch":1124,"log_start":"1097'3000","ondisk_log_start":"1097'3000","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9735,"log_dups_size":0,"ondisk_log_size":9735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93913088,"num_objects":1433,"num_object_clones":0,"num_object_copies":4299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12735,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1461,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1100,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:41.496938+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T09:59:41.496938+0000","last_peered":"2024-09-21T09:59:41.496938+0000","last_clean":"2024-09-21T09:59:41.496938+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T09:59:41.496938+0000","last_undegraded":"2024-09-21T09:59:41.496938+0000","last_fullsized":"2024-09-21T09:59:41.496938+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1420,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.101262+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T09:59:51.101262+0000","last_peered":"2024-09-21T09:59:51.101262+0000","last_clean":"2024-09-21T09:59:51.101262+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T09:59:51.101262+0000","last_undegraded":"2024-09-21T09:59:51.101262+0000","last_fullsized":"2024-09-21T09:59:51.101262+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.153640+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T10:00:03.153640+0000","last_peered":"2024-09-21T10:00:03.153640+0000","last_clean":"2024-09-21T10:00:03.153640+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T10:00:03.153640+0000","last_undegraded":"2024-09-21T10:00:03.153640+0000","last_fullsized":"2024-09-21T10:00:03.153640+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1123'13060","reported_seq":13146,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T10:00:02.115565+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T10:00:02.115565+0000","last_peered":"2024-09-21T10:00:02.115565+0000","last_clean":"2024-09-21T10:00:02.115565+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T10:00:02.115565+0000","last_undegraded":"2024-09-21T10:00:02.115565+0000","last_fullsized":"2024-09-21T10:00:02.115565+0000","mapping_epoch":1086,"log_start":"1101'4400","ondisk_log_start":"1101'4400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8660,"log_dups_size":0,"ondisk_log_size":8660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99352576,"num_objects":1516,"num_object_clones":0,"num_object_copies":4548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13060,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1397,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.104650+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T09:59:51.104650+0000","last_peered":"2024-09-21T09:59:51.104650+0000","last_clean":"2024-09-21T09:59:51.104650+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T09:59:51.104650+0000","last_undegraded":"2024-09-21T09:59:51.104650+0000","last_fullsized":"2024-09-21T09:59:51.104650+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1066,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142242+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T10:00:04.142242+0000","last_peered":"2024-09-21T10:00:04.142242+0000","last_clean":"2024-09-21T10:00:04.142242+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T10:00:04.142242+0000","last_undegraded":"2024-09-21T10:00:04.142242+0000","last_fullsized":"2024-09-21T10:00:04.142242+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1416,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.153697+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T10:00:03.153697+0000","last_peered":"2024-09-21T10:00:03.153697+0000","last_clean":"2024-09-21T10:00:03.153697+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T10:00:03.153697+0000","last_undegraded":"2024-09-21T10:00:03.153697+0000","last_fullsized":"2024-09-21T10:00:03.153697+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1126'12949","reported_seq":13007,"reported_epoch":1126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:00:06.368879+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T10:00:06.368879+0000","last_peered":"2024-09-21T10:00:06.368879+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T10:00:06.368879+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1102'5000","ondisk_log_start":"1102'5000","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7949,"log_dups_size":0,"ondisk_log_size":7949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90898432,"num_objects":1387,"num_object_clones":0,"num_object_copies":4161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12949,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-25}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":79,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.158630+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T10:00:03.158630+0000","last_peered":"2024-09-21T10:00:03.158630+0000","last_clean":"2024-09-21T10:00:03.158630+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T10:00:03.158630+0000","last_undegraded":"2024-09-21T10:00:03.158630+0000","last_fullsized":"2024-09-21T10:00:03.158630+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1397,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.096815+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T09:59:51.096815+0000","last_peered":"2024-09-21T09:59:51.096815+0000","last_clean":"2024-09-21T09:59:51.096815+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T09:59:51.096815+0000","last_undegraded":"2024-09-21T09:59:51.096815+0000","last_fullsized":"2024-09-21T09:59:51.096815+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.155219+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T10:00:03.155219+0000","last_peered":"2024-09-21T10:00:03.155219+0000","last_clean":"2024-09-21T10:00:03.155219+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T10:00:03.155219+0000","last_undegraded":"2024-09-21T10:00:03.155219+0000","last_fullsized":"2024-09-21T10:00:03.155219+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1126'12976","reported_seq":13026,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:06.384128+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T10:00:06.384128+0000","last_peered":"2024-09-21T10:00:06.384128+0000","last_clean":"2024-09-21T10:00:06.384128+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T10:00:06.384128+0000","last_undegraded":"2024-09-21T10:00:06.384128+0000","last_fullsized":"2024-09-21T10:00:06.384128+0000","mapping_epoch":1086,"log_start":"1101'4800","ondisk_log_start":"1101'4800","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85196800,"num_objects":1300,"num_object_clones":0,"num_object_copies":3900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12976,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1430,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154380+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T10:00:03.154380+0000","last_peered":"2024-09-21T10:00:03.154380+0000","last_clean":"2024-09-21T10:00:03.154380+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T10:00:03.154380+0000","last_undegraded":"2024-09-21T10:00:03.154380+0000","last_fullsized":"2024-09-21T10:00:03.154380+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1070,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212428+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T10:00:04.212428+0000","last_peered":"2024-09-21T10:00:04.212428+0000","last_clean":"2024-09-21T10:00:04.212428+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T10:00:04.212428+0000","last_undegraded":"2024-09-21T10:00:04.212428+0000","last_fullsized":"2024-09-21T10:00:04.212428+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1060,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205571+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T10:00:04.205571+0000","last_peered":"2024-09-21T10:00:04.205571+0000","last_clean":"2024-09-21T10:00:04.205571+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T10:00:04.205571+0000","last_undegraded":"2024-09-21T10:00:04.205571+0000","last_fullsized":"2024-09-21T10:00:04.205571+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1073,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205646+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T10:00:04.205646+0000","last_peered":"2024-09-21T10:00:04.205646+0000","last_clean":"2024-09-21T10:00:04.205646+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T10:00:04.205646+0000","last_undegraded":"2024-09-21T10:00:04.205646+0000","last_fullsized":"2024-09-21T10:00:04.205646+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1083,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.160099+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T10:00:05.160099+0000","last_peered":"2024-09-21T10:00:05.160099+0000","last_clean":"2024-09-21T10:00:05.160099+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T10:00:05.160099+0000","last_undegraded":"2024-09-21T10:00:05.160099+0000","last_fullsized":"2024-09-21T10:00:05.160099+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1397,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.094461+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T09:59:51.094461+0000","last_peered":"2024-09-21T09:59:51.094461+0000","last_clean":"2024-09-21T09:59:51.094461+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T09:59:51.094461+0000","last_undegraded":"2024-09-21T09:59:51.094461+0000","last_fullsized":"2024-09-21T09:59:51.094461+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1085,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142161+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T10:00:04.142161+0000","last_peered":"2024-09-21T10:00:04.142161+0000","last_clean":"2024-09-21T10:00:04.142161+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T10:00:04.142161+0000","last_undegraded":"2024-09-21T10:00:04.142161+0000","last_fullsized":"2024-09-21T10:00:04.142161+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1065,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962895+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T09:59:33.962895+0000","last_peered":"2024-09-21T09:59:33.962895+0000","last_clean":"2024-09-21T09:59:33.962895+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T09:59:33.962895+0000","last_undegraded":"2024-09-21T09:59:33.962895+0000","last_fullsized":"2024-09-21T09:59:33.962895+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.152715+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T10:00:03.152715+0000","last_peered":"2024-09-21T10:00:03.152715+0000","last_clean":"2024-09-21T10:00:03.152715+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T10:00:03.152715+0000","last_undegraded":"2024-09-21T10:00:03.152715+0000","last_fullsized":"2024-09-21T10:00:03.152715+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1438,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154670+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T10:00:03.154670+0000","last_peered":"2024-09-21T10:00:03.154670+0000","last_clean":"2024-09-21T10:00:03.154670+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T10:00:03.154670+0000","last_undegraded":"2024-09-21T10:00:03.154670+0000","last_fullsized":"2024-09-21T10:00:03.154670+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.155364+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T10:00:03.155364+0000","last_peered":"2024-09-21T10:00:03.155364+0000","last_clean":"2024-09-21T10:00:03.155364+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T10:00:03.155364+0000","last_undegraded":"2024-09-21T10:00:03.155364+0000","last_fullsized":"2024-09-21T10:00:03.155364+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1066,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212371+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T10:00:04.212371+0000","last_peered":"2024-09-21T10:00:04.212371+0000","last_clean":"2024-09-21T10:00:04.212371+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T10:00:04.212371+0000","last_undegraded":"2024-09-21T10:00:04.212371+0000","last_fullsized":"2024-09-21T10:00:04.212371+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.155135+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T10:00:03.155135+0000","last_peered":"2024-09-21T10:00:03.155135+0000","last_clean":"2024-09-21T10:00:03.155135+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T10:00:03.155135+0000","last_undegraded":"2024-09-21T10:00:03.155135+0000","last_fullsized":"2024-09-21T10:00:03.155135+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1441,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154178+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T10:00:03.154178+0000","last_peered":"2024-09-21T10:00:03.154178+0000","last_clean":"2024-09-21T10:00:03.154178+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T10:00:03.154178+0000","last_undegraded":"2024-09-21T10:00:03.154178+0000","last_fullsized":"2024-09-21T10:00:03.154178+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1410,"reported_epoch":1123,"state":"active+clean","last_fresh":"2024-09-21T09:59:51.099030+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T09:59:51.099030+0000","last_peered":"2024-09-21T09:59:51.099030+0000","last_clean":"2024-09-21T09:59:51.099030+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T09:59:51.099030+0000","last_undegraded":"2024-09-21T09:59:51.099030+0000","last_fullsized":"2024-09-21T09:59:51.099030+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154851+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:00:03.154851+0000","last_peered":"2024-09-21T10:00:03.154851+0000","last_clean":"2024-09-21T10:00:03.154851+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:00:03.154851+0000","last_undegraded":"2024-09-21T10:00:03.154851+0000","last_fullsized":"2024-09-21T10:00:03.154851+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1090,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205602+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T10:00:04.205602+0000","last_peered":"2024-09-21T10:00:04.205602+0000","last_clean":"2024-09-21T10:00:04.205602+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T10:00:04.205602+0000","last_undegraded":"2024-09-21T10:00:04.205602+0000","last_fullsized":"2024-09-21T10:00:04.205602+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1398,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.171421+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T10:00:05.171421+0000","last_peered":"2024-09-21T10:00:05.171421+0000","last_clean":"2024-09-21T10:00:05.171421+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T10:00:05.171421+0000","last_undegraded":"2024-09-21T10:00:05.171421+0000","last_fullsized":"2024-09-21T10:00:05.171421+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1417,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.153782+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T10:00:03.153782+0000","last_peered":"2024-09-21T10:00:03.153782+0000","last_clean":"2024-09-21T10:00:03.153782+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T10:00:03.153782+0000","last_undegraded":"2024-09-21T10:00:03.153782+0000","last_fullsized":"2024-09-21T10:00:03.153782+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1106,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205565+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T10:00:04.205565+0000","last_peered":"2024-09-21T10:00:04.205565+0000","last_clean":"2024-09-21T10:00:04.205565+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T10:00:04.205565+0000","last_undegraded":"2024-09-21T10:00:04.205565+0000","last_fullsized":"2024-09-21T10:00:04.205565+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1067,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962609+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T09:59:33.962609+0000","last_peered":"2024-09-21T09:59:33.962609+0000","last_clean":"2024-09-21T09:59:33.962609+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T09:59:33.962609+0000","last_undegraded":"2024-09-21T09:59:33.962609+0000","last_fullsized":"2024-09-21T09:59:33.962609+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1106,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205459+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T10:00:04.205459+0000","last_peered":"2024-09-21T10:00:04.205459+0000","last_clean":"2024-09-21T10:00:04.205459+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T10:00:04.205459+0000","last_undegraded":"2024-09-21T10:00:04.205459+0000","last_fullsized":"2024-09-21T10:00:04.205459+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1052,"reported_epoch":1122,"state":"active+clean","last_fresh":"2024-09-21T09:59:33.962844+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T09:59:33.962844+0000","last_peered":"2024-09-21T09:59:33.962844+0000","last_clean":"2024-09-21T09:59:33.962844+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T09:59:33.962844+0000","last_undegraded":"2024-09-21T09:59:33.962844+0000","last_fullsized":"2024-09-21T09:59:33.962844+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.152842+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T10:00:03.152842+0000","last_peered":"2024-09-21T10:00:03.152842+0000","last_clean":"2024-09-21T10:00:03.152842+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T10:00:03.152842+0000","last_undegraded":"2024-09-21T10:00:03.152842+0000","last_fullsized":"2024-09-21T10:00:03.152842+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":75,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.157356+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T10:00:03.157356+0000","last_peered":"2024-09-21T10:00:03.157356+0000","last_clean":"2024-09-21T10:00:03.157356+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T10:00:03.157356+0000","last_undegraded":"2024-09-21T10:00:03.157356+0000","last_fullsized":"2024-09-21T10:00:03.157356+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1082,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205237+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T10:00:04.205237+0000","last_peered":"2024-09-21T10:00:04.205237+0000","last_clean":"2024-09-21T10:00:04.205237+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T10:00:04.205237+0000","last_undegraded":"2024-09-21T10:00:04.205237+0000","last_fullsized":"2024-09-21T10:00:04.205237+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1433,"reported_epoch":1124,"state":"active+clean","last_fresh":"2024-09-21T10:00:03.154763+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T10:00:03.154763+0000","last_peered":"2024-09-21T10:00:03.154763+0000","last_clean":"2024-09-21T10:00:03.154763+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T10:00:03.154763+0000","last_undegraded":"2024-09-21T10:00:03.154763+0000","last_fullsized":"2024-09-21T10:00:03.154763+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1504968728,"num_objects":22965,"num_object_clones":0,"num_object_copies":68895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22965,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":204843,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11459,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5033832448,"data_stored":5033820232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142243,"ondisk_log_size":142243,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234291,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":784248,"kb_used_data":614512,"kb_used_omap":1,"kb_used_meta":169726,"kb_avail":92960904,"statfs":{"total":95995035648,"available":95191965696,"internally_reserved":0,"allocated":629260288,"data_stored":627184684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":173799883},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495928,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906928,"kb_used_data":735592,"kb_used_omap":1,"kb_used_meta":171326,"kb_avail":92838224,"statfs":{"total":95995035648,"available":95066341376,"internally_reserved":0,"allocated":753246208,"data_stored":751178669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":175438279},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757565,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":766468,"kb_used_data":591036,"kb_used_omap":1,"kb_used_meta":175422,"kb_avail":92978684,"statfs":{"total":95995035648,"available":95210172416,"internally_reserved":0,"allocated":605220864,"data_stored":603168618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":179632582},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019202,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":621044,"kb_used_data":455400,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":93124108,"statfs":{"total":95995035648,"available":95359086592,"internally_reserved":0,"allocated":466329600,"data_stored":464261574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":169605574},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888498,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1197624,"kb_used_data":944880,"kb_used_omap":34,"kb_used_meta":252701,"kb_avail":92547528,"statfs":{"total":95995035648,"available":94768668672,"internally_reserved":0,"allocated":967557120,"data_stored":965516569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":258766835},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967737,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990876,"kb_used_data":832460,"kb_used_omap":41,"kb_used_meta":158358,"kb_avail":92754276,"statfs":{"total":95995035648,"available":94980378624,"internally_reserved":0,"allocated":852439040,"data_stored":850387068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":162159007},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046975,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594812,"kb_used_data":490548,"kb_used_omap":41,"kb_used_meta":104214,"kb_avail":93150340,"statfs":{"total":95995035648,"available":95385948160,"internally_reserved":0,"allocated":502321152,"data_stored":500276458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":106716073},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126214,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":488788,"kb_used_data":375560,"kb_used_omap":72,"kb_used_meta":113143,"kb_avail":93256364,"statfs":{"total":95995035648,"available":95494516736,"internally_reserved":0,"allocated":384573440,"data_stored":382517768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":115858475},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":377229312,"data_stored":377225240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":494538752,"data_stored":494534680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":831586304,"data_stored":831586304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":943980544,"data_stored":943980544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":443547648,"data_stored":443547648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":595857408,"data_stored":595853336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":733675520,"data_stored":733675520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":613416960,"data_stored":613416960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:00:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:08 smithi191 ceph-mon[102206]: pgmap v1293: 89 pgs: 1 active+recovery_wait, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 2888/68973 objects degraded (4.187%); 0 B/s, 25 objects/s recovering 2024-09-21T10:00:08.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:08.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:08.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:00:08.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:08.797 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:08.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:08.935 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:08.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:09.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:09.063 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:09.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:09.215 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:09.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:09.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:09.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:09.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:09.480 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:09.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:09.513 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:09 smithi191 ceph-mon[102206]: from='client.34896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:09.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 2888/68973 objects degraded (4.187%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:09.532 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:09 smithi076 ceph-mon[106482]: from='client.34896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:09.532 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 2888/68973 objects degraded (4.187%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:09.532 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:09 smithi076 ceph-mon[110963]: from='client.34896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:09.532 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 2888/68973 objects degraded (4.187%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:09.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:09.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:09.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:09.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:09.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:09.870 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:09.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:09.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:09.998 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:10.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:10.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:10.233 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:10.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:10.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:10.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:10.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:10.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:10.508 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:10.545 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:10.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:10.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:10 smithi191 ceph-mon[102206]: pgmap v1294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 2633/65316 objects degraded (4.031%); 0 B/s, 67 objects/s recovering 2024-09-21T10:00:10.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:10 smithi076 ceph-mon[106482]: pgmap v1294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 2633/65316 objects degraded (4.031%); 0 B/s, 67 objects/s recovering 2024-09-21T10:00:10.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:10 smithi076 ceph-mon[110963]: pgmap v1294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 2633/65316 objects degraded (4.031%); 0 B/s, 67 objects/s recovering 2024-09-21T10:00:10.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:10.673 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:10.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:10.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:10.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:10.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:10.930 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:10.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:10.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:11.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:11.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:11.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:11.247 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:11.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:11.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:11.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:11.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:11.521 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:11.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:00:11.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:11.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:11.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:11.799 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:11 smithi191 ceph-mon[102206]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2633/61842 objects degraded (4.258%); 0 B/s, 55 objects/s recovering 2024-09-21T10:00:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:11 smithi076 ceph-mon[106482]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2633/61842 objects degraded (4.258%); 0 B/s, 55 objects/s recovering 2024-09-21T10:00:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:11.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:11 smithi076 ceph-mon[110963]: pgmap v1295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 2633/61842 objects degraded (4.258%); 0 B/s, 55 objects/s recovering 2024-09-21T10:00:11.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:11.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:11.901 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:11.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:11.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:11.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:12.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:12.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:12.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:12.201 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:12.303 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:12.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:12.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:12.455 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:12.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:12.557 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:12.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:12.699 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:12.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:12.833 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:12.882 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:12.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:12.963 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:00:12.963 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:00:12.985 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:13.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:13.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:13.145 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:13.246 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:13.283 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:13.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.412 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:13.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:13.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.553 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:13.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:13.560 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:00:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:00:13] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.51.0" 2024-09-21T10:00:13.655 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:13.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:13.787 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:13.828 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:13.835 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:13 smithi076 ceph-mon[106482]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2445/53667 objects degraded (4.556%); 0 B/s, 52 objects/s recovering 2024-09-21T10:00:13.835 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:13 smithi076 ceph-mon[110963]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2445/53667 objects degraded (4.556%); 0 B/s, 52 objects/s recovering 2024-09-21T10:00:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:13 smithi191 ceph-mon[102206]: pgmap v1296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 2445/53667 objects degraded (4.556%); 0 B/s, 52 objects/s recovering 2024-09-21T10:00:13.930 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:13.977 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:13.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:13.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:14.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:14.114 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:14.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:14.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:14.361 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.397 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:14.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:14.542 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:14.644 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:14.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:14.687 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:14.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:00:14.779 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:14.837 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 2445/53667 objects degraded (4.556%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:14.837 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 2445/53667 objects degraded (4.556%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.842 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 2445/53667 objects degraded (4.556%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:14.943 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:14.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:14.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:14.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:15.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:15.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:15.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:15.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:15.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:15.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:15.385 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:15.486 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.529 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:15.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:15.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:15.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:15.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:15.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:15.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:15 smithi191 ceph-mon[102206]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2445/50787 objects degraded (4.814%); 0 B/s, 44 objects/s recovering 2024-09-21T10:00:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:15 smithi076 ceph-mon[106482]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2445/50787 objects degraded (4.814%); 0 B/s, 44 objects/s recovering 2024-09-21T10:00:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:15 smithi076 ceph-mon[110963]: pgmap v1297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2445/50787 objects degraded (4.814%); 0 B/s, 44 objects/s recovering 2024-09-21T10:00:15.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:15.985 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:15.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:16.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:16.113 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:16.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:16.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:16.353 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:16.400 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:16.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:16.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:16.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:16.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:16.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:16.769 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:16.814 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:16.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:16.832 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:00:16.832 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:00:16.832 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:00:16.832 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:00:16.833 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:00:16.834 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:16.834 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:16.834 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:16.834 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:16.834 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1924496054' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:16.834 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:16.835 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1924496054' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:16.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1924496054' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:16.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:16.960 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:16.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:17.062 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.109 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:17.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:17.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:17.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:17.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:17.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:17.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:17.521 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:17.622 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.668 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:17.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:17.769 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:17.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.807 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:17.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:17.840 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:17 smithi076 ceph-mon[110963]: pgmap v1298: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 963 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 2445/46296 objects degraded (5.281%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:17.840 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:17 smithi076 ceph-mon[106482]: pgmap v1298: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 963 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 2445/46296 objects degraded (5.281%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:17 smithi191 ceph-mon[102206]: pgmap v1298: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 963 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 2445/46296 objects degraded (5.281%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:17.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:17.954 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:17.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:18.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:18.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:18.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:18.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:18.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:18.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:18.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:18.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.465 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:18.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:18.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:18.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:18.618 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:00:18.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:18.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:18.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:18.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2862015913' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:18.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:18 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2862015913' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:18 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2862015913' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:18.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:18 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:18.875 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:18.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:18.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:18.977 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:19.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:19.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.141 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:19.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:19.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:19.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:19.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:19.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.448 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:19.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:19.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:19.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:19.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:19.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:19.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:19.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:19.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:19 smithi191 ceph-mon[102206]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 874 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 2246/42018 objects degraded (5.345%); 0 B/s, 53 objects/s recovering 2024-09-21T10:00:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:19 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:19 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:19 smithi191 ceph-mon[102206]: osdmap e1127: 8 total, 8 up, 8 in 2024-09-21T10:00:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 2246/42018 objects degraded (5.345%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[106482]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 874 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 2246/42018 objects degraded (5.345%); 0 B/s, 53 objects/s recovering 2024-09-21T10:00:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[106482]: osdmap e1127: 8 total, 8 up, 8 in 2024-09-21T10:00:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 2246/42018 objects degraded (5.345%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[110963]: pgmap v1299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 874 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 2246/42018 objects degraded (5.345%); 0 B/s, 53 objects/s recovering 2024-09-21T10:00:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[110963]: osdmap e1127: 8 total, 8 up, 8 in 2024-09-21T10:00:19.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 2246/42018 objects degraded (5.345%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:19.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:00:19.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:19.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:19.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:19.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:19.994 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:20.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:20.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:20.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:20.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:20.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:20.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:20.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.394 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:20.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:20.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:20.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:20.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:20.659 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:20.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:20.800 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:20.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:20.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:20.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:21.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.096 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:21.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:21.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:21.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:21.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:21.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:21.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:21.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:21.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:21.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:21.638 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:21.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:21.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:21.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:21 smithi191 ceph-mon[102206]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 800 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2246/38436 objects degraded (5.843%); 0 B/s, 38 objects/s recovering 2024-09-21T10:00:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:21 smithi076 ceph-mon[110963]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 800 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2246/38436 objects degraded (5.843%); 0 B/s, 38 objects/s recovering 2024-09-21T10:00:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:21 smithi076 ceph-mon[106482]: pgmap v1301: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 800 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2246/38436 objects degraded (5.843%); 0 B/s, 38 objects/s recovering 2024-09-21T10:00:21.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:21.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:21.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:21.990 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:00:22.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:22.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:22.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:22.161 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:22.263 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:00:22.276 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:00:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:22.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:22.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:22.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.445 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:22.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:22.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:22.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:22.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:22.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:22.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:22 smithi191 ceph-mon[102206]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:22 smithi076 ceph-mon[106482]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:22 smithi076 ceph-mon[110963]: from='client.34911 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:22.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:22.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:22.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:23.024 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:23.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:23.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:23.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:23.201 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1301,"stamp":"2024-09-21T10:00:20.526143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":838473725,"num_objects":12812,"num_object_clones":0,"num_object_copies":38436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2246,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12812,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":215301,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12115,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142430,"ondisk_log_size":142430,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4401252,"kb_used_data":3079004,"kb_used_omap":195,"kb_used_meta":1321852,"kb_avail":745559964,"statfs":{"total":767960285184,"available":763453403136,"internally_reserved":0,"allocated":3152900096,"data_stored":3136388695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1353576580},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-511311872,"num_objects":-7802,"num_object_clones":0,"num_object_copies":-23406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-387,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7802,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":387,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006993"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.624743+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T10:00:18.624743+0000","last_peered":"2024-09-21T10:00:18.624743+0000","last_clean":"2024-09-21T10:00:18.624743+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T10:00:18.624743+0000","last_undegraded":"2024-09-21T10:00:18.624743+0000","last_fullsized":"2024-09-21T10:00:18.624743+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1055,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.164496+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T10:00:05.164496+0000","last_peered":"2024-09-21T10:00:05.164496+0000","last_clean":"2024-09-21T10:00:05.164496+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T10:00:05.164496+0000","last_undegraded":"2024-09-21T10:00:05.164496+0000","last_fullsized":"2024-09-21T10:00:05.164496+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1085,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139165+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T10:00:04.139165+0000","last_peered":"2024-09-21T10:00:04.139165+0000","last_clean":"2024-09-21T10:00:04.139165+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T10:00:04.139165+0000","last_undegraded":"2024-09-21T10:00:04.139165+0000","last_fullsized":"2024-09-21T10:00:04.139165+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1445,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.630489+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T10:00:18.630489+0000","last_peered":"2024-09-21T10:00:18.630489+0000","last_clean":"2024-09-21T10:00:18.630489+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T10:00:18.630489+0000","last_undegraded":"2024-09-21T10:00:18.630489+0000","last_fullsized":"2024-09-21T10:00:18.630489+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1445,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.621469+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T10:00:18.621469+0000","last_peered":"2024-09-21T10:00:18.621469+0000","last_clean":"2024-09-21T10:00:18.621469+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T10:00:18.621469+0000","last_undegraded":"2024-09-21T10:00:18.621469+0000","last_fullsized":"2024-09-21T10:00:18.621469+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1038,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.141626+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T10:00:04.141626+0000","last_peered":"2024-09-21T10:00:04.141626+0000","last_clean":"2024-09-21T10:00:04.141626+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T10:00:04.141626+0000","last_undegraded":"2024-09-21T10:00:04.141626+0000","last_fullsized":"2024-09-21T10:00:04.141626+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.634629+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T10:00:18.634629+0000","last_peered":"2024-09-21T10:00:18.634629+0000","last_clean":"2024-09-21T10:00:18.634629+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T10:00:18.634629+0000","last_undegraded":"2024-09-21T10:00:18.634629+0000","last_fullsized":"2024-09-21T10:00:18.634629+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1065,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212176+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T10:00:04.212176+0000","last_peered":"2024-09-21T10:00:04.212176+0000","last_clean":"2024-09-21T10:00:04.212176+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T10:00:04.212176+0000","last_undegraded":"2024-09-21T10:00:04.212176+0000","last_fullsized":"2024-09-21T10:00:04.212176+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1126'13291","reported_seq":13341,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:16.402099+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T10:00:16.402099+0000","last_peered":"2024-09-21T10:00:16.402099+0000","last_clean":"2024-09-21T10:00:16.402099+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T10:00:16.402099+0000","last_undegraded":"2024-09-21T10:00:16.402099+0000","last_fullsized":"2024-09-21T10:00:16.402099+0000","mapping_epoch":1086,"log_start":"1097'3200","ondisk_log_start":"1097'3200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10091,"log_dups_size":0,"ondisk_log_size":10091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53280768,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13291,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1054,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142052+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T10:00:04.142052+0000","last_peered":"2024-09-21T10:00:04.142052+0000","last_clean":"2024-09-21T10:00:04.142052+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T10:00:04.142052+0000","last_undegraded":"2024-09-21T10:00:04.142052+0000","last_fullsized":"2024-09-21T10:00:04.142052+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1053,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139024+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T10:00:04.139024+0000","last_peered":"2024-09-21T10:00:04.139024+0000","last_clean":"2024-09-21T10:00:04.139024+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T10:00:04.139024+0000","last_undegraded":"2024-09-21T10:00:04.139024+0000","last_fullsized":"2024-09-21T10:00:04.139024+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1127'13395","reported_seq":16363,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.783556+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T10:00:18.783556+0000","last_peered":"2024-09-21T10:00:18.783556+0000","last_clean":"2024-09-21T10:00:18.783556+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T10:00:18.783556+0000","last_undegraded":"2024-09-21T10:00:18.783556+0000","last_fullsized":"2024-09-21T10:00:18.783556+0000","mapping_epoch":1108,"log_start":"1106'6000","ondisk_log_start":"1106'6000","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7395,"log_dups_size":0,"ondisk_log_size":7395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47251456,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13395,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1055,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142056+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T10:00:04.142056+0000","last_peered":"2024-09-21T10:00:04.142056+0000","last_clean":"2024-09-21T10:00:04.142056+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T10:00:04.142056+0000","last_undegraded":"2024-09-21T10:00:04.142056+0000","last_fullsized":"2024-09-21T10:00:04.142056+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1107,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205320+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T10:00:04.205320+0000","last_peered":"2024-09-21T10:00:04.205320+0000","last_clean":"2024-09-21T10:00:04.205320+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T10:00:04.205320+0000","last_undegraded":"2024-09-21T10:00:04.205320+0000","last_fullsized":"2024-09-21T10:00:04.205320+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1126'13573","reported_seq":16507,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:16.377239+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T10:00:16.377239+0000","last_peered":"2024-09-21T10:00:16.377239+0000","last_clean":"2024-09-21T10:00:16.377239+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T10:00:16.377239+0000","last_undegraded":"2024-09-21T10:00:16.377239+0000","last_fullsized":"2024-09-21T10:00:16.377239+0000","mapping_epoch":1100,"log_start":"1103'5400","ondisk_log_start":"1103'5400","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53673984,"num_objects":819,"num_object_clones":0,"num_object_copies":2457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13573,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1039,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142247+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T10:00:04.142247+0000","last_peered":"2024-09-21T10:00:04.142247+0000","last_clean":"2024-09-21T10:00:04.142247+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T10:00:04.142247+0000","last_undegraded":"2024-09-21T10:00:04.142247+0000","last_fullsized":"2024-09-21T10:00:04.142247+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1056,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212236+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T10:00:04.212236+0000","last_peered":"2024-09-21T10:00:04.212236+0000","last_clean":"2024-09-21T10:00:04.212236+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T10:00:04.212236+0000","last_undegraded":"2024-09-21T10:00:04.212236+0000","last_fullsized":"2024-09-21T10:00:04.212236+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1127'13589","reported_seq":13683,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:19.066289+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T10:00:19.066289+0000","last_peered":"2024-09-21T10:00:19.066289+0000","last_clean":"2024-09-21T10:00:19.066289+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T10:00:19.066289+0000","last_undegraded":"2024-09-21T10:00:19.066289+0000","last_fullsized":"2024-09-21T10:00:19.066289+0000","mapping_epoch":1086,"log_start":"1098'3900","ondisk_log_start":"1098'3900","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9689,"log_dups_size":0,"ondisk_log_size":9689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47906816,"num_objects":731,"num_object_clones":0,"num_object_copies":2193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13589,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1087,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139178+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T10:00:04.139178+0000","last_peered":"2024-09-21T10:00:04.139178+0000","last_clean":"2024-09-21T10:00:04.139178+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T10:00:04.139178+0000","last_undegraded":"2024-09-21T10:00:04.139178+0000","last_fullsized":"2024-09-21T10:00:04.139178+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1434,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.621071+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T10:00:18.621071+0000","last_peered":"2024-09-21T10:00:18.621071+0000","last_clean":"2024-09-21T10:00:18.621071+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T10:00:18.621071+0000","last_undegraded":"2024-09-21T10:00:18.621071+0000","last_fullsized":"2024-09-21T10:00:18.621071+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1126'13293","reported_seq":13343,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:16.407844+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T10:00:16.407844+0000","last_peered":"2024-09-21T10:00:16.407844+0000","last_clean":"2024-09-21T10:00:16.407844+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T10:00:16.407844+0000","last_undegraded":"2024-09-21T10:00:16.407844+0000","last_fullsized":"2024-09-21T10:00:16.407844+0000","mapping_epoch":1086,"log_start":"1103'5300","ondisk_log_start":"1103'5300","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7993,"log_dups_size":0,"ondisk_log_size":7993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13293,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":85,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.627092+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T10:00:18.627092+0000","last_peered":"2024-09-21T10:00:18.627092+0000","last_clean":"2024-09-21T10:00:18.627092+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T10:00:18.627092+0000","last_undegraded":"2024-09-21T10:00:18.627092+0000","last_fullsized":"2024-09-21T10:00:18.627092+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1092,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205424+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T10:00:04.205424+0000","last_peered":"2024-09-21T10:00:04.205424+0000","last_clean":"2024-09-21T10:00:04.205424+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T10:00:04.205424+0000","last_undegraded":"2024-09-21T10:00:04.205424+0000","last_fullsized":"2024-09-21T10:00:04.205424+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1127'13663","reported_seq":16589,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:20.244877+0000","last_change":"2024-09-21T09:59:03.867059+0000","last_active":"2024-09-21T10:00:20.244877+0000","last_peered":"2024-09-21T10:00:20.244877+0000","last_clean":"2024-09-21T10:00:20.244877+0000","last_became_active":"2024-09-21T09:59:03.844371+0000","last_became_peered":"2024-09-21T09:59:03.844371+0000","last_unstale":"2024-09-21T10:00:20.244877+0000","last_undegraded":"2024-09-21T10:00:20.244877+0000","last_fullsized":"2024-09-21T10:00:20.244877+0000","mapping_epoch":1117,"log_start":"1098'3600","ondisk_log_start":"1098'3600","created":1086,"last_epoch_clean":1118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10063,"log_dups_size":0,"ondisk_log_size":10063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:34:30.990234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45678592,"num_objects":697,"num_object_clones":0,"num_object_copies":2091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13663,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1057,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212311+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T10:00:04.212311+0000","last_peered":"2024-09-21T10:00:04.212311+0000","last_clean":"2024-09-21T10:00:04.212311+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T10:00:04.212311+0000","last_undegraded":"2024-09-21T10:00:04.212311+0000","last_fullsized":"2024-09-21T10:00:04.212311+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1089,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205773+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T10:00:04.205773+0000","last_peered":"2024-09-21T10:00:04.205773+0000","last_clean":"2024-09-21T10:00:04.205773+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T10:00:04.205773+0000","last_undegraded":"2024-09-21T10:00:04.205773+0000","last_fullsized":"2024-09-21T10:00:04.205773+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1126'13302","reported_seq":14642,"reported_epoch":1126,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:00:16.536649+0000","last_change":"2024-09-21T10:00:02.603096+0000","last_active":"2024-09-21T10:00:16.536649+0000","last_peered":"2024-09-21T10:00:16.536649+0000","last_clean":"2024-09-21T09:56:44.195205+0000","last_became_active":"2024-09-21T09:56:46.369383+0000","last_became_peered":"2024-09-21T09:56:46.369383+0000","last_unstale":"2024-09-21T10:00:16.536649+0000","last_undegraded":"2024-09-21T09:56:46.252394+0000","last_fullsized":"2024-09-21T09:56:46.237315+0000","mapping_epoch":1093,"log_start":"1097'3300","ondisk_log_start":"1097'3300","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:55.199459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":834,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13302,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":642,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":38}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1075,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205420+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T10:00:04.205420+0000","last_peered":"2024-09-21T10:00:04.205420+0000","last_clean":"2024-09-21T10:00:04.205420+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T10:00:04.205420+0000","last_undegraded":"2024-09-21T10:00:04.205420+0000","last_fullsized":"2024-09-21T10:00:04.205420+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1083,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205935+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T10:00:04.205935+0000","last_peered":"2024-09-21T10:00:04.205935+0000","last_clean":"2024-09-21T10:00:04.205935+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T10:00:04.205935+0000","last_undegraded":"2024-09-21T10:00:04.205935+0000","last_fullsized":"2024-09-21T10:00:04.205935+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1126'13439","reported_seq":13490,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:16.511320+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T10:00:16.511320+0000","last_peered":"2024-09-21T10:00:16.511320+0000","last_clean":"2024-09-21T10:00:16.511320+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T10:00:16.511320+0000","last_undegraded":"2024-09-21T10:00:16.511320+0000","last_fullsized":"2024-09-21T10:00:16.511320+0000","mapping_epoch":1086,"log_start":"1097'3400","ondisk_log_start":"1097'3400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492056,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13439,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.633425+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T10:00:18.633425+0000","last_peered":"2024-09-21T10:00:18.633425+0000","last_clean":"2024-09-21T10:00:18.633425+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T10:00:18.633425+0000","last_undegraded":"2024-09-21T10:00:18.633425+0000","last_fullsized":"2024-09-21T10:00:18.633425+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1417,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.166027+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:00:05.166027+0000","last_peered":"2024-09-21T10:00:05.166027+0000","last_clean":"2024-09-21T10:00:05.166027+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:00:05.166027+0000","last_undegraded":"2024-09-21T10:00:05.166027+0000","last_fullsized":"2024-09-21T10:00:05.166027+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1126'13177","reported_seq":16015,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:16.534594+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T10:00:16.534594+0000","last_peered":"2024-09-21T10:00:16.534594+0000","last_clean":"2024-09-21T10:00:16.534594+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T10:00:16.534594+0000","last_undegraded":"2024-09-21T10:00:16.534594+0000","last_fullsized":"2024-09-21T10:00:16.534594+0000","mapping_epoch":1113,"log_start":"1097'3100","ondisk_log_start":"1097'3100","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10077,"log_dups_size":0,"ondisk_log_size":10077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13177,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1384,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.141621+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T10:00:04.141621+0000","last_peered":"2024-09-21T10:00:04.141621+0000","last_clean":"2024-09-21T10:00:04.141621+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T10:00:04.141621+0000","last_undegraded":"2024-09-21T10:00:04.141621+0000","last_fullsized":"2024-09-21T10:00:04.141621+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1083,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.138999+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T10:00:04.138999+0000","last_peered":"2024-09-21T10:00:04.138999+0000","last_clean":"2024-09-21T10:00:04.138999+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T10:00:04.138999+0000","last_undegraded":"2024-09-21T10:00:04.138999+0000","last_fullsized":"2024-09-21T10:00:04.138999+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.641204+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T10:00:18.641204+0000","last_peered":"2024-09-21T10:00:18.641204+0000","last_clean":"2024-09-21T10:00:18.641204+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T10:00:18.641204+0000","last_undegraded":"2024-09-21T10:00:18.641204+0000","last_fullsized":"2024-09-21T10:00:18.641204+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1042,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212296+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T10:00:04.212296+0000","last_peered":"2024-09-21T10:00:04.212296+0000","last_clean":"2024-09-21T10:00:04.212296+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T10:00:04.212296+0000","last_undegraded":"2024-09-21T10:00:04.212296+0000","last_fullsized":"2024-09-21T10:00:04.212296+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1126'13361","reported_seq":16421,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:17.032003+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T10:00:17.032003+0000","last_peered":"2024-09-21T10:00:17.032003+0000","last_clean":"2024-09-21T10:00:17.032003+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T10:00:17.032003+0000","last_undegraded":"2024-09-21T10:00:17.032003+0000","last_fullsized":"2024-09-21T10:00:17.032003+0000","mapping_epoch":1105,"log_start":"1101'4700","ondisk_log_start":"1101'4700","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52232192,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13361,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1082,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.141538+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T10:00:04.141538+0000","last_peered":"2024-09-21T10:00:04.141538+0000","last_clean":"2024-09-21T10:00:04.141538+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T10:00:04.141538+0000","last_undegraded":"2024-09-21T10:00:04.141538+0000","last_fullsized":"2024-09-21T10:00:04.141538+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1446,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.625804+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T10:00:18.625804+0000","last_peered":"2024-09-21T10:00:18.625804+0000","last_clean":"2024-09-21T10:00:18.625804+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T10:00:18.625804+0000","last_undegraded":"2024-09-21T10:00:18.625804+0000","last_fullsized":"2024-09-21T10:00:18.625804+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1444,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.621309+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T10:00:18.621309+0000","last_peered":"2024-09-21T10:00:18.621309+0000","last_clean":"2024-09-21T10:00:18.621309+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T10:00:18.621309+0000","last_undegraded":"2024-09-21T10:00:18.621309+0000","last_fullsized":"2024-09-21T10:00:18.621309+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1126'13366","reported_seq":16239,"reported_epoch":1126,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:00:16.403447+0000","last_change":"2024-09-21T09:59:29.091758+0000","last_active":"2024-09-21T10:00:16.403447+0000","last_peered":"2024-09-21T10:00:16.403447+0000","last_clean":"2024-09-21T09:56:44.212270+0000","last_became_active":"2024-09-21T09:59:29.047032+0000","last_became_peered":"2024-09-21T09:59:29.047032+0000","last_unstale":"2024-09-21T10:00:16.403447+0000","last_undegraded":"2024-09-21T10:00:16.403447+0000","last_fullsized":"2024-09-21T10:00:16.403447+0000","mapping_epoch":1120,"log_start":"1103'5200","ondisk_log_start":"1103'5200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8166,"log_dups_size":0,"ondisk_log_size":8166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:33.224146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51249152,"num_objects":782,"num_object_clones":0,"num_object_copies":2346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13366,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1405,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":["7","2","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":782}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1424,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.622249+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T10:00:18.622249+0000","last_peered":"2024-09-21T10:00:18.622249+0000","last_clean":"2024-09-21T10:00:18.622249+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T10:00:18.622249+0000","last_undegraded":"2024-09-21T10:00:18.622249+0000","last_fullsized":"2024-09-21T10:00:18.622249+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1404,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.165602+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T10:00:05.165602+0000","last_peered":"2024-09-21T10:00:05.165602+0000","last_clean":"2024-09-21T10:00:05.165602+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T10:00:05.165602+0000","last_undegraded":"2024-09-21T10:00:05.165602+0000","last_fullsized":"2024-09-21T10:00:05.165602+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1411,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.165839+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T10:00:05.165839+0000","last_peered":"2024-09-21T10:00:05.165839+0000","last_clean":"2024-09-21T10:00:05.165839+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T10:00:05.165839+0000","last_undegraded":"2024-09-21T10:00:05.165839+0000","last_fullsized":"2024-09-21T10:00:05.165839+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1127'13466","reported_seq":16454,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.778996+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T10:00:18.778996+0000","last_peered":"2024-09-21T10:00:18.778996+0000","last_clean":"2024-09-21T10:00:18.778996+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T10:00:18.778996+0000","last_undegraded":"2024-09-21T10:00:18.778996+0000","last_fullsized":"2024-09-21T10:00:18.778996+0000","mapping_epoch":1096,"log_start":"1106'6100","ondisk_log_start":"1106'6100","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7366,"log_dups_size":0,"ondisk_log_size":7366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48365568,"num_objects":738,"num_object_clones":0,"num_object_copies":2214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13466,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1084,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.138965+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T10:00:04.138965+0000","last_peered":"2024-09-21T10:00:04.138965+0000","last_clean":"2024-09-21T10:00:04.138965+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T10:00:04.138965+0000","last_undegraded":"2024-09-21T10:00:04.138965+0000","last_fullsized":"2024-09-21T10:00:04.138965+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1076,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205721+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T10:00:04.205721+0000","last_peered":"2024-09-21T10:00:04.205721+0000","last_clean":"2024-09-21T10:00:04.205721+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T10:00:04.205721+0000","last_undegraded":"2024-09-21T10:00:04.205721+0000","last_fullsized":"2024-09-21T10:00:04.205721+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1429,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.166315+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:00:05.166315+0000","last_peered":"2024-09-21T10:00:05.166315+0000","last_clean":"2024-09-21T10:00:05.166315+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:00:05.166315+0000","last_undegraded":"2024-09-21T10:00:05.166315+0000","last_fullsized":"2024-09-21T10:00:05.166315+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1127'13449","reported_seq":16474,"reported_epoch":1127,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:00:19.053546+0000","last_change":"2024-09-21T10:00:04.262897+0000","last_active":"2024-09-21T10:00:19.053546+0000","last_peered":"2024-09-21T10:00:19.053546+0000","last_clean":"2024-09-21T09:56:44.213840+0000","last_became_active":"2024-09-21T10:00:04.162832+0000","last_became_peered":"2024-09-21T10:00:04.162832+0000","last_unstale":"2024-09-21T10:00:19.053546+0000","last_undegraded":"2024-09-21T10:00:19.053546+0000","last_fullsized":"2024-09-21T10:00:19.053546+0000","mapping_epoch":1124,"log_start":"1098'3700","ondisk_log_start":"1098'3700","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9749,"log_dups_size":0,"ondisk_log_size":9749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47120384,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13449,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1462,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":["0","2","3","5"],"object_location_counts":[{"shards":"0,2,3","objects":719}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1103,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139212+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T10:00:04.139212+0000","last_peered":"2024-09-21T10:00:04.139212+0000","last_clean":"2024-09-21T10:00:04.139212+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T10:00:04.139212+0000","last_undegraded":"2024-09-21T10:00:04.139212+0000","last_fullsized":"2024-09-21T10:00:04.139212+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1426,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.166395+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T10:00:05.166395+0000","last_peered":"2024-09-21T10:00:05.166395+0000","last_clean":"2024-09-21T10:00:05.166395+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T10:00:05.166395+0000","last_undegraded":"2024-09-21T10:00:05.166395+0000","last_fullsized":"2024-09-21T10:00:05.166395+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.624814+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T10:00:18.624814+0000","last_peered":"2024-09-21T10:00:18.624814+0000","last_clean":"2024-09-21T10:00:18.624814+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T10:00:18.624814+0000","last_undegraded":"2024-09-21T10:00:18.624814+0000","last_fullsized":"2024-09-21T10:00:18.624814+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1126'13745","reported_seq":13837,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:17.150673+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T10:00:17.150673+0000","last_peered":"2024-09-21T10:00:17.150673+0000","last_clean":"2024-09-21T10:00:17.150673+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T10:00:17.150673+0000","last_undegraded":"2024-09-21T10:00:17.150673+0000","last_fullsized":"2024-09-21T10:00:17.150673+0000","mapping_epoch":1086,"log_start":"1102'5100","ondisk_log_start":"1102'5100","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8645,"log_dups_size":0,"ondisk_log_size":8645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13745,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1403,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.166637+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T10:00:05.166637+0000","last_peered":"2024-09-21T10:00:05.166637+0000","last_clean":"2024-09-21T10:00:05.166637+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T10:00:05.166637+0000","last_undegraded":"2024-09-21T10:00:05.166637+0000","last_fullsized":"2024-09-21T10:00:05.166637+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1066,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142242+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T10:00:04.142242+0000","last_peered":"2024-09-21T10:00:04.142242+0000","last_clean":"2024-09-21T10:00:04.142242+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T10:00:04.142242+0000","last_undegraded":"2024-09-21T10:00:04.142242+0000","last_fullsized":"2024-09-21T10:00:04.142242+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1422,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.621031+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T10:00:18.621031+0000","last_peered":"2024-09-21T10:00:18.621031+0000","last_clean":"2024-09-21T10:00:18.621031+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T10:00:18.621031+0000","last_undegraded":"2024-09-21T10:00:18.621031+0000","last_fullsized":"2024-09-21T10:00:18.621031+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1126'13440","reported_seq":13498,"reported_epoch":1126,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:00:16.367398+0000","last_change":"2024-09-21T09:56:46.337582+0000","last_active":"2024-09-21T10:00:16.367398+0000","last_peered":"2024-09-21T10:00:16.367398+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T10:00:16.367398+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1103'5500","ondisk_log_start":"1103'5500","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7940,"log_dups_size":0,"ondisk_log_size":7940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58720256,"num_objects":896,"num_object_clones":0,"num_object_copies":2688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13440,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-516}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":85,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.634713+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T10:00:18.634713+0000","last_peered":"2024-09-21T10:00:18.634713+0000","last_clean":"2024-09-21T10:00:18.634713+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T10:00:18.634713+0000","last_undegraded":"2024-09-21T10:00:18.634713+0000","last_fullsized":"2024-09-21T10:00:18.634713+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1403,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.166706+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T10:00:05.166706+0000","last_peered":"2024-09-21T10:00:05.166706+0000","last_clean":"2024-09-21T10:00:05.166706+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T10:00:05.166706+0000","last_undegraded":"2024-09-21T10:00:05.166706+0000","last_fullsized":"2024-09-21T10:00:05.166706+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.632671+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T10:00:18.632671+0000","last_peered":"2024-09-21T10:00:18.632671+0000","last_clean":"2024-09-21T10:00:18.632671+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T10:00:18.632671+0000","last_undegraded":"2024-09-21T10:00:18.632671+0000","last_fullsized":"2024-09-21T10:00:18.632671+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1126'13473","reported_seq":13523,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:16.407915+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T10:00:16.407915+0000","last_peered":"2024-09-21T10:00:16.407915+0000","last_clean":"2024-09-21T10:00:16.407915+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T10:00:16.407915+0000","last_undegraded":"2024-09-21T10:00:16.407915+0000","last_fullsized":"2024-09-21T10:00:16.407915+0000","mapping_epoch":1086,"log_start":"1103'5300","ondisk_log_start":"1103'5300","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52625408,"num_objects":803,"num_object_clones":0,"num_object_copies":2409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13473,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1436,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.621555+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T10:00:18.621555+0000","last_peered":"2024-09-21T10:00:18.621555+0000","last_clean":"2024-09-21T10:00:18.621555+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T10:00:18.621555+0000","last_undegraded":"2024-09-21T10:00:18.621555+0000","last_fullsized":"2024-09-21T10:00:18.621555+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1070,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212428+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T10:00:04.212428+0000","last_peered":"2024-09-21T10:00:04.212428+0000","last_clean":"2024-09-21T10:00:04.212428+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T10:00:04.212428+0000","last_undegraded":"2024-09-21T10:00:04.212428+0000","last_fullsized":"2024-09-21T10:00:04.212428+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1060,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205571+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T10:00:04.205571+0000","last_peered":"2024-09-21T10:00:04.205571+0000","last_clean":"2024-09-21T10:00:04.205571+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T10:00:04.205571+0000","last_undegraded":"2024-09-21T10:00:04.205571+0000","last_fullsized":"2024-09-21T10:00:04.205571+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1074,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.159960+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T10:00:05.159960+0000","last_peered":"2024-09-21T10:00:05.159960+0000","last_clean":"2024-09-21T10:00:05.159960+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T10:00:05.159960+0000","last_undegraded":"2024-09-21T10:00:05.159960+0000","last_fullsized":"2024-09-21T10:00:05.159960+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1083,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.160099+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T10:00:05.160099+0000","last_peered":"2024-09-21T10:00:05.160099+0000","last_clean":"2024-09-21T10:00:05.160099+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T10:00:05.160099+0000","last_undegraded":"2024-09-21T10:00:05.160099+0000","last_fullsized":"2024-09-21T10:00:05.160099+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1403,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.165554+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T10:00:05.165554+0000","last_peered":"2024-09-21T10:00:05.165554+0000","last_clean":"2024-09-21T10:00:05.165554+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T10:00:05.165554+0000","last_undegraded":"2024-09-21T10:00:05.165554+0000","last_fullsized":"2024-09-21T10:00:05.165554+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1085,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.142161+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T10:00:04.142161+0000","last_peered":"2024-09-21T10:00:04.142161+0000","last_clean":"2024-09-21T10:00:04.142161+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T10:00:04.142161+0000","last_undegraded":"2024-09-21T10:00:04.142161+0000","last_fullsized":"2024-09-21T10:00:04.142161+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1068,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139273+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T10:00:04.139273+0000","last_peered":"2024-09-21T10:00:04.139273+0000","last_clean":"2024-09-21T10:00:04.139273+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T10:00:04.139273+0000","last_undegraded":"2024-09-21T10:00:04.139273+0000","last_fullsized":"2024-09-21T10:00:04.139273+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.623454+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T10:00:18.623454+0000","last_peered":"2024-09-21T10:00:18.623454+0000","last_clean":"2024-09-21T10:00:18.623454+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T10:00:18.623454+0000","last_undegraded":"2024-09-21T10:00:18.623454+0000","last_fullsized":"2024-09-21T10:00:18.623454+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1444,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.622588+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T10:00:18.622588+0000","last_peered":"2024-09-21T10:00:18.622588+0000","last_clean":"2024-09-21T10:00:18.622588+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T10:00:18.622588+0000","last_undegraded":"2024-09-21T10:00:18.622588+0000","last_fullsized":"2024-09-21T10:00:18.622588+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.634017+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T10:00:18.634017+0000","last_peered":"2024-09-21T10:00:18.634017+0000","last_clean":"2024-09-21T10:00:18.634017+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T10:00:18.634017+0000","last_undegraded":"2024-09-21T10:00:18.634017+0000","last_fullsized":"2024-09-21T10:00:18.634017+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1066,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.212371+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T10:00:04.212371+0000","last_peered":"2024-09-21T10:00:04.212371+0000","last_clean":"2024-09-21T10:00:04.212371+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T10:00:04.212371+0000","last_undegraded":"2024-09-21T10:00:04.212371+0000","last_fullsized":"2024-09-21T10:00:04.212371+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.631872+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T10:00:18.631872+0000","last_peered":"2024-09-21T10:00:18.631872+0000","last_clean":"2024-09-21T10:00:18.631872+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T10:00:18.631872+0000","last_undegraded":"2024-09-21T10:00:18.631872+0000","last_fullsized":"2024-09-21T10:00:18.631872+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1447,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.621390+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T10:00:18.621390+0000","last_peered":"2024-09-21T10:00:18.621390+0000","last_clean":"2024-09-21T10:00:18.621390+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T10:00:18.621390+0000","last_undegraded":"2024-09-21T10:00:18.621390+0000","last_fullsized":"2024-09-21T10:00:18.621390+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1416,"reported_epoch":1126,"state":"active+clean","last_fresh":"2024-09-21T10:00:05.169773+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:00:05.169773+0000","last_peered":"2024-09-21T10:00:05.169773+0000","last_clean":"2024-09-21T10:00:05.169773+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:00:05.169773+0000","last_undegraded":"2024-09-21T10:00:05.169773+0000","last_fullsized":"2024-09-21T10:00:05.169773+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.625309+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:00:18.625309+0000","last_peered":"2024-09-21T10:00:18.625309+0000","last_clean":"2024-09-21T10:00:18.625309+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:00:18.625309+0000","last_undegraded":"2024-09-21T10:00:18.625309+0000","last_fullsized":"2024-09-21T10:00:18.625309+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1090,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205602+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T10:00:04.205602+0000","last_peered":"2024-09-21T10:00:04.205602+0000","last_clean":"2024-09-21T10:00:04.205602+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T10:00:04.205602+0000","last_undegraded":"2024-09-21T10:00:04.205602+0000","last_fullsized":"2024-09-21T10:00:04.205602+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1401,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.620402+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T10:00:18.620402+0000","last_peered":"2024-09-21T10:00:18.620402+0000","last_clean":"2024-09-21T10:00:18.620402+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T10:00:18.620402+0000","last_undegraded":"2024-09-21T10:00:18.620402+0000","last_fullsized":"2024-09-21T10:00:18.620402+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1423,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.622019+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T10:00:18.622019+0000","last_peered":"2024-09-21T10:00:18.622019+0000","last_clean":"2024-09-21T10:00:18.622019+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T10:00:18.622019+0000","last_undegraded":"2024-09-21T10:00:18.622019+0000","last_fullsized":"2024-09-21T10:00:18.622019+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1106,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205565+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T10:00:04.205565+0000","last_peered":"2024-09-21T10:00:04.205565+0000","last_clean":"2024-09-21T10:00:04.205565+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T10:00:04.205565+0000","last_undegraded":"2024-09-21T10:00:04.205565+0000","last_fullsized":"2024-09-21T10:00:04.205565+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1070,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139064+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T10:00:04.139064+0000","last_peered":"2024-09-21T10:00:04.139064+0000","last_clean":"2024-09-21T10:00:04.139064+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T10:00:04.139064+0000","last_undegraded":"2024-09-21T10:00:04.139064+0000","last_fullsized":"2024-09-21T10:00:04.139064+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1106,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205459+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T10:00:04.205459+0000","last_peered":"2024-09-21T10:00:04.205459+0000","last_clean":"2024-09-21T10:00:04.205459+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T10:00:04.205459+0000","last_undegraded":"2024-09-21T10:00:04.205459+0000","last_fullsized":"2024-09-21T10:00:04.205459+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1055,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.139131+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T10:00:04.139131+0000","last_peered":"2024-09-21T10:00:04.139131+0000","last_clean":"2024-09-21T10:00:04.139131+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T10:00:04.139131+0000","last_undegraded":"2024-09-21T10:00:04.139131+0000","last_fullsized":"2024-09-21T10:00:04.139131+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.623578+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T10:00:18.623578+0000","last_peered":"2024-09-21T10:00:18.623578+0000","last_clean":"2024-09-21T10:00:18.623578+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T10:00:18.623578+0000","last_undegraded":"2024-09-21T10:00:18.623578+0000","last_fullsized":"2024-09-21T10:00:18.623578+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":81,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.625512+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T10:00:18.625512+0000","last_peered":"2024-09-21T10:00:18.625512+0000","last_clean":"2024-09-21T10:00:18.625512+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T10:00:18.625512+0000","last_undegraded":"2024-09-21T10:00:18.625512+0000","last_fullsized":"2024-09-21T10:00:18.625512+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1082,"reported_epoch":1125,"state":"active+clean","last_fresh":"2024-09-21T10:00:04.205237+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T10:00:04.205237+0000","last_peered":"2024-09-21T10:00:04.205237+0000","last_clean":"2024-09-21T10:00:04.205237+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T10:00:04.205237+0000","last_undegraded":"2024-09-21T10:00:04.205237+0000","last_fullsized":"2024-09-21T10:00:04.205237+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1439,"reported_epoch":1127,"state":"active+clean","last_fresh":"2024-09-21T10:00:18.622668+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T10:00:18.622668+0000","last_peered":"2024-09-21T10:00:18.622668+0000","last_clean":"2024-09-21T10:00:18.622668+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T10:00:18.622668+0000","last_undegraded":"2024-09-21T10:00:18.622668+0000","last_fullsized":"2024-09-21T10:00:18.622668+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":837877784,"num_objects":12786,"num_object_clones":0,"num_object_copies":38358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2246,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12786,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":215022,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12102,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3009359872,"data_stored":3009347656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142222,"ondisk_log_size":142222,"up":48,"acting":46,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234293,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588024,"kb_used_data":426480,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":93157128,"statfs":{"total":95995035648,"available":95392899072,"internally_reserved":0,"allocated":436715520,"data_stored":434639916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":165411275},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495930,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":742640,"kb_used_data":563112,"kb_used_omap":1,"kb_used_meta":179518,"kb_avail":93002512,"statfs":{"total":95995035648,"available":95234572288,"internally_reserved":0,"allocated":576626688,"data_stored":574559149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":183826887},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757568,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542056,"kb_used_data":370720,"kb_used_omap":1,"kb_used_meta":171326,"kb_avail":93203096,"statfs":{"total":95995035648,"available":95439970304,"internally_reserved":0,"allocated":379617280,"data_stored":377553239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":175438278},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019204,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":456048,"kb_used_data":298600,"kb_used_omap":1,"kb_used_meta":157438,"kb_avail":93289104,"statfs":{"total":95995035648,"available":95528042496,"internally_reserved":0,"allocated":305766400,"data_stored":303698374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161216966},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888501,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":760860,"kb_used_data":498260,"kb_used_omap":34,"kb_used_meta":262429,"kb_avail":92984292,"statfs":{"total":95995035648,"available":95215915008,"internally_reserved":0,"allocated":510218240,"data_stored":508165894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":268728307},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967740,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":639536,"kb_used_data":458916,"kb_used_omap":41,"kb_used_meta":180566,"kb_avail":93105616,"statfs":{"total":95995035648,"available":95340150784,"internally_reserved":0,"allocated":469929984,"data_stored":467870124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":184899999},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046978,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369376,"kb_used_data":269272,"kb_used_omap":41,"kb_used_meta":100054,"kb_avail":93375776,"statfs":{"total":95995035648,"available":95616794624,"internally_reserved":0,"allocated":275734528,"data_stored":273678039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":102456233},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126217,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302712,"kb_used_data":193644,"kb_used_omap":72,"kb_used_meta":108983,"kb_avail":93442440,"statfs":{"total":95995035648,"available":95685058560,"internally_reserved":0,"allocated":198291456,"data_stored":196223960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":111598635},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":187699200,"data_stored":187695128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":267980800,"data_stored":267976728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":441122816,"data_stored":441122816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":480837632,"data_stored":480837632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":284753920,"data_stored":284753920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":371200000,"data_stored":371195928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":555155456,"data_stored":555155456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":420610048,"data_stored":420610048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:00:23.205 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:00:23.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:23.304 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:23.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:23.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:23.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:23.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:23.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:23.564 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:00:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:00:23] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.51.0" 2024-09-21T10:00:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:23.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:23.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:23.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:23.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:23.826 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:23 smithi076 ceph-mon[106482]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 629 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 2043/30249 objects degraded (6.754%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:23.826 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:23 smithi076 ceph-mon[110963]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 629 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 2043/30249 objects degraded (6.754%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:23.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:23.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:23.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:23.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:23 smithi191 ceph-mon[102206]: pgmap v1302: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 629 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 2043/30249 objects degraded (6.754%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:23.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:23.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:24.101 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:24.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:24.198 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:00:24.198 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:00:24.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:220250 6.f 6:fecdef85:::benchmark_data_smithi076_154645_object106344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.168570+0000", 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.027112707, 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.027142426000000001, 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 220250 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.199 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.168570+0000", 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.168570+0000", 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.168572+0000", 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5969999999999999e-06 2024-09-21T10:00:24.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.168577+0000", 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9049999999999996e-06 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.168589+0000", 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2377e-05 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.168597+0000", 2024-09-21T10:00:24.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7479999999999993e-06 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.170045+0000", 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001448469 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.170149+0000", 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103417 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.170252+0000", 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010353699999999999 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.175122+0000", 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0048701259999999998 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.177281+0000", 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002158662 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:220261 6.0 e1127/1124 6:0c2d298c:::benchmark_data_smithi076_154645_object106355:head v 1127'13684, mlcod=1127'13682)", 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.179945+0000", 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.015737491999999999, 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015842814, 2024-09-21T10:00:24.204 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.179945+0000", 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.179945+0000", 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.179946+0000", 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6000000000000002e-07 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.179952+0000", 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.835e-06 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.179963+0000", 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0917999999999999e-05 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.179970+0000", 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9399999999999996e-06 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182287+0000", 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023175019999999999 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182309+0000", 2024-09-21T10:00:24.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1721000000000001e-05 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:220263 6.e e1127/1086 6:769fba69:::benchmark_data_smithi076_154645_object106357:head v 1127'13716, mlcod=1127'13713)", 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.182229+0000", 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.01345385, 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013606824999999999, 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182229+0000", 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182229+0000", 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182230+0000", 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9999999999999996e-07 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182236+0000", 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0209999999999996e-06 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182257+0000", 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1163e-05 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.182264+0000", 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5850000000000002e-06 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.183155+0000", 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00089095299999999999 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.183251+0000", 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5940999999999995e-05 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.211 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:220268 6.f 6:fe0961ea:::benchmark_data_smithi076_154645_object106362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.184219+0000", 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.011463713, 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011664133, 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 220268 2024-09-21T10:00:24.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.213 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184219+0000", 2024-09-21T10:00:24.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184219+0000", 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184221+0000", 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.677e-06 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184226+0000", 2024-09-21T10:00:24.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.237e-06 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184245+0000", 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8586000000000001e-05 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184254+0000", 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4040000000000001e-06 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184479+0000", 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00022471500000000001 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184670+0000", 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000191258 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.184785+0000", 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000115142 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:24.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.190279+0000", 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054934670000000001 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:220266 6.0 e1127/1124 6:02520916:::benchmark_data_smithi076_154645_object106360:head v 1127'13685, mlcod=1127'13682)", 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.185929+0000", 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0097535580000000007, 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010025295, 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:00:24.217 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.185929+0000", 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.185929+0000", 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.185930+0000", 2024-09-21T10:00:24.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1299999999999999e-07 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.185936+0000", 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7579999999999998e-06 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.185947+0000", 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1379999999999999e-05 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.185954+0000", 2024-09-21T10:00:24.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0600000000000002e-06 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.187036+0000", 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010817579999999999 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.187134+0000", 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7717e-05 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:220267 6.e e1127/1086 6:7714e9db:::benchmark_data_smithi076_154645_object106361:head v 1127'13717, mlcod=1127'13713)", 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.189044+0000", 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0066391339999999997, 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069567090000000002, 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189044+0000", 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189044+0000", 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189044+0000", 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1299999999999999e-07 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189051+0000", 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4330000000000001e-06 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189062+0000", 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1308000000000001e-05 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.223 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189069+0000", 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.973e-06 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.193020+0000", 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039506999999999997 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.193082+0000", 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1748000000000006e-05 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.224 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:220269 6.9 e1127/1086 6:9bd532db:::benchmark_data_smithi076_154645_object106363:head v 1127'13843, mlcod=1127'13841)", 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.189094+0000", 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0065892700000000004, 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069518840000000002, 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189094+0000", 2024-09-21T10:00:24.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189094+0000", 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189094+0000", 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4799999999999998e-07 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189099+0000", 2024-09-21T10:00:24.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4619999999999998e-06 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189106+0000", 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3719999999999999e-06 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.189111+0000", 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0379999999999999e-06 2024-09-21T10:00:24.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.190368+0000", 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012566159999999999 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.190385+0000", 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7716000000000001e-05 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.228 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:220272 6.f 6:f078bb3b:::benchmark_data_smithi076_154645_object106366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:00:24.191673+0000", 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0040099580000000001, 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0044215050000000001, 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 220272 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:24.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.191673+0000", 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.191673+0000", 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.191674+0000", 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.581e-06 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.191680+0000", 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3890000000000004e-06 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.191698+0000", 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7805999999999999e-05 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.191706+0000", 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4330000000000004e-06 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.231 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.194745+0000", 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030393130000000001 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.194867+0000", 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012125000000000001 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:00:24.194996+0000", 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012940099999999999 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:24.232 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.233 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:24.233 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T10:00:24.233 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 8 2024-09-21T10:00:24.233 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:00:24.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:24.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:24.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:24.439 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:24.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:24.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:24.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:24.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:24.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:24.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:24.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:24.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:24.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:24.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:25.004 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 2043/30249 objects degraded (6.754%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:25.005 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 2043/30249 objects degraded (6.754%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:25.076 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 2043/30249 objects degraded (6.754%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:25.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:25.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:25.249 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:25.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:25.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:25.452 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:25.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:25.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:00:25.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:25.520 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:25.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:25.632 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:25.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:25.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:25.766 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:25.800 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:25 smithi076 ceph-mon[106482]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 568 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 2043/27315 objects degraded (7.479%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:25.801 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:25.802 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:25 smithi076 ceph-mon[110963]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 568 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 2043/27315 objects degraded (7.479%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:25.802 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:25.803 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:25.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:25.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:25.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:25.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:26.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:26.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:25 smithi191 ceph-mon[102206]: pgmap v1303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 568 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 2043/27315 objects degraded (7.479%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:26.175 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:26.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:26.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:26.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:26.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:26.502 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:26.604 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:26.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:26.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.794 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:26.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:26.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:26.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:27.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:27.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:27.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:27.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:27.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:27.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:27.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:27.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:27.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:28.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:27 smithi076 ceph-mon[106482]: pgmap v1304: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 478 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2043/23013 objects degraded (8.878%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:28.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:27 smithi076 ceph-mon[110963]: pgmap v1304: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 478 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2043/23013 objects degraded (8.878%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:28.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:27 smithi191 ceph-mon[102206]: pgmap v1304: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 478 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 2043/23013 objects degraded (8.878%); 0 B/s, 40 objects/s recovering 2024-09-21T10:00:28.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:28.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:28.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:28.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:28.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:28.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:28.510 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:28.612 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:28.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:28.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:28.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:28.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:28.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:28.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:28.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:28.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:29.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:29.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:29.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:29.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:29.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:29.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:29.330 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:29.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:29.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:29.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:29.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:29.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:29.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:29.653 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:00:29.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:29.745 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:29.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:29.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[106482]: pgmap v1305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 390 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 1850/18777 objects degraded (9.852%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 1850/18777 objects degraded (9.852%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/802888461' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[110963]: pgmap v1305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 390 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 1850/18777 objects degraded (9.852%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 1850/18777 objects degraded (9.852%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/802888461' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:29 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:29.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:29.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:29.990 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:30.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:29 smithi191 ceph-mon[102206]: pgmap v1305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 390 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 1850/18777 objects degraded (9.852%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 1850/18777 objects degraded (9.852%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:29 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/802888461' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:29 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:00:30.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:30.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:30.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:30.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:30.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:30.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:30.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:30.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:30.566 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:30.668 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:30.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:30.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:30.832 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:30 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:30 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:00:30.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:30 smithi076 ceph-mon[106482]: osdmap e1128: 8 total, 8 up, 8 in 2024-09-21T10:00:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:30 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:30 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:00:30.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:30 smithi076 ceph-mon[110963]: osdmap e1128: 8 total, 8 up, 8 in 2024-09-21T10:00:30.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:30.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:30.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:31.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:31.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:30 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:00:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:30 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:00:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:30 smithi191 ceph-mon[102206]: osdmap e1128: 8 total, 8 up, 8 in 2024-09-21T10:00:31.136 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:31.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:31.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:31.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:31.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:31.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:31.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:31.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:31.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:31.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:31.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.637 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:31.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:31.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:31.739 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:31 smithi076 ceph-mon[106482]: pgmap v1307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 314 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 1850/15126 objects degraded (12.231%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:31.739 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:31 smithi076 ceph-mon[110963]: pgmap v1307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 314 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 1850/15126 objects degraded (12.231%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:31.786 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:31.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:31.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:31.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:31.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:32.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:32.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.069 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:32.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:31 smithi191 ceph-mon[102206]: pgmap v1307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 314 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 1850/15126 objects degraded (12.231%); 0 B/s, 39 objects/s recovering 2024-09-21T10:00:32.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:32.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:32.269 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:00:32.269 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1850/15126 objects degraded (12.231%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":935,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1128,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":8,"osd_in_since":1726912604,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":85},{"state_name":"active+recovery_wait","count":2},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":5042,"data_bytes":329259005,"bytes_used":2974388224,"bytes_avail":764985896960,"bytes_total":767960285184,"degraded_objects":1850,"degraded_total":15126,"degraded_ratio":0.12230596324209969,"recovering_objects_per_sec":39,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":396,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":776},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":25,"modified":"2024-09-21T09:58:56.477511+0000","services":{}},"progress_events":{}} 2024-09-21T10:00:32.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.344 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:32.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:32.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:32.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:32.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:32.613 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:32.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:32.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:32.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:32.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:32.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:32.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:32.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:00:32.940 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:00:32.941 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:32.941 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:32 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2072555826' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:00:32.941 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:32 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2072555826' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:00:32.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:32.983 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:00:32.983 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:33.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:33.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:33.018 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:00:33.018 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:33.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:32 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2072555826' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:00:33.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:33.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:33.132 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:00:33.132 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:33.164 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:00:33.164 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:33.197 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:00:33.197 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:33.227 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:00:33.227 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:33.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:33.258 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:00:33.258 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:33.275 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:33.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.409 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:33.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:33.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:33.550 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:33.605 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:00:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:00:33] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.51.0" 2024-09-21T10:00:33.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:33.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:33.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:33.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:33.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:33 smithi076 ceph-mon[106482]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 147 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 1632/7101 objects degraded (22.983%); 0 B/s, 41 objects/s recovering 2024-09-21T10:00:33.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:33 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/227701254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:33 smithi076 ceph-mon[110963]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 147 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 1632/7101 objects degraded (22.983%); 0 B/s, 41 objects/s recovering 2024-09-21T10:00:33.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:33 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/227701254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:33.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:33.942 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Removed 113903 objects 2024-09-21T10:00:33.942 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Clean up completed and total clean up time :147.001 2024-09-21T10:00:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:33.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:33.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:34.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:34.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.082 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:33 smithi191 ceph-mon[102206]: pgmap v1308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 147 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 1632/7101 objects degraded (22.983%); 0 B/s, 41 objects/s recovering 2024-09-21T10:00:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:33 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/227701254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:34.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:34.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:34.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:34.329 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:34.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:34.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:34.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:34.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:34.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:34.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:34.721 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:00:34.721 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:00:34.803 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:34.820 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-21T10:00:34.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-21T10:00:34.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:34.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:34.947 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:34.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:34.951 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 1632/7101 objects degraded (22.983%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:34.952 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 1632/7101 objects degraded (22.983%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:35.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:35.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 1632/7101 objects degraded (22.983%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:35.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:35.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:35.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:35.317 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:35.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:35.442 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:35.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:35.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:35.564 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:35.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:35.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:35.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:35.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:35 smithi076 ceph-mon[106482]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 87 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1632/4245 objects degraded (38.445%); 0 B/s, 41 objects/s recovering 2024-09-21T10:00:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:35 smithi076 ceph-mon[106482]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-21T10:00:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:35 smithi076 ceph-mon[110963]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 87 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1632/4245 objects degraded (38.445%); 0 B/s, 41 objects/s recovering 2024-09-21T10:00:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:35 smithi076 ceph-mon[110963]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-21T10:00:35.918 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:35.940 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:35.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:36.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:36.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:36.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:35 smithi191 ceph-mon[102206]: pgmap v1309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait, 85 active+clean; 87 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 1632/4245 objects degraded (38.445%); 0 B/s, 41 objects/s recovering 2024-09-21T10:00:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:35 smithi191 ceph-mon[102206]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-21T10:00:36.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:36.186 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:36.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:36.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:36.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:36.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:36.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:36.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:36.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:36.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:36.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:36.680 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:36.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:36.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:36 smithi076 ceph-mon[106482]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-21T10:00:36.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:36 smithi076 ceph-mon[110963]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-21T10:00:36.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:36.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:36.905 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:36.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:36.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:37.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:37.049 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:36 smithi191 ceph-mon[102206]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-21T10:00:37.151 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:37.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:37.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:37.214 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:00:37.214 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:00:37.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:37.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:37.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:37.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:37.519 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:37.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:37.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:00:37.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.664 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:37.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:37.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:37.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:37.887 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:37.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:37.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:38.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:38.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:38.045 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:37 smithi076 ceph-mon[106482]: pgmap v1312: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 86 active+clean; 39 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 659/1929 objects degraded (34.163%); 0 B/s, 172 objects/s recovering 2024-09-21T10:00:38.045 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:37 smithi076 ceph-mon[110963]: pgmap v1312: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 86 active+clean; 39 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 659/1929 objects degraded (34.163%); 0 B/s, 172 objects/s recovering 2024-09-21T10:00:38.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:00:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:37 smithi191 ceph-mon[102206]: pgmap v1312: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 86 active+clean; 39 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 659/1929 objects degraded (34.163%); 0 B/s, 172 objects/s recovering 2024-09-21T10:00:38.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:38.157 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:38.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:38.280 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:38.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:38.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:38.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:38.420 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:38.444 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:38.447 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:38.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:38.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:38.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:38.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:38.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:38.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:38.772 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:38.822 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:38 smithi076 ceph-mon[106482]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-21T10:00:38.823 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:38 smithi076 ceph-mon[110963]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-21T10:00:38.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:38.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:38.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:38.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:39.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:38 smithi191 ceph-mon[102206]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-21T10:00:39.021 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:39.122 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:39.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:39.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:39.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:39.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:39.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:39.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:39.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:39.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:39.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:39.620 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:39.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:39.745 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:39.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:39.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:39.893 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:39.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:39.995 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:40.019 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:39 smithi191 ceph-mon[102206]: pgmap v1314: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 86 active+clean; 8.3 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 659/450 objects degraded (146.444%); 0 B/s, 162 objects/s recovering 2024-09-21T10:00:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:39 smithi191 ceph-mon[102206]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-21T10:00:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 659/450 objects degraded (146.444%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:40.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:40.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:39 smithi076 ceph-mon[110963]: pgmap v1314: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 86 active+clean; 8.3 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 659/450 objects degraded (146.444%); 0 B/s, 162 objects/s recovering 2024-09-21T10:00:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:39 smithi076 ceph-mon[110963]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-21T10:00:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 659/450 objects degraded (146.444%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:39 smithi076 ceph-mon[106482]: pgmap v1314: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 86 active+clean; 8.3 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 369 op/s; 659/450 objects degraded (146.444%); 0 B/s, 162 objects/s recovering 2024-09-21T10:00:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:39 smithi076 ceph-mon[106482]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-21T10:00:40.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 659/450 objects degraded (146.444%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:00:40.143 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:40.244 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:40.267 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:40.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:40.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:40.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:40.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:40.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:40.609 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:40.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:40.621 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:40.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:40.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:40.742 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:40.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:40.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:40.794 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:40.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:40.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:40.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:40.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:40.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:40.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:41.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:41.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:41.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:41.133 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:41.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:41.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.257 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:41.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:41.281 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:00:41.283 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:00:41.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:41.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:41.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:41.481 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:41.503 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:41.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:41.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:41.728 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:41.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:41.850 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:41.872 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:41.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:41.996 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:41.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:41.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:42.085 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:41 smithi076 ceph-mon[106482]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 237 op/s; 439/78 objects degraded (562.821%); 0 B/s, 204 objects/s recovering 2024-09-21T10:00:42.086 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:41 smithi076 ceph-mon[106482]: from='client.34929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:42.086 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:41 smithi076 ceph-mon[110963]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 237 op/s; 439/78 objects degraded (562.821%); 0 B/s, 204 objects/s recovering 2024-09-21T10:00:42.086 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:41 smithi076 ceph-mon[110963]: from='client.34929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:42.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:41 smithi191 ceph-mon[102206]: pgmap v1316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 237 op/s; 439/78 objects degraded (562.821%); 0 B/s, 204 objects/s recovering 2024-09-21T10:00:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:41 smithi191 ceph-mon[102206]: from='client.34929 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:42.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:42.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:42.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:42.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:42.371 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:42.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:42.497 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:42.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:42.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:42.621 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:42.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:42.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:42.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:42.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:42.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:42.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:42.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:42.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:42.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:43.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:43.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:42 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 439/78 objects degraded (562.821%), 1 pg degraded, 1 pg undersized) 2024-09-21T10:00:43.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:43.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:42 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 439/78 objects degraded (562.821%), 1 pg degraded, 1 pg undersized) 2024-09-21T10:00:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:42 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 439/78 objects degraded (562.821%), 1 pg degraded, 1 pg undersized) 2024-09-21T10:00:43.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:43.247 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1316,"stamp":"2024-09-21T10:00:40.532361+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":228087,"num_write_kb":7292242,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13922,"num_bytes_recovered":489490196,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142316,"ondisk_log_size":142316,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1917836,"kb_used_data":561116,"kb_used_omap":195,"kb_used_meta":1356412,"kb_avail":748043380,"statfs":{"total":767960285184,"available":765996421120,"internally_reserved":0,"allocated":574582784,"data_stored":557953134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1388966020},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-90963992,"num_objects":-1389,"num_object_clones":0,"num_object_copies":-4167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1193,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1389,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1193,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.846822"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718882+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T10:00:38.718882+0000","last_peered":"2024-09-21T10:00:38.718882+0000","last_clean":"2024-09-21T10:00:38.718882+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T10:00:38.718882+0000","last_undegraded":"2024-09-21T10:00:38.718882+0000","last_fullsized":"2024-09-21T10:00:38.718882+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1059,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692672+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T10:00:34.692672+0000","last_peered":"2024-09-21T10:00:34.692672+0000","last_clean":"2024-09-21T10:00:34.692672+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T10:00:34.692672+0000","last_undegraded":"2024-09-21T10:00:34.692672+0000","last_fullsized":"2024-09-21T10:00:34.692672+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1089,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692541+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T10:00:34.692541+0000","last_peered":"2024-09-21T10:00:34.692541+0000","last_clean":"2024-09-21T10:00:34.692541+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T10:00:34.692541+0000","last_undegraded":"2024-09-21T10:00:34.692541+0000","last_fullsized":"2024-09-21T10:00:34.692541+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1455,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713364+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T10:00:38.713364+0000","last_peered":"2024-09-21T10:00:38.713364+0000","last_clean":"2024-09-21T10:00:38.713364+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T10:00:38.713364+0000","last_undegraded":"2024-09-21T10:00:38.713364+0000","last_fullsized":"2024-09-21T10:00:38.713364+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1455,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713301+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T10:00:38.713301+0000","last_peered":"2024-09-21T10:00:38.713301+0000","last_clean":"2024-09-21T10:00:38.713301+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T10:00:38.713301+0000","last_undegraded":"2024-09-21T10:00:38.713301+0000","last_fullsized":"2024-09-21T10:00:38.713301+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1042,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692989+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T10:00:34.692989+0000","last_peered":"2024-09-21T10:00:34.692989+0000","last_clean":"2024-09-21T10:00:34.692989+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T10:00:34.692989+0000","last_undegraded":"2024-09-21T10:00:34.692989+0000","last_fullsized":"2024-09-21T10:00:34.692989+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718893+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T10:00:38.718893+0000","last_peered":"2024-09-21T10:00:38.718893+0000","last_clean":"2024-09-21T10:00:38.718893+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T10:00:38.718893+0000","last_undegraded":"2024-09-21T10:00:38.718893+0000","last_fullsized":"2024-09-21T10:00:38.718893+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1069,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691440+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T10:00:34.691440+0000","last_peered":"2024-09-21T10:00:34.691440+0000","last_clean":"2024-09-21T10:00:34.691440+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T10:00:34.691440+0000","last_undegraded":"2024-09-21T10:00:34.691440+0000","last_fullsized":"2024-09-21T10:00:34.691440+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1128'14104","reported_seq":14158,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691029+0000","last_change":"2024-09-21T09:56:26.056273+0000","last_active":"2024-09-21T10:00:34.691029+0000","last_peered":"2024-09-21T10:00:34.691029+0000","last_clean":"2024-09-21T10:00:34.691029+0000","last_became_active":"2024-09-21T09:56:26.055737+0000","last_became_peered":"2024-09-21T09:56:26.055737+0000","last_unstale":"2024-09-21T10:00:34.691029+0000","last_undegraded":"2024-09-21T10:00:34.691029+0000","last_fullsized":"2024-09-21T10:00:34.691029+0000","mapping_epoch":1086,"log_start":"1099'4100","ondisk_log_start":"1099'4100","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10004,"log_dups_size":0,"ondisk_log_size":10004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:11.924554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14104,"num_write_kb":451328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1058,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692921+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T10:00:34.692921+0000","last_peered":"2024-09-21T10:00:34.692921+0000","last_clean":"2024-09-21T10:00:34.692921+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T10:00:34.692921+0000","last_undegraded":"2024-09-21T10:00:34.692921+0000","last_fullsized":"2024-09-21T10:00:34.692921+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1058,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691102+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T10:00:34.691102+0000","last_peered":"2024-09-21T10:00:34.691102+0000","last_clean":"2024-09-21T10:00:34.691102+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T10:00:34.691102+0000","last_undegraded":"2024-09-21T10:00:34.691102+0000","last_fullsized":"2024-09-21T10:00:34.691102+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1128'14116","reported_seq":17094,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718423+0000","last_change":"2024-09-21T09:58:04.151016+0000","last_active":"2024-09-21T10:00:38.718423+0000","last_peered":"2024-09-21T10:00:38.718423+0000","last_clean":"2024-09-21T10:00:38.718423+0000","last_became_active":"2024-09-21T09:58:04.141290+0000","last_became_peered":"2024-09-21T09:58:04.141290+0000","last_unstale":"2024-09-21T10:00:38.718423+0000","last_undegraded":"2024-09-21T10:00:38.718423+0000","last_fullsized":"2024-09-21T10:00:38.718423+0000","mapping_epoch":1108,"log_start":"1107'6700","ondisk_log_start":"1107'6700","created":1086,"last_epoch_clean":1109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7416,"log_dups_size":0,"ondisk_log_size":7416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:49.328616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14116,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1441,"num_bytes_recovered":94437376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1059,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691777+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T10:00:34.691777+0000","last_peered":"2024-09-21T10:00:34.691777+0000","last_clean":"2024-09-21T10:00:34.691777+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T10:00:34.691777+0000","last_undegraded":"2024-09-21T10:00:34.691777+0000","last_fullsized":"2024-09-21T10:00:34.691777+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1111,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686512+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T10:00:34.686512+0000","last_peered":"2024-09-21T10:00:34.686512+0000","last_clean":"2024-09-21T10:00:34.686512+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T10:00:34.686512+0000","last_undegraded":"2024-09-21T10:00:34.686512+0000","last_fullsized":"2024-09-21T10:00:34.686512+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1128'14392","reported_seq":17330,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692265+0000","last_change":"2024-09-21T09:57:18.606016+0000","last_active":"2024-09-21T10:00:34.692265+0000","last_peered":"2024-09-21T10:00:34.692265+0000","last_clean":"2024-09-21T10:00:34.692265+0000","last_became_active":"2024-09-21T09:57:18.600088+0000","last_became_peered":"2024-09-21T09:57:18.600088+0000","last_unstale":"2024-09-21T10:00:34.692265+0000","last_undegraded":"2024-09-21T10:00:34.692265+0000","last_fullsized":"2024-09-21T10:00:34.692265+0000","mapping_epoch":1100,"log_start":"1106'6200","ondisk_log_start":"1106'6200","created":1086,"last_epoch_clean":1101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:23.030218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14392,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1434,"num_bytes_recovered":93978624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1043,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691581+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T10:00:34.691581+0000","last_peered":"2024-09-21T10:00:34.691581+0000","last_clean":"2024-09-21T10:00:34.691581+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T10:00:34.691581+0000","last_undegraded":"2024-09-21T10:00:34.691581+0000","last_fullsized":"2024-09-21T10:00:34.691581+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1060,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692110+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T10:00:34.692110+0000","last_peered":"2024-09-21T10:00:34.692110+0000","last_clean":"2024-09-21T10:00:34.692110+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T10:00:34.692110+0000","last_undegraded":"2024-09-21T10:00:34.692110+0000","last_fullsized":"2024-09-21T10:00:34.692110+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1128'14320","reported_seq":14424,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.712953+0000","last_change":"2024-09-21T09:56:26.051567+0000","last_active":"2024-09-21T10:00:38.712953+0000","last_peered":"2024-09-21T10:00:38.712953+0000","last_clean":"2024-09-21T10:00:38.712953+0000","last_became_active":"2024-09-21T09:56:26.051289+0000","last_became_peered":"2024-09-21T09:56:26.051289+0000","last_unstale":"2024-09-21T10:00:38.712953+0000","last_undegraded":"2024-09-21T10:00:38.712953+0000","last_fullsized":"2024-09-21T10:00:38.712953+0000","mapping_epoch":1086,"log_start":"1101'4600","ondisk_log_start":"1101'4600","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9720,"log_dups_size":0,"ondisk_log_size":9720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:10:49.406949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14320,"num_write_kb":458240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1091,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.696486+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T10:00:34.696486+0000","last_peered":"2024-09-21T10:00:34.696486+0000","last_clean":"2024-09-21T10:00:34.696486+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T10:00:34.696486+0000","last_undegraded":"2024-09-21T10:00:34.696486+0000","last_fullsized":"2024-09-21T10:00:34.696486+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1444,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713045+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T10:00:38.713045+0000","last_peered":"2024-09-21T10:00:38.713045+0000","last_clean":"2024-09-21T10:00:38.713045+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T10:00:38.713045+0000","last_undegraded":"2024-09-21T10:00:38.713045+0000","last_fullsized":"2024-09-21T10:00:38.713045+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1128'14166","reported_seq":14220,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686907+0000","last_change":"2024-09-21T09:56:26.049806+0000","last_active":"2024-09-21T10:00:34.686907+0000","last_peered":"2024-09-21T10:00:34.686907+0000","last_clean":"2024-09-21T10:00:34.686907+0000","last_became_active":"2024-09-21T09:56:26.049476+0000","last_became_peered":"2024-09-21T09:56:26.049476+0000","last_unstale":"2024-09-21T10:00:34.686907+0000","last_undegraded":"2024-09-21T10:00:34.686907+0000","last_fullsized":"2024-09-21T10:00:34.686907+0000","mapping_epoch":1086,"log_start":"1106'6200","ondisk_log_start":"1106'6200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7966,"log_dups_size":0,"ondisk_log_size":7966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:59.529244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14166,"num_write_kb":453312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":95,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718522+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T10:00:38.718522+0000","last_peered":"2024-09-21T10:00:38.718522+0000","last_clean":"2024-09-21T10:00:38.718522+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T10:00:38.718522+0000","last_undegraded":"2024-09-21T10:00:38.718522+0000","last_fullsized":"2024-09-21T10:00:38.718522+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1096,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686769+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T10:00:34.686769+0000","last_peered":"2024-09-21T10:00:34.686769+0000","last_clean":"2024-09-21T10:00:34.686769+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T10:00:34.686769+0000","last_undegraded":"2024-09-21T10:00:34.686769+0000","last_fullsized":"2024-09-21T10:00:34.686769+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1128'14360","reported_seq":17296,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718170+0000","last_change":"2024-09-21T09:59:03.867059+0000","last_active":"2024-09-21T10:00:38.718170+0000","last_peered":"2024-09-21T10:00:38.718170+0000","last_clean":"2024-09-21T10:00:38.718170+0000","last_became_active":"2024-09-21T09:59:03.844371+0000","last_became_peered":"2024-09-21T09:59:03.844371+0000","last_unstale":"2024-09-21T10:00:38.718170+0000","last_undegraded":"2024-09-21T10:00:38.718170+0000","last_fullsized":"2024-09-21T10:00:38.718170+0000","mapping_epoch":1117,"log_start":"1100'4300","ondisk_log_start":"1100'4300","created":1086,"last_epoch_clean":1118,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10060,"log_dups_size":0,"ondisk_log_size":10060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:34:30.990234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14360,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1410,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1062,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.687882+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T10:00:35.687882+0000","last_peered":"2024-09-21T10:00:35.687882+0000","last_clean":"2024-09-21T10:00:35.687882+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T10:00:35.687882+0000","last_undegraded":"2024-09-21T10:00:35.687882+0000","last_fullsized":"2024-09-21T10:00:35.687882+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1093,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686916+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T10:00:34.686916+0000","last_peered":"2024-09-21T10:00:34.686916+0000","last_clean":"2024-09-21T10:00:34.686916+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T10:00:34.686916+0000","last_undegraded":"2024-09-21T10:00:34.686916+0000","last_fullsized":"2024-09-21T10:00:34.686916+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1128'14174","reported_seq":16935,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718968+0000","last_change":"2024-09-21T10:00:35.749479+0000","last_active":"2024-09-21T10:00:38.718968+0000","last_peered":"2024-09-21T10:00:38.718968+0000","last_clean":"2024-09-21T10:00:38.718968+0000","last_became_active":"2024-09-21T10:00:35.688592+0000","last_became_peered":"2024-09-21T10:00:35.688592+0000","last_unstale":"2024-09-21T10:00:38.718968+0000","last_undegraded":"2024-09-21T10:00:38.718968+0000","last_fullsized":"2024-09-21T10:00:38.718968+0000","mapping_epoch":1129,"log_start":"1099'4100","ondisk_log_start":"1099'4100","created":1086,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:47.754927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14174,"num_write_kb":453568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1476,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1080,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.685371+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T10:00:35.685371+0000","last_peered":"2024-09-21T10:00:35.685371+0000","last_clean":"2024-09-21T10:00:35.685371+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T10:00:35.685371+0000","last_undegraded":"2024-09-21T10:00:35.685371+0000","last_fullsized":"2024-09-21T10:00:35.685371+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1087,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.687115+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T10:00:34.687115+0000","last_peered":"2024-09-21T10:00:34.687115+0000","last_clean":"2024-09-21T10:00:34.687115+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T10:00:34.687115+0000","last_undegraded":"2024-09-21T10:00:34.687115+0000","last_fullsized":"2024-09-21T10:00:34.687115+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1128'14302","reported_seq":14357,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692395+0000","last_change":"2024-09-21T09:56:26.055238+0000","last_active":"2024-09-21T10:00:34.692395+0000","last_peered":"2024-09-21T10:00:34.692395+0000","last_clean":"2024-09-21T10:00:34.692395+0000","last_became_active":"2024-09-21T09:56:26.054923+0000","last_became_peered":"2024-09-21T09:56:26.054923+0000","last_unstale":"2024-09-21T10:00:34.692395+0000","last_undegraded":"2024-09-21T10:00:34.692395+0000","last_fullsized":"2024-09-21T10:00:34.692395+0000","mapping_epoch":1086,"log_start":"1101'4300","ondisk_log_start":"1101'4300","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:33:53.447607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14302,"num_write_kb":457601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718144+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T10:00:38.718144+0000","last_peered":"2024-09-21T10:00:38.718144+0000","last_clean":"2024-09-21T10:00:38.718144+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T10:00:38.718144+0000","last_undegraded":"2024-09-21T10:00:38.718144+0000","last_fullsized":"2024-09-21T10:00:38.718144+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1425,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850212+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:00:35.850212+0000","last_peered":"2024-09-21T10:00:35.850212+0000","last_clean":"2024-09-21T10:00:35.850212+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:00:35.850212+0000","last_undegraded":"2024-09-21T10:00:35.850212+0000","last_fullsized":"2024-09-21T10:00:35.850212+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"1128'14008","reported_seq":16850,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692103+0000","last_change":"2024-09-21T09:58:29.726533+0000","last_active":"2024-09-21T10:00:34.692103+0000","last_peered":"2024-09-21T10:00:34.692103+0000","last_clean":"2024-09-21T10:00:34.692103+0000","last_became_active":"2024-09-21T09:58:29.712721+0000","last_became_peered":"2024-09-21T09:58:29.712721+0000","last_unstale":"2024-09-21T10:00:34.692103+0000","last_undegraded":"2024-09-21T10:00:34.692103+0000","last_fullsized":"2024-09-21T10:00:34.692103+0000","mapping_epoch":1113,"log_start":"1099'4000","ondisk_log_start":"1099'4000","created":1086,"last_epoch_clean":1114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:01.409158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14008,"num_write_kb":448256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1385,"num_bytes_recovered":90767360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1388,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692348+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T10:00:34.692348+0000","last_peered":"2024-09-21T10:00:34.692348+0000","last_clean":"2024-09-21T10:00:34.692348+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T10:00:34.692348+0000","last_undegraded":"2024-09-21T10:00:34.692348+0000","last_fullsized":"2024-09-21T10:00:34.692348+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1088,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692170+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T10:00:34.692170+0000","last_peered":"2024-09-21T10:00:34.692170+0000","last_clean":"2024-09-21T10:00:34.692170+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T10:00:34.692170+0000","last_undegraded":"2024-09-21T10:00:34.692170+0000","last_fullsized":"2024-09-21T10:00:34.692170+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718685+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T10:00:38.718685+0000","last_peered":"2024-09-21T10:00:38.718685+0000","last_clean":"2024-09-21T10:00:38.718685+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T10:00:38.718685+0000","last_undegraded":"2024-09-21T10:00:38.718685+0000","last_fullsized":"2024-09-21T10:00:38.718685+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1046,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692252+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T10:00:34.692252+0000","last_peered":"2024-09-21T10:00:34.692252+0000","last_clean":"2024-09-21T10:00:34.692252+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T10:00:34.692252+0000","last_undegraded":"2024-09-21T10:00:34.692252+0000","last_fullsized":"2024-09-21T10:00:34.692252+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1128'14158","reported_seq":17227,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850173+0000","last_change":"2024-09-21T09:57:44.801102+0000","last_active":"2024-09-21T10:00:35.850173+0000","last_peered":"2024-09-21T10:00:35.850173+0000","last_clean":"2024-09-21T10:00:35.850173+0000","last_became_active":"2024-09-21T09:57:44.794543+0000","last_became_peered":"2024-09-21T09:57:44.794543+0000","last_unstale":"2024-09-21T10:00:35.850173+0000","last_undegraded":"2024-09-21T10:00:35.850173+0000","last_fullsized":"2024-09-21T10:00:35.850173+0000","mapping_epoch":1105,"log_start":"1103'5500","ondisk_log_start":"1103'5500","created":1086,"last_epoch_clean":1106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8658,"log_dups_size":0,"ondisk_log_size":8658,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:36:08.731511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14158,"num_write_kb":453056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1478,"num_bytes_recovered":96862208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1086,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692188+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T10:00:34.692188+0000","last_peered":"2024-09-21T10:00:34.692188+0000","last_clean":"2024-09-21T10:00:34.692188+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T10:00:34.692188+0000","last_undegraded":"2024-09-21T10:00:34.692188+0000","last_fullsized":"2024-09-21T10:00:34.692188+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1456,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.712998+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T10:00:38.712998+0000","last_peered":"2024-09-21T10:00:38.712998+0000","last_clean":"2024-09-21T10:00:38.712998+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T10:00:38.712998+0000","last_undegraded":"2024-09-21T10:00:38.712998+0000","last_fullsized":"2024-09-21T10:00:38.712998+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1454,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713139+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T10:00:38.713139+0000","last_peered":"2024-09-21T10:00:38.713139+0000","last_clean":"2024-09-21T10:00:38.713139+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T10:00:38.713139+0000","last_undegraded":"2024-09-21T10:00:38.713139+0000","last_fullsized":"2024-09-21T10:00:38.713139+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1128'14148","reported_seq":17030,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.843489+0000","last_change":"2024-09-21T10:00:34.319251+0000","last_active":"2024-09-21T10:00:35.843489+0000","last_peered":"2024-09-21T10:00:35.843489+0000","last_clean":"2024-09-21T10:00:35.843489+0000","last_became_active":"2024-09-21T09:59:29.047032+0000","last_became_peered":"2024-09-21T09:59:29.047032+0000","last_unstale":"2024-09-21T10:00:35.843489+0000","last_undegraded":"2024-09-21T10:00:35.843489+0000","last_fullsized":"2024-09-21T10:00:35.843489+0000","mapping_epoch":1120,"log_start":"1106'6000","ondisk_log_start":"1106'6000","created":1086,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:33.224146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14148,"num_write_kb":452736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1405,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1435,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718022+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T10:00:38.718022+0000","last_peered":"2024-09-21T10:00:38.718022+0000","last_clean":"2024-09-21T10:00:38.718022+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T10:00:38.718022+0000","last_undegraded":"2024-09-21T10:00:38.718022+0000","last_fullsized":"2024-09-21T10:00:38.718022+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1412,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.849941+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T10:00:35.849941+0000","last_peered":"2024-09-21T10:00:35.849941+0000","last_clean":"2024-09-21T10:00:35.849941+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T10:00:35.849941+0000","last_undegraded":"2024-09-21T10:00:35.849941+0000","last_fullsized":"2024-09-21T10:00:35.849941+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1420,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850066+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T10:00:35.850066+0000","last_peered":"2024-09-21T10:00:35.850066+0000","last_clean":"2024-09-21T10:00:35.850066+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T10:00:35.850066+0000","last_undegraded":"2024-09-21T10:00:35.850066+0000","last_fullsized":"2024-09-21T10:00:35.850066+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1128'14204","reported_seq":17203,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718428+0000","last_change":"2024-09-21T09:57:01.634070+0000","last_active":"2024-09-21T10:00:38.718428+0000","last_peered":"2024-09-21T10:00:38.718428+0000","last_clean":"2024-09-21T10:00:38.718428+0000","last_became_active":"2024-09-21T09:57:01.631093+0000","last_became_peered":"2024-09-21T09:57:01.631093+0000","last_unstale":"2024-09-21T10:00:38.718428+0000","last_undegraded":"2024-09-21T10:00:38.718428+0000","last_fullsized":"2024-09-21T10:00:38.718428+0000","mapping_epoch":1096,"log_start":"1107'6800","ondisk_log_start":"1107'6800","created":1086,"last_epoch_clean":1097,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:52:47.969850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14204,"num_write_kb":454528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1089,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692256+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T10:00:34.692256+0000","last_peered":"2024-09-21T10:00:34.692256+0000","last_clean":"2024-09-21T10:00:34.692256+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T10:00:34.692256+0000","last_undegraded":"2024-09-21T10:00:34.692256+0000","last_fullsized":"2024-09-21T10:00:34.692256+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1080,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686845+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T10:00:34.686845+0000","last_peered":"2024-09-21T10:00:34.686845+0000","last_clean":"2024-09-21T10:00:34.686845+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T10:00:34.686845+0000","last_undegraded":"2024-09-21T10:00:34.686845+0000","last_fullsized":"2024-09-21T10:00:34.686845+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1437,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850202+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:00:35.850202+0000","last_peered":"2024-09-21T10:00:35.850202+0000","last_clean":"2024-09-21T10:00:35.850202+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:00:35.850202+0000","last_undegraded":"2024-09-21T10:00:35.850202+0000","last_fullsized":"2024-09-21T10:00:35.850202+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1128'14168","reported_seq":17209,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713037+0000","last_change":"2024-09-21T10:00:37.486035+0000","last_active":"2024-09-21T10:00:38.713037+0000","last_peered":"2024-09-21T10:00:38.713037+0000","last_clean":"2024-09-21T10:00:38.713037+0000","last_became_active":"2024-09-21T10:00:04.162832+0000","last_became_peered":"2024-09-21T10:00:04.162832+0000","last_unstale":"2024-09-21T10:00:38.713037+0000","last_undegraded":"2024-09-21T10:00:38.713037+0000","last_fullsized":"2024-09-21T10:00:38.713037+0000","mapping_epoch":1124,"log_start":"1101'4400","ondisk_log_start":"1101'4400","created":1086,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":9768,"log_dups_size":0,"ondisk_log_size":9768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:16:18.798286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14168,"num_write_kb":453376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1462,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1107,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692690+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T10:00:34.692690+0000","last_peered":"2024-09-21T10:00:34.692690+0000","last_clean":"2024-09-21T10:00:34.692690+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T10:00:34.692690+0000","last_undegraded":"2024-09-21T10:00:34.692690+0000","last_fullsized":"2024-09-21T10:00:34.692690+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1434,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850253+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T10:00:35.850253+0000","last_peered":"2024-09-21T10:00:35.850253+0000","last_clean":"2024-09-21T10:00:35.850253+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T10:00:35.850253+0000","last_undegraded":"2024-09-21T10:00:35.850253+0000","last_fullsized":"2024-09-21T10:00:35.850253+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.712998+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T10:00:38.712998+0000","last_peered":"2024-09-21T10:00:38.712998+0000","last_clean":"2024-09-21T10:00:38.712998+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T10:00:38.712998+0000","last_undegraded":"2024-09-21T10:00:38.712998+0000","last_fullsized":"2024-09-21T10:00:38.712998+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"1128'14576","reported_seq":14677,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.849966+0000","last_change":"2024-09-21T09:56:26.049677+0000","last_active":"2024-09-21T10:00:35.849966+0000","last_peered":"2024-09-21T10:00:35.849966+0000","last_clean":"2024-09-21T10:00:35.849966+0000","last_became_active":"2024-09-21T09:56:26.049395+0000","last_became_peered":"2024-09-21T09:56:26.049395+0000","last_unstale":"2024-09-21T10:00:35.849966+0000","last_undegraded":"2024-09-21T10:00:35.849966+0000","last_fullsized":"2024-09-21T10:00:35.849966+0000","mapping_epoch":1086,"log_start":"1104'6000","ondisk_log_start":"1104'6000","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8576,"log_dups_size":0,"ondisk_log_size":8576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:40:42.319934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14576,"num_write_kb":466432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1411,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850383+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T10:00:35.850383+0000","last_peered":"2024-09-21T10:00:35.850383+0000","last_clean":"2024-09-21T10:00:35.850383+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T10:00:35.850383+0000","last_undegraded":"2024-09-21T10:00:35.850383+0000","last_fullsized":"2024-09-21T10:00:35.850383+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1071,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691719+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T10:00:34.691719+0000","last_peered":"2024-09-21T10:00:34.691719+0000","last_clean":"2024-09-21T10:00:34.691719+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T10:00:34.691719+0000","last_undegraded":"2024-09-21T10:00:34.691719+0000","last_fullsized":"2024-09-21T10:00:34.691719+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1432,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713175+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T10:00:38.713175+0000","last_peered":"2024-09-21T10:00:38.713175+0000","last_clean":"2024-09-21T10:00:38.713175+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T10:00:38.713175+0000","last_undegraded":"2024-09-21T10:00:38.713175+0000","last_fullsized":"2024-09-21T10:00:38.713175+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1128'14336","reported_seq":16346,"reported_epoch":1130,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:00:36.446255+0000","last_change":"2024-09-21T10:00:34.319694+0000","last_active":"2024-09-21T10:00:36.446255+0000","last_peered":"2024-09-21T10:00:36.446255+0000","last_clean":"2024-09-21T09:56:44.221744+0000","last_became_active":"2024-09-21T09:56:46.335379+0000","last_became_peered":"2024-09-21T09:56:46.335379+0000","last_unstale":"2024-09-21T10:00:36.446255+0000","last_undegraded":"2024-09-21T09:56:46.244519+0000","last_fullsized":"2024-09-21T09:56:46.234659+0000","mapping_epoch":1093,"log_start":"1106'6400","ondisk_log_start":"1106'6400","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":7936,"log_dups_size":0,"ondisk_log_size":7936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:18.699773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14336,"num_write_kb":458752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":973,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-439}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":95,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718950+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T10:00:38.718950+0000","last_peered":"2024-09-21T10:00:38.718950+0000","last_clean":"2024-09-21T10:00:38.718950+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T10:00:38.718950+0000","last_undegraded":"2024-09-21T10:00:38.718950+0000","last_fullsized":"2024-09-21T10:00:38.718950+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1412,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850380+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T10:00:35.850380+0000","last_peered":"2024-09-21T10:00:35.850380+0000","last_clean":"2024-09-21T10:00:35.850380+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T10:00:35.850380+0000","last_undegraded":"2024-09-21T10:00:35.850380+0000","last_fullsized":"2024-09-21T10:00:35.850380+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718017+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T10:00:38.718017+0000","last_peered":"2024-09-21T10:00:38.718017+0000","last_clean":"2024-09-21T10:00:38.718017+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T10:00:38.718017+0000","last_undegraded":"2024-09-21T10:00:38.718017+0000","last_fullsized":"2024-09-21T10:00:38.718017+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1128'14276","reported_seq":14331,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.843325+0000","last_change":"2024-09-21T09:56:26.056148+0000","last_active":"2024-09-21T10:00:35.843325+0000","last_peered":"2024-09-21T10:00:35.843325+0000","last_clean":"2024-09-21T10:00:35.843325+0000","last_became_active":"2024-09-21T09:56:26.055872+0000","last_became_peered":"2024-09-21T09:56:26.055872+0000","last_unstale":"2024-09-21T10:00:35.843325+0000","last_undegraded":"2024-09-21T10:00:35.843325+0000","last_fullsized":"2024-09-21T10:00:35.843325+0000","mapping_epoch":1086,"log_start":"1106'6100","ondisk_log_start":"1106'6100","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:56:25.036245+0000","last_clean_scrub_stamp":"2024-09-21T09:56:25.036245+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:23:56.312740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14276,"num_write_kb":456832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1446,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713334+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T10:00:38.713334+0000","last_peered":"2024-09-21T10:00:38.713334+0000","last_clean":"2024-09-21T10:00:38.713334+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T10:00:38.713334+0000","last_undegraded":"2024-09-21T10:00:38.713334+0000","last_fullsized":"2024-09-21T10:00:38.713334+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1074,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691833+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T10:00:34.691833+0000","last_peered":"2024-09-21T10:00:34.691833+0000","last_clean":"2024-09-21T10:00:34.691833+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T10:00:34.691833+0000","last_undegraded":"2024-09-21T10:00:34.691833+0000","last_fullsized":"2024-09-21T10:00:34.691833+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1064,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686600+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T10:00:34.686600+0000","last_peered":"2024-09-21T10:00:34.686600+0000","last_clean":"2024-09-21T10:00:34.686600+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T10:00:34.686600+0000","last_undegraded":"2024-09-21T10:00:34.686600+0000","last_fullsized":"2024-09-21T10:00:34.686600+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1078,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686782+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T10:00:34.686782+0000","last_peered":"2024-09-21T10:00:34.686782+0000","last_clean":"2024-09-21T10:00:34.686782+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T10:00:34.686782+0000","last_undegraded":"2024-09-21T10:00:34.686782+0000","last_fullsized":"2024-09-21T10:00:34.686782+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1087,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686866+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T10:00:34.686866+0000","last_peered":"2024-09-21T10:00:34.686866+0000","last_clean":"2024-09-21T10:00:34.686866+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T10:00:34.686866+0000","last_undegraded":"2024-09-21T10:00:34.686866+0000","last_fullsized":"2024-09-21T10:00:34.686866+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1411,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850033+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T10:00:35.850033+0000","last_peered":"2024-09-21T10:00:35.850033+0000","last_clean":"2024-09-21T10:00:35.850033+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T10:00:35.850033+0000","last_undegraded":"2024-09-21T10:00:35.850033+0000","last_fullsized":"2024-09-21T10:00:35.850033+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1089,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691439+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T10:00:34.691439+0000","last_peered":"2024-09-21T10:00:34.691439+0000","last_clean":"2024-09-21T10:00:34.691439+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T10:00:34.691439+0000","last_undegraded":"2024-09-21T10:00:34.691439+0000","last_fullsized":"2024-09-21T10:00:34.691439+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1072,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692325+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T10:00:34.692325+0000","last_peered":"2024-09-21T10:00:34.692325+0000","last_clean":"2024-09-21T10:00:34.692325+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T10:00:34.692325+0000","last_undegraded":"2024-09-21T10:00:34.692325+0000","last_fullsized":"2024-09-21T10:00:34.692325+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718431+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T10:00:38.718431+0000","last_peered":"2024-09-21T10:00:38.718431+0000","last_clean":"2024-09-21T10:00:38.718431+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T10:00:38.718431+0000","last_undegraded":"2024-09-21T10:00:38.718431+0000","last_fullsized":"2024-09-21T10:00:38.718431+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1454,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713527+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T10:00:38.713527+0000","last_peered":"2024-09-21T10:00:38.713527+0000","last_clean":"2024-09-21T10:00:38.713527+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T10:00:38.713527+0000","last_undegraded":"2024-09-21T10:00:38.713527+0000","last_fullsized":"2024-09-21T10:00:38.713527+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718236+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T10:00:38.718236+0000","last_peered":"2024-09-21T10:00:38.718236+0000","last_clean":"2024-09-21T10:00:38.718236+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T10:00:38.718236+0000","last_undegraded":"2024-09-21T10:00:38.718236+0000","last_fullsized":"2024-09-21T10:00:38.718236+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1070,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.691761+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T10:00:34.691761+0000","last_peered":"2024-09-21T10:00:34.691761+0000","last_clean":"2024-09-21T10:00:34.691761+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T10:00:34.691761+0000","last_undegraded":"2024-09-21T10:00:34.691761+0000","last_fullsized":"2024-09-21T10:00:34.691761+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.717890+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T10:00:38.717890+0000","last_peered":"2024-09-21T10:00:38.717890+0000","last_clean":"2024-09-21T10:00:38.717890+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T10:00:38.717890+0000","last_undegraded":"2024-09-21T10:00:38.717890+0000","last_fullsized":"2024-09-21T10:00:38.717890+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1457,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713238+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T10:00:38.713238+0000","last_peered":"2024-09-21T10:00:38.713238+0000","last_clean":"2024-09-21T10:00:38.713238+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T10:00:38.713238+0000","last_undegraded":"2024-09-21T10:00:38.713238+0000","last_fullsized":"2024-09-21T10:00:38.713238+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1424,"reported_epoch":1130,"state":"active+clean","last_fresh":"2024-09-21T10:00:35.850011+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:00:35.850011+0000","last_peered":"2024-09-21T10:00:35.850011+0000","last_clean":"2024-09-21T10:00:35.850011+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:00:35.850011+0000","last_undegraded":"2024-09-21T10:00:35.850011+0000","last_fullsized":"2024-09-21T10:00:35.850011+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718771+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:00:38.718771+0000","last_peered":"2024-09-21T10:00:38.718771+0000","last_clean":"2024-09-21T10:00:38.718771+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:00:38.718771+0000","last_undegraded":"2024-09-21T10:00:38.718771+0000","last_fullsized":"2024-09-21T10:00:38.718771+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1094,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686678+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T10:00:34.686678+0000","last_peered":"2024-09-21T10:00:34.686678+0000","last_clean":"2024-09-21T10:00:34.686678+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T10:00:34.686678+0000","last_undegraded":"2024-09-21T10:00:34.686678+0000","last_fullsized":"2024-09-21T10:00:34.686678+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1411,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.717955+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T10:00:38.717955+0000","last_peered":"2024-09-21T10:00:38.717955+0000","last_clean":"2024-09-21T10:00:38.717955+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T10:00:38.717955+0000","last_undegraded":"2024-09-21T10:00:38.717955+0000","last_fullsized":"2024-09-21T10:00:38.717955+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1433,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713145+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T10:00:38.713145+0000","last_peered":"2024-09-21T10:00:38.713145+0000","last_clean":"2024-09-21T10:00:38.713145+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T10:00:38.713145+0000","last_undegraded":"2024-09-21T10:00:38.713145+0000","last_fullsized":"2024-09-21T10:00:38.713145+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1110,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686750+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T10:00:34.686750+0000","last_peered":"2024-09-21T10:00:34.686750+0000","last_clean":"2024-09-21T10:00:34.686750+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T10:00:34.686750+0000","last_undegraded":"2024-09-21T10:00:34.686750+0000","last_fullsized":"2024-09-21T10:00:34.686750+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1074,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.696426+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T10:00:34.696426+0000","last_peered":"2024-09-21T10:00:34.696426+0000","last_clean":"2024-09-21T10:00:34.696426+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T10:00:34.696426+0000","last_undegraded":"2024-09-21T10:00:34.696426+0000","last_fullsized":"2024-09-21T10:00:34.696426+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1110,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686573+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T10:00:34.686573+0000","last_peered":"2024-09-21T10:00:34.686573+0000","last_clean":"2024-09-21T10:00:34.686573+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T10:00:34.686573+0000","last_undegraded":"2024-09-21T10:00:34.686573+0000","last_fullsized":"2024-09-21T10:00:34.686573+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1059,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.692619+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T10:00:34.692619+0000","last_peered":"2024-09-21T10:00:34.692619+0000","last_clean":"2024-09-21T10:00:34.692619+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T10:00:34.692619+0000","last_undegraded":"2024-09-21T10:00:34.692619+0000","last_fullsized":"2024-09-21T10:00:34.692619+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718524+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T10:00:38.718524+0000","last_peered":"2024-09-21T10:00:38.718524+0000","last_clean":"2024-09-21T10:00:38.718524+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T10:00:38.718524+0000","last_undegraded":"2024-09-21T10:00:38.718524+0000","last_fullsized":"2024-09-21T10:00:38.718524+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718262+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T10:00:38.718262+0000","last_peered":"2024-09-21T10:00:38.718262+0000","last_clean":"2024-09-21T10:00:38.718262+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T10:00:38.718262+0000","last_undegraded":"2024-09-21T10:00:38.718262+0000","last_fullsized":"2024-09-21T10:00:38.718262+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1086,"reported_epoch":1129,"state":"active+clean","last_fresh":"2024-09-21T10:00:34.686420+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T10:00:34.686420+0000","last_peered":"2024-09-21T10:00:34.686420+0000","last_clean":"2024-09-21T10:00:34.686420+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T10:00:34.686420+0000","last_undegraded":"2024-09-21T10:00:34.686420+0000","last_fullsized":"2024-09-21T10:00:34.686420+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1449,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713551+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T10:00:38.713551+0000","last_peered":"2024-09-21T10:00:38.713551+0000","last_clean":"2024-09-21T10:00:38.713551+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T10:00:38.713551+0000","last_undegraded":"2024-09-21T10:00:38.713551+0000","last_fullsized":"2024-09-21T10:00:38.713551+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":227808,"num_write_kb":7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13909,"num_bytes_recovered":487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":516227072,"data_stored":516227072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142108,"ondisk_log_size":142108,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234297,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":267360,"kb_used_data":101716,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":93477792,"statfs":{"total":95995035648,"available":95721259008,"internally_reserved":0,"allocated":104157184,"data_stored":102069748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":169605579},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495934,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":430488,"kb_used_data":246860,"kb_used_omap":1,"kb_used_meta":183614,"kb_avail":93314664,"statfs":{"total":95995035648,"available":95554215936,"internally_reserved":0,"allocated":252784640,"data_stored":250705269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188021191},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757572,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":271516,"kb_used_data":100172,"kb_used_omap":1,"kb_used_meta":171326,"kb_avail":93473636,"statfs":{"total":95995035648,"available":95717003264,"internally_reserved":0,"allocated":102576128,"data_stored":100500362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":175438278},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019208,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":164132,"kb_used_data":6680,"kb_used_omap":1,"kb_used_meta":157438,"kb_avail":93581020,"statfs":{"total":95995035648,"available":95826964480,"internally_reserved":0,"allocated":6840320,"data_stored":4756497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161216966},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888505,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277468,"kb_used_data":6672,"kb_used_omap":34,"kb_used_meta":270749,"kb_avail":93467684,"statfs":{"total":95995035648,"available":95710908416,"internally_reserved":0,"allocated":6832128,"data_stored":4760012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":277247987},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967744,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":269876,"kb_used_data":85088,"kb_used_omap":41,"kb_used_meta":184726,"kb_avail":93475276,"statfs":{"total":95995035648,"available":95718682624,"internally_reserved":0,"allocated":87130112,"data_stored":85050432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":189159839},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046982,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":124816,"kb_used_data":6660,"kb_used_omap":41,"kb_used_meta":118102,"kb_avail":93620336,"statfs":{"total":95995035648,"available":95867224064,"internally_reserved":0,"allocated":6819840,"data_stored":4751626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":120937385},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126221,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112180,"kb_used_data":7268,"kb_used_omap":72,"kb_used_meta":104823,"kb_avail":93632972,"statfs":{"total":95995035648,"available":95880163328,"internally_reserved":0,"allocated":7442432,"data_stored":5359188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":107338795},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":80281600,"data_stored":80281600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":95748096,"data_stored":95748096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":245366784,"data_stored":245366784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":94830592,"data_stored":94830592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:00:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:43.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:43.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:43.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:43.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:43.479 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:00:43] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.51.0" 2024-09-21T10:00:43.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:43.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:43.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:43.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:43.730 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:43.751 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:43.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:43.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:43.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:43.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:43.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:44.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:44.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-mon[106482]: pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 0 B/s, 238 objects/s recovering 2024-09-21T10:00:44.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:44.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:44.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-mon[110963]: pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 0 B/s, 238 objects/s recovering 2024-09-21T10:00:44.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:44.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:43 smithi191 ceph-mon[102206]: pgmap v1317: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 0 B/s, 238 objects/s recovering 2024-09-21T10:00:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:44.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:44.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:44.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:44.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:44.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:44.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:44.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:44.474 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:44.495 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:44.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:44.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:44.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:44.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:44.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:44.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:44.870 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:44.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:44.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:44.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:44.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:45.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:45.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:45.016 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:45.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:45.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:45.016 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:45.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:45.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:45.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:45.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:45.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:45.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:45.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:45.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:45.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:45.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:45.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:45.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:45.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:45.632 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:45.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:45.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:45.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:45.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:45.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:45.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:45.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:46.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:46.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:46.131 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:46.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:46.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:00:46.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:46.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:46.275 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:46 smithi076 ceph-mon[106482]: pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 0 B/s, 82 objects/s recovering 2024-09-21T10:00:46.276 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:46 smithi076 ceph-mon[110963]: pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 0 B/s, 82 objects/s recovering 2024-09-21T10:00:46.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:46.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:46 smithi191 ceph-mon[102206]: pgmap v1318: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 0 B/s, 82 objects/s recovering 2024-09-21T10:00:46.378 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:46.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:46.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:46.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:46.633 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:46.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:46.760 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:46.862 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:46.886 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:46.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:46.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:47.009 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:47.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.255 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:47.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:47.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:47.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:47.510 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:47.611 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.633 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:47.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.641 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:00:47.641 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:00:47.650 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:00:47.651 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:00:47.667 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:00:47.667 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:00:47.684 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:00:47.684 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:00:47.701 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:00:47.701 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:00:47.718 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:00:47.718 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:00:47.734 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:00:47.734 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:00:47.735 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:47.751 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:00:47.751 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:00:47.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:47.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:47.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:47.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:47.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:47.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:48.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:48.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.149 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:48.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:48.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:48.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:48 smithi191 ceph-mon[102206]: pgmap v1319: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 0 B/s, 74 objects/s recovering 2024-09-21T10:00:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/897913162' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:48 smithi076 ceph-mon[106482]: pgmap v1319: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 0 B/s, 74 objects/s recovering 2024-09-21T10:00:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/897913162' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:48 smithi076 ceph-mon[110963]: pgmap v1319: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 14 op/s; 0 B/s, 74 objects/s recovering 2024-09-21T10:00:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/897913162' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:00:48.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:48.403 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:48.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:48.505 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.531 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:48.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:48.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:48.656 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:48.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:48.782 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:48.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:48.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:48.906 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:48.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:49.008 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:49.029 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:49.131 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:49.152 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:49.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:49.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:49.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:49.408 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:49.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:49.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:49.532 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:49.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:49.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:49.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:49.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:49.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:49.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:49.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:49.890 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:49.906 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:49.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:49.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:50.008 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:50.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:50.123 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:00:50.133 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:50.156 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:50.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:50.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:50.279 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:50 smithi191 ceph-mon[102206]: pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 0 B/s, 65 objects/s recovering 2024-09-21T10:00:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:50 smithi076 ceph-mon[106482]: pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 0 B/s, 65 objects/s recovering 2024-09-21T10:00:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:50 smithi076 ceph-mon[110963]: pgmap v1320: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 0 B/s, 65 objects/s recovering 2024-09-21T10:00:50.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:50.404 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:50.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:50.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:50.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:50.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:50.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:50.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:50.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:50.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:50.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:50.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:50.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:50.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:50.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:51.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:51.024 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:51.125 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_1' removed 2024-09-21T10:00:51.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:51.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:51.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:00:51.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:51.278 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3290534218' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1062746375' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3290534218' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:00:51.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:51.280 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-21T10:00:51.280 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1062746375' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:51.287 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3290534218' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-21T10:00:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1062746375' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:00:51.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:51.410 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:51.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:51.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:51.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:51.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:51.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:51.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:51.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:51.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:51.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:51.922 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:51.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:51.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:52.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:52.044 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:52.146 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:52.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:52.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:52.290 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:52.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:52 smithi076 ceph-mon[110963]: pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 43 objects/s recovering 2024-09-21T10:00:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1062746375' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:00:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:52 smithi076 ceph-mon[110963]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-21T10:00:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:52 smithi076 ceph-mon[106482]: pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 43 objects/s recovering 2024-09-21T10:00:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1062746375' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:00:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:52 smithi076 ceph-mon[106482]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-21T10:00:52.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:52.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:52.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:52.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:52.536 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:52 smithi191 ceph-mon[102206]: pgmap v1322: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s, 43 objects/s recovering 2024-09-21T10:00:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1062746375' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:00:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:52 smithi191 ceph-mon[102206]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-21T10:00:52.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:52.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:52.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:52.782 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:52.793 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:00:52.793 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:00:52.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:130958 6.f 6:fa59f2a5:::benchmark_data_smithi076_154645_object17052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1112)", 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:26.593738+0000", 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 144.67298940500001, 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.118238289, 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.794 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 130958 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593738+0000", 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593738+0000", 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593739+0000", 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.522e-06 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593743+0000", 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.58e-06 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593753+0000", 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0207000000000001e-05 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593759+0000", 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1639999999999998e-06 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593814+0000", 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4435000000000003e-05 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.593964+0000", 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000150049 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.594718+0000", 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00075450599999999995 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.709847+0000", 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.11512869100000001 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.709899+0000", 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2487000000000001e-05 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.710224+0000", 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000324977 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.710275+0000", 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0547000000000002e-05 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.711976+0000", 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001701124 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:130964 6.f 6:f0d00d5b:::benchmark_data_smithi076_154645_object17058:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1112)", 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:26.597908+0000", 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 144.66881912900001, 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.11917784300000001, 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.800 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 130964 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.597908+0000", 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.597908+0000", 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.597909+0000", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.161e-06 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.597939+0000", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9669999999999999e-05 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.597950+0000", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1457999999999999e-05 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.597957+0000", 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.075e-06 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705688+0000", 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.10773031700000001 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705841+0000", 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000153284 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705992+0000", 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015096400000000001 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.710205+0000", 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042136059999999999 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.712441+0000", 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022359369999999999 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.716383+0000", 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039418190000000001 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.716471+0000", 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8065999999999993e-05 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.717086+0000", 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00061448600000000003 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:130963 6.9 e1112/1086 6:937659e1:::benchmark_data_smithi076_154645_object17057:head v 1112'8213, mlcod=1112'8211)", 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:26.602038+0000", 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 144.664689372, 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.13169451600000001, 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.602038+0000", 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.602038+0000", 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.602038+0000", 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0200000000000003e-07 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.602041+0000", 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3139999999999999e-06 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.602048+0000", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2310000000000001e-06 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.602052+0000", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0130000000000004e-06 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705715+0000", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.10366376200000001 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705736+0000", 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0557e-05 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.732614+0000", 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.026878368999999999 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.733732+0000", 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011178679999999999 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.809 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34674.0:130968 6.1 e1112/1086 6:8e3e636b:::benchmark_data_smithi076_154645_object17062:head v 1112'8397, mlcod=1112'8395)", 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:26.609233+0000", 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 144.65749429900001, 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.12337841500000001, 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.609233+0000", 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.609233+0000", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.609234+0000", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2979999999999999e-06 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.609242+0000", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3229999999999998e-06 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705606+0000", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.096363481000000001 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.705621+0000", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5176e-05 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.706154+0000", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00053270800000000001 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.706173+0000", 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9276e-05 2024-09-21T10:00:52.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.732115+0000", 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.025942073999999999 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:26.732611+0000", 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00049607899999999996 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.813 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159005 6.f 6:f6a4ad1c:::benchmark_data_smithi076_154645_object45099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.851352+0000", 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.415374746, 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0199435720000001, 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159005 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851352+0000", 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851352+0000", 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851354+0000", 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.327e-06 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851359+0000", 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9660000000000004e-06 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851368+0000", 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6369999999999994e-06 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851375+0000", 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1409999999999999e-06 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.870075+0000", 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018699335000000001 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.870086+0000", 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1508999999999999e-05 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.862921+0000", 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.99283502099999998 2024-09-21T10:00:52.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.863032+0000", 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110525 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.863134+0000", 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010220100000000001 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871068+0000", 2024-09-21T10:00:52.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079340639999999994 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871112+0000", 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3816000000000002e-05 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871230+0000", 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118262 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871266+0000", 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6143000000000003e-05 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871296+0000", 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9624999999999998e-05 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.820 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159036 6.f 6:fa619a2f:::benchmark_data_smithi076_154645_object45130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.909283+0000", 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.357444328, 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96219932799999996, 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159036 2024-09-21T10:00:52.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909283+0000", 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909283+0000", 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909284+0000", 2024-09-21T10:00:52.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4100000000000001e-06 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909287+0000", 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2339999999999999e-06 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909300+0000", 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2184999999999999e-05 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909305+0000", 2024-09-21T10:00:52.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6790000000000002e-06 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.911068+0000", 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017625690000000001 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.911077+0000", 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6190000000000006e-06 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.867281+0000", 2024-09-21T10:00:52.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95620355999999995 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.867405+0000", 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012382699999999999 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.867537+0000", 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013225199999999999 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871344+0000", 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038068329999999999 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871353+0000", 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9789999999999999e-06 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871390+0000", 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6910999999999999e-05 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871443+0000", 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3619000000000003e-05 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871482+0000", 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8651e-05 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159051 6.f 6:fab518e4:::benchmark_data_smithi076_154645_object45145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.827 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.926074+0000", 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.340653002, 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96485169299999995, 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159051 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.828 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926074+0000", 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926074+0000", 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926076+0000", 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.578e-06 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.829 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926082+0000", 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5220000000000004e-06 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926099+0000", 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7258999999999999e-05 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926107+0000", 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.148e-06 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.830 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926290+0000", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018331599999999999 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926314+0000", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3791e-05 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871532+0000", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94521803500000001 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871646+0000", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000114288 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871744+0000", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8425999999999996e-05 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881091+0000", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.009346732 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881104+0000", 2024-09-21T10:00:52.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.294e-05 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890814+0000", 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097100260000000001 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890872+0000", 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7636999999999997e-05 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.833 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890926+0000", 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3995000000000001e-05 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159079 6.f 6:fabef842:::benchmark_data_smithi076_154645_object45173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.961015+0000", 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.305712504, 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.93010314400000005, 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.834 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159079 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961015+0000", 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961015+0000", 2024-09-21T10:00:52.835 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961016+0000", 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7099999999999999e-06 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961022+0000", 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6710000000000004e-06 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961041+0000", 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8695e-05 2024-09-21T10:00:52.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961049+0000", 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0709999999999995e-06 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961120+0000", 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1664999999999999e-05 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961133+0000", 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2649999999999999e-05 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.876285+0000", 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91515186100000001 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.876390+0000", 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010521700000000001 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.876517+0000", 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012682500000000001 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.838 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881191+0000", 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046744860000000003 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890750+0000", 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0095585719999999996 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891032+0000", 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00028154299999999998 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.839 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891081+0000", 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9280000000000003e-05 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891118+0000", 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6897999999999998e-05 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159136 6.f 6:f59b07bd:::benchmark_data_smithi076_154645_object45230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.840 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.028209+0000", 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.23851846700001, 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88298491899999998, 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159136 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028209+0000", 2024-09-21T10:00:52.841 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028209+0000", 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028210+0000", 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.437e-06 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028214+0000", 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4989999999999999e-06 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028223+0000", 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6080000000000002e-06 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028229+0000", 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8420000000000001e-06 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.843 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030110+0000", 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018811570000000001 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030121+0000", 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1229e-05 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881239+0000", 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85111755600000005 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.844 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881336+0000", 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7343999999999995e-05 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881437+0000", 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000100758 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890973+0000", 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0095361449999999993 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.846 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.846 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891198+0000", 2024-09-21T10:00:52.846 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000224405 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911101+0000", 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.019903677000000002 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911146+0000", 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5164000000000003e-05 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911193+0000", 2024-09-21T10:00:52.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7098000000000002e-05 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159137 6.f 6:f9e69ffb:::benchmark_data_smithi076_154645_object45231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.238091416, 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88267890299999996, 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.848 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159137 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5300000000000003e-07 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028640+0000", 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6780000000000002e-06 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028651+0000", 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0752e-05 2024-09-21T10:00:52.850 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028657+0000", 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4989999999999999e-06 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030209+0000", 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001551796 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030219+0000", 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5149999999999995e-06 2024-09-21T10:00:52.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.886339+0000", 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85612032900000001 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.886444+0000", 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.886554+0000", 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011025 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891158+0000", 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046040780000000002 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911230+0000", 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020071848 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911240+0000", 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7829999999999993e-06 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911282+0000", 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2216999999999999e-05 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911315+0000", 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2404999999999999e-05 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159138 6.f 6:ff6bf720:::benchmark_data_smithi076_154645_object45232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.854 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.032925+0000", 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.23380211600001, 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87857126900000004, 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159138 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032925+0000", 2024-09-21T10:00:52.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032925+0000", 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032927+0000", 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9690000000000001e-06 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032932+0000", 2024-09-21T10:00:52.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5470000000000004e-06 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032964+0000", 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1260000000000002e-05 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032972+0000", 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1499999999999999e-06 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.033026+0000", 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4296999999999999e-05 2024-09-21T10:00:52.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.033036+0000", 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9280000000000005e-06 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891238+0000", 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85820230900000005 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891379+0000", 2024-09-21T10:00:52.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000140586 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891507+0000", 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012846900000000001 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901276+0000", 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097683360000000007 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911320+0000", 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010044136 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911407+0000", 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6948999999999997e-05 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911462+0000", 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5572999999999998e-05 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911496+0000", 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3760000000000002e-05 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159153 6.f 6:fc7c58f9:::benchmark_data_smithi076_154645_object45247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.047418+0000", 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.219309534, 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86424884499999999, 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.861 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159153 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047418+0000", 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047418+0000", 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047420+0000", 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1449999999999998e-06 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047425+0000", 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5879999999999997e-06 2024-09-21T10:00:52.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047441+0000", 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5801999999999999e-05 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047452+0000", 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0648999999999999e-05 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.049292+0000", 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018407250000000001 2024-09-21T10:00:52.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.049304+0000", 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2035000000000001e-05 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.896156+0000", 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84685153999999996 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.896260+0000", 2024-09-21T10:00:52.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103546 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.896370+0000", 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011090399999999999 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911089+0000", 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.014718491 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911325+0000", 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000235691 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911585+0000", 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000260277 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911618+0000", 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3068999999999998e-05 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.867 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911666+0000", 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8383000000000001e-05 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159165 6.f 6:fe126f68:::benchmark_data_smithi076_154645_object45259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.063179+0000", 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.203548023, 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85356281499999997, 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.868 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159165 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063179+0000", 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063179+0000", 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063181+0000", 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.691e-06 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063187+0000", 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3559999999999998e-06 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063204+0000", 2024-09-21T10:00:52.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6668e-05 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063212+0000", 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9000000000000006e-06 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063477+0000", 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026537700000000002 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063496+0000", 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8651000000000002e-05 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901305+0000", 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83780932399999997 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901415+0000", 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110479 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901531+0000", 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011521799999999999 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911329+0000", 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097980210000000005 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911358+0000", 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8867999999999999e-05 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916649+0000", 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052918260000000003 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916695+0000", 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5355999999999997e-05 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916742+0000", 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7080000000000003e-05 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159170 6.f 6:f90e3041:::benchmark_data_smithi076_154645_object45264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.070623+0000", 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.196104084, 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84630223000000004, 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159170 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.875 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070623+0000", 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070623+0000", 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070625+0000", 2024-09-21T10:00:52.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9920000000000002e-06 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070630+0000", 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4179999999999996e-06 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070647+0000", 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6229e-05 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070654+0000", 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8639999999999996e-06 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070720+0000", 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5827000000000005e-05 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070730+0000", 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9650000000000003e-06 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.906055+0000", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83532494199999996 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.906178+0000", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012309300000000001 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.906307+0000", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012857600000000001 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911367+0000", 2024-09-21T10:00:52.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050601049999999996 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911532+0000", 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000165315 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916803+0000", 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052708980000000004 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916882+0000", 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8366000000000002e-05 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916925+0000", 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3640000000000002e-05 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159194 6.f 6:f0fd59f5:::benchmark_data_smithi076_154645_object45288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.102922+0000", 2024-09-21T10:00:52.881 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.16380492499999, 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84748727099999999, 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159194 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102922+0000", 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102922+0000", 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102924+0000", 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9649999999999998e-06 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102930+0000", 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2199999999999997e-06 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102949+0000", 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8638999999999999e-05 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102957+0000", 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0860000000000002e-06 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.103021+0000", 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3607000000000003e-05 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.103031+0000", 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0549e-05 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911710+0000", 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.80867835600000004 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911782+0000", 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2279000000000004e-05 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911899+0000", 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116721 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.926988+0000", 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015089111000000001 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.926999+0000", 2024-09-21T10:00:52.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1079e-05 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950320+0000", 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023321278000000001 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950370+0000", 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9959000000000001e-05 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950409+0000", 2024-09-21T10:00:52.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9422000000000003e-05 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159224 6.f 6:f634f675:::benchmark_data_smithi076_154645_object45318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.158844+0000", 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 108.10788352599999, 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.79173849500000004, 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.888 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159224 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158844+0000", 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158844+0000", 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158845+0000", 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7689999999999999e-06 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158851+0000", 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.998e-06 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158869+0000", 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7356000000000001e-05 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158876+0000", 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6229999999999997e-06 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158948+0000", 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1767000000000006e-05 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158958+0000", 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9879999999999999e-06 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916959+0000", 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75800074799999995 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.917095+0000", 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013579900000000001 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.917204+0000", 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109182 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927003+0000", 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097995760000000008 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927174+0000", 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000170786 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950512+0000", 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023337994000000001 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950551+0000", 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9227999999999998e-05 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950582+0000", 2024-09-21T10:00:52.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0681e-05 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159255 6.f 6:fe2e5df9:::benchmark_data_smithi076_154645_object45349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.342273+0000", 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 107.924453546, 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.60849325200000004, 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.895 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159255 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342273+0000", 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342273+0000", 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342275+0000", 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.575e-06 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342280+0000", 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5299999999999998e-06 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342292+0000", 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2115e-05 2024-09-21T10:00:52.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342297+0000", 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3800000000000002e-06 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342353+0000", 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6195e-05 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342360+0000", 2024-09-21T10:00:52.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5629999999999997e-06 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.922043+0000", 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.57968352300000003 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.922229+0000", 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018597700000000001 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.922339+0000", 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010936499999999999 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937451+0000", 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015112038 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.943758+0000", 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006307511 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950664+0000", 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069058239999999996 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950704+0000", 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9446999999999997e-05 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950767+0000", 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3208999999999997e-05 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.901 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159258 6.f 6:f50607f5:::benchmark_data_smithi076_154645_object45352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.345695+0000", 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 107.92103197900001, 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.60567683400000005, 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159258 2024-09-21T10:00:52.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345695+0000", 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345695+0000", 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345696+0000", 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3289999999999999e-06 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345702+0000", 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1379999999999998e-06 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345714+0000", 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2772000000000001e-05 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345721+0000", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7379999999999997e-06 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345776+0000", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4976999999999999e-05 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345783+0000", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4949999999999996e-06 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927211+0000", 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.58142859400000002 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927328+0000", 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011688699999999999 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927452+0000", 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012365299999999999 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.943763+0000", 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016311811999999998 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950452+0000", 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066881589999999999 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951082+0000", 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00063082299999999997 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951227+0000", 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014471699999999999 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951372+0000", 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014474 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159302 6.f 6:f3d805b1:::benchmark_data_smithi076_154645_object45396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.450435+0000", 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 107.816291726, 2024-09-21T10:00:52.908 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.50150189700000003, 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159302 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450435+0000", 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450435+0000", 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450437+0000", 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.443e-06 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450442+0000", 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3820000000000003e-06 2024-09-21T10:00:52.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450455+0000", 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2689e-05 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450462+0000", 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9739999999999996e-06 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450527+0000", 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5129999999999995e-05 2024-09-21T10:00:52.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450539+0000", 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2199e-05 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.932309+0000", 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.48176940899999998 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.932411+0000", 2024-09-21T10:00:52.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010220799999999999 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.932534+0000", 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012294300000000001 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.943768+0000", 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011234177999999999 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950630+0000", 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0068617390000000004 2024-09-21T10:00:52.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951676+0000", 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010461310000000001 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951819+0000", 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014361300000000001 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951937+0000", 2024-09-21T10:00:52.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000117859 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159325 6.f 6:f4e968b7:::benchmark_data_smithi076_154645_object45419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.552683+0000", 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 107.71404440400001, 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.39968836299999999, 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:00:52.915 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159325 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552683+0000", 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552683+0000", 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552684+0000", 2024-09-21T10:00:52.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.39e-06 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552689+0000", 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.583e-06 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552700+0000", 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1615e-05 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552706+0000", 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3879999999999999e-06 2024-09-21T10:00:52.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552758+0000", 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.189e-05 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552766+0000", 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9090000000000006e-06 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937484+0000", 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.38471757299999998 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937595+0000", 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110933 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937707+0000", 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011215200000000001 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950419+0000", 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012711524 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950927+0000", 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00050878800000000004 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.952090+0000", 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011627440000000001 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.952240+0000", 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000149479 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:00:52.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.952371+0000", 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000131395 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:00:52.921 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:00:52.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:52.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:52.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:53.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:53.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:53.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:53.188 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:53.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:53.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:53.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:53.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.440 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:53.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:53.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:53.563 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:00:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:00:53] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.51.0" 2024-09-21T10:00:53.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:53.687 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:53.737 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:00:53.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:53.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:53.915 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:53.936 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:53.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:54.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:54.060 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:54.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:54.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:54.183 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:54.285 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:54.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:54.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:54.370 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:00:54.371 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:54 smithi076 ceph-mon[106482]: pgmap v1324: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:54.371 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:54 smithi076 ceph-mon[110963]: pgmap v1324: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:54.373 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:00:54.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:54.433 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:54.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.559 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:54.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:54 smithi191 ceph-mon[102206]: pgmap v1324: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:54.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:54.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:54.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:54.793 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:00:54.793 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:00:54.794 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-21T10:00:54.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_2 16 2024-09-21T10:00:54.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:54.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:54.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:54.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:54.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:55.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:55.063 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:55.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:55.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:55.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:55.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:55.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:55.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:55.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:55.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:55.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:55.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:55.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:55.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:55.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:55.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:55.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:55.993 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1324,"stamp":"2024-09-21T10:00:52.535303+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":1771284,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1727004,"kb_used_data":366204,"kb_used_omap":195,"kb_used_meta":1360508,"kb_avail":748234212,"statfs":{"total":767960285184,"available":766191833088,"internally_reserved":0,"allocated":374992896,"data_stored":358283301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":200572,"internal_metadata":1393160324},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-227808,"num_write_kb":-7289793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-14348,"num_bytes_recovered":-487718912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001837"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718882+0000","last_change":"2024-09-21T09:56:45.291826+0000","last_active":"2024-09-21T10:00:38.718882+0000","last_peered":"2024-09-21T10:00:38.718882+0000","last_clean":"2024-09-21T10:00:38.718882+0000","last_became_active":"2024-09-21T09:56:45.291477+0000","last_became_peered":"2024-09-21T09:56:45.291477+0000","last_unstale":"2024-09-21T10:00:38.718882+0000","last_undegraded":"2024-09-21T10:00:38.718882+0000","last_fullsized":"2024-09-21T10:00:38.718882+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:46.771815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1062,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.096819+0000","last_change":"2024-09-21T09:50:11.807934+0000","last_active":"2024-09-21T10:00:39.096819+0000","last_peered":"2024-09-21T10:00:39.096819+0000","last_clean":"2024-09-21T10:00:39.096819+0000","last_became_active":"2024-09-21T09:50:11.807561+0000","last_became_peered":"2024-09-21T09:50:11.807561+0000","last_unstale":"2024-09-21T10:00:39.096819+0000","last_undegraded":"2024-09-21T10:00:39.096819+0000","last_fullsized":"2024-09-21T10:00:39.096819+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:01:42.357680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1094,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.399068+0000","last_change":"2024-09-21T09:56:45.267420+0000","last_active":"2024-09-21T10:00:51.399068+0000","last_peered":"2024-09-21T10:00:51.399068+0000","last_clean":"2024-09-21T10:00:51.399068+0000","last_became_active":"2024-09-21T09:56:45.266666+0000","last_became_peered":"2024-09-21T09:56:45.266666+0000","last_unstale":"2024-09-21T10:00:51.399068+0000","last_undegraded":"2024-09-21T10:00:51.399068+0000","last_fullsized":"2024-09-21T10:00:51.399068+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:00.929766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1456,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713403+0000","last_change":"2024-09-21T09:56:45.265541+0000","last_active":"2024-09-21T10:00:38.713403+0000","last_peered":"2024-09-21T10:00:38.713403+0000","last_clean":"2024-09-21T10:00:38.713403+0000","last_became_active":"2024-09-21T09:56:45.265196+0000","last_became_peered":"2024-09-21T09:56:45.265196+0000","last_unstale":"2024-09-21T10:00:38.713403+0000","last_undegraded":"2024-09-21T10:00:38.713403+0000","last_fullsized":"2024-09-21T10:00:38.713403+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:59.046416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1455,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713301+0000","last_change":"2024-09-21T09:56:45.265507+0000","last_active":"2024-09-21T10:00:38.713301+0000","last_peered":"2024-09-21T10:00:38.713301+0000","last_clean":"2024-09-21T10:00:38.713301+0000","last_became_active":"2024-09-21T09:56:45.265087+0000","last_became_peered":"2024-09-21T09:56:45.265087+0000","last_unstale":"2024-09-21T10:00:38.713301+0000","last_undegraded":"2024-09-21T10:00:38.713301+0000","last_fullsized":"2024-09-21T10:00:38.713301+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:03.295116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1045,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.096553+0000","last_change":"2024-09-21T09:36:39.879222+0000","last_active":"2024-09-21T10:00:39.096553+0000","last_peered":"2024-09-21T10:00:39.096553+0000","last_clean":"2024-09-21T10:00:39.096553+0000","last_became_active":"2024-09-21T09:36:39.878983+0000","last_became_peered":"2024-09-21T09:36:39.878983+0000","last_unstale":"2024-09-21T10:00:39.096553+0000","last_undegraded":"2024-09-21T10:00:39.096553+0000","last_fullsized":"2024-09-21T10:00:39.096553+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:46:09.167018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718893+0000","last_change":"2024-09-21T09:56:45.288726+0000","last_active":"2024-09-21T10:00:38.718893+0000","last_peered":"2024-09-21T10:00:38.718893+0000","last_clean":"2024-09-21T10:00:38.718893+0000","last_became_active":"2024-09-21T09:56:45.288290+0000","last_became_peered":"2024-09-21T09:56:45.288290+0000","last_unstale":"2024-09-21T10:00:38.718893+0000","last_undegraded":"2024-09-21T10:00:38.718893+0000","last_fullsized":"2024-09-21T10:00:38.718893+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:58:23.113043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1072,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097109+0000","last_change":"2024-09-21T09:49:05.802908+0000","last_active":"2024-09-21T10:00:39.097109+0000","last_peered":"2024-09-21T10:00:39.097109+0000","last_clean":"2024-09-21T10:00:39.097109+0000","last_became_active":"2024-09-21T09:49:05.802533+0000","last_became_peered":"2024-09-21T09:49:05.802533+0000","last_unstale":"2024-09-21T10:00:39.097109+0000","last_undegraded":"2024-09-21T10:00:39.097109+0000","last_fullsized":"2024-09-21T10:00:39.097109+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:16.668000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1061,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.096862+0000","last_change":"2024-09-21T09:49:05.804446+0000","last_active":"2024-09-21T10:00:39.096862+0000","last_peered":"2024-09-21T10:00:39.096862+0000","last_clean":"2024-09-21T10:00:39.096862+0000","last_became_active":"2024-09-21T09:49:05.804097+0000","last_became_peered":"2024-09-21T09:49:05.804097+0000","last_unstale":"2024-09-21T10:00:39.096862+0000","last_undegraded":"2024-09-21T10:00:39.096862+0000","last_fullsized":"2024-09-21T10:00:39.096862+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:45.455565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1062,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117441+0000","last_change":"2024-09-21T09:36:40.084129+0000","last_active":"2024-09-21T10:00:50.117441+0000","last_peered":"2024-09-21T10:00:50.117441+0000","last_clean":"2024-09-21T10:00:50.117441+0000","last_became_active":"2024-09-21T09:36:40.083760+0000","last_became_peered":"2024-09-21T09:36:40.083760+0000","last_unstale":"2024-09-21T10:00:50.117441+0000","last_undegraded":"2024-09-21T10:00:50.117441+0000","last_fullsized":"2024-09-21T10:00:50.117441+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:43:30.065970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1062,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097521+0000","last_change":"2024-09-21T09:49:38.834887+0000","last_active":"2024-09-21T10:00:39.097521+0000","last_peered":"2024-09-21T10:00:39.097521+0000","last_clean":"2024-09-21T10:00:39.097521+0000","last_became_active":"2024-09-21T09:49:38.834566+0000","last_became_peered":"2024-09-21T09:49:38.834566+0000","last_unstale":"2024-09-21T10:00:39.097521+0000","last_undegraded":"2024-09-21T10:00:39.097521+0000","last_fullsized":"2024-09-21T10:00:39.097521+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:33.594036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1116,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431708+0000","last_change":"2024-09-21T09:56:45.289496+0000","last_active":"2024-09-21T10:00:50.431708+0000","last_peered":"2024-09-21T10:00:50.431708+0000","last_clean":"2024-09-21T10:00:50.431708+0000","last_became_active":"2024-09-21T09:56:45.289150+0000","last_became_peered":"2024-09-21T09:56:45.289150+0000","last_unstale":"2024-09-21T10:00:50.431708+0000","last_undegraded":"2024-09-21T10:00:50.431708+0000","last_fullsized":"2024-09-21T10:00:50.431708+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:35.564432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1046,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097680+0000","last_change":"2024-09-21T09:36:39.873655+0000","last_active":"2024-09-21T10:00:39.097680+0000","last_peered":"2024-09-21T10:00:39.097680+0000","last_clean":"2024-09-21T10:00:39.097680+0000","last_became_active":"2024-09-21T09:36:39.873355+0000","last_became_peered":"2024-09-21T09:36:39.873355+0000","last_unstale":"2024-09-21T10:00:39.097680+0000","last_undegraded":"2024-09-21T10:00:39.097680+0000","last_fullsized":"2024-09-21T10:00:39.097680+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:52.014555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1064,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:41.421925+0000","last_change":"2024-09-21T09:49:38.836641+0000","last_active":"2024-09-21T10:00:41.421925+0000","last_peered":"2024-09-21T10:00:41.421925+0000","last_clean":"2024-09-21T10:00:41.421925+0000","last_became_active":"2024-09-21T09:49:38.835882+0000","last_became_peered":"2024-09-21T09:49:38.835882+0000","last_unstale":"2024-09-21T10:00:41.421925+0000","last_undegraded":"2024-09-21T10:00:41.421925+0000","last_fullsized":"2024-09-21T10:00:41.421925+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:08.237430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1095,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117721+0000","last_change":"2024-09-21T09:56:45.267734+0000","last_active":"2024-09-21T10:00:50.117721+0000","last_peered":"2024-09-21T10:00:50.117721+0000","last_clean":"2024-09-21T10:00:50.117721+0000","last_became_active":"2024-09-21T09:56:45.266258+0000","last_became_peered":"2024-09-21T09:56:45.266258+0000","last_unstale":"2024-09-21T10:00:50.117721+0000","last_undegraded":"2024-09-21T10:00:50.117721+0000","last_fullsized":"2024-09-21T10:00:50.117721+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:21.291294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1444,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713045+0000","last_change":"2024-09-21T09:49:05.814788+0000","last_active":"2024-09-21T10:00:38.713045+0000","last_peered":"2024-09-21T10:00:38.713045+0000","last_clean":"2024-09-21T10:00:38.713045+0000","last_became_active":"2024-09-21T09:49:05.811208+0000","last_became_peered":"2024-09-21T09:49:05.811208+0000","last_unstale":"2024-09-21T10:00:38.713045+0000","last_undegraded":"2024-09-21T10:00:38.713045+0000","last_fullsized":"2024-09-21T10:00:38.713045+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:30:55.870928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":95,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718522+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T10:00:38.718522+0000","last_peered":"2024-09-21T10:00:38.718522+0000","last_clean":"2024-09-21T10:00:38.718522+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T10:00:38.718522+0000","last_undegraded":"2024-09-21T10:00:38.718522+0000","last_fullsized":"2024-09-21T10:00:38.718522+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1100,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431829+0000","last_change":"2024-09-21T09:56:45.266387+0000","last_active":"2024-09-21T10:00:50.431829+0000","last_peered":"2024-09-21T10:00:50.431829+0000","last_clean":"2024-09-21T10:00:50.431829+0000","last_became_active":"2024-09-21T09:56:45.266056+0000","last_became_peered":"2024-09-21T09:56:45.266056+0000","last_unstale":"2024-09-21T10:00:50.431829+0000","last_undegraded":"2024-09-21T10:00:50.431829+0000","last_fullsized":"2024-09-21T10:00:50.431829+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:20.977292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1065,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097188+0000","last_change":"2024-09-21T09:49:38.836465+0000","last_active":"2024-09-21T10:00:39.097188+0000","last_peered":"2024-09-21T10:00:39.097188+0000","last_clean":"2024-09-21T10:00:39.097188+0000","last_became_active":"2024-09-21T09:49:38.835217+0000","last_became_peered":"2024-09-21T09:49:38.835217+0000","last_unstale":"2024-09-21T10:00:39.097188+0000","last_undegraded":"2024-09-21T10:00:39.097188+0000","last_fullsized":"2024-09-21T10:00:39.097188+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:21.413224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1097,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.432111+0000","last_change":"2024-09-21T09:50:11.801642+0000","last_active":"2024-09-21T10:00:50.432111+0000","last_peered":"2024-09-21T10:00:50.432111+0000","last_clean":"2024-09-21T10:00:50.432111+0000","last_became_active":"2024-09-21T09:50:11.801209+0000","last_became_peered":"2024-09-21T09:50:11.801209+0000","last_unstale":"2024-09-21T10:00:50.432111+0000","last_undegraded":"2024-09-21T10:00:50.432111+0000","last_fullsized":"2024-09-21T10:00:50.432111+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:02.467767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1084,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431796+0000","last_change":"2024-09-21T09:50:11.811259+0000","last_active":"2024-09-21T10:00:50.431796+0000","last_peered":"2024-09-21T10:00:50.431796+0000","last_clean":"2024-09-21T10:00:50.431796+0000","last_became_active":"2024-09-21T09:50:11.810950+0000","last_became_peered":"2024-09-21T09:50:11.810950+0000","last_unstale":"2024-09-21T10:00:50.431796+0000","last_undegraded":"2024-09-21T10:00:50.431796+0000","last_fullsized":"2024-09-21T10:00:50.431796+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:03:55.741045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1091,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431577+0000","last_change":"2024-09-21T09:56:42.218271+0000","last_active":"2024-09-21T10:00:50.431577+0000","last_peered":"2024-09-21T10:00:50.431577+0000","last_clean":"2024-09-21T10:00:50.431577+0000","last_became_active":"2024-09-21T09:56:42.216707+0000","last_became_peered":"2024-09-21T09:56:42.216707+0000","last_unstale":"2024-09-21T10:00:50.431577+0000","last_undegraded":"2024-09-21T10:00:50.431577+0000","last_fullsized":"2024-09-21T10:00:50.431577+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:04:06.388884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718144+0000","last_change":"2024-09-21T09:56:45.290221+0000","last_active":"2024-09-21T10:00:38.718144+0000","last_peered":"2024-09-21T10:00:38.718144+0000","last_clean":"2024-09-21T10:00:38.718144+0000","last_became_active":"2024-09-21T09:56:45.289027+0000","last_became_peered":"2024-09-21T09:56:45.289027+0000","last_unstale":"2024-09-21T10:00:38.718144+0000","last_undegraded":"2024-09-21T10:00:38.718144+0000","last_fullsized":"2024-09-21T10:00:38.718144+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:19.983960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1433,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.126889+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:00:51.126889+0000","last_peered":"2024-09-21T10:00:51.126889+0000","last_clean":"2024-09-21T10:00:51.126889+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:00:51.126889+0000","last_undegraded":"2024-09-21T10:00:51.126889+0000","last_fullsized":"2024-09-21T10:00:51.126889+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1391,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.096655+0000","last_change":"2024-09-21T09:49:05.797479+0000","last_active":"2024-09-21T10:00:39.096655+0000","last_peered":"2024-09-21T10:00:39.096655+0000","last_clean":"2024-09-21T10:00:39.096655+0000","last_became_active":"2024-09-21T09:49:05.797007+0000","last_became_peered":"2024-09-21T09:49:05.797007+0000","last_unstale":"2024-09-21T10:00:39.096655+0000","last_undegraded":"2024-09-21T10:00:39.096655+0000","last_fullsized":"2024-09-21T10:00:39.096655+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:33.847497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1092,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117576+0000","last_change":"2024-09-21T09:50:11.808263+0000","last_active":"2024-09-21T10:00:50.117576+0000","last_peered":"2024-09-21T10:00:50.117576+0000","last_clean":"2024-09-21T10:00:50.117576+0000","last_became_active":"2024-09-21T09:50:11.807889+0000","last_became_peered":"2024-09-21T09:50:11.807889+0000","last_unstale":"2024-09-21T10:00:50.117576+0000","last_undegraded":"2024-09-21T10:00:50.117576+0000","last_fullsized":"2024-09-21T10:00:50.117576+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:40:41.548614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718685+0000","last_change":"2024-09-21T09:56:45.263055+0000","last_active":"2024-09-21T10:00:38.718685+0000","last_peered":"2024-09-21T10:00:38.718685+0000","last_clean":"2024-09-21T10:00:38.718685+0000","last_became_active":"2024-09-21T09:56:45.262483+0000","last_became_peered":"2024-09-21T09:56:45.262483+0000","last_unstale":"2024-09-21T10:00:38.718685+0000","last_undegraded":"2024-09-21T10:00:38.718685+0000","last_fullsized":"2024-09-21T10:00:38.718685+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:20.334643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1049,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097542+0000","last_change":"2024-09-21T09:36:40.085479+0000","last_active":"2024-09-21T10:00:39.097542+0000","last_peered":"2024-09-21T10:00:39.097542+0000","last_clean":"2024-09-21T10:00:39.097542+0000","last_became_active":"2024-09-21T09:36:40.084782+0000","last_became_peered":"2024-09-21T09:36:40.084782+0000","last_unstale":"2024-09-21T10:00:39.097542+0000","last_undegraded":"2024-09-21T10:00:39.097542+0000","last_fullsized":"2024-09-21T10:00:39.097542+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:25.507451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1089,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.096544+0000","last_change":"2024-09-21T09:49:38.834637+0000","last_active":"2024-09-21T10:00:39.096544+0000","last_peered":"2024-09-21T10:00:39.096544+0000","last_clean":"2024-09-21T10:00:39.096544+0000","last_became_active":"2024-09-21T09:49:38.834335+0000","last_became_peered":"2024-09-21T09:49:38.834335+0000","last_unstale":"2024-09-21T10:00:39.096544+0000","last_undegraded":"2024-09-21T10:00:39.096544+0000","last_fullsized":"2024-09-21T10:00:39.096544+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:47.158223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1456,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.712998+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T10:00:38.712998+0000","last_peered":"2024-09-21T10:00:38.712998+0000","last_clean":"2024-09-21T10:00:38.712998+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T10:00:38.712998+0000","last_undegraded":"2024-09-21T10:00:38.712998+0000","last_fullsized":"2024-09-21T10:00:38.712998+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1454,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713139+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T10:00:38.713139+0000","last_peered":"2024-09-21T10:00:38.713139+0000","last_clean":"2024-09-21T10:00:38.713139+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T10:00:38.713139+0000","last_undegraded":"2024-09-21T10:00:38.713139+0000","last_fullsized":"2024-09-21T10:00:38.713139+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1435,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718022+0000","last_change":"2024-09-21T09:56:45.285422+0000","last_active":"2024-09-21T10:00:38.718022+0000","last_peered":"2024-09-21T10:00:38.718022+0000","last_clean":"2024-09-21T10:00:38.718022+0000","last_became_active":"2024-09-21T09:56:45.285108+0000","last_became_peered":"2024-09-21T09:56:45.285108+0000","last_unstale":"2024-09-21T10:00:38.718022+0000","last_undegraded":"2024-09-21T10:00:38.718022+0000","last_fullsized":"2024-09-21T10:00:38.718022+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:52:29.239502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1419,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.125634+0000","last_change":"2024-09-21T09:50:11.807559+0000","last_active":"2024-09-21T10:00:51.125634+0000","last_peered":"2024-09-21T10:00:51.125634+0000","last_clean":"2024-09-21T10:00:51.125634+0000","last_became_active":"2024-09-21T09:50:11.807251+0000","last_became_peered":"2024-09-21T09:50:11.807251+0000","last_unstale":"2024-09-21T10:00:51.125634+0000","last_undegraded":"2024-09-21T10:00:51.125634+0000","last_fullsized":"2024-09-21T10:00:51.125634+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:15.424417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1427,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.125316+0000","last_change":"2024-09-21T09:49:38.840781+0000","last_active":"2024-09-21T10:00:51.125316+0000","last_peered":"2024-09-21T10:00:51.125316+0000","last_clean":"2024-09-21T10:00:51.125316+0000","last_became_active":"2024-09-21T09:49:38.839550+0000","last_became_peered":"2024-09-21T09:49:38.839550+0000","last_unstale":"2024-09-21T10:00:51.125316+0000","last_undegraded":"2024-09-21T10:00:51.125316+0000","last_fullsized":"2024-09-21T10:00:51.125316+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:56.886726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1093,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117553+0000","last_change":"2024-09-21T09:49:38.835783+0000","last_active":"2024-09-21T10:00:50.117553+0000","last_peered":"2024-09-21T10:00:50.117553+0000","last_clean":"2024-09-21T10:00:50.117553+0000","last_became_active":"2024-09-21T09:49:38.835458+0000","last_became_peered":"2024-09-21T09:49:38.835458+0000","last_unstale":"2024-09-21T10:00:50.117553+0000","last_undegraded":"2024-09-21T10:00:50.117553+0000","last_fullsized":"2024-09-21T10:00:50.117553+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:36:05.789483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094558999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1084,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.432098+0000","last_change":"2024-09-21T09:49:05.813921+0000","last_active":"2024-09-21T10:00:50.432098+0000","last_peered":"2024-09-21T10:00:50.432098+0000","last_clean":"2024-09-21T10:00:50.432098+0000","last_became_active":"2024-09-21T09:49:05.813706+0000","last_became_peered":"2024-09-21T09:49:05.813706+0000","last_unstale":"2024-09-21T10:00:50.432098+0000","last_undegraded":"2024-09-21T10:00:50.432098+0000","last_fullsized":"2024-09-21T10:00:50.432098+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:57.820276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1444,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.127936+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:00:51.127936+0000","last_peered":"2024-09-21T10:00:51.127936+0000","last_clean":"2024-09-21T10:00:51.127936+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:00:51.127936+0000","last_undegraded":"2024-09-21T10:00:51.127936+0000","last_fullsized":"2024-09-21T10:00:51.127936+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1111,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117849+0000","last_change":"2024-09-21T09:56:45.286828+0000","last_active":"2024-09-21T10:00:50.117849+0000","last_peered":"2024-09-21T10:00:50.117849+0000","last_clean":"2024-09-21T10:00:50.117849+0000","last_became_active":"2024-09-21T09:56:45.286587+0000","last_became_peered":"2024-09-21T09:56:45.286587+0000","last_unstale":"2024-09-21T10:00:50.117849+0000","last_undegraded":"2024-09-21T10:00:50.117849+0000","last_fullsized":"2024-09-21T10:00:50.117849+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:46:53.911632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1441,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.125544+0000","last_change":"2024-09-21T09:49:38.837316+0000","last_active":"2024-09-21T10:00:51.125544+0000","last_peered":"2024-09-21T10:00:51.125544+0000","last_clean":"2024-09-21T10:00:51.125544+0000","last_became_active":"2024-09-21T09:49:38.836734+0000","last_became_peered":"2024-09-21T09:49:38.836734+0000","last_unstale":"2024-09-21T10:00:51.125544+0000","last_undegraded":"2024-09-21T10:00:51.125544+0000","last_fullsized":"2024-09-21T10:00:51.125544+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:21.005673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.712998+0000","last_change":"2024-09-21T09:56:42.217967+0000","last_active":"2024-09-21T10:00:38.712998+0000","last_peered":"2024-09-21T10:00:38.712998+0000","last_clean":"2024-09-21T10:00:38.712998+0000","last_became_active":"2024-09-21T09:56:42.217638+0000","last_became_peered":"2024-09-21T09:56:42.217638+0000","last_unstale":"2024-09-21T10:00:38.712998+0000","last_undegraded":"2024-09-21T10:00:38.712998+0000","last_fullsized":"2024-09-21T10:00:38.712998+0000","mapping_epoch":1090,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1091,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:36.717644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1418,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.128524+0000","last_change":"2024-09-21T09:50:11.806829+0000","last_active":"2024-09-21T10:00:51.128524+0000","last_peered":"2024-09-21T10:00:51.128524+0000","last_clean":"2024-09-21T10:00:51.128524+0000","last_became_active":"2024-09-21T09:50:11.806514+0000","last_became_peered":"2024-09-21T09:50:11.806514+0000","last_unstale":"2024-09-21T10:00:51.128524+0000","last_undegraded":"2024-09-21T10:00:51.128524+0000","last_fullsized":"2024-09-21T10:00:51.128524+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:04:40.718302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1074,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097710+0000","last_change":"2024-09-21T09:49:05.802235+0000","last_active":"2024-09-21T10:00:39.097710+0000","last_peered":"2024-09-21T10:00:39.097710+0000","last_clean":"2024-09-21T10:00:39.097710+0000","last_became_active":"2024-09-21T09:49:05.801878+0000","last_became_peered":"2024-09-21T09:49:05.801878+0000","last_unstale":"2024-09-21T10:00:39.097710+0000","last_undegraded":"2024-09-21T10:00:39.097710+0000","last_fullsized":"2024-09-21T10:00:39.097710+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:05.550942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1432,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713175+0000","last_change":"2024-09-21T09:49:05.814443+0000","last_active":"2024-09-21T10:00:38.713175+0000","last_peered":"2024-09-21T10:00:38.713175+0000","last_clean":"2024-09-21T10:00:38.713175+0000","last_became_active":"2024-09-21T09:49:05.810754+0000","last_became_peered":"2024-09-21T09:49:05.810754+0000","last_unstale":"2024-09-21T10:00:38.713175+0000","last_undegraded":"2024-09-21T10:00:38.713175+0000","last_fullsized":"2024-09-21T10:00:38.713175+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:01.710795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":95,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718950+0000","last_change":"2024-09-21T09:56:45.269959+0000","last_active":"2024-09-21T10:00:38.718950+0000","last_peered":"2024-09-21T10:00:38.718950+0000","last_clean":"2024-09-21T10:00:38.718950+0000","last_became_active":"2024-09-21T09:56:45.265291+0000","last_became_peered":"2024-09-21T09:56:45.265291+0000","last_unstale":"2024-09-21T10:00:38.718950+0000","last_undegraded":"2024-09-21T10:00:38.718950+0000","last_fullsized":"2024-09-21T10:00:38.718950+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:58.470574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1419,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.125855+0000","last_change":"2024-09-21T09:49:38.839784+0000","last_active":"2024-09-21T10:00:51.125855+0000","last_peered":"2024-09-21T10:00:51.125855+0000","last_clean":"2024-09-21T10:00:51.125855+0000","last_became_active":"2024-09-21T09:49:38.839396+0000","last_became_peered":"2024-09-21T09:49:38.839396+0000","last_unstale":"2024-09-21T10:00:51.125855+0000","last_undegraded":"2024-09-21T10:00:51.125855+0000","last_fullsized":"2024-09-21T10:00:51.125855+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:10.051228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718058+0000","last_change":"2024-09-21T09:56:45.265712+0000","last_active":"2024-09-21T10:00:38.718058+0000","last_peered":"2024-09-21T10:00:38.718058+0000","last_clean":"2024-09-21T10:00:38.718058+0000","last_became_active":"2024-09-21T09:56:45.264515+0000","last_became_peered":"2024-09-21T09:56:45.264515+0000","last_unstale":"2024-09-21T10:00:38.718058+0000","last_undegraded":"2024-09-21T10:00:38.718058+0000","last_fullsized":"2024-09-21T10:00:38.718058+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:09.359131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1446,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713334+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T10:00:38.713334+0000","last_peered":"2024-09-21T10:00:38.713334+0000","last_clean":"2024-09-21T10:00:38.713334+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T10:00:38.713334+0000","last_undegraded":"2024-09-21T10:00:38.713334+0000","last_fullsized":"2024-09-21T10:00:38.713334+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1077,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097438+0000","last_change":"2024-09-21T09:50:11.808758+0000","last_active":"2024-09-21T10:00:39.097438+0000","last_peered":"2024-09-21T10:00:39.097438+0000","last_clean":"2024-09-21T10:00:39.097438+0000","last_became_active":"2024-09-21T09:50:11.807565+0000","last_became_peered":"2024-09-21T09:50:11.807565+0000","last_unstale":"2024-09-21T10:00:39.097438+0000","last_undegraded":"2024-09-21T10:00:39.097438+0000","last_fullsized":"2024-09-21T10:00:39.097438+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:48.078915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1068,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.432146+0000","last_change":"2024-09-21T09:36:39.896248+0000","last_active":"2024-09-21T10:00:50.432146+0000","last_peered":"2024-09-21T10:00:50.432146+0000","last_clean":"2024-09-21T10:00:50.432146+0000","last_became_active":"2024-09-21T09:36:39.895969+0000","last_became_peered":"2024-09-21T09:36:39.895969+0000","last_unstale":"2024-09-21T10:00:50.432146+0000","last_undegraded":"2024-09-21T10:00:50.432146+0000","last_fullsized":"2024-09-21T10:00:50.432146+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:52.059876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1082,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.432101+0000","last_change":"2024-09-21T09:49:38.830819+0000","last_active":"2024-09-21T10:00:50.432101+0000","last_peered":"2024-09-21T10:00:50.432101+0000","last_clean":"2024-09-21T10:00:50.432101+0000","last_became_active":"2024-09-21T09:49:38.830493+0000","last_became_peered":"2024-09-21T09:49:38.830493+0000","last_unstale":"2024-09-21T10:00:50.432101+0000","last_undegraded":"2024-09-21T10:00:50.432101+0000","last_fullsized":"2024-09-21T10:00:50.432101+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:25.633624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1091,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431930+0000","last_change":"2024-09-21T09:36:39.884176+0000","last_active":"2024-09-21T10:00:50.431930+0000","last_peered":"2024-09-21T10:00:50.431930+0000","last_clean":"2024-09-21T10:00:50.431930+0000","last_became_active":"2024-09-21T09:36:39.883873+0000","last_became_peered":"2024-09-21T09:36:39.883873+0000","last_unstale":"2024-09-21T10:00:50.431930+0000","last_undegraded":"2024-09-21T10:00:50.431930+0000","last_fullsized":"2024-09-21T10:00:50.431930+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:28.789044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1418,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.126004+0000","last_change":"2024-09-21T09:50:11.806827+0000","last_active":"2024-09-21T10:00:51.126004+0000","last_peered":"2024-09-21T10:00:51.126004+0000","last_clean":"2024-09-21T10:00:51.126004+0000","last_became_active":"2024-09-21T09:50:11.806518+0000","last_became_peered":"2024-09-21T09:50:11.806518+0000","last_unstale":"2024-09-21T10:00:51.126004+0000","last_undegraded":"2024-09-21T10:00:51.126004+0000","last_fullsized":"2024-09-21T10:00:51.126004+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:56.665524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1093,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:46.433573+0000","last_change":"2024-09-21T09:56:45.271279+0000","last_active":"2024-09-21T10:00:46.433573+0000","last_peered":"2024-09-21T10:00:46.433573+0000","last_clean":"2024-09-21T10:00:46.433573+0000","last_became_active":"2024-09-21T09:56:45.264968+0000","last_became_peered":"2024-09-21T09:56:45.264968+0000","last_unstale":"2024-09-21T10:00:46.433573+0000","last_undegraded":"2024-09-21T10:00:46.433573+0000","last_fullsized":"2024-09-21T10:00:46.433573+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:31:28.288992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1077,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117753+0000","last_change":"2024-09-21T09:49:38.837077+0000","last_active":"2024-09-21T10:00:50.117753+0000","last_peered":"2024-09-21T10:00:50.117753+0000","last_clean":"2024-09-21T10:00:50.117753+0000","last_became_active":"2024-09-21T09:49:38.836787+0000","last_became_peered":"2024-09-21T09:49:38.836787+0000","last_unstale":"2024-09-21T10:00:50.117753+0000","last_undegraded":"2024-09-21T10:00:50.117753+0000","last_fullsized":"2024-09-21T10:00:50.117753+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:52:16.378796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718431+0000","last_change":"2024-09-21T09:56:45.266989+0000","last_active":"2024-09-21T10:00:38.718431+0000","last_peered":"2024-09-21T10:00:38.718431+0000","last_clean":"2024-09-21T10:00:38.718431+0000","last_became_active":"2024-09-21T09:56:45.266582+0000","last_became_peered":"2024-09-21T09:56:45.266582+0000","last_unstale":"2024-09-21T10:00:38.718431+0000","last_undegraded":"2024-09-21T10:00:38.718431+0000","last_fullsized":"2024-09-21T10:00:38.718431+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:33.985970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1454,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713527+0000","last_change":"2024-09-21T09:56:45.266434+0000","last_active":"2024-09-21T10:00:38.713527+0000","last_peered":"2024-09-21T10:00:38.713527+0000","last_clean":"2024-09-21T10:00:38.713527+0000","last_became_active":"2024-09-21T09:56:45.266046+0000","last_became_peered":"2024-09-21T09:56:45.266046+0000","last_unstale":"2024-09-21T10:00:38.713527+0000","last_undegraded":"2024-09-21T10:00:38.713527+0000","last_fullsized":"2024-09-21T10:00:38.713527+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:22.729731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718236+0000","last_change":"2024-09-21T09:56:45.287659+0000","last_active":"2024-09-21T10:00:38.718236+0000","last_peered":"2024-09-21T10:00:38.718236+0000","last_clean":"2024-09-21T10:00:38.718236+0000","last_became_active":"2024-09-21T09:56:45.287253+0000","last_became_peered":"2024-09-21T09:56:45.287253+0000","last_unstale":"2024-09-21T10:00:38.718236+0000","last_undegraded":"2024-09-21T10:00:38.718236+0000","last_fullsized":"2024-09-21T10:00:38.718236+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:46:34.075024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1073,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:39.097220+0000","last_change":"2024-09-21T09:50:11.808910+0000","last_active":"2024-09-21T10:00:39.097220+0000","last_peered":"2024-09-21T10:00:39.097220+0000","last_clean":"2024-09-21T10:00:39.097220+0000","last_became_active":"2024-09-21T09:50:11.807784+0000","last_became_peered":"2024-09-21T09:50:11.807784+0000","last_unstale":"2024-09-21T10:00:39.097220+0000","last_undegraded":"2024-09-21T10:00:39.097220+0000","last_fullsized":"2024-09-21T10:00:39.097220+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:32:30.410606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.717890+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T10:00:38.717890+0000","last_peered":"2024-09-21T10:00:38.717890+0000","last_clean":"2024-09-21T10:00:38.717890+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T10:00:38.717890+0000","last_undegraded":"2024-09-21T10:00:38.717890+0000","last_fullsized":"2024-09-21T10:00:38.717890+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1457,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713238+0000","last_change":"2024-09-21T09:56:45.265747+0000","last_active":"2024-09-21T10:00:38.713238+0000","last_peered":"2024-09-21T10:00:38.713238+0000","last_clean":"2024-09-21T10:00:38.713238+0000","last_became_active":"2024-09-21T09:56:45.265374+0000","last_became_peered":"2024-09-21T09:56:45.265374+0000","last_unstale":"2024-09-21T10:00:38.713238+0000","last_undegraded":"2024-09-21T10:00:38.713238+0000","last_fullsized":"2024-09-21T10:00:38.713238+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:07.525394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1431,"reported_epoch":1134,"state":"active+clean","last_fresh":"2024-09-21T10:00:51.126837+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:00:51.126837+0000","last_peered":"2024-09-21T10:00:51.126837+0000","last_clean":"2024-09-21T10:00:51.126837+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:00:51.126837+0000","last_undegraded":"2024-09-21T10:00:51.126837+0000","last_fullsized":"2024-09-21T10:00:51.126837+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718771+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:00:38.718771+0000","last_peered":"2024-09-21T10:00:38.718771+0000","last_clean":"2024-09-21T10:00:38.718771+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:00:38.718771+0000","last_undegraded":"2024-09-21T10:00:38.718771+0000","last_fullsized":"2024-09-21T10:00:38.718771+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1098,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.432026+0000","last_change":"2024-09-21T09:50:11.810904+0000","last_active":"2024-09-21T10:00:50.432026+0000","last_peered":"2024-09-21T10:00:50.432026+0000","last_clean":"2024-09-21T10:00:50.432026+0000","last_became_active":"2024-09-21T09:50:11.810559+0000","last_became_peered":"2024-09-21T09:50:11.810559+0000","last_unstale":"2024-09-21T10:00:50.432026+0000","last_undegraded":"2024-09-21T10:00:50.432026+0000","last_fullsized":"2024-09-21T10:00:50.432026+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:30.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1411,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.717955+0000","last_change":"2024-09-21T09:56:45.289794+0000","last_active":"2024-09-21T10:00:38.717955+0000","last_peered":"2024-09-21T10:00:38.717955+0000","last_clean":"2024-09-21T10:00:38.717955+0000","last_became_active":"2024-09-21T09:56:45.289424+0000","last_became_peered":"2024-09-21T09:56:45.289424+0000","last_unstale":"2024-09-21T10:00:38.717955+0000","last_undegraded":"2024-09-21T10:00:38.717955+0000","last_fullsized":"2024-09-21T10:00:38.717955+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:22.927871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1433,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713145+0000","last_change":"2024-09-21T09:49:05.815313+0000","last_active":"2024-09-21T10:00:38.713145+0000","last_peered":"2024-09-21T10:00:38.713145+0000","last_clean":"2024-09-21T10:00:38.713145+0000","last_became_active":"2024-09-21T09:49:05.811876+0000","last_became_peered":"2024-09-21T09:49:05.811876+0000","last_unstale":"2024-09-21T10:00:38.713145+0000","last_undegraded":"2024-09-21T10:00:38.713145+0000","last_fullsized":"2024-09-21T10:00:38.713145+0000","mapping_epoch":730,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":731,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.211385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1114,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431993+0000","last_change":"2024-09-21T09:56:45.286288+0000","last_active":"2024-09-21T10:00:50.431993+0000","last_peered":"2024-09-21T10:00:50.431993+0000","last_clean":"2024-09-21T10:00:50.431993+0000","last_became_active":"2024-09-21T09:56:45.286089+0000","last_became_peered":"2024-09-21T09:56:45.286089+0000","last_unstale":"2024-09-21T10:00:50.431993+0000","last_undegraded":"2024-09-21T10:00:50.431993+0000","last_fullsized":"2024-09-21T10:00:50.431993+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:33:25.362669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1078,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117641+0000","last_change":"2024-09-21T09:49:38.835003+0000","last_active":"2024-09-21T10:00:50.117641+0000","last_peered":"2024-09-21T10:00:50.117641+0000","last_clean":"2024-09-21T10:00:50.117641+0000","last_became_active":"2024-09-21T09:49:38.834672+0000","last_became_peered":"2024-09-21T09:49:38.834672+0000","last_unstale":"2024-09-21T10:00:50.117641+0000","last_undegraded":"2024-09-21T10:00:50.117641+0000","last_fullsized":"2024-09-21T10:00:50.117641+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:01:52.463595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1114,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431715+0000","last_change":"2024-09-21T09:56:45.266452+0000","last_active":"2024-09-21T10:00:50.431715+0000","last_peered":"2024-09-21T10:00:50.431715+0000","last_clean":"2024-09-21T10:00:50.431715+0000","last_became_active":"2024-09-21T09:56:45.266041+0000","last_became_peered":"2024-09-21T09:56:45.266041+0000","last_unstale":"2024-09-21T10:00:50.431715+0000","last_undegraded":"2024-09-21T10:00:50.431715+0000","last_fullsized":"2024-09-21T10:00:50.431715+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:44.737887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1063,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.117762+0000","last_change":"2024-09-21T09:36:39.875917+0000","last_active":"2024-09-21T10:00:50.117762+0000","last_peered":"2024-09-21T10:00:50.117762+0000","last_clean":"2024-09-21T10:00:50.117762+0000","last_became_active":"2024-09-21T09:36:39.875598+0000","last_became_peered":"2024-09-21T09:36:39.875598+0000","last_unstale":"2024-09-21T10:00:50.117762+0000","last_undegraded":"2024-09-21T10:00:50.117762+0000","last_fullsized":"2024-09-21T10:00:50.117762+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:51.746140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":91,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718524+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T10:00:38.718524+0000","last_peered":"2024-09-21T10:00:38.718524+0000","last_clean":"2024-09-21T10:00:38.718524+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T10:00:38.718524+0000","last_undegraded":"2024-09-21T10:00:38.718524+0000","last_fullsized":"2024-09-21T10:00:38.718524+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":92,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.718316+0000","last_change":"2024-09-21T09:56:45.291445+0000","last_active":"2024-09-21T10:00:38.718316+0000","last_peered":"2024-09-21T10:00:38.718316+0000","last_clean":"2024-09-21T10:00:38.718316+0000","last_became_active":"2024-09-21T09:56:45.291119+0000","last_became_peered":"2024-09-21T09:56:45.291119+0000","last_unstale":"2024-09-21T10:00:38.718316+0000","last_undegraded":"2024-09-21T10:00:38.718316+0000","last_fullsized":"2024-09-21T10:00:38.718316+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:53.887710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1090,"reported_epoch":1133,"state":"active+clean","last_fresh":"2024-09-21T10:00:50.431625+0000","last_change":"2024-09-21T09:50:11.810982+0000","last_active":"2024-09-21T10:00:50.431625+0000","last_peered":"2024-09-21T10:00:50.431625+0000","last_clean":"2024-09-21T10:00:50.431625+0000","last_became_active":"2024-09-21T09:50:11.810644+0000","last_became_peered":"2024-09-21T09:50:11.810644+0000","last_unstale":"2024-09-21T10:00:50.431625+0000","last_undegraded":"2024-09-21T10:00:50.431625+0000","last_fullsized":"2024-09-21T10:00:50.431625+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:54:50.216384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1449,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-09-21T10:00:38.713551+0000","last_change":"2024-09-21T09:50:11.815604+0000","last_active":"2024-09-21T10:00:38.713551+0000","last_peered":"2024-09-21T10:00:38.713551+0000","last_clean":"2024-09-21T10:00:38.713551+0000","last_became_active":"2024-09-21T09:50:11.815246+0000","last_became_peered":"2024-09-21T09:50:11.815246+0000","last_unstale":"2024-09-21T10:00:38.713551+0000","last_undegraded":"2024-09-21T10:00:38.713551+0000","last_fullsized":"2024-09-21T10:00:38.713551+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:29.113507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234300,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236736,"kb_used_data":71096,"kb_used_omap":1,"kb_used_meta":165630,"kb_avail":93508416,"statfs":{"total":95995035648,"available":95752617984,"internally_reserved":0,"allocated":72802304,"data_stored":70702936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":169605579},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495937,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378744,"kb_used_data":195120,"kb_used_omap":1,"kb_used_meta":183614,"kb_avail":93366408,"statfs":{"total":95995035648,"available":95607201792,"internally_reserved":0,"allocated":199802880,"data_stored":197711577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188021191},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757575,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":182160,"kb_used_data":6720,"kb_used_omap":1,"kb_used_meta":175422,"kb_avail":93562992,"statfs":{"total":95995035648,"available":95808503808,"internally_reserved":0,"allocated":6881280,"data_stored":4785455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":179632582},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019211,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":164168,"kb_used_data":6716,"kb_used_omap":1,"kb_used_meta":157438,"kb_avail":93580984,"statfs":{"total":95995035648,"available":95826927616,"internally_reserved":0,"allocated":6877184,"data_stored":4781392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161216966},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888507,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277484,"kb_used_data":6684,"kb_used_omap":34,"kb_used_meta":270749,"kb_avail":93467668,"statfs":{"total":95995035648,"available":95710892032,"internally_reserved":0,"allocated":6844416,"data_stored":4768306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":277247987},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967746,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":250664,"kb_used_data":65888,"kb_used_omap":41,"kb_used_meta":184726,"kb_avail":93494488,"statfs":{"total":95995035648,"available":95738355712,"internally_reserved":0,"allocated":67469312,"data_stored":65389632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":189159839},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060046985,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":124868,"kb_used_data":6712,"kb_used_omap":41,"kb_used_meta":118102,"kb_avail":93620284,"statfs":{"total":95995035648,"available":95867170816,"internally_reserved":0,"allocated":6873088,"data_stored":4784815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":120937385},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126223,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112180,"kb_used_data":7268,"kb_used_omap":72,"kb_used_meta":104823,"kb_avail":93632972,"statfs":{"total":95995035648,"available":95880163328,"internally_reserved":0,"allocated":7442432,"data_stored":5359188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":107338795},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:00:55.996 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:00:55.996 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:00:55.996 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 4, 5, 6, 7, 3] 2024-09-21T10:00:55.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 6 2024-09-21T10:00:55.998 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:56.053 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:56.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:56.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:56.201 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:56.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:56.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[106482]: from='client.44855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:56.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[106482]: pgmap v1325: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:56.254 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:56.254 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:56.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[110963]: from='client.44855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:56.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[110963]: pgmap v1325: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:56.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:56.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:56.267 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:56.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:56.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:56.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:56.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:56.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:56.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:56.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:56 smithi191 ceph-mon[102206]: from='client.44855 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:00:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:56 smithi191 ceph-mon[102206]: pgmap v1325: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:00:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:56.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:56.702 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:56.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:56.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:56.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:56.954 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:56.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:57.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:57.081 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:57.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:57.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:57.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:00:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.205 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:57.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:57.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:57.333 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:57.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:00:57.460 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:00:57.562 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:57.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:57.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:57.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:57.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:57.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:57.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:57.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:57.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:57.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:57.963 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:57.963 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:58.063 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:58.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:58.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:58.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:58.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:58.314 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:58.340 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:58.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:58.341 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:00:58 smithi076 ceph-mon[106482]: pgmap v1326: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:58.341 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:00:58 smithi076 ceph-mon[110963]: pgmap v1326: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:58.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:58.467 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:58.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:58.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:58.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:58.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:00:58 smithi191 ceph-mon[102206]: pgmap v1326: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:00:58.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:00:58.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:58.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:00:58.731 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:00:58.731 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:00:58.731 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:00:58.731 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:00:58.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:58.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:58.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:00:58.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:58.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:58.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:00:59.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:59.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:59.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:00:59.218 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:00:59.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:59.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:59.446 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:00:59.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:00:59.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:00:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.597 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:59.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:00:59.698 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:59.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:00:59.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:00:59.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:00:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:00:59.948 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:00:59.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:00:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:00:59.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:00.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:00.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:00.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:00.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:00.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:00.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:01:00.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:00.349 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:00.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:00.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:00.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:00.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:00.475 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:00 smithi076 ceph-mon[106482]: pgmap v1327: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:00.476 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:00 smithi076 ceph-mon[110963]: pgmap v1327: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:00.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:00.600 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:00.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:00.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:00 smithi191 ceph-mon[102206]: pgmap v1327: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:00.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:00.725 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:00.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:00.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:00.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:00.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:00.950 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:00.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:00.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:01.072 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:01.093 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:01.195 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:01.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:01.320 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:01.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:01.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:01.469 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:01.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:01.571 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:01.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:01.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:01.722 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:01.823 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:01.845 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:01.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:01.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:01.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:02.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:02.086 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:02.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:02.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:02.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:02.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:02.350 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:02.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:02.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:02.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:02.542 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:02 smithi076 ceph-mon[110963]: pgmap v1328: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:02.543 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:02 smithi076 ceph-mon[106482]: pgmap v1328: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:02.543 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:02 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:01:02.165+0000 7f240edc2640 -1 mon.a@0(leader).osd e1134 definitely_dead 0 2024-09-21T10:01:02.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:02.600 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:02.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:02 smithi191 ceph-mon[102206]: pgmap v1328: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:02.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:02.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:02.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:02.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:02.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:02.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:02.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:02.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:03.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:03.102 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:03.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:03.146 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.6. 2024-09-21T10:01:03.204 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:03.226 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:03.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:03.328 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:03.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:03.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:03 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2733540286' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:01:03.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:03 smithi076 ceph-mon[106482]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:01:03.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:03 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2733540286' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:01:03.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:03 smithi076 ceph-mon[110963]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:01:03.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:01:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:01:03] "GET /metrics HTTP/1.1" 200 33973 "" "Prometheus/2.51.0" 2024-09-21T10:01:03.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:03.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:03.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:03.603 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:03.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:03 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2733540286' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:01:03.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:03 smithi191 ceph-mon[102206]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:01:03.705 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.728 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:03.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:03.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.857 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:03.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:03.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:03.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:03.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:03.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:04.088 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:04.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:04.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:04.232 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:04.334 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:04.356 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:04.357 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[106482]: pgmap v1329: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:04.357 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2733540286' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:01:04.357 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[106482]: osdmap e1135: 8 total, 8 up, 7 in 2024-09-21T10:01:04.358 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[106482]: osdmap e1136: 8 total, 8 up, 7 in 2024-09-21T10:01:04.358 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[110963]: pgmap v1329: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:04.358 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2733540286' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:01:04.358 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[110963]: osdmap e1135: 8 total, 8 up, 7 in 2024-09-21T10:01:04.358 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:04 smithi076 ceph-mon[110963]: osdmap e1136: 8 total, 8 up, 7 in 2024-09-21T10:01:04.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:04.479 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:04.510 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:04.581 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:04.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:04 smithi191 ceph-mon[102206]: pgmap v1329: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:01:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2733540286' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:01:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:04 smithi191 ceph-mon[102206]: osdmap e1135: 8 total, 8 up, 7 in 2024-09-21T10:01:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:04 smithi191 ceph-mon[102206]: osdmap e1136: 8 total, 8 up, 7 in 2024-09-21T10:01:04.704 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:04.726 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:04.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:04.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:04.954 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_2' created 2024-09-21T10:01:04.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:04.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:04.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:05.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:05.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:05.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:05.230 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:05.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:05.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:05.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:05.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2505165809' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[110963]: osdmap e1137: 8 total, 8 up, 7 in 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2505165809' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:01:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:05 smithi076 ceph-mon[106482]: osdmap e1137: 8 total, 8 up, 7 in 2024-09-21T10:01:05.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:05.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:05.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:05.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:05.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:05.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:05.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:05 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2505165809' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:01:05.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:05 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-21T10:01:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:05 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-21T10:01:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:05 smithi191 ceph-mon[102206]: osdmap e1137: 8 total, 8 up, 7 in 2024-09-21T10:01:05.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:05.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:05.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:05.810 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:01:05.811 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:05.824 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:01:05.824 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:05.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:05.840 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:01:05.841 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:05.857 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:01:05.857 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:05.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:05.873 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:01:05.873 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:05.889 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:01:05.890 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:05.906 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:01:05.906 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:05.923 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:01:05.923 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:05.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:05.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:05.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:06.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:06.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:06.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:06.233 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:06.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:06.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:06 smithi076 ceph-mon[110963]: pgmap v1332: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:01:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2549542971' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:06 smithi076 ceph-mon[110963]: osdmap e1138: 8 total, 8 up, 7 in 2024-09-21T10:01:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:06 smithi076 ceph-mon[106482]: pgmap v1332: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:01:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2549542971' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:06.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:06 smithi076 ceph-mon[106482]: osdmap e1138: 8 total, 8 up, 7 in 2024-09-21T10:01:06.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:06.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:06.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:06.607 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:06 smithi191 ceph-mon[102206]: pgmap v1332: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:01:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2549542971' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:06 smithi191 ceph-mon[102206]: osdmap e1138: 8 total, 8 up, 7 in 2024-09-21T10:01:06.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:06.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:06.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:06.858 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:06.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:06.981 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:06.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:06.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:07.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:07.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:07.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:07.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:07.233 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:07.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:07.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:07.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:07.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:07.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:07.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:07.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:07.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:07.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:07.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:07.831 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:07.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:07.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:07.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-21T10:01:07.980 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:07.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:08.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:08.104 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:08.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:08.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:08.238 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:07 smithi076 ceph-mon[110963]: pgmap v1335: 89 pgs: 2 creating+peering, 12 unknown, 5 peering, 70 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 145 KiB/s, 2 objects/s recovering 2024-09-21T10:01:08.238 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:07 smithi076 ceph-mon[110963]: osdmap e1139: 8 total, 8 up, 7 in 2024-09-21T10:01:08.238 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:07 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:01:08.238 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:07 smithi076 ceph-mon[106482]: pgmap v1335: 89 pgs: 2 creating+peering, 12 unknown, 5 peering, 70 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 145 KiB/s, 2 objects/s recovering 2024-09-21T10:01:08.238 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:07 smithi076 ceph-mon[106482]: osdmap e1139: 8 total, 8 up, 7 in 2024-09-21T10:01:08.238 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:07 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:01:08.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:08.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:08.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:07 smithi191 ceph-mon[102206]: pgmap v1335: 89 pgs: 2 creating+peering, 12 unknown, 5 peering, 70 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 145 KiB/s, 2 objects/s recovering 2024-09-21T10:01:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:07 smithi191 ceph-mon[102206]: osdmap e1139: 8 total, 8 up, 7 in 2024-09-21T10:01:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:07 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:01:08.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:08.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:08.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:08.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:08.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:08.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:08.728 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:08.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:08.785 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:08.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:08.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:08.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:08.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:08.961 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:01:08.980 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:08.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:09.076 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:08 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1036001988' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:09.077 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:08 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:09.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:08 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1036001988' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:09.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:08 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:09.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:09.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:09.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:09.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:09.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:01:09.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:09.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:09.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:08 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1036001988' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:09.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:08 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:01:09.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:09.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:09.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:09.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:09.482 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:09.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:09.506 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:01:09.506 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:01:09.507 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:01:09.507 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2024-09-21T10:01:09.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 2 0 2024-09-21T10:01:09.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:09.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:09.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:09.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:09.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:09.832 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:09.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:09.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:09.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:09.976 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:09.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:10.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[106482]: pgmap v1337: 89 pgs: 2 creating+peering, 9 unknown, 5 peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 127 KiB/s, 2 objects/s recovering 2024-09-21T10:01:10.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:10.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:01:10.019 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[106482]: osdmap e1140: 8 total, 8 up, 7 in 2024-09-21T10:01:10.019 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[110963]: pgmap v1337: 89 pgs: 2 creating+peering, 9 unknown, 5 peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 127 KiB/s, 2 objects/s recovering 2024-09-21T10:01:10.019 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:10.019 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:01:10.019 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:09 smithi076 ceph-mon[110963]: osdmap e1140: 8 total, 8 up, 7 in 2024-09-21T10:01:10.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:10.099 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:10.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:10.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:10.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.346 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:10.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:09 smithi191 ceph-mon[102206]: pgmap v1337: 89 pgs: 2 creating+peering, 9 unknown, 5 peering, 73 active+clean; 582 KiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 127 KiB/s, 2 objects/s recovering 2024-09-21T10:01:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:09 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:09 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:01:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:09 smithi191 ceph-mon[102206]: osdmap e1140: 8 total, 8 up, 7 in 2024-09-21T10:01:10.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:10.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:10.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:10.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:10.591 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:10.693 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:10.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:10.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:10.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:10.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:10.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:10.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:11.065 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:11.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:11.144 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:11.144 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:11.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:11.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:11.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:11.337 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:11.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:11.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:11.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:11.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:11.588 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:11.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:11.713 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:11.745 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:01:11.777 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:11 smithi076 ceph-mon[106482]: pgmap v1339: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 104 KiB/s, 1 objects/s recovering 2024-09-21T10:01:11.777 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:11.777 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:11.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:11 smithi076 ceph-mon[110963]: pgmap v1339: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 104 KiB/s, 1 objects/s recovering 2024-09-21T10:01:11.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:11.777 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:11.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:11.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:01:11.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:11.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:11 smithi191 ceph-mon[102206]: pgmap v1339: 89 pgs: 1 peering, 88 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 104 KiB/s, 1 objects/s recovering 2024-09-21T10:01:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:11.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:11.970 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:11.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:12.072 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:12.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:12.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:12.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:12.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:12.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:12.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:12.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:12.382 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:12.406 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:12.409 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:12.418 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:12.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:12.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:12.561 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-21T10:01:12.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:12.599 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:12.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2566431972' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:01:12.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:12 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:01:12.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2566431972' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:01:12.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:12 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:01:12.701 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:12.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:12.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:12.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:12.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2566431972' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:01:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:12 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:01:12.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:12.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:12.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:13.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:13.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:13.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:13.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:13.328 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:13.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:13.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:13.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:13.559 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:01:13] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T10:01:13.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:13.604 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:13.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:13.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:13.833 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:13.855 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:13 smithi191 ceph-mon[102206]: pgmap v1340: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 88 KiB/s, 1 objects/s recovering 2024-09-21T10:01:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:13 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:01:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:13 smithi191 ceph-mon[102206]: osdmap e1141: 8 total, 8 up, 7 in 2024-09-21T10:01:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-mon[110963]: pgmap v1340: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 88 KiB/s, 1 objects/s recovering 2024-09-21T10:01:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:01:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-mon[110963]: osdmap e1141: 8 total, 8 up, 7 in 2024-09-21T10:01:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-mon[106482]: pgmap v1340: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 88 KiB/s, 1 objects/s recovering 2024-09-21T10:01:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:01:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:13 smithi076 ceph-mon[106482]: osdmap e1141: 8 total, 8 up, 7 in 2024-09-21T10:01:13.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:13.981 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:13.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:14.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:14.109 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:14.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:14.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:14.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:14.362 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:14.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:14.485 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:14.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:14.609 DEBUG:teuthology.orchestra.run.smithi076:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-21T10:01:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:14.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:14.652 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:01:14.652 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:01:14.701 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:hints = 1 2024-09-21T10:01:14.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:01:14.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object prefix: benchmark_data_smithi076_190198 2024-09-21T10:01:14.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:01:14.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:01:14.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:14.748 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:14.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:14.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:14 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:01:14.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:14 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:01:14.850 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:14 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:01:14.887 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:14.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:14.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:15.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:15.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:15.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:15.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:15.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:15.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:15.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:15.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:15.423 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:15.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:15.552 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:15.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:15.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:15.680 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:15.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 1 16 3221 3205 200.301 200.312 0.00433427 0.00497383 2024-09-21T10:01:15.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:15.822 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:15.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:15 smithi191 ceph-mon[102206]: pgmap v1342: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-21T10:01:15.869 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:15 smithi076 ceph-mon[106482]: pgmap v1342: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-21T10:01:15.869 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:15 smithi076 ceph-mon[110963]: pgmap v1342: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-09-21T10:01:15.924 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:15.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:15.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:15.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:16.070 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:16.108 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:16.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:16.235 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:16.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:16.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:16.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:16.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:16.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:16.511 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:16.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:16.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:16.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 2 16 6270 6254 195.423 190.562 0.00534995 0.00510192 2024-09-21T10:01:16.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:16.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:16.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:16.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:16.906 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:16.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:16.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:17.008 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:17.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:17.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:17.148 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:17.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:17.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:17.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:17.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:17.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:17.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:17.570 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:17.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:17.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:17.701 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:17 smithi076 ceph-mon[110963]: pgmap v1343: 89 pgs: 89 active+clean; 62 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 7.7 MiB/s wr, 123 op/s 2024-09-21T10:01:17.702 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:17 smithi076 ceph-mon[106482]: pgmap v1343: 89 pgs: 89 active+clean; 62 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 7.7 MiB/s wr, 123 op/s 2024-09-21T10:01:17.703 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 3 16 9076 9060 188.733 175.375 0.0043722 0.00528808 2024-09-21T10:01:17.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:17.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:17.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:17.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:17.966 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:17.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:17.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:18.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:18.092 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:18.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:17 smithi191 ceph-mon[102206]: pgmap v1343: 89 pgs: 89 active+clean; 62 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 7.7 MiB/s wr, 123 op/s 2024-09-21T10:01:18.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:18.219 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:18.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:18.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:18.451 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:01:18.451 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:18.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:18.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:18.491 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:01:18.491 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:18.525 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:01:18.525 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:18.560 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:01:18.561 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:18.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:18.595 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:01:18.595 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:18.613 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:18.628 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:01:18.628 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:18.659 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:01:18.660 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:18.692 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:01:18.692 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:18.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 4 16 11672 11656 182.109 162.25 0.00683554 0.005482 2024-09-21T10:01:18.712 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/224381230' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:18.713 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/224381230' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:18.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:18.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:18.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:18.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:18.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:18.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:18.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:19.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:19.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:19.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/224381230' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:19.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:19.163 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:19.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:19.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:19.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:19.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:19.521 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:19.559 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:19.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:19.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:19.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:19.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 5 16 14128 14112 176.384 153.5 0.00576782 0.00566133 2024-09-21T10:01:19.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:19.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:19.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:19.957 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:19 smithi076 ceph-mon[106482]: pgmap v1344: 89 pgs: 89 active+clean; 145 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 15 MiB/s wr, 240 op/s 2024-09-21T10:01:19.957 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:19 smithi076 ceph-mon[110963]: pgmap v1344: 89 pgs: 89 active+clean; 145 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 15 MiB/s wr, 240 op/s 2024-09-21T10:01:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:19.987 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:19.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:20.082 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:19 smithi191 ceph-mon[102206]: pgmap v1344: 89 pgs: 89 active+clean; 145 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 15 MiB/s wr, 240 op/s 2024-09-21T10:01:20.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:20.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:20.214 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:20.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:20.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:20.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:20.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:20.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:20.507 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:20.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:20.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:20.702 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 6 16 16291 16275 169.516 135.188 0.0129397 0.00589029 2024-09-21T10:01:20.749 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:20.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:20.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:20.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:20.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:20.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:21.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:21.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:21.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:21.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:21.185 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:21.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:21.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:21.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:21.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:21.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:21.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:21.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:21.660 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:01:21.667 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.2 primary-affinity to 0 (802) 2024-09-21T10:01:21.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:21.703 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 7 16 18396 18380 164.092 131.562 0.00442182 0.0060752 2024-09-21T10:01:21.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:21.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:21.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:21.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:21.942 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:21.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:21.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:21.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:22.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.087 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:22.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:21 smithi191 ceph-mon[102206]: pgmap v1345: 89 pgs: 89 active+clean; 683 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:01:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3721296411' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1246261485' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-21T10:01:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[106482]: pgmap v1345: 89 pgs: 89 active+clean; 683 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3721296411' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1246261485' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[110963]: pgmap v1345: 89 pgs: 89 active+clean; 683 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3721296411' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:22.122 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1246261485' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-21T10:01:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.0} : dispatch 2024-09-21T10:01:22.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:22.209 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:22.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:22.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:22.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:22.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:22.559 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:22.582 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:22.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:22.704 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 8 16 18553 18537 144.787 9.8125 0.00713892 0.006899 2024-09-21T10:01:22.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:22.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:22.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[106482]: osdmap e1142: 8 total, 8 up, 7 in 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-09-21T10:01:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:22 smithi076 ceph-mon[110963]: osdmap e1142: 8 total, 8 up, 7 in 2024-09-21T10:01:22.962 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:22.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:22.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:23.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:22 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2024-09-21T10:01:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:22 smithi191 ceph-mon[102206]: osdmap e1142: 8 total, 8 up, 7 in 2024-09-21T10:01:23.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:23.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:23.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:23.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:23.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:23.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:01:23] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.51.0" 2024-09-21T10:01:23.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:23.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:01:23.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:23.482 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:01:23.482 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:01:23.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:46985 6.6 6:6e1f852b:::benchmark_data_smithi076_154645_object46984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.669767+0000", 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.80786295899998, 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96641414199999998, 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.483 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 46985 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669767+0000", 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669767+0000", 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.484 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669768+0000", 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.716e-06 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669789+0000", 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0208e-05 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669800+0000", 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1205e-05 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669806+0000", 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8370000000000002e-06 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669871+0000", 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5026999999999999e-05 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.669881+0000", 2024-09-21T10:01:23.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0001e-05 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.632690+0000", 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96280914299999998 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.632809+0000", 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011861700000000001 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.632916+0000", 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000107295 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636065+0000", 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031496459999999999 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636088+0000", 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2756999999999999e-05 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636123+0000", 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5277999999999997e-05 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636153+0000", 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9638999999999999e-05 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636181+0000", 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7773e-05 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47011 6.6 6:61779813:::benchmark_data_smithi076_154645_object47010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.686871+0000", 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.79075902099999, 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95140228800000004, 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.491 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47011 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686871+0000", 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686871+0000", 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686872+0000", 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7010000000000001e-06 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686894+0000", 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1946e-05 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686909+0000", 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4918e-05 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.686925+0000", 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6084000000000001e-05 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.687084+0000", 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015839200000000001 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.687093+0000", 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8470000000000002e-06 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.634228+0000", 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94713563499999998 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.634305+0000", 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6805999999999996e-05 2024-09-21T10:01:23.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.634388+0000", 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3156000000000001e-05 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636245+0000", 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018568479999999999 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638112+0000", 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018669030000000001 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638166+0000", 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3739000000000002e-05 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638229+0000", 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3026000000000003e-05 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638273+0000", 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4286999999999998e-05 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47019 6.6 6:6051a9e9:::benchmark_data_smithi076_154645_object47018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.499 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.691688+0000", 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.78594153099999, 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95238085900000002, 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47019 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.500 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691688+0000", 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691688+0000", 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691690+0000", 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.61e-06 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691714+0000", 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4204000000000001e-05 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691729+0000", 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5175e-05 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691736+0000", 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.764e-06 2024-09-21T10:01:23.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691797+0000", 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1158e-05 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.691806+0000", 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.166e-06 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.635098+0000", 2024-09-21T10:01:23.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94329137500000004 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.635175+0000", 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7442999999999997e-05 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.635267+0000", 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2093000000000001e-05 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.504 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638073+0000", 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028060369999999999 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638324+0000", 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025033400000000001 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.643961+0000", 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056372310000000004 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.505 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644024+0000", 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3589000000000004e-05 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644069+0000", 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4679999999999999e-05 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47049 6.6 6:62e62db7:::benchmark_data_smithi076_154645_object47048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.715047+0000", 2024-09-21T10:01:23.506 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.76258261200002, 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92919887800000001, 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47049 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715047+0000", 2024-09-21T10:01:23.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715047+0000", 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715049+0000", 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1090000000000001e-06 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715073+0000", 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3575999999999999e-05 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715099+0000", 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5953999999999999e-05 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715112+0000", 2024-09-21T10:01:23.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2958e-05 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715214+0000", 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102301 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.715776+0000", 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056210400000000001 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.510 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636260+0000", 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92048376300000001 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636333+0000", 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2977000000000003e-05 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.636424+0000", 2024-09-21T10:01:23.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1558000000000007e-05 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638124+0000", 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0016996400000000001 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.643778+0000", 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056541059999999999 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644140+0000", 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00036158099999999998 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644197+0000", 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6860000000000001e-05 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644246+0000", 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9391000000000002e-05 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.513 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47057 6.6 6:69a8df47:::benchmark_data_smithi076_154645_object47056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.716332+0000", 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.761297539, 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.93103870899999996, 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.514 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47057 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716332+0000", 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716332+0000", 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716334+0000", 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.364e-06 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716355+0000", 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1871000000000001e-05 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716371+0000", 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5804999999999999e-05 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716378+0000", 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0079999999999996e-06 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716445+0000", 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6452999999999995e-05 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.716454+0000", 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7860000000000008e-06 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.637102+0000", 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92064744700000001 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.637176+0000", 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3728999999999998e-05 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.637375+0000", 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000199655 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.518 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.643917+0000", 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065415359999999997 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644327+0000", 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00041007899999999999 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647275+0000", 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029477190000000001 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.519 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647325+0000", 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9966999999999998e-05 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647371+0000", 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6289999999999999e-05 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.520 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47076 6.6 6:6ee40c86:::benchmark_data_smithi076_154645_object47075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.732163+0000", 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.74546719599999, 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91528458199999996, 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47076 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.521 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732163+0000", 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732163+0000", 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732164+0000", 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7120000000000001e-06 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.522 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732187+0000", 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2736000000000001e-05 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732200+0000", 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2661e-05 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732208+0000", 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3990000000000004e-06 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.523 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732281+0000", 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2748000000000002e-05 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.732291+0000", 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0692e-05 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.638367+0000", 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90607561700000006 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.524 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.640014+0000", 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001647019 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.640171+0000", 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015642600000000001 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644376+0000", 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042058360000000001 2024-09-21T10:01:23.525 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644415+0000", 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9106000000000001e-05 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647379+0000", 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029633319999999999 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.526 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647417+0000", 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7871999999999999e-05 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647447+0000", 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0426e-05 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47083 6.6 6:643c6a0e:::benchmark_data_smithi076_154645_object47082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.736858+0000", 2024-09-21T10:01:23.527 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.74077193300002, 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91066441899999995, 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47083 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736858+0000", 2024-09-21T10:01:23.528 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736858+0000", 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736859+0000", 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5009999999999999e-06 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736882+0000", 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2955999999999998e-05 2024-09-21T10:01:23.529 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736898+0000", 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5270000000000001e-05 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.736906+0000", 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1689999999999992e-06 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.530 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737064+0000", 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000157844 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737074+0000", 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0335e-05 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.642009+0000", 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90493507699999998 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.531 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.642168+0000", 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000158698 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.642312+0000", 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014451500000000001 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644458+0000", 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021454870000000002 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.532 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647260+0000", 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002801984 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647454+0000", 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0001939 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647491+0000", 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7478999999999998e-05 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.533 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647522+0000", 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1204000000000001e-05 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47084 6.6 6:6c69cbb1:::benchmark_data_smithi076_154645_object47083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.737229+0000", 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.74040074700002, 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91724743799999997, 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.534 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47084 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737229+0000", 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.535 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737229+0000", 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737230+0000", 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.066e-06 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737274+0000", 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4033999999999999e-05 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.536 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737286+0000", 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.182e-05 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737293+0000", 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8480000000000003e-06 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737346+0000", 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3501999999999997e-05 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.537 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.737356+0000", 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5380000000000008e-06 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644489+0000", 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90713360899999995 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644593+0000", 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010351700000000001 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.539 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.644716+0000", 2024-09-21T10:01:23.539 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000122936 2024-09-21T10:01:23.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.540 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.540 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.650977+0000", 2024-09-21T10:01:23.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006261656 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.650988+0000", 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0947e-05 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654373+0000", 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0033849190000000001 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654433+0000", 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9237999999999998e-05 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654476+0000", 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3807999999999998e-05 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.542 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47102 6.6 6:62017bac:::benchmark_data_smithi076_154645_object47101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.748493+0000", 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.72913703400002, 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90618986199999996, 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.543 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47102 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748493+0000", 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748493+0000", 2024-09-21T10:01:23.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748495+0000", 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0470000000000001e-06 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748517+0000", 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2297e-05 2024-09-21T10:01:23.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748536+0000", 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8678e-05 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748544+0000", 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8720000000000002e-06 2024-09-21T10:01:23.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748694+0000", 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015076900000000001 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.748783+0000", 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8314999999999997e-05 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647598+0000", 2024-09-21T10:01:23.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89881569400000005 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.647899+0000", 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000300759 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.648048+0000", 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014875700000000001 2024-09-21T10:01:23.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.650993+0000", 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029446709999999998 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654524+0000", 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0035316089999999998 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654604+0000", 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9351000000000006e-05 2024-09-21T10:01:23.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654649+0000", 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5185000000000002e-05 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654683+0000", 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3858e-05 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47104 6.6 6:65f7465e:::benchmark_data_smithi076_154645_object47103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.749569+0000", 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.72806051700002, 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90523379999999998, 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47104 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.551 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749569+0000", 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749569+0000", 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.552 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749571+0000", 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.39e-06 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749589+0000", 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8525e-05 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749600+0000", 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0804e-05 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749608+0000", 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6709999999999999e-06 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749707+0000", 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9166000000000006e-05 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.749714+0000", 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7560000000000002e-06 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.649434+0000", 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89972096499999998 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.649544+0000", 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109367 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.649674+0000", 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000129958 2024-09-21T10:01:23.555 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654534+0000", 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004860596 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654574+0000", 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9669999999999998e-05 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654718+0000", 2024-09-21T10:01:23.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014418699999999999 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654760+0000", 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1433000000000001e-05 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654803+0000", 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3312000000000001e-05 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.557 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47112 6.6 6:61680d0b:::benchmark_data_smithi076_154645_object47111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.755556+0000", 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.72207342399997, 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90315495099999998, 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.558 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47112 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755556+0000", 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755556+0000", 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755558+0000", 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2470000000000001e-06 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755596+0000", 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8209e-05 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755607+0000", 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0753e-05 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755612+0000", 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6470000000000003e-06 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755663+0000", 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1007999999999997e-05 2024-09-21T10:01:23.561 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.755671+0000", 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7510000000000008e-06 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.651036+0000", 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89536528599999998 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.651179+0000", 2024-09-21T10:01:23.562 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000142874 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.651304+0000", 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124945 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654540+0000", 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003235833 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658565+0000", 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040251549999999999 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658607+0000", 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1953000000000003e-05 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658671+0000", 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3708999999999996e-05 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.564 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658711+0000", 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0581000000000003e-05 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47117 6.6 6:61d9a68f:::benchmark_data_smithi076_154645_object47116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.759560+0000", 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.71806992, 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90283510499999997, 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.565 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47117 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759560+0000", 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759560+0000", 2024-09-21T10:01:23.566 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759562+0000", 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7940000000000001e-06 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759607+0000", 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5479000000000003e-05 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759621+0000", 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3978000000000001e-05 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759628+0000", 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7769999999999997e-06 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759687+0000", 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9446e-05 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759696+0000", 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9120000000000004e-06 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.652733+0000", 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89303699599999997 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.652851+0000", 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118035 2024-09-21T10:01:23.569 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.652976+0000", 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000125241 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654544+0000", 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001567613 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662253+0000", 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0077084429999999997 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662297+0000", 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4849999999999999e-05 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662349+0000", 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1956000000000002e-05 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662395+0000", 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5584999999999998e-05 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:47118 6.6 6:63619ed9:::benchmark_data_smithi076_154645_object47117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:57:00.759968+0000", 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 262.71766138499999, 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90271875099999999, 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.572 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 47118 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759968+0000", 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759968+0000", 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759969+0000", 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6600000000000005e-07 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.759988+0000", 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9284999999999999e-05 2024-09-21T10:01:23.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760000+0000", 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1949e-05 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760008+0000", 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3289999999999996e-06 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760133+0000", 2024-09-21T10:01:23.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012529300000000001 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:00.760141+0000", 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3999999999999992e-06 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.654839+0000", 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89469742399999996 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.655030+0000", 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019161199999999999 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,7", 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.655554+0000", 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00052317800000000001 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.658576+0000", 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030221060000000001 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662437+0000", 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038615670000000002 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662593+0000", 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015548499999999999 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662646+0000", 2024-09-21T10:01:23.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3599e-05 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:57:01.662687+0000", 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0658000000000003e-05 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111335 6.b 6:d43ac681:::benchmark_data_smithi076_154645_object111334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.158293+0000", 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.319337058, 2024-09-21T10:01:23.579 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.99914799300000001, 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111335 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158293+0000", 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.580 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158293+0000", 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158294+0000", 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4470000000000001e-06 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158318+0000", 2024-09-21T10:01:23.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4261999999999999e-05 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158329+0000", 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.095e-05 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158339+0000", 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5019999999999998e-06 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158900+0000", 2024-09-21T10:01:23.583 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056092700000000002 2024-09-21T10:01:23.583 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.583 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.583 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.583 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:23.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:23.584 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.158909+0000", 2024-09-21T10:01:23.584 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6260000000000007e-06 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.147702+0000", 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.98879247100000001 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.147851+0000", 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014964199999999999 2024-09-21T10:01:23.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.147993+0000", 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000141953 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154220+0000", 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0062264310000000002 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157142+0000", 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029221329999999999 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157336+0000", 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000194153 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157403+0000", 2024-09-21T10:01:23.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6457000000000004e-05 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157441+0000", 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8039e-05 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111356 6.b 6:d2723c0a:::benchmark_data_smithi076_154645_object111355:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.200592+0000", 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.277037638, 2024-09-21T10:01:23.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96187469699999995, 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111356 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200592+0000", 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200592+0000", 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200593+0000", 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.311e-06 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200614+0000", 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0341000000000001e-05 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200625+0000", 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1280999999999999e-05 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200632+0000", 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1049999999999997e-06 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.591 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200712+0000", 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0185999999999994e-05 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.200732+0000", 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9264000000000001e-05 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.151228+0000", 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95049628399999997 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.151368+0000", 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013990199999999999 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.151532+0000", 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000164253 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157285+0000", 2024-09-21T10:01:23.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005753073 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157454+0000", 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000169276 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162351+0000", 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004897022 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.594 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162416+0000", 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4478000000000002e-05 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162467+0000", 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0921000000000003e-05 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111358 6.b 6:d2bb22f9:::benchmark_data_smithi076_154645_object111357:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.595 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.204700+0000", 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.27292929699999, 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.958391628, 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111358 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.596 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204700+0000", 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204700+0000", 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204702+0000", 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.33e-06 2024-09-21T10:01:23.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204723+0000", 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1506999999999999e-05 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204734+0000", 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0671e-05 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204741+0000", 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1749999999999999e-06 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204827+0000", 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5944000000000002e-05 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.204838+0000", 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1168999999999999e-05 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154255+0000", 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94941701300000003 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154360+0000", 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105165 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.154460+0000", 2024-09-21T10:01:23.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000100055 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157459+0000", 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029986240000000001 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157512+0000", 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2787999999999997e-05 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.162998+0000", 2024-09-21T10:01:23.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054861119999999996 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163050+0000", 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1573000000000002e-05 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163092+0000", 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2502000000000001e-05 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111371 6.b 6:d2bd86e5:::benchmark_data_smithi076_154645_object111370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.224867+0000", 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.25276320500001, 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95080495399999998, 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111371 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.603 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224867+0000", 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224867+0000", 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224868+0000", 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8870000000000001e-06 2024-09-21T10:01:23.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224894+0000", 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5743999999999999e-05 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224905+0000", 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0577000000000001e-05 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224917+0000", 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2252000000000001e-05 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.224995+0000", 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7725000000000005e-05 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.225007+0000", 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2738000000000001e-05 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157624+0000", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.932616691 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157763+0000", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000139295 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.157928+0000", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016422799999999999 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163177+0000", 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052494389999999998 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163218+0000", 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1242999999999998e-05 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175584+0000", 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012365309999999999 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.608 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175627+0000", 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3164000000000002e-05 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175671+0000", 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4660999999999998e-05 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111382 6.b 6:dca8587a:::benchmark_data_smithi076_154645_object111381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.609 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.240220+0000", 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.23740971699999, 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.93556396500000005, 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111382 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240220+0000", 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240220+0000", 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240222+0000", 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5820000000000001e-06 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240253+0000", 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1544000000000002e-05 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240268+0000", 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5145e-05 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240281+0000", 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3077e-05 2024-09-21T10:01:23.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240358+0000", 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6327000000000003e-05 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.240369+0000", 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0949e-05 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163248+0000", 2024-09-21T10:01:23.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92287976999999999 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163365+0000", 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011615700000000001 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.163502+0000", 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000137214 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175571+0000", 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012069623999999999 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175678+0000", 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106986 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175715+0000", 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6426e-05 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175755+0000", 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9758000000000001e-05 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175784+0000", 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9405999999999999e-05 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111385 6.b 6:dec80fb3:::benchmark_data_smithi076_154645_object111384:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.245780+0000", 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.231850165, 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94135651099999995, 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111385 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.617 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245780+0000", 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245780+0000", 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245781+0000", 2024-09-21T10:01:23.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4440000000000001e-06 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245806+0000", 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4955000000000001e-05 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245816+0000", 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0173999999999999e-05 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245822+0000", 2024-09-21T10:01:23.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5990000000000001e-06 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245896+0000", 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4368000000000004e-05 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.245906+0000", 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4569999999999994e-06 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.168204+0000", 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92229890699999995 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.168319+0000", 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000114707 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.168450+0000", 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130569 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175683+0000", 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072328649999999998 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175818+0000", 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013559100000000001 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.186999+0000", 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011180456 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187089+0000", 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0130999999999995e-05 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187136+0000", 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7287999999999999e-05 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:111417 6.b 6:d8513038:::benchmark_data_smithi076_154645_object111416:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1108)", 2024-09-21T10:01:23.623 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:58:03.286159+0000", 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 200.19147042399999, 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90118947800000004, 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 111417 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286159+0000", 2024-09-21T10:01:23.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286159+0000", 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286161+0000", 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6360000000000001e-06 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:01:23.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286186+0000", 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4655e-05 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286203+0000", 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7e-05 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286215+0000", 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2395000000000001e-05 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286431+0000", 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021601500000000001 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:03.286445+0000", 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4331000000000001e-05 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.171860+0000", 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88541457300000004 2024-09-21T10:01:23.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.172020+0000", 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015997400000000001 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,7", 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.172166+0000", 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014658600000000001 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.175827+0000", 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0036610240000000001 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187185+0000", 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011357631999999999 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187222+0000", 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6575999999999997e-05 2024-09-21T10:01:23.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187309+0000", 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7380999999999996e-05 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:58:04.187349+0000", 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9700000000000003e-05 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:01:23.630 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:01:23.631 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:01:23.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:23.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 9 16 20753 20737 143.962 137.5 0.00576638 0.00694012 2024-09-21T10:01:23.728 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:23.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:23.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:23.857 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-mon[106482]: pgmap v1347: 89 pgs: 1 peering, 88 active+clean; 912 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:01:23.857 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:01:23.857 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-mon[106482]: osdmap e1143: 8 total, 8 up, 7 in 2024-09-21T10:01:23.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-mon[110963]: pgmap v1347: 89 pgs: 1 peering, 88 active+clean; 912 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:01:23.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:01:23.858 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:23 smithi076 ceph-mon[110963]: osdmap e1143: 8 total, 8 up, 7 in 2024-09-21T10:01:23.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.866 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:23.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:23.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:23.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:23.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:24.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:23 smithi191 ceph-mon[102206]: pgmap v1347: 89 pgs: 1 peering, 88 active+clean; 912 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:01:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:23 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:01:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:23 smithi191 ceph-mon[102206]: osdmap e1143: 8 total, 8 up, 7 in 2024-09-21T10:01:24.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:24.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:24.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:24.243 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:24.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:24.380 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:24.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:24.507 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:24.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:24.644 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:24.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:24.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 10 16 22451 22435 140.178 106.125 0.00169678 0.00708487 2024-09-21T10:01:24.746 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:24.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.784 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:24.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:24.885 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:24.909 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:24.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:24.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:25.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:25.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:25.134 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:25.159 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:01:25.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:25.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:25.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:25.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:25.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:25.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:25.428 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:25.530 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:25.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:25.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:25.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:25.680 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:25.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:25.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 11 16 24175 24159 137.23 107.75 0.00569031 0.00728168 2024-09-21T10:01:25.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:25.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:25.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:25.941 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:25.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:26.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:26.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:25 smithi191 ceph-mon[102206]: pgmap v1349: 89 pgs: 1 peering, 88 active+clean; 1.2 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-21T10:01:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:25 smithi076 ceph-mon[106482]: pgmap v1349: 89 pgs: 1 peering, 88 active+clean; 1.2 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-21T10:01:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:25 smithi076 ceph-mon[110963]: pgmap v1349: 89 pgs: 1 peering, 88 active+clean; 1.2 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-21T10:01:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:26.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:26.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:26.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:26.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:26.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:26.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:26.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:26.448 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:26.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:26.591 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:26.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:26.692 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:26.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 12 16 25836 25820 134.445 103.812 0.00737762 0.00743161 2024-09-21T10:01:26.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:26.831 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:26.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:26.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:26.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:26.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:27.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:27.103 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:27.204 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:27.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:27.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:27.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:27.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:27.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:27.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:27.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:27.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:27.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:27.704 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:27.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 13 16 27151 27135 130.425 82.1875 0.00465315 0.00766155 2024-09-21T10:01:27.739 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:27.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:27.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:27.866 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:27.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:27.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:27.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:27.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:28.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:27 smithi191 ceph-mon[102206]: pgmap v1350: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:01:28.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:27 smithi076 ceph-mon[106482]: pgmap v1350: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:01:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:27 smithi076 ceph-mon[110963]: pgmap v1350: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:01:28.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:28.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:28.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:28.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:28.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:28.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:28.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:28.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:28.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:28.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 14 16 28633 28617 127.725 92.625 0.014889 0.00782333 2024-09-21T10:01:28.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:28.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:28.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:28.899 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 7, 3] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:01:28.899 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:01:28.899 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 4, 5, 7, 3] 2024-09-21T10:01:28.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 7 2024-09-21T10:01:28.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:28.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:28.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:29.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:29.030 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:28 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:01:29.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:29.503 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:28 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:01:29.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:28 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:01:29.543 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:29.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:29.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:29.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 15 16 30074 30058 125.213 90.0625 0.00230754 0.00793388 2024-09-21T10:01:29.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:29.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:29.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:29.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:29.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:30.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.066 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:30.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:30.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:30.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:30.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:30.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:30.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:29 smithi191 ceph-mon[102206]: pgmap v1351: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:01:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:29 smithi076 ceph-mon[106482]: pgmap v1351: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:01:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:29 smithi076 ceph-mon[110963]: pgmap v1351: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:01:30.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:30.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:30.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:30.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:30.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:30.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:30.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:30.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 16 16 31595 31579 123.329 95.0625 0.0104351 0.00810367 2024-09-21T10:01:30.721 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:30.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:30.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:30.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:30.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:30.950 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:31.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:31.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:31.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:31.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:31.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:31.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:31.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:31.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:31.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:31.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:31.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:31.705 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 17 16 33296 33280 122.327 106.312 0.014414 0.00816839 2024-09-21T10:01:31.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.799 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:31.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:31.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:31.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:31.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:31.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:32.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:32.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:01:32.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:32.085 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:32.128 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:32.155 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:32.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:32.227 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:32.242 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:31 smithi076 ceph-mon[110963]: pgmap v1352: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s; 7.2 KiB/s, 0 objects/s recovering 2024-09-21T10:01:32.242 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:31 smithi076 ceph-mon[106482]: pgmap v1352: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s; 7.2 KiB/s, 0 objects/s recovering 2024-09-21T10:01:32.279 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:32.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:32.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:31 smithi191 ceph-mon[102206]: pgmap v1352: 89 pgs: 89 active+clean; 1.7 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s; 7.2 KiB/s, 0 objects/s recovering 2024-09-21T10:01:32.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:32.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:32.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:32.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:32.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 18 16 35002 34986 121.454 106.625 0.0134261 0.0082266 2024-09-21T10:01:32.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:32.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:32.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:32.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:32.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:33.002 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:33.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:33.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:33.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:33.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:33.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:33.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:33.387 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:01:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:01:33] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-21T10:01:33.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:33.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:33.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:33.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:33.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:33.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 19 16 36753 36737 120.821 109.438 0.00638482 0.0082709 2024-09-21T10:01:33.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:33.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:33.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:33.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:33.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:34.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:34.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:34.126 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:33 smithi191 ceph-mon[102206]: pgmap v1353: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s; 6.4 KiB/s, 0 objects/s recovering 2024-09-21T10:01:34.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:34.239 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:34.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:34.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:33 smithi076 ceph-mon[106482]: pgmap v1353: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s; 6.4 KiB/s, 0 objects/s recovering 2024-09-21T10:01:34.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:33 smithi076 ceph-mon[110963]: pgmap v1353: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s; 6.4 KiB/s, 0 objects/s recovering 2024-09-21T10:01:34.379 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:01:34.379 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:01:34.379 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:01:34.379 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:01:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.393 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:34.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:34.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:34.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:34.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:34.673 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:34.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:01:34.704977+0000 min lat: 0.00130696 max lat: 1.01112 avg lat: 0.00835382 2024-09-21T10:01:34.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:01:34.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 20 16 38302 38286 119.621 96.8125 0.00556536 0.00835382 2024-09-21T10:01:34.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:34.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:34.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:34.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:34.957 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:34.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:35.059 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:35.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:01:35.114 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:35.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:35.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:35.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:35.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:35.348 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:35.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:35.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:35.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:35.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:35.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:35.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:35.638 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:35.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:35.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 21 16 39846 39830 118.519 96.5 0.0198841 0.00843053 2024-09-21T10:01:35.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:35.765 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:35.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:35.867 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:35.904 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:35.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:35.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:36.003 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:35 smithi191 ceph-mon[102206]: pgmap v1354: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 5.4 KiB/s, 0 objects/s recovering 2024-09-21T10:01:36.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:36.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.048 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:36.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:36.071 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:35 smithi076 ceph-mon[106482]: pgmap v1354: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 5.4 KiB/s, 0 objects/s recovering 2024-09-21T10:01:36.071 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:35 smithi076 ceph-mon[110963]: pgmap v1354: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s; 5.4 KiB/s, 0 objects/s recovering 2024-09-21T10:01:36.148 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:36.188 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:36.256 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:01:36.257 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:36.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:36.296 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:01:36.296 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:36.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:36.334 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:01:36.334 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:36.369 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:01:36.369 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:36.400 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:01:36.400 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:36.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:36.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:01:36.432 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:36.441 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:36.466 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:01:36.466 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:36.497 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:01:36.497 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:36.542 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:36.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:36.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:36.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 22 16 41194 41178 116.961 84.25 0.00478156 0.00854286 2024-09-21T10:01:36.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:36.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:36.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:36.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:36.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:36.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:36.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:37.070 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:37.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:37.139 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:36 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1388794240' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:37.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:36 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1388794240' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:37.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:37.239 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:36 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1388794240' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:37.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:37.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:37.357 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:37.357 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:37.384 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:37.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:37.524 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:37.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:37.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:37.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 23 16 42729 42713 116.047 95.9375 0.0117207 0.00861129 2024-09-21T10:01:37.751 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:37.775 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:37.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:37.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:37.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:37.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:38.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:38.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:01:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:38.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:38.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:38.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:38.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:38.247 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:37 smithi076 ceph-mon[106482]: pgmap v1355: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:01:38.247 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:37 smithi076 ceph-mon[110963]: pgmap v1355: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:01:38.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:38.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:38.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:38.336 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:37 smithi191 ceph-mon[102206]: pgmap v1355: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:01:38.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:38.439 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:38.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:38.565 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:38.667 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:38.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 24 16 44231 44215 115.123 93.875 0.0114204 0.00867961 2024-09-21T10:01:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.709 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:38.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:38.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:38.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:38.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:38.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:38.988 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:38.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:39.012 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.7. 2024-09-21T10:01:39.090 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:39.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/360164499' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:01:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:38 smithi191 ceph-mon[102206]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-21T10:01:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/360164499' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:01:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:38 smithi076 ceph-mon[110963]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-21T10:01:39.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:01:38.937+0000 7f240edc2640 -1 mon.a@0(leader).osd e1143 definitely_dead 0 2024-09-21T10:01:39.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/360164499' entity='client.admin' cmd={"prefix": "osd out", "ids": ["7"]} : dispatch 2024-09-21T10:01:39.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:38 smithi076 ceph-mon[106482]: Client client.admin marked osd.7 out, while it was still marked up 2024-09-21T10:01:39.214 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:39.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:39.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:39.338 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:39.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:39.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:39.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:39.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:39.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:39.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:39.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:39.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:39.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 25 16 44641 44625 111.543 25.625 0.00284071 0.008721 2024-09-21T10:01:39.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:39.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:39.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:39.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:39.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:39.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:01:39.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:40.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:40.012 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:01:40.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:40.231 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:40.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:40.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:40 smithi191 ceph-mon[102206]: pgmap v1356: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:01:40.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/360164499' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-21T10:01:40.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:40 smithi191 ceph-mon[102206]: osdmap e1144: 8 total, 8 up, 6 in 2024-09-21T10:01:40.309 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1857867146' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:40.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:40.355 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[106482]: pgmap v1356: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/360164499' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[106482]: osdmap e1144: 8 total, 8 up, 6 in 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1857867146' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[110963]: pgmap v1356: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/360164499' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[110963]: osdmap e1144: 8 total, 8 up, 6 in 2024-09-21T10:01:40.356 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1857867146' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:01:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:40.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:40.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:40.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:40.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:40.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:40.700 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:01:40.700 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:01:40.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 26 16 44641 44625 107.253 0 - 0.008721 2024-09-21T10:01:40.726 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:40.748 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:40.793 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:40.850 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:40.872 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:40.973 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:40.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:40.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:41.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:41 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:41 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1857867146' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:01:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:41 smithi191 ceph-mon[102206]: osdmap e1145: 8 total, 8 up, 6 in 2024-09-21T10:01:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:41.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:41 smithi191 ceph-mon[102206]: osdmap e1146: 8 total, 8 up, 6 in 2024-09-21T10:01:41.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:41.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:41.246 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1857867146' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[106482]: osdmap e1145: 8 total, 8 up, 6 in 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[106482]: osdmap e1146: 8 total, 8 up, 6 in 2024-09-21T10:01:41.259 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:01:41.260 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1857867146' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:01:41.260 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[110963]: osdmap e1145: 8 total, 8 up, 6 in 2024-09-21T10:01:41.260 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:41.260 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:41.260 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:41 smithi076 ceph-mon[110963]: osdmap e1146: 8 total, 8 up, 6 in 2024-09-21T10:01:41.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:41.368 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:41.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:41.490 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:41.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:41.614 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:41.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:41.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 27 16 44744 44728 103.52 3.21875 0.0022569 0.0092868 2024-09-21T10:01:41.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:41.737 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:41.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:41.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:41.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:01:41.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:41.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:41.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:42.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:42.124 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:42.137 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:42.226 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:42.250 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:42.258 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:42 smithi191 ceph-mon[102206]: pgmap v1359: 89 pgs: 8 peering, 2 unknown, 79 active+clean; 2.6 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:01:42.258 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:42 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:01:42.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:42.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:42 smithi076 ceph-mon[110963]: pgmap v1359: 89 pgs: 8 peering, 2 unknown, 79 active+clean; 2.6 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:01:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:42 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:01:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:42 smithi076 ceph-mon[106482]: pgmap v1359: 89 pgs: 8 peering, 2 unknown, 79 active+clean; 2.6 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:01:42.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:42 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:01:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:42.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:42.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:42.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:42.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:42.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:42.633 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:01:42.633 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:01:42.647 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:01:42.648 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:01:42.667 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:01:42.667 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:01:42.686 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:01:42.686 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:01:42.692 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:01:42.703 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:01:42.703 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:01:42.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 28 16 44744 44728 99.8229 0 - 0.0092868 2024-09-21T10:01:42.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:42.723 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:01:42.723 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:01:42.737 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:01:42.737 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:01:42.743 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:42.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:42.754 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:01:42.754 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:01:42.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:42.866 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:42.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:42.989 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:42.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:43.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:43.112 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:43.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:43.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:43.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:43.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:43 smithi191 ceph-mon[102206]: osdmap e1147: 8 total, 8 up, 6 in 2024-09-21T10:01:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/468044607' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:43.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:01:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:01:43] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-21T10:01:43.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:43 smithi076 ceph-mon[106482]: osdmap e1147: 8 total, 8 up, 6 in 2024-09-21T10:01:43.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/468044607' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:43.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:43 smithi076 ceph-mon[110963]: osdmap e1147: 8 total, 8 up, 6 in 2024-09-21T10:01:43.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/468044607' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:01:43.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:43.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:43.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:43.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:43.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:43.706 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 29 16 44779 44763 96.4564 1.09375 0.00240857 0.00968452 2024-09-21T10:01:43.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:43.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:43.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:43.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:43.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:43.957 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:01:43.957 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:01:43.957 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:01:43.957 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:01:43.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:43.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:43.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:44.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:44.102 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:44.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:44.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:44.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:44 smithi191 ceph-mon[102206]: pgmap v1362: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 unknown, 78 active+clean; 2.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 88 MiB/s wr, 1.41k op/s; 2755/134013 objects degraded (2.056%); 0 B/s, 2 keys/s, 1 objects/s recovering 2024-09-21T10:01:44.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:44 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 2755/134013 objects degraded (2.056%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:01:44.331 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:44 smithi191 ceph-mon[102206]: osdmap e1148: 8 total, 8 up, 6 in 2024-09-21T10:01:44.332 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:44 smithi076 ceph-mon[106482]: pgmap v1362: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 unknown, 78 active+clean; 2.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 88 MiB/s wr, 1.41k op/s; 2755/134013 objects degraded (2.056%); 0 B/s, 2 keys/s, 1 objects/s recovering 2024-09-21T10:01:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:44 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 2755/134013 objects degraded (2.056%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:01:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:44 smithi076 ceph-mon[106482]: osdmap e1148: 8 total, 8 up, 6 in 2024-09-21T10:01:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:44 smithi076 ceph-mon[110963]: pgmap v1362: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 unknown, 78 active+clean; 2.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 88 MiB/s wr, 1.41k op/s; 2755/134013 objects degraded (2.056%); 0 B/s, 2 keys/s, 1 objects/s recovering 2024-09-21T10:01:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:44 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 2755/134013 objects degraded (2.056%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:01:44.376 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:44 smithi076 ceph-mon[110963]: osdmap e1148: 8 total, 8 up, 6 in 2024-09-21T10:01:44.380 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:44.483 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:44.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:44.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:44.628 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:44.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:44.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:44.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:01:44.707 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 30 16 45401 45385 94.5368 38.875 0.0210373 0.0105692 2024-09-21T10:01:44.746 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:01:44.746 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:01:44.747 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:01:44.747 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2024-09-21T10:01:44.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 4 0 2024-09-21T10:01:44.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:44.804 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:44.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:44.905 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:44.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:44.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:45.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:45.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:45.195 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:45.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.247 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:45.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:45.349 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:45.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:45 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:45.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:45 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:45 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:45.388 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:45.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:45.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:45.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:45.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:45.640 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:45.707 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 31 16 46450 46434 93.602 65.5625 0.00637346 0.0106771 2024-09-21T10:01:45.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:45.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:45.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:45.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:45.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:45.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:46.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:46.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:46.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:46.151 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:46.176 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:46.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.186 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:46.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:46.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:46.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:46 smithi076 ceph-mon[106482]: pgmap v1364: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 20 MiB/s wr, 325 op/s; 11147/134463 objects degraded (8.290%); 2387/134463 objects misplaced (1.775%); 5.9 MiB/s, 2 keys/s, 95 objects/s recovering 2024-09-21T10:01:46.304 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:46 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:01:46.305 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:46 smithi076 ceph-mon[110963]: pgmap v1364: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 20 MiB/s wr, 325 op/s; 11147/134463 objects degraded (8.290%); 2387/134463 objects misplaced (1.775%); 5.9 MiB/s, 2 keys/s, 95 objects/s recovering 2024-09-21T10:01:46.305 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:46 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:01:46.323 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:46 smithi191 ceph-mon[102206]: pgmap v1364: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 20 MiB/s wr, 325 op/s; 11147/134463 objects degraded (8.290%); 2387/134463 objects misplaced (1.775%); 5.9 MiB/s, 2 keys/s, 95 objects/s recovering 2024-09-21T10:01:46.323 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:46 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:01:46.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:46.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:46.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.471 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:46.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:46.472 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:01:46.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:46.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:46.696 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:46.708 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 32 16 47465 47449 92.6569 63.4375 0.0143013 0.0107861 2024-09-21T10:01:46.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.748 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:46.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:46.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:46.894 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:46.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:46.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:47.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:47.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:47.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:47.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:47.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:47.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:47.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:47.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:47.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:47.531 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:47.633 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:47.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:47.710 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 33 16 48453 48437 91.7142 61.75 0.0125731 0.0108967 2024-09-21T10:01:47.777 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:47.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:47.923 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:47.946 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:47.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:47.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:48.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:48.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:48.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:48.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:48.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:48.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:48.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:48.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:48 smithi191 ceph-mon[102206]: pgmap v1365: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 18 MiB/s wr, 294 op/s; 11147/135339 objects degraded (8.236%); 2387/135339 objects misplaced (1.764%); 4.5 MiB/s, 1 keys/s, 72 objects/s recovering 2024-09-21T10:01:48.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:48 smithi076 ceph-mon[106482]: pgmap v1365: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 18 MiB/s wr, 294 op/s; 11147/135339 objects degraded (8.236%); 2387/135339 objects misplaced (1.764%); 4.5 MiB/s, 1 keys/s, 72 objects/s recovering 2024-09-21T10:01:48.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:48 smithi076 ceph-mon[110963]: pgmap v1365: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 18 MiB/s wr, 294 op/s; 11147/135339 objects degraded (8.236%); 2387/135339 objects misplaced (1.764%); 4.5 MiB/s, 1 keys/s, 72 objects/s recovering 2024-09-21T10:01:48.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:48.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:48.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:48.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.615 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:48.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:48.710 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 34 16 49493 49477 90.9284 65 0.0148589 0.0109913 2024-09-21T10:01:48.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:48.743 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:48.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:48.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:48.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:48.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:48.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:49.015 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:49.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:49.169 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:49.254 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 11147/135339 objects degraded (8.236%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:49.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 11147/135339 objects degraded (8.236%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:49.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:49.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:01:49.323 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:01:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 11147/135339 objects degraded (8.236%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:49.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:49.420 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:49.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:49.545 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:49.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:49.671 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:49.713 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 35 16 50582 50566 90.2674 68.0625 0.0129493 0.0110717 2024-09-21T10:01:49.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:49.796 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:49.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:49.944 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:49.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:50.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:01:50.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:50.102 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:50.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.103 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:50.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:50.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:50.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:50.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:50.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:50 smithi076 ceph-mon[106482]: pgmap v1366: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 21 MiB/s wr, 335 op/s; 11147/137610 objects degraded (8.100%); 2387/137610 objects misplaced (1.735%); 3.6 MiB/s, 1 keys/s, 57 objects/s recovering 2024-09-21T10:01:50.240 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:50 smithi076 ceph-mon[110963]: pgmap v1366: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 21 MiB/s wr, 335 op/s; 11147/137610 objects degraded (8.100%); 2387/137610 objects misplaced (1.735%); 3.6 MiB/s, 1 keys/s, 57 objects/s recovering 2024-09-21T10:01:50.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:50.359 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:50 smithi191 ceph-mon[102206]: pgmap v1366: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 21 MiB/s wr, 335 op/s; 11147/137610 objects degraded (8.100%); 2387/137610 objects misplaced (1.735%); 3.6 MiB/s, 1 keys/s, 57 objects/s recovering 2024-09-21T10:01:50.461 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:50.506 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:50.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:50.608 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:50.647 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:50.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:50.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:50.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 36 16 51480 51464 89.3154 56.125 0.0228352 0.011189 2024-09-21T10:01:50.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.814 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:50.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:50.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:50.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:50.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:50.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:51.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:51.082 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:51.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:51.105 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:01:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1340688721' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1340688721' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:51.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:51.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:51.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:51.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:51.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:51.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1340688721' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:51.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:01:51.363 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:01:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=cleanup t=2024-09-21T10:01:51.324310231Z level=info msg="Completed cleanup jobs" duration=2.875068ms 2024-09-21T10:01:51.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:51.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.494 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:51.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:51.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:51.618 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:51.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 37 16 52357 52341 88.383 54.8125 0.0133054 0.011308 2024-09-21T10:01:51.719 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:51.757 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:51.759 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:01:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugins.update.checker t=2024-09-21T10:01:51.474194209Z level=info msg="Update check succeeded" duration=66.00335ms 2024-09-21T10:01:51.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:51.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:51.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:51.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:51.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:52.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:52.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:52.162 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:52.264 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:52.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:52.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[106482]: pgmap v1367: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 35 MiB/s wr, 559 op/s; 11147/147168 objects degraded (7.574%); 2172/147168 objects misplaced (1.476%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[106482]: osdmap e1149: 8 total, 8 up, 6 in 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1273919171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 0.0} : dispatch 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[110963]: pgmap v1367: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 35 MiB/s wr, 559 op/s; 11147/147168 objects degraded (7.574%); 2172/147168 objects misplaced (1.476%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:01:52.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[110963]: osdmap e1149: 8 total, 8 up, 6 in 2024-09-21T10:01:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1273919171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 0.0} : dispatch 2024-09-21T10:01:52.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:52.416 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:52.493 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:52 smithi191 ceph-mon[102206]: pgmap v1367: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 35 MiB/s wr, 559 op/s; 11147/147168 objects degraded (7.574%); 2172/147168 objects misplaced (1.476%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:01:52.494 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:52 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:01:52.494 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:52 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:01:52.494 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:52 smithi191 ceph-mon[102206]: osdmap e1149: 8 total, 8 up, 6 in 2024-09-21T10:01:52.494 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1273919171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 0.0} : dispatch 2024-09-21T10:01:52.517 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:52.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:52.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:52.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:52.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 38 16 52845 52829 86.86 30.5 0.00233546 0.0113629 2024-09-21T10:01:52.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:52.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:52.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:52.910 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:52.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:52.948 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:01:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:01:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:01:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:01:52.948 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132669 6.7 6:ee3236d4:::benchmark_data_smithi076_154645_object18763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.701663+0000", 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.24277815900001, 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0345916079999999, 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132669 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.949 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701663+0000", 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701663+0000", 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701662+0000", 2024-09-21T10:01:52.950 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701668+0000", 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5389999999999998e-06 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701685+0000", 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6798000000000001e-05 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.951 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701693+0000", 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2679999999999994e-06 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701770+0000", 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7224000000000005e-05 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701782+0000", 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2174000000000001e-05 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.952 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723044+0000", 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0212620189999999 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723136+0000", 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1699000000000004e-05 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723211+0000", 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5343e-05 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.953 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729587+0000", 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063752499999999998 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732288+0000", 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027012820000000002 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736194+0000", 2024-09-21T10:01:52.954 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003905974 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736229+0000", 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5385000000000001e-05 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736255+0000", 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5789e-05 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.955 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132674 6.7 6:e6149bab:::benchmark_data_smithi076_154645_object18768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.709017+0000", 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.23542421900001, 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.027360359, 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132674 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.956 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709017+0000", 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709017+0000", 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709016+0000", 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.957 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709023+0000", 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1690000000000003e-06 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709040+0000", 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7295999999999998e-05 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709048+0000", 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0309999999999998e-06 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.958 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709157+0000", 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109302 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709170+0000", 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3006999999999999e-05 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726637+0000", 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.017466998 2024-09-21T10:01:52.959 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726724+0000", 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6982000000000002e-05 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726801+0000", 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6508999999999995e-05 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.960 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736264+0000", 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094625979999999991 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736288+0000", 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4314000000000001e-05 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736328+0000", 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0627000000000003e-05 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.961 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736358+0000", 2024-09-21T10:01:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9179999999999998e-05 2024-09-21T10:01:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.962 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736378+0000", 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9995000000000001e-05 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132678 6.7 6:e83a6585:::benchmark_data_smithi076_154645_object18772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.716074+0000", 2024-09-21T10:01:52.963 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.22836759800001, 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0204681069999999, 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132678 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716074+0000", 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.964 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716074+0000", 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716072+0000", 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.965 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716080+0000", 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2679999999999997e-06 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716092+0000", 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2102e-05 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.966 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716100+0000", 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9829999999999996e-06 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716143+0000", 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2979999999999998e-05 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716151+0000", 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9750000000000007e-06 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.967 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729620+0000", 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.013469213 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729708+0000", 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7615999999999996e-05 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729777+0000", 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9509000000000001e-05 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.968 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736385+0000", 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006608198 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736425+0000", 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0061999999999997e-05 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736481+0000", 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5494000000000001e-05 2024-09-21T10:01:52.969 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736512+0000", 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1439000000000002e-05 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736542+0000", 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9791e-05 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.970 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132696 6.7 6:e3a68bfc:::benchmark_data_smithi076_154645_object18790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.735329+0000", 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.209112046, 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.001302906, 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132696 2024-09-21T10:01:52.971 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735329+0000", 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735329+0000", 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.972 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735328+0000", 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735335+0000", 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4699999999999999e-06 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.973 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735349+0000", 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4493e-05 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735357+0000", 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5259999999999996e-06 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735402+0000", 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5572999999999999e-05 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.974 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735420+0000", 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7569999999999999e-05 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732376+0000", 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99695626999999998 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732509+0000", 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013274200000000001 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.975 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732636+0000", 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000127346 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736458+0000", 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038217440000000002 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736462+0000", 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5299999999999998e-06 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.976 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736583+0000", 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120709 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736612+0000", 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9308000000000001e-05 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736632+0000", 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.001e-05 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.977 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132704 6.7 6:ee4531e9:::benchmark_data_smithi076_154645_object18798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.744678+0000", 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.19976333, 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.004240655, 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132704 2024-09-21T10:01:52.978 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744678+0000", 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744678+0000", 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744676+0000", 2024-09-21T10:01:52.979 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744685+0000", 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5040000000000002e-06 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744702+0000", 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7399000000000001e-05 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.980 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744711+0000", 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4200000000000007e-06 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744768+0000", 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6725999999999999e-05 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744776+0000", 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5329999999999994e-06 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.981 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736669+0000", 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99189255799999998 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736774+0000", 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105721 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736877+0000", 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010238699999999999 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.982 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748781+0000", 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011904068 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748790+0000", 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.679e-06 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748855+0000", 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4892000000000001e-05 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.983 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748889+0000", 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3206e-05 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748919+0000", 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0226000000000002e-05 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.984 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132719 6.7 6:ecb28ace:::benchmark_data_smithi076_154645_object18813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.758432+0000", 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.18600946399999, 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99870091400000005, 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132719 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.985 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758432+0000", 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758432+0000", 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758430+0000", 2024-09-21T10:01:52.986 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758438+0000", 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7360000000000001e-06 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758453+0000", 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4759999999999999e-05 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.987 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758460+0000", 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6350000000000006e-06 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758522+0000", 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2273000000000005e-05 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758533+0000", 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0237e-05 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.988 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741029+0000", 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98249609900000001 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741177+0000", 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014844599999999999 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741300+0000", 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122748 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.989 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748825+0000", 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075247990000000004 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748931+0000", 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010597799999999999 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757064+0000", 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008132841 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.990 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757105+0000", 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1198000000000001e-05 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757133+0000", 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8116999999999999e-05 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.991 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132729 6.7 6:ec0c304a:::benchmark_data_smithi076_154645_object18823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.776958+0000", 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.167483667, 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98031006499999995, 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132729 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.992 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776958+0000", 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776958+0000", 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776957+0000", 2024-09-21T10:01:52.993 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776962+0000", 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4190000000000001e-06 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776974+0000", 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2303999999999999e-05 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:52.994 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776982+0000", 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.486e-06 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.777032+0000", 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9852999999999999e-05 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.777039+0000", 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4900000000000003e-06 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.995 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745236+0000", 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96819714999999995 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745349+0000", 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112273 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:52.996 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745455+0000", 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106005 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757033+0000", 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011578564 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757190+0000", 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156661 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.997 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757223+0000", 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3413e-05 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757249+0000", 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5522000000000001e-05 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:52.998 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757268+0000", 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9046999999999999e-05 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132747 6.7 6:ef3db1a5:::benchmark_data_smithi076_154645_object18841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.807098+0000", 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.13734343100001, 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95029417199999999, 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:52.999 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132747 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807098+0000", 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807098+0000", 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.000 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807097+0000", 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807101+0000", 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4649999999999999e-06 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807107+0000", 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.207e-06 2024-09-21T10:01:53.001 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807111+0000", 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7620000000000001e-06 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807144+0000", 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2768999999999997e-05 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.002 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807150+0000", 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9540000000000001e-06 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748957+0000", 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94180735199999999 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.749051+0000", 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3607e-05 2024-09-21T10:01:53.003 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.749151+0000", 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100497 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757182+0000", 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080311150000000001 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757197+0000", 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4562000000000001e-05 2024-09-21T10:01:53.004 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757311+0000", 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011408000000000001 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757358+0000", 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7392999999999998e-05 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.005 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757392+0000", 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4044000000000001e-05 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132752 6.7 6:e393e829:::benchmark_data_smithi076_154645_object18846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.808248+0000", 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.136193593, 2024-09-21T10:01:53.006 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96938841399999998, 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132752 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808248+0000", 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.007 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808248+0000", 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808247+0000", 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808251+0000", 2024-09-21T10:01:53.008 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6559999999999998e-06 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808259+0000", 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8590000000000005e-06 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808263+0000", 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4039999999999997e-06 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.009 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808297+0000", 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3346000000000002e-05 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808302+0000", 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.964e-06 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753045+0000", 2024-09-21T10:01:53.010 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94474228999999998 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753148+0000", 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103346 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753251+0000", 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010297 2024-09-21T10:01:53.011 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757404+0000", 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041525709999999999 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757440+0000", 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6028999999999997e-05 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777567+0000", 2024-09-21T10:01:53.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020127794000000001 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777606+0000", 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8336e-05 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777636+0000", 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0482000000000001e-05 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.013 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132759 6.7 6:ec6776a2:::benchmark_data_smithi076_154645_object18853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.822499+0000", 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.12194259099999, 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95524379100000001, 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132759 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.014 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822499+0000", 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822499+0000", 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822498+0000", 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:53.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822504+0000", 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2910000000000004e-06 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822515+0000", 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1122e-05 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822521+0000", 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8850000000000004e-06 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822576+0000", 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5012000000000001e-05 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822586+0000", 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0198e-05 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757477+0000", 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93489060499999999 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757603+0000", 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000125699 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757727+0000", 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012464799999999999 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.769569+0000", 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011841539 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777648+0000", 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080794749999999992 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777682+0000", 2024-09-21T10:01:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3782000000000003e-05 2024-09-21T10:01:53.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:53.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777710+0000", 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8453e-05 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777743+0000", 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2187000000000002e-05 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132762 6.7 6:e3ba75f7:::benchmark_data_smithi076_154645_object18856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.11839621199999, 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97741979800000001, 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132762 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:53.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826049+0000", 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7199999999999997e-06 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826058+0000", 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8810000000000002e-06 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826063+0000", 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5870000000000003e-06 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826098+0000", 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5222999999999998e-05 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826104+0000", 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1140000000000002e-06 2024-09-21T10:01:53.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761541+0000", 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93543742500000004 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761637+0000", 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5283000000000002e-05 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761731+0000", 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3795999999999994e-05 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777779+0000", 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016048020999999999 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777784+0000", 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1710000000000001e-06 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803357+0000", 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025573056 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803440+0000", 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3496000000000002e-05 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803465+0000", 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4794999999999999e-05 2024-09-21T10:01:53.029 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132777 6.7 6:e0ee62be:::benchmark_data_smithi076_154645_object18871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.868119+0000", 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 204.07632271400001, 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94012129, 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:01:53.030 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132777 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868119+0000", 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868119+0000", 2024-09-21T10:01:53.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868117+0000", 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868125+0000", 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4200000000000001e-06 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868139+0000", 2024-09-21T10:01:53.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4076999999999999e-05 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868146+0000", 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7950000000000002e-06 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868192+0000", 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.693e-05 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868201+0000", 2024-09-21T10:01:53.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6580000000000008e-06 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765747+0000", 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89754568400000001 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765878+0000", 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013122700000000001 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:01:53.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765999+0000", 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120541 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777788+0000", 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011788997000000001 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777826+0000", 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8155999999999999e-05 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808156+0000", 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030329964000000001 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808203+0000", 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7398000000000003e-05 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808240+0000", 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6976e-05 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.036 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18337 7.e 7:790a4a7c:::benchmark_data_smithi076_190198_object18336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661783+0000", 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.282658215000001, 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020676100000001, 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18337 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661783+0000", 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661783+0000", 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661782+0000", 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:53.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661814+0000", 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2070999999999998e-05 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661825+0000", 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0596e-05 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661833+0000", 2024-09-21T10:01:53.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3480000000000004e-06 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661880+0000", 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6983000000000001e-05 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661891+0000", 2024-09-21T10:01:53.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0540000000000001e-05 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663450+0000", 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015596760000000001 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663456+0000", 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4750000000000001e-06 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.041 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663740+0000", 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0002838890000001 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663851+0000", 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011123799999999999 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18338 7.e 7:7ab9ca86:::benchmark_data_smithi076_190198_object18337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.042 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.282596399999999, 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0022534350000001, 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18338 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:01:53.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661868+0000", 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2954000000000001e-05 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661873+0000", 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4399999999999996e-06 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661876+0000", 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4039999999999999e-06 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661943+0000", 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6816000000000005e-05 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661949+0000", 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5720000000000002e-06 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663476+0000", 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015270419999999999 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663486+0000", 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9089999999999995e-06 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664045+0000", 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.000559376 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664099+0000", 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3440999999999999e-05 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.047 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18343 7.e 7:715bc4a5:::benchmark_data_smithi076_190198_object18342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.282560520000001, 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0044391960000001, 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.048 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18343 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661905+0000", 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4547000000000001e-05 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661912+0000", 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3829999999999997e-06 2024-09-21T10:01:53.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661916+0000", 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6299999999999997e-06 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661985+0000", 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8645000000000004e-05 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661989+0000", 2024-09-21T10:01:53.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3619999999999999e-06 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663500+0000", 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001511075 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663504+0000", 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0199999999999996e-06 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664142+0000", 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0006375890000001 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664250+0000", 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010797 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666217+0000", 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019669169999999999 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666320+0000", 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010336 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18376 7.e 7:7e9a0281:::benchmark_data_smithi076_190198_object18375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.686399+0000", 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.258042526000001, 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98100495300000001, 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.054 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18376 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686399+0000", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686399+0000", 2024-09-21T10:01:53.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686398+0000", 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686477+0000", 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9285999999999999e-05 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686488+0000", 2024-09-21T10:01:53.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0686999999999999e-05 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686493+0000", 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3340000000000001e-06 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686542+0000", 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8446000000000003e-05 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686549+0000", 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9330000000000004e-06 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664284+0000", 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97773493499999997 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664406+0000", 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122318 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:01:53.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664521+0000", 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114911 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667148+0000", 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026273590000000001 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667234+0000", 2024-09-21T10:01:53.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5628000000000001e-05 2024-09-21T10:01:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667278+0000", 2024-09-21T10:01:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4450999999999998e-05 2024-09-21T10:01:53.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667313+0000", 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4839000000000003e-05 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667404+0000", 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0903999999999996e-05 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18381 7.e 7:744d80e9:::benchmark_data_smithi076_190198_object18380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.692250+0000", 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.252191202999999, 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97926354900000001, 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18381 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692250+0000", 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692250+0000", 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692249+0000", 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:53.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692293+0000", 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3548999999999999e-05 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692302+0000", 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.516e-06 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692308+0000", 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.203e-06 2024-09-21T10:01:53.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692342+0000", 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4851999999999998e-05 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692349+0000", 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6209999999999998e-06 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666352+0000", 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97400270799999999 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666462+0000", 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110207 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666578+0000", 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115673 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669202+0000", 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026243019999999998 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669317+0000", 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011520599999999999 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671370+0000", 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020528909999999998 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671430+0000", 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9469999999999998e-05 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671514+0000", 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4266999999999998e-05 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.068 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18429 7.e 7:714a56fb:::benchmark_data_smithi076_190198_object18428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.725346+0000", 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.219095601999999, 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94707176999999998, 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.069 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18429 2024-09-21T10:01:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725346+0000", 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725346+0000", 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725345+0000", 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725445+0000", 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9802000000000004e-05 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725456+0000", 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1443999999999999e-05 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725463+0000", 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3149999999999997e-06 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725502+0000", 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9968000000000001e-05 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725509+0000", 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3489999999999997e-06 2024-09-21T10:01:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667477+0000", 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94196792100000004 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667605+0000", 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000127985 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667718+0000", 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113751 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669324+0000", 2024-09-21T10:01:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001605277 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671567+0000", 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002242881 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.672272+0000", 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070508799999999998 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.672318+0000", 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6419999999999999e-05 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.672418+0000", 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9507999999999997e-05 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18445 7.e 7:7dbc857c:::benchmark_data_smithi076_190198_object18444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.734496+0000", 2024-09-21T10:01:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.209945375, 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93906263300000004, 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18445 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734496+0000", 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734496+0000", 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734495+0000", 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734564+0000", 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8957999999999999e-05 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734573+0000", 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9080000000000001e-06 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734578+0000", 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8049999999999997e-06 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734622+0000", 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4554000000000001e-05 2024-09-21T10:01:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734629+0000", 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3539999999999996e-06 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.668318+0000", 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93368945000000003 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.668449+0000", 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013118200000000001 2024-09-21T10:01:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.668550+0000", 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100407 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671162+0000", 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026126690000000002 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671617+0000", 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000454504 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673479+0000", 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018617390000000001 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673524+0000", 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5828000000000002e-05 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673559+0000", 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4276999999999997e-05 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18460 7.e 7:785ec73b:::benchmark_data_smithi076_190198_object18459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.743709+0000", 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 31.200732195000001, 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92995887700000002, 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:01:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18460 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743709+0000", 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743709+0000", 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743708+0000", 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743754+0000", 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6079999999999999e-05 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743764+0000", 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0465000000000001e-05 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743770+0000", 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2340000000000002e-06 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743825+0000", 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5464999999999998e-05 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743840+0000", 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4936999999999999e-05 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669358+0000", 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92551827799999997 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669491+0000", 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013267300000000001 2024-09-21T10:01:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669636+0000", 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014477000000000001 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671527+0000", 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018911889999999999 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671656+0000", 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129417 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673600+0000", 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019433020000000001 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673640+0000", 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9900000000000001e-05 2024-09-21T10:01:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673668+0000", 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8639000000000001e-05 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:01:53.093 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:01:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.095 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:53.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:53.195 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:53.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:53.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:53.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:53.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:53.446 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:53.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:53.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:53.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:53 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1273919171' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2024-09-21T10:01:53.561 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:53 smithi191 ceph-mon[102206]: osdmap e1150: 8 total, 8 up, 6 in 2024-09-21T10:01:53.562 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:53 smithi191 ceph-mon[102206]: osdmap e1151: 8 total, 8 up, 6 in 2024-09-21T10:01:53.571 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:53.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:01:53] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-21T10:01:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1273919171' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2024-09-21T10:01:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-mon[106482]: osdmap e1150: 8 total, 8 up, 6 in 2024-09-21T10:01:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-mon[106482]: osdmap e1151: 8 total, 8 up, 6 in 2024-09-21T10:01:53.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1273919171' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2024-09-21T10:01:53.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-mon[110963]: osdmap e1150: 8 total, 8 up, 6 in 2024-09-21T10:01:53.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:53 smithi076 ceph-mon[110963]: osdmap e1151: 8 total, 8 up, 6 in 2024-09-21T10:01:53.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:53.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 39 16 52921 52905 84.7552 4.75 0.00280637 0.0114757 2024-09-21T10:01:53.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:53.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:53.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:53.843 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:53.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:01:53.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:53.966 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:53.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:54.067 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:54.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:54.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:54.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:54.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:54.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:54.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:54.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:54.425 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:54 smithi191 ceph-mon[102206]: pgmap v1370: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 11147/152436 objects degraded (7.313%); 2172/152436 objects misplaced (1.425%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:01:54.425 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:54 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-21T10:01:54.425 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 11147/152436 objects degraded (7.313%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:54.426 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:54 smithi191 ceph-mon[102206]: osdmap e1152: 8 total, 8 up, 6 in 2024-09-21T10:01:54.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:54.460 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:54.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:54.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:54.585 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[110963]: pgmap v1370: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 11147/152436 objects degraded (7.313%); 2172/152436 objects misplaced (1.425%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:01:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-21T10:01:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 11147/152436 objects degraded (7.313%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[110963]: osdmap e1152: 8 total, 8 up, 6 in 2024-09-21T10:01:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[106482]: pgmap v1370: 89 pgs: 7 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 47 MiB/s wr, 748 op/s; 11147/152436 objects degraded (7.313%); 2172/152436 objects misplaced (1.425%); 1.7 MiB/s, 26 objects/s recovering 2024-09-21T10:01:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-21T10:01:54.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 11147/152436 objects degraded (7.313%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:54.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:54 smithi076 ceph-mon[106482]: osdmap e1152: 8 total, 8 up, 6 in 2024-09-21T10:01:54.632 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:54.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:54.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:54.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:01:54.714295+0000 min lat: 0.00130696 max lat: 5.04823 avg lat: 0.0114757 2024-09-21T10:01:54.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:01:54.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 40 16 52921 52905 82.6368 0 - 0.0114757 2024-09-21T10:01:54.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:54.831 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:54.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:54.956 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:54.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:54.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:55.058 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:55.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:55.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:55.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:55.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:55.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:55.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:55.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:55.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:55.436 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:55.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:55.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:55.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:55.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:55.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 41 16 53191 53175 81.0331 8.4375 0.00481462 0.0122929 2024-09-21T10:01:55.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:55.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:55.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:55.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:55.982 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:55.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:56.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:56.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:56.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:56.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:56.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:56.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:56.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:56.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:56.410 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:56 smithi191 ceph-mon[102206]: pgmap v1373: 89 pgs: 8 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 941 op/s; 5479/158469 objects degraded (3.457%); 4490/158469 objects misplaced (2.833%); 9.3 MiB/s, 146 objects/s recovering 2024-09-21T10:01:56.410 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:56.410 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:56.424 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:56 smithi076 ceph-mon[106482]: pgmap v1373: 89 pgs: 8 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 941 op/s; 5479/158469 objects degraded (3.457%); 4490/158469 objects misplaced (2.833%); 9.3 MiB/s, 146 objects/s recovering 2024-09-21T10:01:56.425 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:56.425 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:56.425 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:56 smithi076 ceph-mon[110963]: pgmap v1373: 89 pgs: 8 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 941 op/s; 5479/158469 objects degraded (3.457%); 4490/158469 objects misplaced (2.833%); 9.3 MiB/s, 146 objects/s recovering 2024-09-21T10:01:56.425 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:56.425 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:56.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:56.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:56.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:56.600 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:56.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:56.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 42 16 53749 53733 79.9343 34.875 0.00495258 0.0124736 2024-09-21T10:01:56.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:56.761 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:56.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:56.840 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:01:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:01:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:01:56.841 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:01:56.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:56.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:56.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:56.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:57.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:57.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:57.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:57.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:57.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:57.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:57.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:57.396 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:57.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:57.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:01:57.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:01:57.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:57.644 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:57.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:01:57.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 43 15 54302 54287 78.8808 34.625 0.00611905 0.0126246 2024-09-21T10:01:57.746 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:01:57.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:01:57.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:57.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:57.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:57.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:58.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:58.036 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:01:58.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.049 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:58.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:58.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:58.173 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:58.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:58.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:58.302 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:58.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:58.404 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:01:58.432 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:58.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:01:58.484 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:58 smithi191 ceph-mon[102206]: pgmap v1374: 89 pgs: 8 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 43 MiB/s wr, 690 op/s; 5479/158469 objects degraded (3.457%); 4490/158469 objects misplaced (2.833%); 6.8 MiB/s, 107 objects/s recovering 2024-09-21T10:01:58.485 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:58.485 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:58.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:58.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:58 smithi076 ceph-mon[106482]: pgmap v1374: 89 pgs: 8 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 43 MiB/s wr, 690 op/s; 5479/158469 objects degraded (3.457%); 4490/158469 objects misplaced (2.833%); 6.8 MiB/s, 107 objects/s recovering 2024-09-21T10:01:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:58.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:58 smithi076 ceph-mon[110963]: pgmap v1374: 89 pgs: 8 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 43 MiB/s wr, 690 op/s; 5479/158469 objects degraded (3.457%); 4490/158469 objects misplaced (2.833%); 6.8 MiB/s, 107 objects/s recovering 2024-09-21T10:01:58.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:58.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:01:58.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:58.687 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:58.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:58.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 44 16 54767 54751 77.7473 29 0.005036 0.012805 2024-09-21T10:01:58.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:01:58.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:01:58.917 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.4 primary-affinity to 0 (802) 2024-09-21T10:01:58.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:58.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:58.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:59.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:59.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:59.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:01:59.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:01:59.238 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:01:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 5479/158469 objects degraded (3.457%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:59.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:01:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 5479/158469 objects degraded (3.457%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:59.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:59.316 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:59.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:59.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:59.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:59.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:59.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:01:59.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.563 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:59.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:01:59.611 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:01:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 5479/158469 objects degraded (3.457%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:01:59.665 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:59.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:01:59.715 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 45 16 55209 55193 76.6336 27.625 0.00527399 0.0130031 2024-09-21T10:01:59.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:01:59.818 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:01:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:01:59.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:01:59.952 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:01:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:01:59.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:00.053 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:00.138 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:00.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:00.202 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:02:00.202 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:02:00.202 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:02:00.202 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:02:00.202 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:02:00.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:00 smithi191 ceph-mon[102206]: pgmap v1375: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 28 MiB/s wr, 451 op/s; 5479/159438 objects degraded (3.436%); 4490/159438 objects misplaced (2.816%); 5.8 MiB/s, 91 objects/s recovering 2024-09-21T10:02:00.211 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:00 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-21T10:02:00.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:00.263 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:00.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:00.364 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:00.393 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:00.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:00.522 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:00.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:00 smithi076 ceph-mon[110963]: pgmap v1375: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 28 MiB/s wr, 451 op/s; 5479/159438 objects degraded (3.436%); 4490/159438 objects misplaced (2.816%); 5.8 MiB/s, 91 objects/s recovering 2024-09-21T10:02:00.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:00 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-21T10:02:00.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:00 smithi076 ceph-mon[106482]: pgmap v1375: 89 pgs: 1 peering, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 28 MiB/s wr, 451 op/s; 5479/159438 objects degraded (3.436%); 4490/159438 objects misplaced (2.816%); 5.8 MiB/s, 91 objects/s recovering 2024-09-21T10:02:00.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:00 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-21T10:02:00.623 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:00.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:00.716 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 46 16 55612 55596 75.5154 25.1875 0.00266872 0.0131899 2024-09-21T10:02:00.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:00.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:00.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:00.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:00.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:00.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:01.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:01.029 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:01.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:01.131 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:01.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:01.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:01.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:02:01.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:01.394 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:01.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:01.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:01.473 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:02:01.516 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:01.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:01.639 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:01.660 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:01.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:01.716 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 47 16 55990 55974 74.4116 23.625 0.00318728 0.0133788 2024-09-21T10:02:01.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:01.783 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:01.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:01.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:01.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:01.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:02.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:02.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:02.148 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:02.169 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:02.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:02.302 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:02.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:02.425 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:02.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:02.527 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:02.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:02.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:02 smithi191 ceph-mon[102206]: pgmap v1376: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 30 MiB/s wr, 482 op/s; 8234/164022 objects degraded (5.020%); 3514/164022 objects misplaced (2.142%); 12 MiB/s, 195 objects/s recovering 2024-09-21T10:02:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:02 smithi076 ceph-mon[106482]: pgmap v1376: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 30 MiB/s wr, 482 op/s; 8234/164022 objects degraded (5.020%); 3514/164022 objects misplaced (2.142%); 12 MiB/s, 195 objects/s recovering 2024-09-21T10:02:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:02 smithi076 ceph-mon[110963]: pgmap v1376: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 30 MiB/s wr, 482 op/s; 8234/164022 objects degraded (5.020%); 3514/164022 objects misplaced (2.142%); 12 MiB/s, 195 objects/s recovering 2024-09-21T10:02:02.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:02.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:02.716 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 48 16 56395 56379 73.3889 25.3125 0.0410477 0.0135762 2024-09-21T10:02:02.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:02.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:02.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:02.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:02.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:02.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:03.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:03.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:03.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:03.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:03.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:03.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:03.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:03.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:36602 6.b 6:d2655bf9:::benchmark_data_smithi076_154645_object36601:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-09-21T10:02:03.350 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:53.960818+0000", 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 309.329846207, 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.065386888000000004, 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 36602 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960818+0000", 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960818+0000", 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960816+0000", 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960892+0000", 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6422999999999997e-05 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960907+0000", 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4487e-05 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960914+0000", 2024-09-21T10:02:03.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2620000000000001e-06 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.960956+0000", 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2141999999999997e-05 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.961105+0000", 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000149154 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.961273+0000", 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016793 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.962574+0000", 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013011050000000001 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.963446+0000", 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00087137499999999997 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.968687+0000", 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052408200000000002 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:54.025236+0000", 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.056548911 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:54.025296+0000", 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0612000000000002e-05 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:54.026205+0000", 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000908354 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:36609 6.b 6:d167ef22:::benchmark_data_smithi076_154645_object36608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1095)", 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:56:53.969251+0000", 2024-09-21T10:02:03.357 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 309.32141281399998, 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057537866999999999, 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 36609 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969251+0000", 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969251+0000", 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969250+0000", 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969304+0000", 2024-09-21T10:02:03.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4385000000000002e-05 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969316+0000", 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1745e-05 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969321+0000", 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.921e-06 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969373+0000", 2024-09-21T10:02:03.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1814000000000002e-05 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969526+0000", 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015340899999999999 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.969761+0000", 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00023542999999999999 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.971156+0000", 2024-09-21T10:02:03.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013948509999999999 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.973507+0000", 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002350881 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:53.974175+0000", 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066739100000000003 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:54.026251+0000", 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052076029000000003 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:54.026298+0000", 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7741000000000003e-05 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:56:54.026789+0000", 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00049057899999999999 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80267 6.b 6:d6537f18:::benchmark_data_smithi076_154645_object80266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.469628+0000", 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 273.82103562200001, 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.068112300000000001, 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80267 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.364 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469628+0000", 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469628+0000", 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469626+0000", 2024-09-21T10:02:03.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469674+0000", 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7642999999999998e-05 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469691+0000", 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7020999999999998e-05 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469699+0000", 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5859999999999999e-06 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469757+0000", 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8186000000000002e-05 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.469901+0000", 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014452399999999999 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470052+0000", 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015051600000000001 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.475829+0000", 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0057775839999999997 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.475968+0000", 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000138827 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.511695+0000", 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.035726536000000003 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.537049+0000", 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025354003 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.537088+0000", 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9146000000000001e-05 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.537741+0000", 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00065265799999999997 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80269 6.b 6:da79640e:::benchmark_data_smithi076_154645_object80268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.470739+0000", 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 273.81992464699999, 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.067390564, 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.370 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80269 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470739+0000", 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470739+0000", 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470738+0000", 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470792+0000", 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4472000000000003e-05 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470805+0000", 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2597e-05 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.470812+0000", 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9330000000000004e-06 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.472699+0000", 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001887365 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.472846+0000", 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014652600000000001 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.473018+0000", 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017249600000000001 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.478871+0000", 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005853123 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.479851+0000", 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00097926700000000003 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.511753+0000", 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.031902162999999997 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.537784+0000", 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026030886999999999 2024-09-21T10:02:03.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.537820+0000", 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6486999999999998e-05 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.538130+0000", 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00030964099999999999 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.376 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80273 6.b 6:dadbd41d:::benchmark_data_smithi076_154645_object80272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.475924+0000", 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 273.81474028999997, 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.062565351000000005, 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80273 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.475924+0000", 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.475924+0000", 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.475922+0000", 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.475992+0000", 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9839000000000007e-05 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.476003+0000", 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.066e-05 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.476009+0000", 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2280000000000003e-06 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.476050+0000", 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0822999999999999e-05 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.476193+0000", 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014320199999999999 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.476354+0000", 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016105899999999999 2024-09-21T10:02:03.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.479045+0000", 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026906999999999999 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.481571+0000", 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002526422 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.511797+0000", 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030226541999999999 2024-09-21T10:02:03.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.538160+0000", 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.026362051000000001 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.538189+0000", 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9220000000000001e-05 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.538489+0000", 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00030015199999999998 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80490 6.b 6:d4652148:::benchmark_data_smithi076_154645_object80489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.739119+0000", 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 273.551544353, 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.094209785000000004, 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.383 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80490 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.739119+0000", 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.739119+0000", 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.739118+0000", 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.739184+0000", 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5714000000000001e-05 2024-09-21T10:02:03.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.739200+0000", 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6038000000000001e-05 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.739208+0000", 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.038e-06 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.740116+0000", 2024-09-21T10:02:03.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00090761599999999998 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.740285+0000", 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000169187 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.740454+0000", 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000169347 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.742937+0000", 2024-09-21T10:02:03.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024825590000000001 2024-09-21T10:02:03.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.743748+0000", 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00081155500000000005 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754015+0000", 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010266608 2024-09-21T10:02:03.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.832662+0000", 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.078647132999999994 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.832700+0000", 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7985000000000003e-05 2024-09-21T10:02:03.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.833329+0000", 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00062927399999999998 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80498 6.b 6:d63bbc7e:::benchmark_data_smithi076_154645_object80497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.746518+0000", 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 273.54414542799998, 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.087460258999999999, 2024-09-21T10:02:03.391 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80498 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746518+0000", 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746518+0000", 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746517+0000", 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746583+0000", 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6296000000000004e-05 2024-09-21T10:02:03.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746595+0000", 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2132e-05 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746602+0000", 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5429999999999999e-06 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746662+0000", 2024-09-21T10:02:03.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9855000000000002e-05 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746784+0000", 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122712 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.746927+0000", 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000142872 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.395 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754051+0000", 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071241280000000004 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754433+0000", 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00038215200000000002 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.757427+0000", 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029939010000000002 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.833401+0000", 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.075974086999999996 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.833460+0000", 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9077999999999999e-05 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.833979+0000", 2024-09-21T10:02:03.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00051822599999999999 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:80508 6.b 6:d79df5d1:::benchmark_data_smithi076_154645_object80507:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:29.753904+0000", 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 273.53676023200001, 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.086564358999999994, 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.398 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 80508 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753904+0000", 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753904+0000", 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753902+0000", 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753929+0000", 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7115999999999999e-05 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753941+0000", 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2289e-05 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.753948+0000", 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8519999999999998e-06 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754462+0000", 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00051345099999999995 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754591+0000", 2024-09-21T10:02:03.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129498 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.754750+0000", 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015924799999999999 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.757767+0000", 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030169429999999998 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.759384+0000", 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001616312 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.834015+0000", 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.074631568999999995 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.839632+0000", 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056165269999999996 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.839689+0000", 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6972999999999998e-05 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:29.840468+0000", 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00077939700000000003 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:82562 6.b 6:dd2db2c5:::benchmark_data_smithi076_154645_object82561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:31.769670+0000", 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 271.52099334899998, 2024-09-21T10:02:03.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.049627860000000003, 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 82562 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.769670+0000", 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.769670+0000", 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.769670+0000", 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.769697+0000", 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7106000000000001e-05 2024-09-21T10:02:03.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.769703+0000", 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0900000000000001e-06 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.769707+0000", 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.49e-06 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.775405+0000", 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056984999999999996 2024-09-21T10:02:03.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.775523+0000", 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117885 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.775631+0000", 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010788799999999999 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.778330+0000", 2024-09-21T10:02:03.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026987080000000002 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.778892+0000", 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00056212500000000002 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.786397+0000", 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075053109999999998 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.817062+0000", 2024-09-21T10:02:03.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030665234999999999 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.817125+0000", 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2424999999999999e-05 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:31.819298+0000", 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021735539999999999 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:86563 6.b 6:d57d9d49:::benchmark_data_smithi076_154645_object86562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:35.803091+0000", 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 267.48757322699998, 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.060459672999999998, 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86563 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803091+0000", 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803091+0000", 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803089+0000", 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803143+0000", 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4200999999999998e-05 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803157+0000", 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3309000000000001e-05 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803164+0000", 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5410000000000003e-06 2024-09-21T10:02:03.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803218+0000", 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3533999999999999e-05 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803357+0000", 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013915499999999999 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.803563+0000", 2024-09-21T10:02:03.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020572400000000001 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.809443+0000", 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058807399999999998 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810077+0000", 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00063336999999999996 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.824680+0000", 2024-09-21T10:02:03.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014603115999999999 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.862810+0000", 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038130367999999998 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.862846+0000", 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6063999999999999e-05 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.863550+0000", 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070407800000000002 2024-09-21T10:02:03.416 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:86567 6.b 6:db787eb3:::benchmark_data_smithi076_154645_object86566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:35.804322+0000", 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 267.48634215300001, 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.059902938000000003, 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86567 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.417 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.804322+0000", 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.804322+0000", 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.804321+0000", 2024-09-21T10:02:03.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.804375+0000", 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4369000000000001e-05 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.804390+0000", 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5254e-05 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.804397+0000", 2024-09-21T10:02:03.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8000000000000001e-06 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.806094+0000", 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0016967740000000001 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.806235+0000", 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014127899999999999 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.806395+0000", 2024-09-21T10:02:03.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160247 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.809450+0000", 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00305456 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810142+0000", 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00069264999999999995 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.824721+0000", 2024-09-21T10:02:03.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014578035 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.863611+0000", 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038890658000000002 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.863666+0000", 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5056000000000003e-05 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.864225+0000", 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00055839500000000005 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:86570 6.b 6:d3431770:::benchmark_data_smithi076_154645_object86569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:35.810225+0000", 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 267.48043840000003, 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.054626755999999999, 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.423 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86570 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810225+0000", 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810225+0000", 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810224+0000", 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810286+0000", 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1766999999999993e-05 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810305+0000", 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9731000000000001e-05 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810312+0000", 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0040000000000002e-06 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810352+0000", 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9477999999999997e-05 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810501+0000", 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000148768 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810688+0000", 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000187157 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.815830+0000", 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051422719999999998 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.816057+0000", 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022652600000000001 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.824754+0000", 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086977200000000008 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.864337+0000", 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039582282000000003 2024-09-21T10:02:03.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.864404+0000", 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7249999999999995e-05 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.864852+0000", 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00044833399999999999 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.429 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:86573 6.b 6:d0f9f51f:::benchmark_data_smithi076_154645_object86572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:35.810323+0000", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 267.48034068599998, 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057294664000000002, 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 86573 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810323+0000", 2024-09-21T10:02:03.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810323+0000", 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810323+0000", 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810374+0000", 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0952000000000003e-05 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810382+0000", 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4190000000000002e-06 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.810387+0000", 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1540000000000001e-06 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.813109+0000", 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027215049999999999 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.813246+0000", 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000136969 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.813397+0000", 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015180900000000001 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.816087+0000", 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026894850000000001 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.816094+0000", 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2169999999999997e-06 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.824776+0000", 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086820549999999993 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.434 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.866736+0000", 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041959582000000002 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.866824+0000", 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7891000000000003e-05 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:35.867618+0000", 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00079423199999999997 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.435 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:105158 6.b 6:d3f46154:::benchmark_data_smithi076_154645_object105157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:56.129059+0000", 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 247.16160472000001, 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.071798984999999996, 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 105158 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.436 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.129059+0000", 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.129059+0000", 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.129057+0000", 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.437 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.129129+0000", 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2321000000000001e-05 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.129145+0000", 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5566999999999999e-05 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.129152+0000", 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3119999999999996e-06 2024-09-21T10:02:03.438 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.130122+0000", 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00096984100000000004 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.130223+0000", 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010116299999999999 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.130326+0000", 2024-09-21T10:02:03.439 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102499 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.133899+0000", 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003573498 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.139593+0000", 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056940130000000004 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.143166+0000", 2024-09-21T10:02:03.440 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035731399999999998 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.200471+0000", 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057304961000000001 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.200528+0000", 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6376999999999999e-05 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.200858+0000", 2024-09-21T10:02:03.441 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000330317 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:105188 6.b 6:dc8170fe:::benchmark_data_smithi076_154645_object105187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:57:56.166683+0000", 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 247.12398062099999, 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052053703999999999, 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.442 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 105188 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166683+0000", 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166683+0000", 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.443 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166682+0000", 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166726+0000", 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3565e-05 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166737+0000", 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1803e-05 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.444 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166742+0000", 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0919999999999997e-06 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166780+0000", 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7997999999999998e-05 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.166937+0000", 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015668799999999999 2024-09-21T10:02:03.445 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.167125+0000", 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018782200000000001 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.170102+0000", 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002977155 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.171001+0000", 2024-09-21T10:02:03.446 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00089863699999999998 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.171039+0000", 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8167999999999998e-05 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.218218+0000", 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.047178734 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.447 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.218273+0000", 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5739000000000003e-05 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:57:56.218737+0000", 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000463573 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:110161 6.b 6:dce929c8:::benchmark_data_smithi076_154645_object110160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:01.766192+0000", 2024-09-21T10:02:03.448 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 241.52447231100001, 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.060524306, 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 110161 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766192+0000", 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.449 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766192+0000", 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766190+0000", 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766253+0000", 2024-09-21T10:02:03.450 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3538000000000001e-05 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766264+0000", 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1073000000000001e-05 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766270+0000", 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2270000000000001e-06 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.451 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.766951+0000", 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00068157199999999997 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.767069+0000", 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117664 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.767178+0000", 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010946999999999999 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.452 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.770706+0000", 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035279230000000001 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.771356+0000", 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000650126 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775537+0000", 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041805510000000002 2024-09-21T10:02:03.453 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.826517+0000", 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.050980677000000002 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.826573+0000", 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5775e-05 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.826716+0000", 2024-09-21T10:02:03.454 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014258399999999999 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:110171 6.b 6:dec9c1b2:::benchmark_data_smithi076_154645_object110170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:01.775543+0000", 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 241.515121222, 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.057211621999999997, 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.455 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 110171 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775543+0000", 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775543+0000", 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.456 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775541+0000", 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775584+0000", 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3124000000000002e-05 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775595+0000", 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0795000000000001e-05 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.457 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775600+0000", 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4280000000000004e-06 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775627+0000", 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7182000000000002e-05 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775727+0000", 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9258000000000004e-05 2024-09-21T10:02:03.458 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.775846+0000", 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119144 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.786888+0000", 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011042428999999999 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.786929+0000", 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0883000000000001e-05 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.459 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.786981+0000", 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1820999999999997e-05 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.832521+0000", 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.045539934999999997 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.832607+0000", 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6469000000000002e-05 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.460 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.832754+0000", 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014686199999999999 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:110175 6.b 6:d2a8a7e9:::benchmark_data_smithi076_154645_object110174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:01.777055+0000", 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 241.51360864399999, 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.055962651000000002, 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.461 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 110175 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.777055+0000", 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.777055+0000", 2024-09-21T10:02:03.462 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.777054+0000", 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.777158+0000", 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010347400000000001 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.463 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.777167+0000", 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4690000000000003e-06 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.777173+0000", 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0789999999999997e-06 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.780586+0000", 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034130990000000002 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.464 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.780751+0000", 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016437799999999999 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.780924+0000", 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017340899999999999 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.786990+0000", 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060658839999999997 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.465 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.788976+0000", 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019860289999999998 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.789242+0000", 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000265989 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.832829+0000", 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.043586456000000003 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.466 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.832904+0000", 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5525000000000006e-05 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.833018+0000", 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113746 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.467 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:110177 6.b 6:dc4b1b18:::benchmark_data_smithi076_154645_object110176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:01.781339+0000", 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 241.5093249, 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.053030159, 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 110177 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.468 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.781339+0000", 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.781339+0000", 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.781337+0000", 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.469 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.781403+0000", 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6180999999999995e-05 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.781423+0000", 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9865e-05 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.781429+0000", 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5799999999999999e-06 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.470 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.783695+0000", 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0022667730000000001 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.783790+0000", 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4906000000000002e-05 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,7", 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.783888+0000", 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8243000000000001e-05 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.471 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.786998+0000", 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003109555 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.789167+0000", 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002169434 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.791673+0000", 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025055810000000002 2024-09-21T10:02:03.472 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.834192+0000", 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042519217999999998 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.834242+0000", 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9410999999999998e-05 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:01.834369+0000", 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012746099999999999 2024-09-21T10:02:03.473 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.34674.0:157392 6.3 e1116/1086 6:cd0761c7:::benchmark_data_smithi076_154645_object43486:head v 1116'9902, mlcod=1116'9900)", 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:00.621140+0000", 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 182.66952381300001, 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039250699, 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:02:03.474 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621140+0000", 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621140+0000", 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621139+0000", 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:02:03.475 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621146+0000", 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.528e-06 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621157+0000", 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.081e-05 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621163+0000", 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3929999999999996e-06 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.476 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621205+0000", 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1621e-05 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.621224+0000", 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9335999999999999e-05 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.660355+0000", 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039130915000000002 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:02:03.477 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:00.660391+0000", 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5945000000000002e-05 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:02:03.478 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:02:03.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:03.500 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:03.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:03.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:02:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:02:03] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.51.0" 2024-09-21T10:02:03.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:03.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:03.716 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 49 16 56817 56801 72.4295 26.375 0.00295358 0.0137621 2024-09-21T10:02:03.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:03.759 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:03.842 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:02:03.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:03.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:03.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:03.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:04.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:04.060 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:04.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:04.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:04.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:04.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:04.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:04.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:02:04.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:04.423 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:04 smithi076 ceph-mon[106482]: pgmap v1377: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 29 MiB/s wr, 466 op/s; 8234/165660 objects degraded (4.970%); 3514/165660 objects misplaced (2.121%); 10 MiB/s, 165 objects/s recovering 2024-09-21T10:02:04.423 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8234/165660 objects degraded (4.970%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:04.423 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:04 smithi076 ceph-mon[110963]: pgmap v1377: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 29 MiB/s wr, 466 op/s; 8234/165660 objects degraded (4.970%); 3514/165660 objects misplaced (2.121%); 10 MiB/s, 165 objects/s recovering 2024-09-21T10:02:04.423 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8234/165660 objects degraded (4.970%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:04.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:04.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:04.556 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:04.578 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:02:04.590 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:04.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:04 smithi191 ceph-mon[102206]: pgmap v1377: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 29 MiB/s wr, 466 op/s; 8234/165660 objects degraded (4.970%); 3514/165660 objects misplaced (2.121%); 10 MiB/s, 165 objects/s recovering 2024-09-21T10:02:04.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8234/165660 objects degraded (4.970%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:04.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:04.714 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:04.718 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 50 16 57349 57333 71.6435 33.25 0.00490922 0.0139052 2024-09-21T10:02:04.778 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 3] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:02:04.778 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:02:04.778 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 4, 5, 3] 2024-09-21T10:02:04.778 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 5 2024-09-21T10:02:04.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:04.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:02:04.849 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:04.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:04.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:04.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:04.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:05.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:05.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:05.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:05.223 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:05.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:05.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:05 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2657202956' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:05.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:05.484 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:05.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:05.597 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:05 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2657202956' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:05.597 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:05 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2657202956' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:05.607 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:05.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:05.718 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 51 16 57694 57678 70.661 21.5625 0.00688582 0.0140937 2024-09-21T10:02:05.728 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:05.733 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:05.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:05.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:05.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:05.893 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:05.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:05.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:06.016 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:06.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:06.118 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:06.144 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:06.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:06.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:06.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:06 smithi076 ceph-mon[106482]: pgmap v1378: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 31 MiB/s wr, 491 op/s; 8234/170427 objects degraded (4.831%); 2190/170427 objects misplaced (1.285%); 16 MiB/s, 258 objects/s recovering 2024-09-21T10:02:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:06 smithi076 ceph-mon[106482]: osdmap e1153: 8 total, 8 up, 6 in 2024-09-21T10:02:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:06 smithi076 ceph-mon[110963]: pgmap v1378: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 31 MiB/s wr, 491 op/s; 8234/170427 objects degraded (4.831%); 2190/170427 objects misplaced (1.285%); 16 MiB/s, 258 objects/s recovering 2024-09-21T10:02:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:06 smithi076 ceph-mon[110963]: osdmap e1153: 8 total, 8 up, 6 in 2024-09-21T10:02:06.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:06.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:06.520 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:06 smithi191 ceph-mon[102206]: pgmap v1378: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 31 MiB/s wr, 491 op/s; 8234/170427 objects degraded (4.831%); 2190/170427 objects misplaced (1.285%); 16 MiB/s, 258 objects/s recovering 2024-09-21T10:02:06.520 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:06 smithi191 ceph-mon[102206]: osdmap e1153: 8 total, 8 up, 6 in 2024-09-21T10:02:06.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:06.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:06.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:06.668 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:06.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:06.718 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 52 16 58219 58203 69.9332 32.8125 0.020468 0.0142794 2024-09-21T10:02:06.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:06.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:06.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:06.894 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:06.932 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:06.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:07.002 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:07.003 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:07.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:07.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:07.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:07.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:07.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:07.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:07.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:07 smithi191 ceph-mon[102206]: osdmap e1154: 8 total, 8 up, 6 in 2024-09-21T10:02:07.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:07.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:07.515 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:07 smithi076 ceph-mon[106482]: osdmap e1154: 8 total, 8 up, 6 in 2024-09-21T10:02:07.515 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:07 smithi076 ceph-mon[110963]: osdmap e1154: 8 total, 8 up, 6 in 2024-09-21T10:02:07.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:07.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:07.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:02:07.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:07.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:07.718 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 53 16 58730 58714 69.2165 31.9375 0.00552079 0.014405 2024-09-21T10:02:07.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:02:07.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:07.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:07.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:07.983 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:02:07.986 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:07.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:08.088 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:08.117 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:08.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:08.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:08.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:08 smithi191 ceph-mon[102206]: pgmap v1381: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 29 MiB/s wr, 457 op/s; 8234/170427 objects degraded (4.831%); 2190/170427 objects misplaced (1.285%); 18 MiB/s, 287 objects/s recovering 2024-09-21T10:02:08.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:08.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:08.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:08.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:08.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:08 smithi076 ceph-mon[110963]: pgmap v1381: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 29 MiB/s wr, 457 op/s; 8234/170427 objects degraded (4.831%); 2190/170427 objects misplaced (1.285%); 18 MiB/s, 287 objects/s recovering 2024-09-21T10:02:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:08 smithi076 ceph-mon[106482]: pgmap v1381: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 29 MiB/s wr, 457 op/s; 8234/170427 objects degraded (4.831%); 2190/170427 objects misplaced (1.285%); 18 MiB/s, 287 objects/s recovering 2024-09-21T10:02:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:08.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:08.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 54 16 59278 59262 68.5691 34.25 0.00430234 0.0145475 2024-09-21T10:02:08.727 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:08.761 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:08.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:08.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:08.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:08.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:09.007 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:09.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:09.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:09.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:09.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:09.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8234/170427 objects degraded (4.831%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8234/170427 objects degraded (4.831%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:09.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:09.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:09.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:09.521 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:09.532 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8234/170427 objects degraded (4.831%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:09.622 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:09.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:09.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:09.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 55 16 59782 59766 67.8951 31.5 0.00393833 0.0146792 2024-09-21T10:02:09.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:09.762 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:09.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:09.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:09.785 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:09.785 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:09.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:09 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:02:09.510+0000 7f240edc2640 -1 mon.a@0(leader).osd e1154 definitely_dead 0 2024-09-21T10:02:09.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:09.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:09.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:10.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:10.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:10.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:10.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:10.197 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.5. 2024-09-21T10:02:10.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:10.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:10.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:10.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:10.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.421 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:10.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:10.457 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[106482]: pgmap v1382: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 21 MiB/s wr, 336 op/s; 8234/172110 objects degraded (4.784%); 2190/172110 objects misplaced (1.272%); 10 MiB/s, 165 objects/s recovering 2024-09-21T10:02:10.457 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2936133343' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-09-21T10:02:10.457 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-09-21T10:02:10.457 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[106482]: Client client.admin marked osd.5 out, while it was still marked up 2024-09-21T10:02:10.458 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[110963]: pgmap v1382: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 21 MiB/s wr, 336 op/s; 8234/172110 objects degraded (4.784%); 2190/172110 objects misplaced (1.272%); 10 MiB/s, 165 objects/s recovering 2024-09-21T10:02:10.458 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2936133343' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-09-21T10:02:10.458 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-09-21T10:02:10.458 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:10 smithi076 ceph-mon[110963]: Client client.admin marked osd.5 out, while it was still marked up 2024-09-21T10:02:10.467 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:10 smithi191 ceph-mon[102206]: pgmap v1382: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 21 MiB/s wr, 336 op/s; 8234/172110 objects degraded (4.784%); 2190/172110 objects misplaced (1.272%); 10 MiB/s, 165 objects/s recovering 2024-09-21T10:02:10.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2936133343' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-09-21T10:02:10.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-09-21T10:02:10.468 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:10 smithi191 ceph-mon[102206]: Client client.admin marked osd.5 out, while it was still marked up 2024-09-21T10:02:10.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:02:10.522 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:10.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:10.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:10.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:10.672 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:10.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:10.698 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:02:10.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 56 16 60045 60029 66.9764 16.4375 0.00334473 0.014825 2024-09-21T10:02:10.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:10.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:10.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:10.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:10.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:10.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:11.021 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:11.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:11.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:11.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:11.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:11.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:11.222 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:02:11.223 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-09-21T10:02:11.223 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[106482]: osdmap e1155: 8 total, 8 up, 5 in 2024-09-21T10:02:11.223 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/607704297' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[110963]: osdmap e1155: 8 total, 8 up, 5 in 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/607704297' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:11.224 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:11.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:11.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-09-21T10:02:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:11 smithi191 ceph-mon[102206]: osdmap e1155: 8 total, 8 up, 5 in 2024-09-21T10:02:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/607704297' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:02:11.393 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:11.415 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:11.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:11.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:11.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:11.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:11.664 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:11.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 57 16 60046 60030 65.8027 0.0625 0.944882 0.0148405 2024-09-21T10:02:11.766 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:11.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:11.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:11.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:11.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:11.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:12.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:12.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:12.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:12.138 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:12.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:12.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.298 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:12.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:12.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:12.425 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:12.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:12 smithi191 ceph-mon[102206]: pgmap v1384: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 25 MiB/s wr, 398 op/s; 5503/177606 objects degraded (3.098%); 3867/177606 objects misplaced (2.177%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:02:12.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:12 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:12.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:12 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:02:12.465 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:12 smithi191 ceph-mon[102206]: osdmap e1156: 8 total, 8 up, 5 in 2024-09-21T10:02:12.466 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:12.466 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:12.527 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:12.560 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:12.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[110963]: pgmap v1384: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 25 MiB/s wr, 398 op/s; 5503/177606 objects degraded (3.098%); 3867/177606 objects misplaced (2.177%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:02:12.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:12.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:02:12.582 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[110963]: osdmap e1156: 8 total, 8 up, 5 in 2024-09-21T10:02:12.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[106482]: pgmap v1384: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 25 MiB/s wr, 398 op/s; 5503/177606 objects degraded (3.098%); 3867/177606 objects misplaced (2.177%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:02:12.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:12.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:02:12.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:12 smithi076 ceph-mon[106482]: osdmap e1156: 8 total, 8 up, 5 in 2024-09-21T10:02:12.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:12.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:12.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:12.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 58 16 60156 60140 64.7869 6.875 0.00306207 0.0152654 2024-09-21T10:02:12.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:12.808 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:12.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:12.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:12.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:12.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:13.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:13.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:13.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:13.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:13.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:13.235 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:02:13.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:13.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:13.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:13.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:13.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:13.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-mon[106482]: osdmap e1157: 8 total, 8 up, 5 in 2024-09-21T10:02:13.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:13.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:13.528 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:02:13] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-21T10:02:13.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-mon[110963]: osdmap e1157: 8 total, 8 up, 5 in 2024-09-21T10:02:13.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:13.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:13.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:13.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:13.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:13 smithi191 ceph-mon[102206]: osdmap e1157: 8 total, 8 up, 5 in 2024-09-21T10:02:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:13.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:13.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:13.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 59 16 60156 60140 63.6891 0 - 0.0152654 2024-09-21T10:02:13.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:13.802 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:13.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:13.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:13.926 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:13.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:13.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:14.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:14.048 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:14.150 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:14.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:14.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:14.250 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:14.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:14.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:14.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:14.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:14.421 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:14.523 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:14.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:14.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:14 smithi191 ceph-mon[102206]: pgmap v1387: 89 pgs: 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 28 MiB/s wr, 454 op/s; 12970/178611 objects degraded (7.262%); 3867/178611 objects misplaced (2.165%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:02:14.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 12970/178611 objects degraded (7.262%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:14.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2491907464' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:14.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:14 smithi076 ceph-mon[106482]: pgmap v1387: 89 pgs: 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 28 MiB/s wr, 454 op/s; 12970/178611 objects degraded (7.262%); 3867/178611 objects misplaced (2.165%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:02:14.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 12970/178611 objects degraded (7.262%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:14.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2491907464' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:14.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:14 smithi076 ceph-mon[110963]: pgmap v1387: 89 pgs: 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 28 MiB/s wr, 454 op/s; 12970/178611 objects degraded (7.262%); 3867/178611 objects misplaced (2.165%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:02:14.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 12970/178611 objects degraded (7.262%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:14.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2491907464' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:14.647 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:14.669 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:14.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:14.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:02:14.718380+0000 min lat: 0.00130696 max lat: 5.04823 avg lat: 0.0152654 2024-09-21T10:02:14.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:02:14.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 60 16 60156 60140 62.6278 0 - 0.0152654 2024-09-21T10:02:14.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:14.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:14.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:14.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:14.917 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:14.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:14.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:15.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:15.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:15.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:15.167 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:15.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:15.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:15.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:15.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:15.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:15.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:15.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:15.650 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:15.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:15.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 61 16 60156 60140 61.6013 0 - 0.0152654 2024-09-21T10:02:15.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:15.797 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:15.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:15.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:15.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:15.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:02:16.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:16.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:16.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:16.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:16.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:16.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:16.223 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:16 smithi076 ceph-mon[106482]: pgmap v1388: 89 pgs: 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 25 MiB/s wr, 405 op/s; 17913/179412 objects degraded (9.984%); 2391/179412 objects misplaced (1.333%); 14 MiB/s, 229 objects/s recovering 2024-09-21T10:02:16.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:16.306 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:16.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:16.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:16.484 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:16 smithi076 ceph-mon[110963]: pgmap v1388: 89 pgs: 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 25 MiB/s wr, 405 op/s; 17913/179412 objects degraded (9.984%); 2391/179412 objects misplaced (1.333%); 14 MiB/s, 229 objects/s recovering 2024-09-21T10:02:16.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:16.537 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:02:16.537 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:16.561 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:16.563 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:02:16.563 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:16.596 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:02:16.596 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:16.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:16 smithi191 ceph-mon[102206]: pgmap v1388: 89 pgs: 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 25 MiB/s wr, 405 op/s; 17913/179412 objects degraded (9.984%); 2391/179412 objects misplaced (1.333%); 14 MiB/s, 229 objects/s recovering 2024-09-21T10:02:16.616 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:02:16.617 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:16.636 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:02:16.636 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:02:16.636 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:02:16.636 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:02:16.640 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:02:16.640 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:16.646 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:16.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:16.666 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:02:16.666 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:16.690 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:02:16.690 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:16.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.691 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:16.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:16.708 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:02:16.708 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:16.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 62 16 60156 60140 60.6079 0 - 0.0152654 2024-09-21T10:02:16.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:16.818 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:16.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:16.945 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:16.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:17.047 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:17.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.073 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:17.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:17.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:17.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:17.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:17.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:17.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:17.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:17.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:17.577 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:17.610 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 3] out_osds: [6, 7, 5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:02:17.610 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:02:17.610 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-21T10:02:17.610 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:02:17.610 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:02:17.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-21T10:02:17.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:17.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:17.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 63 16 60156 60140 59.6461 0 - 0.0152654 2024-09-21T10:02:17.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:17.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:17.829 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:02:17.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:17.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:17.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:18.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:18.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:18.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:18.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:18.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:18.339 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:18.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:18.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.464 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:18.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:18.477 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:18 smithi076 ceph-mon[106482]: pgmap v1389: 89 pgs: 1 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 9.2 MiB/s wr, 147 op/s; 22541/180420 objects degraded (12.494%); 1969/180420 objects misplaced (1.091%); 12 MiB/s, 187 objects/s recovering 2024-09-21T10:02:18.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:18 smithi076 ceph-mon[110963]: pgmap v1389: 89 pgs: 1 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 9.2 MiB/s wr, 147 op/s; 22541/180420 objects degraded (12.494%); 1969/180420 objects misplaced (1.091%); 12 MiB/s, 187 objects/s recovering 2024-09-21T10:02:18.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:18.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:18.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:18 smithi191 ceph-mon[102206]: pgmap v1389: 89 pgs: 1 activating+undersized+remapped, 2 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 9.2 MiB/s wr, 147 op/s; 22541/180420 objects degraded (12.494%); 1969/180420 objects misplaced (1.091%); 12 MiB/s, 187 objects/s recovering 2024-09-21T10:02:18.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:18.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:18.719 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 64 16 60562 60546 59.1107 4.22917 0.0603829 0.0168799 2024-09-21T10:02:18.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:18.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:18.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:18.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:18.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:18.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:19.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:19.117 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:19.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:19.255 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:19.314 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 22541/180420 objects degraded (12.494%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:19.314 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 22541/180420 objects degraded (12.494%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:19.357 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 22541/180420 objects degraded (12.494%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:19.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:19.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:19.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:19.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:19.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:19.527 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:02:19.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:19.635 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:02:19.635 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:02:19.635 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:02:19.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:19.720 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 65 15 61056 61041 58.6769 30.9375 0.01393 0.0169965 2024-09-21T10:02:19.732 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:02:19.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:19.818 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:19.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:19.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:19.957 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:19.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:20.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:20.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:20.189 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:20.217 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:20.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:20.340 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:20 smithi191 ceph-mon[102206]: pgmap v1390: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 7.5 MiB/s wr, 120 op/s; 22541/180501 objects degraded (12.488%); 1969/180501 objects misplaced (1.091%); 9.2 MiB/s, 148 objects/s recovering 2024-09-21T10:02:20.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:20.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:20.489 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:20.564 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:20 smithi076 ceph-mon[106482]: pgmap v1390: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 7.5 MiB/s wr, 120 op/s; 22541/180501 objects degraded (12.488%); 1969/180501 objects misplaced (1.091%); 9.2 MiB/s, 148 objects/s recovering 2024-09-21T10:02:20.564 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:20 smithi076 ceph-mon[110963]: pgmap v1390: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 7.5 MiB/s wr, 120 op/s; 22541/180501 objects degraded (12.488%); 1969/180501 objects misplaced (1.091%); 9.2 MiB/s, 148 objects/s recovering 2024-09-21T10:02:20.591 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:20.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:20.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:20.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 66 16 61488 61472 58.196 26.9375 0.00380386 0.0171418 2024-09-21T10:02:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:20.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:20.843 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:20.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.866 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:20.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:20.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:20.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:20.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:21.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:21.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:21.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:21.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:21.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:21.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:21.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:21.510 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:21.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:21.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:21.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3190635092' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:21.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3190635092' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:21.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3190635092' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:21.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:21.663 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:02:21.720 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 67 16 62035 62019 57.8377 34.1875 0.00441539 0.0172582 2024-09-21T10:02:21.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:21.757 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:21.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:21.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:21.890 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:21.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:21.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:21.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:22.013 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:22.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:22.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:22.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:22.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:22.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:22.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:22.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:22.497 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:22.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:22.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:22 smithi191 ceph-mon[102206]: pgmap v1391: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 7.9 MiB/s wr, 126 op/s; 22541/181158 objects degraded (12.443%); 1969/181158 objects misplaced (1.087%); 7.9 MiB/s, 127 objects/s recovering 2024-09-21T10:02:22.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:22 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:22.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3190635092' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:22.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:22 smithi191 ceph-mon[102206]: osdmap e1158: 8 total, 8 up, 5 in 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[106482]: pgmap v1391: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 7.9 MiB/s wr, 126 op/s; 22541/181158 objects degraded (12.443%); 1969/181158 objects misplaced (1.087%); 7.9 MiB/s, 127 objects/s recovering 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3190635092' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[106482]: osdmap e1158: 8 total, 8 up, 5 in 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[110963]: pgmap v1391: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 7.9 MiB/s wr, 126 op/s; 22541/181158 objects degraded (12.443%); 1969/181158 objects misplaced (1.087%); 7.9 MiB/s, 127 objects/s recovering 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:22.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3190635092' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:22.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:22 smithi076 ceph-mon[110963]: osdmap e1158: 8 total, 8 up, 5 in 2024-09-21T10:02:22.632 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.671 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:22.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:22.720 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 68 16 62519 62503 57.4321 30.25 0.00316693 0.0173683 2024-09-21T10:02:22.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:22.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:22.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:22.916 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:22.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:23.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:23.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:23.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:23.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:23.179 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:23.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:23.303 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:23.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:23.434 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:23.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:23.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:23.561 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:02:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:02:23] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-21T10:02:23.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:23.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:23.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:23.723 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 69 16 62991 62975 57.027 29.5 0.0039447 0.0174898 2024-09-21T10:02:23.758 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:23.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:23.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:23.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:23.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:23.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:23.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:24.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:24.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:24.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:24.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:24.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:24.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:24.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:24.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:24.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:24.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:24.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:24.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:24.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:24 smithi191 ceph-mon[102206]: pgmap v1393: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 13 MiB/s wr, 202 op/s; 22539/184692 objects degraded (12.204%); 1368/184692 objects misplaced (0.741%); 11 MiB/s, 179 objects/s recovering 2024-09-21T10:02:24.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 22539/184692 objects degraded (12.204%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:24.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:24.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:24 smithi076 ceph-mon[106482]: pgmap v1393: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 13 MiB/s wr, 202 op/s; 22539/184692 objects degraded (12.204%); 1368/184692 objects misplaced (0.741%); 11 MiB/s, 179 objects/s recovering 2024-09-21T10:02:24.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 22539/184692 objects degraded (12.204%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:24.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:24 smithi076 ceph-mon[110963]: pgmap v1393: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 13 MiB/s wr, 202 op/s; 22539/184692 objects degraded (12.204%); 1368/184692 objects misplaced (0.741%); 11 MiB/s, 179 objects/s recovering 2024-09-21T10:02:24.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 22539/184692 objects degraded (12.204%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:24.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:24.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 70 16 63410 63394 56.5865 26.1875 0.00516861 0.0176277 2024-09-21T10:02:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:24.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:24.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:24.871 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:24.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:24.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:24.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:25.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:25.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:25.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:25.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:25.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:25.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:25.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:25.362 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 3] out_osds: [6, 7, 5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:02:25.363 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:02:25.363 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-21T10:02:25.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 6 2024-09-21T10:02:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:25.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:25.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:25.515 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:25.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:25.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:25.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:25.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:25.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 71 16 63771 63755 56.1073 22.5625 0.00755872 0.0177627 2024-09-21T10:02:25.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:25.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:25.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:25.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:25.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:25.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:26.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:26.035 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:26.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:26.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:26.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:26.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:26.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:26.393 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:26.423 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:26.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:26.546 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:26.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:26 smithi191 ceph-mon[102206]: pgmap v1394: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 17 MiB/s wr, 266 op/s; 22539/187404 objects degraded (12.027%); 1368/187404 objects misplaced (0.730%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:02:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:26 smithi076 ceph-mon[106482]: pgmap v1394: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 17 MiB/s wr, 266 op/s; 22539/187404 objects degraded (12.027%); 1368/187404 objects misplaced (0.730%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:02:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:26 smithi076 ceph-mon[110963]: pgmap v1394: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 17 MiB/s wr, 266 op/s; 22539/187404 objects degraded (12.027%); 1368/187404 objects misplaced (0.730%); 9.2 MiB/s, 147 objects/s recovering 2024-09-21T10:02:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:26.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:26.669 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:26.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:26.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 72 16 64253 64237 55.7465 30.125 0.00958936 0.0178949 2024-09-21T10:02:26.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:26.791 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:26.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:26.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:26.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:27.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:27.048 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:27.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:27.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:27.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:27.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:27.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:27.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:27.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:27.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:27.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:27.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:27.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:27.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 73 16 64800 64784 55.4512 34.1875 0.00956699 0.0180088 2024-09-21T10:02:27.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:27.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:27.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:27.961 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:27.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:28.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:28.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:28.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:28.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:28.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:28.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:28.462 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:28.497 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:28.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:28.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:28 smithi191 ceph-mon[102206]: pgmap v1395: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 20 MiB/s wr, 327 op/s; 22539/190236 objects degraded (11.848%); 797/190236 objects misplaced (0.419%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:28.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:28 smithi076 ceph-mon[106482]: pgmap v1395: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 20 MiB/s wr, 327 op/s; 22539/190236 objects degraded (11.848%); 797/190236 objects misplaced (0.419%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:28.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:28 smithi076 ceph-mon[110963]: pgmap v1395: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 20 MiB/s wr, 327 op/s; 22539/190236 objects degraded (11.848%); 797/190236 objects misplaced (0.419%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:28.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:28.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 74 16 65352 65336 55.168 34.5 0.00337185 0.0181005 2024-09-21T10:02:28.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:28.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:28.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:28.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:28.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:28.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:29.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:29.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:29.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:29.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:29.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:29.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:29.381 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:29.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:29.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:29.483 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 22539/190236 objects degraded (11.848%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:29.483 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 22539/190236 objects degraded (11.848%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:29.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:29.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:29.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 22539/190236 objects degraded (11.848%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:29.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:29.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:29.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 75 16 65777 65761 54.7867 26.5625 0.0502385 0.0182118 2024-09-21T10:02:29.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:29.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:29.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:29.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:29.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:30.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:30.026 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:30.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:30.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:30.213 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:30.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:30.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:30.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:30.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:30.417 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:30.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:30.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:30.593 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:30 smithi076 ceph-mon[106482]: pgmap v1396: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 23 MiB/s wr, 369 op/s; 22539/191589 objects degraded (11.764%); 797/191589 objects misplaced (0.416%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:30.594 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:30 smithi076 ceph-mon[110963]: pgmap v1396: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 23 MiB/s wr, 369 op/s; 22539/191589 objects degraded (11.764%); 797/191589 objects misplaced (0.416%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:30.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:30 smithi191 ceph-mon[102206]: pgmap v1396: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 3.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 23 MiB/s wr, 369 op/s; 22539/191589 objects degraded (11.764%); 797/191589 objects misplaced (0.416%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:30.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:30.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:30.721 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 76 16 66236 66220 54.4433 28.6875 0.00493015 0.0183393 2024-09-21T10:02:30.777 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:30.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:30.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:30.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:31.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:31.087 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:31.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:31.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:31.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:31.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:31.315 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.348 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:31.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:31.372 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:02:31.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:31.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:31.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:31.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:31.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:31.696 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:31.723 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 77 16 66729 66713 54.135 30.8125 0.0096866 0.0184347 2024-09-21T10:02:31.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:31.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:31.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:31.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:31.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:32.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:32.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:32.137 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:32.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:32.260 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:32.362 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:32.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:32.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:32.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.517 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:32.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:32 smithi191 ceph-mon[102206]: pgmap v1397: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 27 MiB/s wr, 435 op/s; 22539/194241 objects degraded (11.604%); 797/194241 objects misplaced (0.410%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:32.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:32 smithi076 ceph-mon[106482]: pgmap v1397: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 27 MiB/s wr, 435 op/s; 22539/194241 objects degraded (11.604%); 797/194241 objects misplaced (0.410%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:32 smithi076 ceph-mon[110963]: pgmap v1397: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 27 MiB/s wr, 435 op/s; 22539/194241 objects degraded (11.604%); 797/194241 objects misplaced (0.410%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:02:32.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:32.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:32.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 78 16 67127 67111 53.7599 24.875 0.0034588 0.0185586 2024-09-21T10:02:32.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:32.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:32.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:32.894 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:32.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:32.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:33.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:33.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:33.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:33.145 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:33.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:33.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:33.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:33.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:33.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:33.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:33.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:33.528 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:33.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:33 smithi191 ceph-mon[102206]: osdmap e1159: 8 total, 8 up, 5 in 2024-09-21T10:02:33.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:33 smithi076 ceph-mon[106482]: osdmap e1159: 8 total, 8 up, 5 in 2024-09-21T10:02:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:02:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:02:33] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-09-21T10:02:33.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:33 smithi076 ceph-mon[110963]: osdmap e1159: 8 total, 8 up, 5 in 2024-09-21T10:02:33.630 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:33.663 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:33.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 79 16 67522 67506 53.392 24.6875 0.0570398 0.0187056 2024-09-21T10:02:33.765 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:33.798 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:33.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:33.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:33.932 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:33.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:34.067 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.169 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:34.199 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:34.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:34.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:34.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:34.453 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:34.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:34 smithi191 ceph-mon[102206]: pgmap v1399: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 30 MiB/s wr, 474 op/s; 22539/198936 objects degraded (11.330%); 84/198936 objects misplaced (0.042%); 8.0 MiB/s, 128 objects/s recovering 2024-09-21T10:02:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:34 smithi191 ceph-mon[102206]: osdmap e1160: 8 total, 8 up, 5 in 2024-09-21T10:02:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 22539/198936 objects degraded (11.330%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:34 smithi076 ceph-mon[106482]: pgmap v1399: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 30 MiB/s wr, 474 op/s; 22539/198936 objects degraded (11.330%); 84/198936 objects misplaced (0.042%); 8.0 MiB/s, 128 objects/s recovering 2024-09-21T10:02:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:34 smithi076 ceph-mon[106482]: osdmap e1160: 8 total, 8 up, 5 in 2024-09-21T10:02:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 22539/198936 objects degraded (11.330%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:34 smithi076 ceph-mon[110963]: pgmap v1399: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 30 MiB/s wr, 474 op/s; 22539/198936 objects degraded (11.330%); 84/198936 objects misplaced (0.042%); 8.0 MiB/s, 128 objects/s recovering 2024-09-21T10:02:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:34 smithi076 ceph-mon[110963]: osdmap e1160: 8 total, 8 up, 5 in 2024-09-21T10:02:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 22539/198936 objects degraded (11.330%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:34.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:34.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:34.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:02:34.722925+0000 min lat: 0.00130696 max lat: 7.83808 avg lat: 0.0187977 2024-09-21T10:02:34.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:02:34.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 80 16 67967 67951 53.0722 27.8125 0.00505045 0.0187977 2024-09-21T10:02:34.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:34.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:34.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:34.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:34.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:34.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:35.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:35.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:35.169 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:02:35.170 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:02:35.170 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:02:35.170 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:02:35.170 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:02:35.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:35.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:35.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:35.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:35.333 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:35.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:35.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:35.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:35.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:35.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:35.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:35.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:35.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 81 16 68439 68423 52.7811 29.5 0.320089 0.0189089 2024-09-21T10:02:35.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:35.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:35.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:35.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:35.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:35.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:36.070 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:36.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:36.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:36.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:36.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:36.366 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:36.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:36.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:36.481 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:36 smithi076 ceph-mon[110963]: pgmap v1401: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 29 MiB/s wr, 467 op/s; 22538/201447 objects degraded (11.188%); 84/201447 objects misplaced (0.042%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:02:36.481 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:36 smithi076 ceph-mon[106482]: pgmap v1401: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 29 MiB/s wr, 467 op/s; 22538/201447 objects degraded (11.188%); 84/201447 objects misplaced (0.042%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:02:36.513 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:36.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:02:36.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:36 smithi191 ceph-mon[102206]: pgmap v1401: 89 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 29 MiB/s wr, 467 op/s; 22538/201447 objects degraded (11.188%); 84/201447 objects misplaced (0.042%); 5.6 MiB/s, 89 objects/s recovering 2024-09-21T10:02:36.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:36.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:36.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:36.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 82 16 68926 68910 52.5086 30.4375 0.00351836 0.0190105 2024-09-21T10:02:36.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:36.791 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:36.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:36.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:36.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:36.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:37.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:37.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:37.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:37.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:37.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:37.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:37.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:37.316 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:02:37.317 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:37.345 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:02:37.345 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:37.375 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:02:37.375 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:37.397 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1393410479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:37.398 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1393410479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:37.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:37.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:02:37.405 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:37.430 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:37.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:37.433 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:02:37.434 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:37.461 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:02:37.461 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:37.487 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:02:37.488 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:37.512 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:37.513 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:02:37.514 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:37.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:37.553 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:37.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:37.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1393410479' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:37.655 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:37.687 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:37.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:37.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 83 16 69325 69309 52.1765 24.9375 0.00951524 0.0191175 2024-09-21T10:02:37.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:37.796 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:37.825 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:37.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:37.966 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:37.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:38.067 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:38.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:02:37.865+0000 7f240edc2640 -1 mon.a@0(leader).osd e1160 definitely_dead 0 2024-09-21T10:02:38.190 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:38.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:38.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:38.315 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.6. 2024-09-21T10:02:38.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:38.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:38.461 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:38.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:38.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:38.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:38 smithi191 ceph-mon[102206]: pgmap v1402: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 33 MiB/s wr, 530 op/s; 18850/204336 objects degraded (9.225%); 3447/204336 objects misplaced (1.687%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:02:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1712562856' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:02:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:38 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:02:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:38 smithi076 ceph-mon[106482]: pgmap v1402: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 33 MiB/s wr, 530 op/s; 18850/204336 objects degraded (9.225%); 3447/204336 objects misplaced (1.687%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:02:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1712562856' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:02:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:38 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:02:38.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:38 smithi076 ceph-mon[110963]: pgmap v1402: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 33 MiB/s wr, 530 op/s; 18850/204336 objects degraded (9.225%); 3447/204336 objects misplaced (1.687%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:02:38.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1712562856' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:02:38.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:38 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:02:38.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:38.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:38.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 84 16 69623 69607 51.7771 18.625 0.00285288 0.0192607 2024-09-21T10:02:38.810 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:38.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:38.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:38.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:38.995 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:38.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:39.097 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:39.128 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:39.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:39.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:39.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:39.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:39.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:39.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:39.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:39.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:39.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:39.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:39 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:02:39.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:39 smithi191 ceph-mon[102206]: osdmap e1161: 8 total, 8 up, 6 in 2024-09-21T10:02:39.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 18850/204336 objects degraded (9.225%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:39.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:39 smithi191 ceph-mon[102206]: osdmap e1162: 8 total, 8 up, 6 in 2024-09-21T10:02:39.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:39.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[110963]: osdmap e1161: 8 total, 8 up, 6 in 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 18850/204336 objects degraded (9.225%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[110963]: osdmap e1162: 8 total, 8 up, 6 in 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[106482]: osdmap e1161: 8 total, 8 up, 6 in 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 18850/204336 objects degraded (9.225%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:39 smithi076 ceph-mon[106482]: osdmap e1162: 8 total, 8 up, 6 in 2024-09-21T10:02:39.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:39.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 85 16 69652 69636 51.1894 1.8125 0.0248142 0.0192936 2024-09-21T10:02:39.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:39.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:39.866 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:39.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:39.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:40.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:40.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:40.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:40.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:40.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.274 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:40.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:40 smithi076 ceph-mon[106482]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 22 MiB/s wr, 352 op/s; 18850/205590 objects degraded (9.169%); 3447/205590 objects misplaced (1.677%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:02:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:40 smithi076 ceph-mon[106482]: osdmap e1163: 8 total, 8 up, 6 in 2024-09-21T10:02:40.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:40 smithi076 ceph-mon[110963]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 22 MiB/s wr, 352 op/s; 18850/205590 objects degraded (9.169%); 3447/205590 objects misplaced (1.677%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:02:40.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:40 smithi076 ceph-mon[110963]: osdmap e1163: 8 total, 8 up, 6 in 2024-09-21T10:02:40.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:40.396 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:40.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:40.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:40.520 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:40.587 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-21T10:02:40.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:40 smithi191 ceph-mon[102206]: pgmap v1404: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 22 MiB/s wr, 352 op/s; 18850/205590 objects degraded (9.169%); 3447/205590 objects misplaced (1.677%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:02:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:40 smithi191 ceph-mon[102206]: osdmap e1163: 8 total, 8 up, 6 in 2024-09-21T10:02:40.622 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:40.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:40.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 86 16 69986 69970 50.837 20.875 0.393347 0.0196387 2024-09-21T10:02:40.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:40.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:40.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:40.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:02:40.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:40.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:40.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:40.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:41.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:41.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:41.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:41.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:41.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:41.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:41.306 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:41.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:41.433 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:41.514 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:41.514 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:41.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:41.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:41.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:41.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.693 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:41.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:41.724 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 87 16 70535 70519 50.647 34.3125 0.00581132 0.0197071 2024-09-21T10:02:41.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:41.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:41.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:41.940 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:41.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:42.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:42.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:42.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:42.203 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:42.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:42.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:42 smithi076 ceph-mon[106482]: pgmap v1407: 89 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 80 active+clean; 4.2 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 21 MiB/s wr, 343 op/s; 19438/207636 objects degraded (9.362%); 6194/207636 objects misplaced (2.983%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T10:02:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:42 smithi076 ceph-mon[110963]: pgmap v1407: 89 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 80 active+clean; 4.2 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 21 MiB/s wr, 343 op/s; 19438/207636 objects degraded (9.362%); 6194/207636 objects misplaced (2.983%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T10:02:42.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:42.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:42.485 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:02:42.485 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:02:42.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:42.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:42 smithi191 ceph-mon[102206]: pgmap v1407: 89 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 80 active+clean; 4.2 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 21 MiB/s wr, 343 op/s; 19438/207636 objects degraded (9.362%); 6194/207636 objects misplaced (2.983%); 3.5 MiB/s, 55 objects/s recovering 2024-09-21T10:02:42.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:42.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:42.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 88 16 71011 70995 50.4096 29.75 0.00469592 0.0198074 2024-09-21T10:02:42.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:42.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:42.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:42.984 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:42.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:43.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:43.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:43.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:43.186 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:02:43.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:43.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:43.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:43.363 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:02:43.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:43.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:43.502 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:43.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:43.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:43.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1518226891' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:02:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:43 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:02:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1518226891' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:02:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:43 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:02:43.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:02:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:02:43] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.51.0" 2024-09-21T10:02:43.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1518226891' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:02:43.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:43 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:02:43.625 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:43.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:43.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 89 16 71500 71484 50.1866 30.5625 0.496781 0.0198909 2024-09-21T10:02:43.727 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:43.765 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:43.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:43.867 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:43.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:43.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:43.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:43.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:44.030 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:44.052 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:44.132 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:44.154 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:44.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:44.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:44.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:44.382 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:44.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:44.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:44.535 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:44.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:44 smithi191 ceph-mon[102206]: pgmap v1408: 89 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 22 MiB/s wr, 352 op/s; 32136/210690 objects degraded (15.253%); 2747/210690 objects misplaced (1.304%); 1.8 MiB/s, 1 keys/s, 30 objects/s recovering 2024-09-21T10:02:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:44 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:44 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:02:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:44 smithi191 ceph-mon[102206]: osdmap e1164: 8 total, 8 up, 6 in 2024-09-21T10:02:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 32136/210690 objects degraded (15.253%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[106482]: pgmap v1408: 89 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 22 MiB/s wr, 352 op/s; 32136/210690 objects degraded (15.253%); 2747/210690 objects misplaced (1.304%); 1.8 MiB/s, 1 keys/s, 30 objects/s recovering 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[106482]: osdmap e1164: 8 total, 8 up, 6 in 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 32136/210690 objects degraded (15.253%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[110963]: pgmap v1408: 89 pgs: 1 remapped+peering, 1 active+recovery_wait+degraded, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 22 MiB/s wr, 352 op/s; 32136/210690 objects degraded (15.253%); 2747/210690 objects misplaced (1.304%); 1.8 MiB/s, 1 keys/s, 30 objects/s recovering 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[110963]: osdmap e1164: 8 total, 8 up, 6 in 2024-09-21T10:02:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 32136/210690 objects degraded (15.253%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:44.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:44.662 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:44.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 90 13 71986 71973 49.9685 30.5625 0.0106001 0.019974 2024-09-21T10:02:44.764 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:44.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:44.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:44.887 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:44.908 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:44.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:45.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:45.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:45.081 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:45.133 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:45.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:45.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:45.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:45.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:45.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:45.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:45.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:45.523 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:45.587 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 3, 6] out_osds: [7, 5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:02:45.587 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:02:45.588 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2024-09-21T10:02:45.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:02:45.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:45.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:45.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total time run: 90.2559 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total writes made: 71986 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Write size: 65536 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object size: 65536 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Bandwidth (MB/sec): 49.8485 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Bandwidth: 47.2352 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max bandwidth (MB/sec): 200.312 2024-09-21T10:02:45.725 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average IOPS: 797 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev IOPS: 755.782 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max IOPS: 3205 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min IOPS: 0 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average Latency(s): 0.0200251 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Latency(s): 0.129422 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max latency(s): 7.83808 2024-09-21T10:02:45.726 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min latency(s): 0.00130696 2024-09-21T10:02:45.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:02:45.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:45.782 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:45.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:45.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:45.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:45.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:45.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:46.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:46.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:46.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:02:46.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:46.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:46.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:46.329 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:46.431 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:46.467 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:46.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:46.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:46.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:46 smithi191 ceph-mon[102206]: pgmap v1410: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 36503/213066 objects degraded (17.132%); 2709/213066 objects misplaced (1.271%); 2.2 MiB/s, 1 keys/s, 36 objects/s recovering 2024-09-21T10:02:46.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:46 smithi076 ceph-mon[106482]: pgmap v1410: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 36503/213066 objects degraded (17.132%); 2709/213066 objects misplaced (1.271%); 2.2 MiB/s, 1 keys/s, 36 objects/s recovering 2024-09-21T10:02:46.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:46 smithi076 ceph-mon[110963]: pgmap v1410: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 36503/213066 objects degraded (17.132%); 2709/213066 objects misplaced (1.271%); 2.2 MiB/s, 1 keys/s, 36 objects/s recovering 2024-09-21T10:02:46.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:46.712 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:46.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:46.835 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:46.925 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:02:46.925 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:02:46.933 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:46.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:46.959 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:02:46.959 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:02:46.974 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:46.977 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:46.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:46.998 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:02:46.998 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:02:47.031 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:02:47.031 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:02:47.056 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:02:47.056 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:02:47.076 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:02:47.076 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:02:47.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:47.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:47.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:47.101 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:02:47.101 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:02:47.123 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:02:47.123 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:02:47.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:47.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:47.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:47.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:47.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:47.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:47.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:47.600 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:47.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:47 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/527777569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/527777569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/527777569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:02:47.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:47.723 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:47.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:47.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:47.846 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:47.948 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:47.977 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:47.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:48.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:48.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:48.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:48.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:48.233 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:48.335 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:48.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:48.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:48.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:48.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:48.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:48.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:48.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:48.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:48 smithi191 ceph-mon[102206]: pgmap v1411: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 134 B/s rd, 27 MiB/s wr, 458 op/s; 36503/214713 objects degraded (17.001%); 2709/214713 objects misplaced (1.262%); 1.7 MiB/s, 1 keys/s, 29 objects/s recovering 2024-09-21T10:02:48.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:48 smithi076 ceph-mon[106482]: pgmap v1411: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 134 B/s rd, 27 MiB/s wr, 458 op/s; 36503/214713 objects degraded (17.001%); 2709/214713 objects misplaced (1.262%); 1.7 MiB/s, 1 keys/s, 29 objects/s recovering 2024-09-21T10:02:48.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:48 smithi076 ceph-mon[110963]: pgmap v1411: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 134 B/s rd, 27 MiB/s wr, 458 op/s; 36503/214713 objects degraded (17.001%); 2709/214713 objects misplaced (1.262%); 1.7 MiB/s, 1 keys/s, 29 objects/s recovering 2024-09-21T10:02:48.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:48.732 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:48.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:48.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:48.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:48.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:48.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:48.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:49.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:49.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:49.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:49.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:49.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:49.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:49.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:49.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:49.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:49.485 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/214713 objects degraded (17.001%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:49.485 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/214713 objects degraded (17.001%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:49.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:49.606 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:49.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:49.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:49.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:49.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:49.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/214713 objects degraded (17.001%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:49.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:49.985 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:49.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:49.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:49.993 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:02:49.994 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:02:49.994 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:02:49.994 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:02:50.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:50.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:50.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:50.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:50.233 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:50.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:50.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:50.356 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:50.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:50.479 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:50.581 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:50.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:50.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:50.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:50 smithi191 ceph-mon[102206]: pgmap v1412: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 118 B/s rd, 20 MiB/s wr, 351 op/s; 36503/214905 objects degraded (16.986%); 2709/214905 objects misplaced (1.261%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2024-09-21T10:02:50.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:50 smithi076 ceph-mon[106482]: pgmap v1412: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 118 B/s rd, 20 MiB/s wr, 351 op/s; 36503/214905 objects degraded (16.986%); 2709/214905 objects misplaced (1.261%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2024-09-21T10:02:50.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:50 smithi076 ceph-mon[110963]: pgmap v1412: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 118 B/s rd, 20 MiB/s wr, 351 op/s; 36503/214905 objects degraded (16.986%); 2709/214905 objects misplaced (1.261%); 1.4 MiB/s, 1 keys/s, 24 objects/s recovering 2024-09-21T10:02:50.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:50.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:50.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:50.860 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:50.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:50.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:50.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:50.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:51.085 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:51.106 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:51.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:02:51.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:51.247 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:51.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:51.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:02:51.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:51.402 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:51.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:51.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:51 smithi076 ceph-mon[106482]: pgmap v1413: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 17 MiB/s wr, 370 op/s; 36503/213312 objects degraded (17.112%); 2642/213312 objects misplaced (1.239%); 1.6 MiB/s, 0 keys/s, 27 objects/s recovering 2024-09-21T10:02:51.471 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:51 smithi076 ceph-mon[110963]: pgmap v1413: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 17 MiB/s wr, 370 op/s; 36503/213312 objects degraded (17.112%); 2642/213312 objects misplaced (1.239%); 1.6 MiB/s, 0 keys/s, 27 objects/s recovering 2024-09-21T10:02:51.504 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:51.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:51.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:51.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:51.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:51.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:51.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:51.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:51.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:51 smithi191 ceph-mon[102206]: pgmap v1413: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 17 MiB/s wr, 370 op/s; 36503/213312 objects degraded (17.112%); 2642/213312 objects misplaced (1.239%); 1.6 MiB/s, 0 keys/s, 27 objects/s recovering 2024-09-21T10:02:51.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:51.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:51.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:51.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:52.017 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:52.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:52.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:52.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:52.270 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:52.298 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:52.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:52.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:52.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:52.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:52.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:52.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:52.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:52.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:52.808 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:52.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:52.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:52.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:52.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:52.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:53.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:53.047 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:02:53.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:53.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:53.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:53.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:53.304 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:53.329 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:02:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:02:53] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.51.0" 2024-09-21T10:02:53.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:53.428 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:53.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:53.559 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:53.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:53.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:53.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:53.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.818 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:53.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:53.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:53.926 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:53 smithi076 ceph-mon[106482]: pgmap v1414: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 11 MiB/s wr, 375 op/s; 36503/210126 objects degraded (17.372%); 2642/210126 objects misplaced (1.257%); 608 KiB/s, 9 objects/s recovering 2024-09-21T10:02:53.926 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:53 smithi076 ceph-mon[110963]: pgmap v1414: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 11 MiB/s wr, 375 op/s; 36503/210126 objects degraded (17.372%); 2642/210126 objects misplaced (1.257%); 608 KiB/s, 9 objects/s recovering 2024-09-21T10:02:53.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:53.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:54.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:54.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:53 smithi191 ceph-mon[102206]: pgmap v1414: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 11 MiB/s wr, 375 op/s; 36503/210126 objects degraded (17.372%); 2642/210126 objects misplaced (1.257%); 608 KiB/s, 9 objects/s recovering 2024-09-21T10:02:54.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:54.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:54.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:54.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:54.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:54.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:54.462 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:54.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:54.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:54.687 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:54.710 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:54.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:54.834 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:54.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:54.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:54.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:54.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:54.981 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/210126 objects degraded (17.372%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:54.982 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2029002064' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:54.982 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:54 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:54.982 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/210126 objects degraded (17.372%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:54.982 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2029002064' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:54.982 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:54 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:54.988 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:02:55.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:55.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/210126 objects degraded (17.372%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:02:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2029002064' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:54 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:02:55.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:55.215 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:55.311 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:02:55.314 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:02:55.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:55.338 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:55.440 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:55.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:55.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:55.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:55.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:55.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:55.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:55.832 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:55.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:55.956 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:55.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:56.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:56.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: pgmap v1415: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 91 B/s rd, 9.9 MiB/s wr, 391 op/s; 36503/208209 objects degraded (17.532%); 2554/208209 objects misplaced (1.227%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: osdmap e1165: 8 total, 8 up, 6 in 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: from='client.44996 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: pgmap v1415: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 91 B/s rd, 9.9 MiB/s wr, 391 op/s; 36503/208209 objects degraded (17.532%); 2554/208209 objects misplaced (1.227%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: osdmap e1165: 8 total, 8 up, 6 in 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: from='client.44996 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: pgmap v1415: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 91 B/s rd, 9.9 MiB/s wr, 391 op/s; 36503/208209 objects degraded (17.532%); 2554/208209 objects misplaced (1.227%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: osdmap e1165: 8 total, 8 up, 6 in 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: from='client.44996 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:02:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:56.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:56.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:56.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:56.353 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:56.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:56.495 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:56.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:56.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:56.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:56.728 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:56.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:56.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:56.852 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:56.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:56.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:56.975 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:57.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:57.133 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:57.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:57.257 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:57.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:57.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:57.481 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:57.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:57.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:57.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:57.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:57.754 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:57.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:57 smithi076 ceph-mon[110963]: pgmap v1417: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 1.4 MiB/s wr, 354 op/s; 36503/205371 objects degraded (17.774%); 2554/205371 objects misplaced (1.244%); 953 KiB/s, 14 objects/s recovering 2024-09-21T10:02:57.852 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:57 smithi076 ceph-mon[106482]: pgmap v1417: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 1.4 MiB/s wr, 354 op/s; 36503/205371 objects degraded (17.774%); 2554/205371 objects misplaced (1.244%); 953 KiB/s, 14 objects/s recovering 2024-09-21T10:02:57.855 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:02:57.876 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:57.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:02:57.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:57.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:57.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:58.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:02:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:57 smithi191 ceph-mon[102206]: pgmap v1417: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 1.4 MiB/s wr, 354 op/s; 36503/205371 objects degraded (17.774%); 2554/205371 objects misplaced (1.244%); 953 KiB/s, 14 objects/s recovering 2024-09-21T10:02:58.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:02:58.237 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:58.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:58.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:02:58.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:02:58.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:58.523 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:58.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:58.646 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:58.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:58.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:58.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:58.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:58.896 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:58.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:58.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:02:59.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:59.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:02:59.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:59.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:59.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:59.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:02:59.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:59.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:02:59.375 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:59.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:02:59.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:59.531 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:59.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:59.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:02:59.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:02:59.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:02:59.776 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:02:59.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:02:59.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:02:59.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:02:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:02:59.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:00.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:00.024 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:00.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:59 smithi191 ceph-mon[102206]: pgmap v1418: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 486 KiB/s wr, 357 op/s; 36503/204627 objects degraded (17.839%); 2554/204627 objects misplaced (1.248%); 953 KiB/s, 14 objects/s recovering 2024-09-21T10:03:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:02:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/204627 objects degraded (17.839%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:59 smithi076 ceph-mon[106482]: pgmap v1418: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 486 KiB/s wr, 357 op/s; 36503/204627 objects degraded (17.839%); 2554/204627 objects misplaced (1.248%); 953 KiB/s, 14 objects/s recovering 2024-09-21T10:03:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:02:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/204627 objects degraded (17.839%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:59 smithi076 ceph-mon[110963]: pgmap v1418: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 486 KiB/s wr, 357 op/s; 36503/204627 objects degraded (17.839%); 2554/204627 objects misplaced (1.248%); 953 KiB/s, 14 objects/s recovering 2024-09-21T10:03:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:02:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/204627 objects degraded (17.839%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:00.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:00.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:00.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:00.270 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:00.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:00.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:00.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:00.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:00.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:00.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:00.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:00.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:00.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:00.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:00.895 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:00.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:00.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:00.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:01.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:01.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:01.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:01.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:01.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:01.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:01.394 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:01.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:01.521 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:01.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:01.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:01.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:01.719 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:03:01.746 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:01.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:01.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:01.861 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:01 smithi076 ceph-mon[106482]: pgmap v1419: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 358 op/s; 36503/202560 objects degraded (18.021%); 2462/202560 objects misplaced (1.215%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:03:01.861 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:01 smithi076 ceph-mon[110963]: pgmap v1419: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 358 op/s; 36503/202560 objects degraded (18.021%); 2462/202560 objects misplaced (1.215%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:03:01.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:01.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:01.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:01.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:02.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:02.380 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:01 smithi191 ceph-mon[102206]: pgmap v1419: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 358 op/s; 36503/202560 objects degraded (18.021%); 2462/202560 objects misplaced (1.215%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:03:02.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:02.504 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:02.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:02.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:02.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:02.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:02.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:02.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:02.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:02.916 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:02.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:03.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:03.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:03.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:03.090 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1416,"stamp":"2024-09-21T10:02:54.982517+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4547221501,"num_objects":69403,"num_object_clones":0,"num_object_copies":208209,"num_objects_missing_on_primary":2554,"num_objects_missing":2554,"num_objects_degraded":36503,"num_objects_misplaced":2554,"num_objects_unfound":0,"num_objects_dirty":69403,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":74876,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8389,"num_bytes_recovered":548811638,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74805,"ondisk_log_size":74805,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":293,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13563796,"kb_used_data":12092772,"kb_used_omap":192,"kb_used_meta":1470783,"kb_avail":548907116,"statfs":{"total":575970213888,"available":562080886784,"internally_reserved":0,"allocated":12382998528,"data_stored":12369388606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":197389,"internal_metadata":1506082035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,1,2,0,1,2,4,4,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-106168296,"num_objects":-1619,"num_object_clones":0,"num_object_copies":-4857,"num_objects_missing_on_primary":-155,"num_objects_missing":-155,"num_objects_degraded":0,"num_objects_misplaced":-155,"num_objects_unfound":0,"num_objects_dirty":-1619,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":3601,"num_write_kb":63361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9764864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.407045"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":163,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.380142+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:02:43.380142+0000","last_peered":"2024-09-21T10:02:43.380142+0000","last_clean":"2024-09-21T10:02:43.380142+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:02:43.380142+0000","last_undegraded":"2024-09-21T10:02:43.380142+0000","last_fullsized":"2024-09-21T10:02:43.380142+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":19,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432864+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:02:41.432864+0000","last_peered":"2024-09-21T10:02:41.432864+0000","last_clean":"2024-09-21T10:02:41.432864+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:02:41.432864+0000","last_undegraded":"2024-09-21T10:02:41.432864+0000","last_fullsized":"2024-09-21T10:02:41.432864+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1141,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.374729+0000","last_change":"2024-09-21T10:02:39.004713+0000","last_active":"2024-09-21T10:02:43.374729+0000","last_peered":"2024-09-21T10:02:43.374729+0000","last_clean":"2024-09-21T10:02:43.374729+0000","last_became_active":"2024-09-21T10:02:39.004389+0000","last_became_peered":"2024-09-21T10:02:39.004389+0000","last_unstale":"2024-09-21T10:02:43.374729+0000","last_undegraded":"2024-09-21T10:02:43.374729+0000","last_fullsized":"2024-09-21T10:02:43.374729+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:50.043186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1527,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.362809+0000","last_change":"2024-09-21T10:01:40.038095+0000","last_active":"2024-09-21T10:02:43.362809+0000","last_peered":"2024-09-21T10:02:43.362809+0000","last_clean":"2024-09-21T10:02:43.362809+0000","last_became_active":"2024-09-21T10:01:40.037789+0000","last_became_peered":"2024-09-21T10:01:40.037789+0000","last_unstale":"2024-09-21T10:02:43.362809+0000","last_undegraded":"2024-09-21T10:02:43.362809+0000","last_fullsized":"2024-09-21T10:02:43.362809+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:55.781742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1527,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.369795+0000","last_change":"2024-09-21T10:01:40.034054+0000","last_active":"2024-09-21T10:02:43.369795+0000","last_peered":"2024-09-21T10:02:43.369795+0000","last_clean":"2024-09-21T10:02:43.369795+0000","last_became_active":"2024-09-21T10:01:40.033685+0000","last_became_peered":"2024-09-21T10:01:40.033685+0000","last_unstale":"2024-09-21T10:02:43.369795+0000","last_undegraded":"2024-09-21T10:02:43.369795+0000","last_fullsized":"2024-09-21T10:02:43.369795+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:47:24.221450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432821+0000","last_change":"2024-09-21T10:02:38.992491+0000","last_active":"2024-09-21T10:02:41.432821+0000","last_peered":"2024-09-21T10:02:41.432821+0000","last_clean":"2024-09-21T10:02:41.432821+0000","last_became_active":"2024-09-21T10:02:38.992051+0000","last_became_peered":"2024-09-21T10:02:38.992051+0000","last_unstale":"2024-09-21T10:02:41.432821+0000","last_undegraded":"2024-09-21T10:02:41.432821+0000","last_fullsized":"2024-09-21T10:02:41.432821+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:41.314858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":163,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.375403+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:02:43.375403+0000","last_peered":"2024-09-21T10:02:43.375403+0000","last_clean":"2024-09-21T10:02:43.375403+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:02:43.375403+0000","last_undegraded":"2024-09-21T10:02:43.375403+0000","last_fullsized":"2024-09-21T10:02:43.375403+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432770+0000","last_change":"2024-09-21T10:02:39.030937+0000","last_active":"2024-09-21T10:02:41.432770+0000","last_peered":"2024-09-21T10:02:41.432770+0000","last_clean":"2024-09-21T10:02:41.432770+0000","last_became_active":"2024-09-21T10:02:39.001617+0000","last_became_peered":"2024-09-21T10:02:39.001617+0000","last_unstale":"2024-09-21T10:02:41.432770+0000","last_undegraded":"2024-09-21T10:02:41.432770+0000","last_fullsized":"2024-09-21T10:02:41.432770+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:23.501776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1164'4686","reported_seq":4758,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:52.134531+0000","last_change":"2024-09-21T10:02:39.142708+0000","last_active":"2024-09-21T10:02:52.134531+0000","last_peered":"2024-09-21T10:02:52.134531+0000","last_clean":"2024-09-21T10:02:10.162551+0000","last_became_active":"2024-09-21T10:02:39.134585+0000","last_became_peered":"2024-09-21T10:02:39.134585+0000","last_unstale":"2024-09-21T10:02:52.134531+0000","last_undegraded":"2024-09-21T10:02:39.015098+0000","last_fullsized":"2024-09-21T10:02:38.977951+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4686,"log_dups_size":0,"ondisk_log_size":4686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285736960,"num_objects":4360,"num_object_clones":0,"num_object_copies":13080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4356,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2","objects":4},{"shards":"1,2,4","objects":627},{"shards":"1,2,5","objects":3566}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":51,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.364015+0000","last_change":"2024-09-21T10:01:53.152342+0000","last_active":"2024-09-21T10:02:43.364015+0000","last_peered":"2024-09-21T10:02:43.364015+0000","last_clean":"2024-09-21T10:02:43.364015+0000","last_became_active":"2024-09-21T10:01:53.151939+0000","last_became_peered":"2024-09-21T10:01:53.151939+0000","last_unstale":"2024-09-21T10:02:43.364015+0000","last_undegraded":"2024-09-21T10:02:43.364015+0000","last_fullsized":"2024-09-21T10:02:43.364015+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:56.840214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432800+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:02:41.432800+0000","last_peered":"2024-09-21T10:02:41.432800+0000","last_clean":"2024-09-21T10:02:41.432800+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:02:41.432800+0000","last_undegraded":"2024-09-21T10:02:41.432800+0000","last_fullsized":"2024-09-21T10:02:41.432800+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1164'4727","reported_seq":4794,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:54.369880+0000","last_change":"2024-09-21T10:02:12.380357+0000","last_active":"2024-09-21T10:02:54.369880+0000","last_peered":"2024-09-21T10:02:54.369880+0000","last_clean":"2024-09-21T10:02:10.175100+0000","last_became_active":"2024-09-21T10:02:12.372954+0000","last_became_peered":"2024-09-21T10:02:12.372954+0000","last_unstale":"2024-09-21T10:02:54.369880+0000","last_undegraded":"2024-09-21T10:02:12.236022+0000","last_fullsized":"2024-09-21T10:02:12.213003+0000","mapping_epoch":1156,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281870336,"num_objects":4301,"num_object_clones":0,"num_object_copies":12903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3787,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4727,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","2","5"],"object_location_counts":[{"shards":"2,3","objects":514},{"shards":"2,3,5","objects":3573}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":51,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387709+0000","last_change":"2024-09-21T10:01:53.139817+0000","last_active":"2024-09-21T10:02:43.387709+0000","last_peered":"2024-09-21T10:02:43.387709+0000","last_clean":"2024-09-21T10:02:43.387709+0000","last_became_active":"2024-09-21T10:01:53.139408+0000","last_became_peered":"2024-09-21T10:01:53.139408+0000","last_unstale":"2024-09-21T10:02:43.387709+0000","last_undegraded":"2024-09-21T10:02:43.387709+0000","last_fullsized":"2024-09-21T10:02:43.387709+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:40:35.257539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1103,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386971+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:02:43.386971+0000","last_peered":"2024-09-21T10:02:43.386971+0000","last_clean":"2024-09-21T10:02:43.386971+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:02:43.386971+0000","last_undegraded":"2024-09-21T10:02:43.386971+0000","last_fullsized":"2024-09-21T10:02:43.386971+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1164'4725","reported_seq":4790,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:51.447077+0000","last_change":"2024-09-21T10:01:05.960487+0000","last_active":"2024-09-21T10:02:51.447077+0000","last_peered":"2024-09-21T10:02:51.447077+0000","last_clean":"2024-09-21T10:02:51.447077+0000","last_became_active":"2024-09-21T10:01:05.960206+0000","last_became_peered":"2024-09-21T10:01:05.960206+0000","last_unstale":"2024-09-21T10:02:51.447077+0000","last_undegraded":"2024-09-21T10:02:51.447077+0000","last_fullsized":"2024-09-21T10:02:51.447077+0000","mapping_epoch":1137,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4725,"log_dups_size":0,"ondisk_log_size":4725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:16.848104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293797888,"num_objects":4483,"num_object_clones":0,"num_object_copies":13449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4725,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":17,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432917+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:02:41.432917+0000","last_peered":"2024-09-21T10:02:41.432917+0000","last_clean":"2024-09-21T10:02:41.432917+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:02:41.432917+0000","last_undegraded":"2024-09-21T10:02:41.432917+0000","last_fullsized":"2024-09-21T10:02:41.432917+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432843+0000","last_change":"2024-09-21T10:02:38.994239+0000","last_active":"2024-09-21T10:02:41.432843+0000","last_peered":"2024-09-21T10:02:41.432843+0000","last_clean":"2024-09-21T10:02:41.432843+0000","last_became_active":"2024-09-21T10:02:38.993866+0000","last_became_peered":"2024-09-21T10:02:38.993866+0000","last_unstale":"2024-09-21T10:02:41.432843+0000","last_undegraded":"2024-09-21T10:02:41.432843+0000","last_fullsized":"2024-09-21T10:02:41.432843+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:23.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1164'4686","reported_seq":5157,"reported_epoch":1164,"state":"active+recovering","last_fresh":"2024-09-21T10:02:54.361194+0000","last_change":"2024-09-21T10:02:39.037166+0000","last_active":"2024-09-21T10:02:54.361194+0000","last_peered":"2024-09-21T10:02:54.361194+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:02:54.361194+0000","last_undegraded":"2024-09-21T10:02:54.361194+0000","last_fullsized":"2024-09-21T10:02:54.361194+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4686,"log_dups_size":0,"ondisk_log_size":4686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804800,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":2554,"num_objects_missing":2554,"num_objects_degraded":0,"num_objects_misplaced":2554,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4686,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":194,"num_bytes_recovered":12713984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":1739},{"shards":"2,3,4,7","objects":7},{"shards":"2,4,7","objects":2554}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1133,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.377920+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:02:43.377920+0000","last_peered":"2024-09-21T10:02:43.377920+0000","last_clean":"2024-09-21T10:02:43.377920+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:02:43.377920+0000","last_undegraded":"2024-09-21T10:02:43.377920+0000","last_fullsized":"2024-09-21T10:02:43.377920+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1516,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.363040+0000","last_change":"2024-09-21T10:02:11.234123+0000","last_active":"2024-09-21T10:02:43.363040+0000","last_peered":"2024-09-21T10:02:43.363040+0000","last_clean":"2024-09-21T10:02:43.363040+0000","last_became_active":"2024-09-21T10:02:11.233791+0000","last_became_peered":"2024-09-21T10:02:11.233791+0000","last_unstale":"2024-09-21T10:02:43.363040+0000","last_undegraded":"2024-09-21T10:02:43.363040+0000","last_fullsized":"2024-09-21T10:02:43.363040+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:00:43.806609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"1164'4662","reported_seq":10354,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:51.440580+0000","last_change":"2024-09-21T10:02:06.188626+0000","last_active":"2024-09-21T10:02:51.440580+0000","last_peered":"2024-09-21T10:02:51.440580+0000","last_clean":"2024-09-21T10:02:51.440580+0000","last_became_active":"2024-09-21T10:02:06.181012+0000","last_became_peered":"2024-09-21T10:02:06.181012+0000","last_unstale":"2024-09-21T10:02:51.440580+0000","last_undegraded":"2024-09-21T10:02:51.440580+0000","last_fullsized":"2024-09-21T10:02:51.440580+0000","mapping_epoch":1153,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4662,"log_dups_size":0,"ondisk_log_size":4662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:42.819309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285212672,"num_objects":4352,"num_object_clones":0,"num_object_copies":13056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4662,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2808,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":160,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.375905+0000","last_change":"2024-09-21T09:56:45.272715+0000","last_active":"2024-09-21T10:02:43.375905+0000","last_peered":"2024-09-21T10:02:43.375905+0000","last_clean":"2024-09-21T10:02:43.375905+0000","last_became_active":"2024-09-21T09:56:45.266211+0000","last_became_peered":"2024-09-21T09:56:45.266211+0000","last_unstale":"2024-09-21T10:02:43.375905+0000","last_undegraded":"2024-09-21T10:02:43.375905+0000","last_fullsized":"2024-09-21T10:02:43.375905+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:04.865828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1148,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.377834+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:02:43.377834+0000","last_peered":"2024-09-21T10:02:43.377834+0000","last_clean":"2024-09-21T10:02:43.377834+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:02:43.377834+0000","last_undegraded":"2024-09-21T10:02:43.377834+0000","last_fullsized":"2024-09-21T10:02:43.377834+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1164'4588","reported_seq":4655,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:51.490006+0000","last_change":"2024-09-21T10:02:18.024476+0000","last_active":"2024-09-21T10:02:51.490006+0000","last_peered":"2024-09-21T10:02:51.490006+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:02:17.996987+0000","last_became_peered":"2024-09-21T10:02:17.996987+0000","last_unstale":"2024-09-21T10:02:51.490006+0000","last_undegraded":"2024-09-21T10:02:16.473692+0000","last_fullsized":"2024-09-21T10:02:12.219366+0000","mapping_epoch":1156,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4588,"log_dups_size":0,"ondisk_log_size":4588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280756224,"num_objects":4284,"num_object_clones":0,"num_object_copies":12852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3686,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4588,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2","objects":598},{"shards":"0,2,5","objects":3534}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432832+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:02:41.432832+0000","last_peered":"2024-09-21T10:02:41.432832+0000","last_clean":"2024-09-21T10:02:41.432832+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:02:41.432832+0000","last_undegraded":"2024-09-21T10:02:41.432832+0000","last_fullsized":"2024-09-21T10:02:41.432832+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":791,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387377+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:02:43.387377+0000","last_peered":"2024-09-21T10:02:43.387377+0000","last_clean":"2024-09-21T10:02:43.387377+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:02:43.387377+0000","last_undegraded":"2024-09-21T10:02:43.387377+0000","last_fullsized":"2024-09-21T10:02:43.387377+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1164'4705","reported_seq":4773,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:51.399055+0000","last_change":"2024-09-21T10:02:40.126603+0000","last_active":"2024-09-21T10:02:51.399055+0000","last_peered":"2024-09-21T10:02:51.399055+0000","last_clean":"2024-09-21T10:02:38.228948+0000","last_became_active":"2024-09-21T10:02:40.117846+0000","last_became_peered":"2024-09-21T10:02:40.117846+0000","last_unstale":"2024-09-21T10:02:51.399055+0000","last_undegraded":"2024-09-21T10:02:40.008548+0000","last_fullsized":"2024-09-21T10:02:39.974178+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291766296,"num_objects":4453,"num_object_clones":0,"num_object_copies":13359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4406,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4453,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4705,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[0,1],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1","objects":47},{"shards":"0,1,3","objects":4280}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1134,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.385891+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:02:43.385891+0000","last_peered":"2024-09-21T10:02:43.385891+0000","last_clean":"2024-09-21T10:02:43.385891+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:02:43.385891+0000","last_undegraded":"2024-09-21T10:02:43.385891+0000","last_fullsized":"2024-09-21T10:02:43.385891+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432927+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:02:41.432927+0000","last_peered":"2024-09-21T10:02:41.432927+0000","last_clean":"2024-09-21T10:02:41.432927+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:02:41.432927+0000","last_undegraded":"2024-09-21T10:02:41.432927+0000","last_fullsized":"2024-09-21T10:02:41.432927+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1164'4850","reported_seq":8444,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:54.220666+0000","last_change":"2024-09-21T10:02:39.200547+0000","last_active":"2024-09-21T10:02:54.220666+0000","last_peered":"2024-09-21T10:02:54.220666+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:02:54.220666+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291373056,"num_objects":4446,"num_object_clones":0,"num_object_copies":13338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4497,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4850,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"0,2,3","objects":1584},{"shards":"0,2,3,7","objects":1556},{"shards":"2,3","objects":-51},{"shards":"2,3,7","objects":1155}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":163,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.411966+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:02:43.411966+0000","last_peered":"2024-09-21T10:02:43.411966+0000","last_clean":"2024-09-21T10:02:43.411966+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:02:43.411966+0000","last_undegraded":"2024-09-21T10:02:43.411966+0000","last_fullsized":"2024-09-21T10:02:43.411966+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1493,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387497+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:02:43.387497+0000","last_peered":"2024-09-21T10:02:43.387497+0000","last_clean":"2024-09-21T10:02:43.387497+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:02:43.387497+0000","last_undegraded":"2024-09-21T10:02:43.387497+0000","last_fullsized":"2024-09-21T10:02:43.387497+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1164'4550","reported_seq":6277,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:51.447108+0000","last_change":"2024-09-21T10:02:39.006738+0000","last_active":"2024-09-21T10:02:51.447108+0000","last_peered":"2024-09-21T10:02:51.447108+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:02:11.352801+0000","last_became_peered":"2024-09-21T10:02:11.352801+0000","last_unstale":"2024-09-21T10:02:51.447108+0000","last_undegraded":"2024-09-21T10:02:39.002670+0000","last_fullsized":"2024-09-21T10:02:11.227814+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281149440,"num_objects":4290,"num_object_clones":0,"num_object_copies":12870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3286,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4550,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":826,"num_bytes_recovered":54132736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3","objects":1004},{"shards":"0,3,5","objects":957},{"shards":"0,3,5,7","objects":23},{"shards":"0,3,7","objects":2306}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":683,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:43.361895+0000","last_change":"2024-09-21T10:02:40.013983+0000","last_active":"2024-09-21T10:02:43.361895+0000","last_peered":"2024-09-21T10:02:43.361895+0000","last_clean":"2024-09-21T10:02:33.294112+0000","last_became_active":"2024-09-21T10:02:40.013674+0000","last_became_peered":"2024-09-21T10:02:40.013674+0000","last_unstale":"2024-09-21T10:02:43.361895+0000","last_undegraded":"2024-09-21T10:02:39.973101+0000","last_fullsized":"2024-09-21T10:02:39.972506+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1140,"reported_epoch":1164,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:02:43.375488+0000","last_change":"2024-09-21T10:02:39.000712+0000","last_active":"2024-09-21T10:02:43.375488+0000","last_peered":"2024-09-21T10:02:43.375488+0000","last_clean":"2024-09-21T10:02:33.315699+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:02:43.375488+0000","last_undegraded":"2024-09-21T10:02:38.970353+0000","last_fullsized":"2024-09-21T10:02:43.375488+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":["3","0","2"],"object_location_counts":[{"shards":"0,2,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":177,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.379390+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:02:43.379390+0000","last_peered":"2024-09-21T10:02:43.379390+0000","last_clean":"2024-09-21T10:02:43.379390+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:02:43.379390+0000","last_undegraded":"2024-09-21T10:02:43.379390+0000","last_fullsized":"2024-09-21T10:02:43.379390+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432907+0000","last_change":"2024-09-21T10:02:38.992338+0000","last_active":"2024-09-21T10:02:41.432907+0000","last_peered":"2024-09-21T10:02:41.432907+0000","last_clean":"2024-09-21T10:02:41.432907+0000","last_became_active":"2024-09-21T10:02:38.992058+0000","last_became_peered":"2024-09-21T10:02:38.992058+0000","last_unstale":"2024-09-21T10:02:41.432907+0000","last_undegraded":"2024-09-21T10:02:41.432907+0000","last_fullsized":"2024-09-21T10:02:41.432907+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:48:40.910171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1164'4608","reported_seq":10178,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:51.493400+0000","last_change":"2024-09-21T10:02:33.305817+0000","last_active":"2024-09-21T10:02:51.493400+0000","last_peered":"2024-09-21T10:02:51.493400+0000","last_clean":"2024-09-21T10:02:51.493400+0000","last_became_active":"2024-09-21T10:02:33.298689+0000","last_became_peered":"2024-09-21T10:02:33.298689+0000","last_unstale":"2024-09-21T10:02:51.493400+0000","last_undegraded":"2024-09-21T10:02:51.493400+0000","last_fullsized":"2024-09-21T10:02:51.493400+0000","mapping_epoch":1159,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4608,"log_dups_size":0,"ondisk_log_size":4608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:44.863551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284295168,"num_objects":4338,"num_object_clones":0,"num_object_copies":13014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4608,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":770,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.394285+0000","last_change":"2024-09-21T10:01:40.044254+0000","last_active":"2024-09-21T10:02:43.394285+0000","last_peered":"2024-09-21T10:02:43.394285+0000","last_clean":"2024-09-21T10:02:43.394285+0000","last_became_active":"2024-09-21T10:01:40.043908+0000","last_became_peered":"2024-09-21T10:01:40.043908+0000","last_unstale":"2024-09-21T10:02:43.394285+0000","last_undegraded":"2024-09-21T10:02:43.394285+0000","last_fullsized":"2024-09-21T10:02:43.394285+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:39.819510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1521,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.363675+0000","last_change":"2024-09-21T09:56:45.265575+0000","last_active":"2024-09-21T10:02:43.363675+0000","last_peered":"2024-09-21T10:02:43.363675+0000","last_clean":"2024-09-21T10:02:43.363675+0000","last_became_active":"2024-09-21T09:56:45.265247+0000","last_became_peered":"2024-09-21T09:56:45.265247+0000","last_unstale":"2024-09-21T10:02:43.363675+0000","last_undegraded":"2024-09-21T10:02:43.363675+0000","last_fullsized":"2024-09-21T10:02:43.363675+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:28.902055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1519,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.369553+0000","last_change":"2024-09-21T09:50:11.815820+0000","last_active":"2024-09-21T10:02:43.369553+0000","last_peered":"2024-09-21T10:02:43.369553+0000","last_clean":"2024-09-21T10:02:43.369553+0000","last_became_active":"2024-09-21T09:50:11.815499+0000","last_became_peered":"2024-09-21T09:50:11.815499+0000","last_unstale":"2024-09-21T10:02:43.369553+0000","last_undegraded":"2024-09-21T10:02:43.369553+0000","last_fullsized":"2024-09-21T10:02:43.369553+0000","mapping_epoch":796,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":797,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:53:05.361801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.4","version":"1164'4422","reported_seq":4487,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:51.507895+0000","last_change":"2024-09-21T10:01:05.959955+0000","last_active":"2024-09-21T10:02:51.507895+0000","last_peered":"2024-09-21T10:02:51.507895+0000","last_clean":"2024-09-21T10:02:51.507895+0000","last_became_active":"2024-09-21T10:01:05.959655+0000","last_became_peered":"2024-09-21T10:01:05.959655+0000","last_unstale":"2024-09-21T10:02:51.507895+0000","last_undegraded":"2024-09-21T10:02:51.507895+0000","last_fullsized":"2024-09-21T10:02:51.507895+0000","mapping_epoch":1137,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:09.797118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275251200,"num_objects":4200,"num_object_clones":0,"num_object_copies":12600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4422,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1400,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.379904+0000","last_change":"2024-09-21T10:01:22.663509+0000","last_active":"2024-09-21T10:02:43.379904+0000","last_peered":"2024-09-21T10:02:43.379904+0000","last_clean":"2024-09-21T10:02:43.379904+0000","last_became_active":"2024-09-21T10:01:22.663167+0000","last_became_peered":"2024-09-21T10:01:22.663167+0000","last_unstale":"2024-09-21T10:02:43.379904+0000","last_undegraded":"2024-09-21T10:02:43.379904+0000","last_fullsized":"2024-09-21T10:02:43.379904+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:51:54.557039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1494,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.394564+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:02:43.394564+0000","last_peered":"2024-09-21T10:02:43.394564+0000","last_clean":"2024-09-21T10:02:43.394564+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:02:43.394564+0000","last_undegraded":"2024-09-21T10:02:43.394564+0000","last_fullsized":"2024-09-21T10:02:43.394564+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1495,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387414+0000","last_change":"2024-09-21T10:02:11.238694+0000","last_active":"2024-09-21T10:02:43.387414+0000","last_peered":"2024-09-21T10:02:43.387414+0000","last_clean":"2024-09-21T10:02:43.387414+0000","last_became_active":"2024-09-21T10:02:11.238393+0000","last_became_peered":"2024-09-21T10:02:11.238393+0000","last_unstale":"2024-09-21T10:02:43.387414+0000","last_undegraded":"2024-09-21T10:02:43.387414+0000","last_fullsized":"2024-09-21T10:02:43.387414+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:27.208780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1164'4643","reported_seq":4710,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:51.403406+0000","last_change":"2024-09-21T10:02:40.163625+0000","last_active":"2024-09-21T10:02:51.403406+0000","last_peered":"2024-09-21T10:02:51.403406+0000","last_clean":"2024-09-21T10:02:38.308681+0000","last_became_active":"2024-09-21T10:02:40.150085+0000","last_became_peered":"2024-09-21T10:02:40.150085+0000","last_unstale":"2024-09-21T10:02:51.403406+0000","last_undegraded":"2024-09-21T10:02:40.008230+0000","last_fullsized":"2024-09-21T10:02:39.973996+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4643,"log_dups_size":0,"ondisk_log_size":4643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286720000,"num_objects":4375,"num_object_clones":0,"num_object_copies":13125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4377,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4643,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":4243},{"shards":"0,3","objects":-2}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":64,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386447+0000","last_change":"2024-09-21T10:02:39.003749+0000","last_active":"2024-09-21T10:02:43.386447+0000","last_peered":"2024-09-21T10:02:43.386447+0000","last_clean":"2024-09-21T10:02:43.386447+0000","last_became_active":"2024-09-21T10:02:38.995779+0000","last_became_peered":"2024-09-21T10:02:38.995779+0000","last_unstale":"2024-09-21T10:02:43.386447+0000","last_undegraded":"2024-09-21T10:02:43.386447+0000","last_fullsized":"2024-09-21T10:02:43.386447+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:12.638249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1132,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.363349+0000","last_change":"2024-09-21T10:02:38.991528+0000","last_active":"2024-09-21T10:02:43.363349+0000","last_peered":"2024-09-21T10:02:43.363349+0000","last_clean":"2024-09-21T10:02:43.363349+0000","last_became_active":"2024-09-21T10:02:38.990795+0000","last_became_peered":"2024-09-21T10:02:38.990795+0000","last_unstale":"2024-09-21T10:02:43.363349+0000","last_undegraded":"2024-09-21T10:02:43.363349+0000","last_fullsized":"2024-09-21T10:02:43.363349+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:46:16.467124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1505,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386345+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:02:43.386345+0000","last_peered":"2024-09-21T10:02:43.386345+0000","last_clean":"2024-09-21T10:02:43.386345+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:02:43.386345+0000","last_undegraded":"2024-09-21T10:02:43.386345+0000","last_fullsized":"2024-09-21T10:02:43.386345+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1164'4751","reported_seq":4812,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:54.335645+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:02:54.335645+0000","last_peered":"2024-09-21T10:02:54.335645+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:02:54.335645+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281346048,"num_objects":4293,"num_object_clones":0,"num_object_copies":12879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4751,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":4138},{"shards":"3,4","objects":-74}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1095,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.374585+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:02:43.374585+0000","last_peered":"2024-09-21T10:02:43.374585+0000","last_clean":"2024-09-21T10:02:43.374585+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:02:43.374585+0000","last_undegraded":"2024-09-21T10:02:43.374585+0000","last_fullsized":"2024-09-21T10:02:43.374585+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1509,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386430+0000","last_change":"2024-09-21T10:01:40.044315+0000","last_active":"2024-09-21T10:02:43.386430+0000","last_peered":"2024-09-21T10:02:43.386430+0000","last_clean":"2024-09-21T10:02:43.386430+0000","last_became_active":"2024-09-21T10:01:40.043896+0000","last_became_peered":"2024-09-21T10:01:40.043896+0000","last_unstale":"2024-09-21T10:02:43.386430+0000","last_undegraded":"2024-09-21T10:02:43.386430+0000","last_fullsized":"2024-09-21T10:02:43.386430+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:23:26.239503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386783+0000","last_change":"2024-09-21T10:02:11.225610+0000","last_active":"2024-09-21T10:02:43.386783+0000","last_peered":"2024-09-21T10:02:43.386783+0000","last_clean":"2024-09-21T10:02:43.386783+0000","last_became_active":"2024-09-21T10:02:11.225203+0000","last_became_peered":"2024-09-21T10:02:11.225203+0000","last_unstale":"2024-09-21T10:02:43.386783+0000","last_undegraded":"2024-09-21T10:02:43.386783+0000","last_fullsized":"2024-09-21T10:02:43.386783+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:34:18.211760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1164'4575","reported_seq":4624,"reported_epoch":1164,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:02:52.218082+0000","last_change":"2024-09-21T10:02:39.010581+0000","last_active":"2024-09-21T10:02:52.218082+0000","last_peered":"2024-09-21T10:02:52.218082+0000","last_clean":"2024-09-21T10:02:10.200616+0000","last_became_active":"2024-09-21T10:02:12.354206+0000","last_became_peered":"2024-09-21T10:02:12.354206+0000","last_unstale":"2024-09-21T10:02:52.218082+0000","last_undegraded":"2024-09-21T10:02:39.007711+0000","last_fullsized":"2024-09-21T10:02:12.213422+0000","mapping_epoch":1156,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4575,"log_dups_size":0,"ondisk_log_size":4575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281870336,"num_objects":4301,"num_object_clones":0,"num_object_copies":12903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3738,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4575,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3],"avail_no_missing":["1","3","5"],"object_location_counts":[{"shards":"1,3","objects":563},{"shards":"1,3,5","objects":3601}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1486,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386685+0000","last_change":"2024-09-21T10:02:11.230174+0000","last_active":"2024-09-21T10:02:43.386685+0000","last_peered":"2024-09-21T10:02:43.386685+0000","last_clean":"2024-09-21T10:02:43.386685+0000","last_became_active":"2024-09-21T10:02:11.229833+0000","last_became_peered":"2024-09-21T10:02:11.229833+0000","last_unstale":"2024-09-21T10:02:43.386685+0000","last_undegraded":"2024-09-21T10:02:43.386685+0000","last_fullsized":"2024-09-21T10:02:43.386685+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:24.905417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":720,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.364470+0000","last_change":"2024-09-21T10:01:40.043209+0000","last_active":"2024-09-21T10:02:43.364470+0000","last_peered":"2024-09-21T10:02:43.364470+0000","last_clean":"2024-09-21T10:02:43.364470+0000","last_became_active":"2024-09-21T10:01:40.042343+0000","last_became_peered":"2024-09-21T10:01:40.042343+0000","last_unstale":"2024-09-21T10:02:43.364470+0000","last_undegraded":"2024-09-21T10:02:43.364470+0000","last_fullsized":"2024-09-21T10:02:43.364470+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:23:52.354686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1511,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.363120+0000","last_change":"2024-09-21T10:02:11.236236+0000","last_active":"2024-09-21T10:02:43.363120+0000","last_peered":"2024-09-21T10:02:43.363120+0000","last_clean":"2024-09-21T10:02:43.363120+0000","last_became_active":"2024-09-21T10:02:11.235926+0000","last_became_peered":"2024-09-21T10:02:11.235926+0000","last_unstale":"2024-09-21T10:02:43.363120+0000","last_undegraded":"2024-09-21T10:02:43.363120+0000","last_fullsized":"2024-09-21T10:02:43.363120+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:16:35.798621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"1164'4791","reported_seq":4856,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:54.373247+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:02:54.373247+0000","last_peered":"2024-09-21T10:02:54.373247+0000","last_clean":"2024-09-21T10:02:54.373247+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:02:54.373247+0000","last_undegraded":"2024-09-21T10:02:54.373247+0000","last_fullsized":"2024-09-21T10:02:54.373247+0000","mapping_epoch":1137,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4791,"log_dups_size":0,"ondisk_log_size":4791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:21.517444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287113216,"num_objects":4381,"num_object_clones":0,"num_object_copies":13143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4791,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":182,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.375492+0000","last_change":"2024-09-21T10:02:11.254815+0000","last_active":"2024-09-21T10:02:43.375492+0000","last_peered":"2024-09-21T10:02:43.375492+0000","last_clean":"2024-09-21T10:02:43.375492+0000","last_became_active":"2024-09-21T10:02:11.242456+0000","last_became_peered":"2024-09-21T10:02:11.242456+0000","last_unstale":"2024-09-21T10:02:43.375492+0000","last_undegraded":"2024-09-21T10:02:43.375492+0000","last_fullsized":"2024-09-21T10:02:43.375492+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:29.112175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1494,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387812+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:02:43.387812+0000","last_peered":"2024-09-21T10:02:43.387812+0000","last_clean":"2024-09-21T10:02:43.387812+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:02:43.387812+0000","last_undegraded":"2024-09-21T10:02:43.387812+0000","last_fullsized":"2024-09-21T10:02:43.387812+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":164,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.411909+0000","last_change":"2024-09-21T10:01:40.046560+0000","last_active":"2024-09-21T10:02:43.411909+0000","last_peered":"2024-09-21T10:02:43.411909+0000","last_clean":"2024-09-21T10:02:43.411909+0000","last_became_active":"2024-09-21T10:01:40.046056+0000","last_became_peered":"2024-09-21T10:01:40.046056+0000","last_unstale":"2024-09-21T10:02:43.411909+0000","last_undegraded":"2024-09-21T10:02:43.411909+0000","last_fullsized":"2024-09-21T10:02:43.411909+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:58:52.805316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1164'4628","reported_seq":4693,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:54.359326+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:02:54.359326+0000","last_peered":"2024-09-21T10:02:54.359326+0000","last_clean":"2024-09-21T10:02:54.359326+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:02:54.359326+0000","last_undegraded":"2024-09-21T10:02:54.359326+0000","last_fullsized":"2024-09-21T10:02:54.359326+0000","mapping_epoch":1137,"log_start":"0'0","ondisk_log_start":"0'0","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4628,"log_dups_size":0,"ondisk_log_size":4628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:26:46.255638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4628,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1511,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.369870+0000","last_change":"2024-09-21T09:49:38.835287+0000","last_active":"2024-09-21T10:02:43.369870+0000","last_peered":"2024-09-21T10:02:43.369870+0000","last_clean":"2024-09-21T10:02:43.369870+0000","last_became_active":"2024-09-21T09:49:38.834964+0000","last_became_peered":"2024-09-21T09:49:38.834964+0000","last_unstale":"2024-09-21T10:02:43.369870+0000","last_undegraded":"2024-09-21T10:02:43.369870+0000","last_fullsized":"2024-09-21T10:02:43.369870+0000","mapping_epoch":763,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":764,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:17.707191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432947+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:02:41.432947+0000","last_peered":"2024-09-21T10:02:41.432947+0000","last_clean":"2024-09-21T10:02:41.432947+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:02:41.432947+0000","last_undegraded":"2024-09-21T10:02:41.432947+0000","last_fullsized":"2024-09-21T10:02:41.432947+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432811+0000","last_change":"2024-09-21T10:02:38.992313+0000","last_active":"2024-09-21T10:02:41.432811+0000","last_peered":"2024-09-21T10:02:41.432811+0000","last_clean":"2024-09-21T10:02:41.432811+0000","last_became_active":"2024-09-21T10:02:38.991604+0000","last_became_peered":"2024-09-21T10:02:38.991604+0000","last_unstale":"2024-09-21T10:02:41.432811+0000","last_undegraded":"2024-09-21T10:02:41.432811+0000","last_fullsized":"2024-09-21T10:02:41.432811+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:39.279472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432853+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:02:41.432853+0000","last_peered":"2024-09-21T10:02:41.432853+0000","last_clean":"2024-09-21T10:02:41.432853+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:02:41.432853+0000","last_undegraded":"2024-09-21T10:02:41.432853+0000","last_fullsized":"2024-09-21T10:02:41.432853+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":23,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432876+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:02:41.432876+0000","last_peered":"2024-09-21T10:02:41.432876+0000","last_clean":"2024-09-21T10:02:41.432876+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:02:41.432876+0000","last_undegraded":"2024-09-21T10:02:41.432876+0000","last_fullsized":"2024-09-21T10:02:41.432876+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1493,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386545+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:02:43.386545+0000","last_peered":"2024-09-21T10:02:43.386545+0000","last_clean":"2024-09-21T10:02:43.386545+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:02:43.386545+0000","last_undegraded":"2024-09-21T10:02:43.386545+0000","last_fullsized":"2024-09-21T10:02:43.386545+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1093,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.374803+0000","last_change":"2024-09-21T10:01:40.057224+0000","last_active":"2024-09-21T10:02:43.374803+0000","last_peered":"2024-09-21T10:02:43.374803+0000","last_clean":"2024-09-21T10:02:43.374803+0000","last_became_active":"2024-09-21T10:01:40.038762+0000","last_became_peered":"2024-09-21T10:01:40.038762+0000","last_unstale":"2024-09-21T10:02:43.374803+0000","last_undegraded":"2024-09-21T10:02:43.374803+0000","last_fullsized":"2024-09-21T10:02:43.374803+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:16.446381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1117,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387901+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:02:43.387901+0000","last_peered":"2024-09-21T10:02:43.387901+0000","last_clean":"2024-09-21T10:02:43.387901+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:02:43.387901+0000","last_undegraded":"2024-09-21T10:02:43.387901+0000","last_fullsized":"2024-09-21T10:02:43.387901+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.379818+0000","last_change":"2024-09-21T10:02:39.005319+0000","last_active":"2024-09-21T10:02:43.379818+0000","last_peered":"2024-09-21T10:02:43.379818+0000","last_clean":"2024-09-21T10:02:43.379818+0000","last_became_active":"2024-09-21T10:02:39.005026+0000","last_became_peered":"2024-09-21T10:02:39.005026+0000","last_unstale":"2024-09-21T10:02:43.379818+0000","last_undegraded":"2024-09-21T10:02:43.379818+0000","last_fullsized":"2024-09-21T10:02:43.379818+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:07.658364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1533,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.364167+0000","last_change":"2024-09-21T10:02:38.999533+0000","last_active":"2024-09-21T10:02:43.364167+0000","last_peered":"2024-09-21T10:02:43.364167+0000","last_clean":"2024-09-21T10:02:43.364167+0000","last_became_active":"2024-09-21T10:02:38.999242+0000","last_became_peered":"2024-09-21T10:02:38.999242+0000","last_unstale":"2024-09-21T10:02:43.364167+0000","last_undegraded":"2024-09-21T10:02:43.364167+0000","last_fullsized":"2024-09-21T10:02:43.364167+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:03:01.501959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":170,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.412121+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:02:43.412121+0000","last_peered":"2024-09-21T10:02:43.412121+0000","last_clean":"2024-09-21T10:02:43.412121+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:02:43.412121+0000","last_undegraded":"2024-09-21T10:02:43.412121+0000","last_fullsized":"2024-09-21T10:02:43.412121+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432937+0000","last_change":"2024-09-21T10:02:38.993292+0000","last_active":"2024-09-21T10:02:41.432937+0000","last_peered":"2024-09-21T10:02:41.432937+0000","last_clean":"2024-09-21T10:02:41.432937+0000","last_became_active":"2024-09-21T10:02:38.992982+0000","last_became_peered":"2024-09-21T10:02:38.992982+0000","last_unstale":"2024-09-21T10:02:41.432937+0000","last_undegraded":"2024-09-21T10:02:41.432937+0000","last_fullsized":"2024-09-21T10:02:41.432937+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:56:34.552861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":156,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.411832+0000","last_change":"2024-09-21T09:56:45.284567+0000","last_active":"2024-09-21T10:02:43.411832+0000","last_peered":"2024-09-21T10:02:43.411832+0000","last_clean":"2024-09-21T10:02:43.411832+0000","last_became_active":"2024-09-21T09:56:45.284231+0000","last_became_peered":"2024-09-21T09:56:45.284231+0000","last_unstale":"2024-09-21T10:02:43.411832+0000","last_undegraded":"2024-09-21T10:02:43.411832+0000","last_fullsized":"2024-09-21T10:02:43.411832+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:38.528617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1529,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.369716+0000","last_change":"2024-09-21T10:01:40.038726+0000","last_active":"2024-09-21T10:02:43.369716+0000","last_peered":"2024-09-21T10:02:43.369716+0000","last_clean":"2024-09-21T10:02:43.369716+0000","last_became_active":"2024-09-21T10:01:40.038416+0000","last_became_peered":"2024-09-21T10:01:40.038416+0000","last_unstale":"2024-09-21T10:02:43.369716+0000","last_undegraded":"2024-09-21T10:02:43.369716+0000","last_fullsized":"2024-09-21T10:02:43.369716+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:33:31.071206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1492,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.387066+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:02:43.387066+0000","last_peered":"2024-09-21T10:02:43.387066+0000","last_clean":"2024-09-21T10:02:43.387066+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:02:43.387066+0000","last_undegraded":"2024-09-21T10:02:43.387066+0000","last_fullsized":"2024-09-21T10:02:43.387066+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.374883+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:02:43.374883+0000","last_peered":"2024-09-21T10:02:43.374883+0000","last_clean":"2024-09-21T10:02:43.374883+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:02:43.374883+0000","last_undegraded":"2024-09-21T10:02:43.374883+0000","last_fullsized":"2024-09-21T10:02:43.374883+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":40,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386685+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:02:43.386685+0000","last_peered":"2024-09-21T10:02:43.386685+0000","last_clean":"2024-09-21T10:02:43.386685+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:02:43.386685+0000","last_undegraded":"2024-09-21T10:02:43.386685+0000","last_fullsized":"2024-09-21T10:02:43.386685+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1376,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386593+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:02:43.386593+0000","last_peered":"2024-09-21T10:02:43.386593+0000","last_clean":"2024-09-21T10:02:43.386593+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:02:43.386593+0000","last_undegraded":"2024-09-21T10:02:43.386593+0000","last_fullsized":"2024-09-21T10:02:43.386593+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1505,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.363199+0000","last_change":"2024-09-21T10:01:40.042635+0000","last_active":"2024-09-21T10:02:43.363199+0000","last_peered":"2024-09-21T10:02:43.363199+0000","last_clean":"2024-09-21T10:02:43.363199+0000","last_became_active":"2024-09-21T10:01:40.042260+0000","last_became_peered":"2024-09-21T10:01:40.042260+0000","last_unstale":"2024-09-21T10:02:43.363199+0000","last_undegraded":"2024-09-21T10:02:43.363199+0000","last_fullsized":"2024-09-21T10:02:43.363199+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:07.993484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1102,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.375834+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:02:43.375834+0000","last_peered":"2024-09-21T10:02:43.375834+0000","last_clean":"2024-09-21T10:02:43.375834+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:02:43.375834+0000","last_undegraded":"2024-09-21T10:02:43.375834+0000","last_fullsized":"2024-09-21T10:02:43.375834+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":56,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.386763+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:02:43.386763+0000","last_peered":"2024-09-21T10:02:43.386763+0000","last_clean":"2024-09-21T10:02:43.386763+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:02:43.386763+0000","last_undegraded":"2024-09-21T10:02:43.386763+0000","last_fullsized":"2024-09-21T10:02:43.386763+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1102,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.379189+0000","last_change":"2024-09-21T10:01:53.141868+0000","last_active":"2024-09-21T10:02:43.379189+0000","last_peered":"2024-09-21T10:02:43.379189+0000","last_clean":"2024-09-21T10:02:43.379189+0000","last_became_active":"2024-09-21T10:01:53.141359+0000","last_became_peered":"2024-09-21T10:01:53.141359+0000","last_unstale":"2024-09-21T10:02:43.379189+0000","last_undegraded":"2024-09-21T10:02:43.379189+0000","last_fullsized":"2024-09-21T10:02:43.379189+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:01:27.166933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432895+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:02:41.432895+0000","last_peered":"2024-09-21T10:02:41.432895+0000","last_clean":"2024-09-21T10:02:41.432895+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:02:41.432895+0000","last_undegraded":"2024-09-21T10:02:41.432895+0000","last_fullsized":"2024-09-21T10:02:41.432895+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":156,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.380002+0000","last_change":"2024-09-21T09:56:45.285280+0000","last_active":"2024-09-21T10:02:43.380002+0000","last_peered":"2024-09-21T10:02:43.380002+0000","last_clean":"2024-09-21T10:02:43.380002+0000","last_became_active":"2024-09-21T09:56:45.284986+0000","last_became_peered":"2024-09-21T09:56:45.284986+0000","last_unstale":"2024-09-21T10:02:43.380002+0000","last_undegraded":"2024-09-21T10:02:43.380002+0000","last_fullsized":"2024-09-21T10:02:43.380002+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:44.292553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":171,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.375618+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:02:43.375618+0000","last_peered":"2024-09-21T10:02:43.375618+0000","last_clean":"2024-09-21T10:02:43.375618+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:02:43.375618+0000","last_undegraded":"2024-09-21T10:02:43.375618+0000","last_fullsized":"2024-09-21T10:02:43.375618+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-09-21T10:02:41.432790+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:02:41.432790+0000","last_peered":"2024-09-21T10:02:41.432790+0000","last_clean":"2024-09-21T10:02:41.432790+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:02:41.432790+0000","last_undegraded":"2024-09-21T10:02:41.432790+0000","last_fullsized":"2024-09-21T10:02:41.432790+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1540,"reported_epoch":1164,"state":"active+clean","last_fresh":"2024-09-21T10:02:43.366754+0000","last_change":"2024-09-21T10:02:39.010452+0000","last_active":"2024-09-21T10:02:43.366754+0000","last_peered":"2024-09-21T10:02:43.366754+0000","last_clean":"2024-09-21T10:02:43.366754+0000","last_became_active":"2024-09-21T10:02:39.000131+0000","last_became_peered":"2024-09-21T10:02:39.000131+0000","last_unstale":"2024-09-21T10:02:43.366754+0000","last_undegraded":"2024-09-21T10:02:43.366754+0000","last_fullsized":"2024-09-21T10:02:43.366754+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:08:15.146003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4546625560,"num_objects":69377,"num_object_clones":0,"num_object_copies":208131,"num_objects_missing_on_primary":2554,"num_objects_missing":2554,"num_objects_degraded":36500,"num_objects_misplaced":2554,"num_objects_unfound":0,"num_objects_dirty":69377,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":74597,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8338,"num_bytes_recovered":546439168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14175186944,"data_stored":14175174728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74597,"ondisk_log_size":74597,"up":48,"acting":39,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":10960,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":2,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234324,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495961,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":246260,"kb_used_data":56876,"kb_used_omap":1,"kb_used_meta":189374,"kb_avail":93498892,"statfs":{"total":95995035648,"available":95742865408,"internally_reserved":0,"allocated":58241024,"data_stored":55968153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":193919431},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757599,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019235,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1957680,"kb_used_data":1723944,"kb_used_omap":1,"kb_used_meta":233726,"kb_avail":91787472,"statfs":{"total":95995035648,"available":93990371328,"internally_reserved":0,"allocated":1765318656,"data_stored":1763049368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":239335878},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888532,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3317408,"kb_used_data":2928600,"kb_used_omap":34,"kb_used_meta":388765,"kb_avail":90427744,"statfs":{"total":95995035648,"available":92598009856,"internally_reserved":0,"allocated":2998886400,"data_stored":2996628715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":398096371},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,0,1,2,4,4,5],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967770,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2660972,"kb_used_data":2367908,"kb_used_omap":41,"kb_used_meta":293014,"kb_avail":91084180,"statfs":{"total":95995035648,"available":93270200320,"internally_reserved":0,"allocated":2424737792,"data_stored":2422473556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":300046751},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047009,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2765476,"kb_used_data":2562652,"kb_used_omap":41,"kb_used_meta":202774,"kb_avail":90979676,"statfs":{"total":95995035648,"available":93163188224,"internally_reserved":0,"allocated":2624155648,"data_stored":2621894297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":207641513},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":730,"seq":3135326126247,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2616000,"kb_used_data":2452792,"kb_used_omap":72,"kb_used_meta":163127,"kb_avail":91129152,"statfs":{"total":95995035648,"available":93316251648,"internally_reserved":0,"allocated":2511659008,"data_stored":2509374517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74709,"internal_metadata":167042091},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2497908736,"data_stored":2497904664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2608533504,"data_stored":2608529432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2413821952,"data_stored":2413821952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2987655168,"data_stored":2987655168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1757741056,"data_stored":1757741056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1069678592,"data_stored":1069678592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":50925568,"data_stored":50921496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":788922368,"data_stored":788922368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:03:03.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_2 pgp_num 16 2024-09-21T10:03:03.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.185 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:03.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:03.211 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:03:03.212 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:03.242 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:03:03.242 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:03.265 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:03:03.265 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:03.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:03.287 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:03:03.287 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:03.312 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:03:03.312 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:03.319 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:03.336 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:03:03.336 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:03.363 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:03:03.363 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:03.393 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:03:03.394 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:03.413 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:03:03.413 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:03:03.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:03.442 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:03.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:03.572 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:03:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:03:03] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-09-21T10:03:03.674 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:03.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:03.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:03.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:03.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:03.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:03.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:03.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:03.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:04.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:04.095 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:03 smithi191 ceph-mon[102206]: pgmap v1420: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 350 op/s; 36503/199617 objects degraded (18.287%); 2462/199617 objects misplaced (1.233%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:03:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:03 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/690142981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:03 smithi076 ceph-mon[106482]: pgmap v1420: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 350 op/s; 36503/199617 objects degraded (18.287%); 2462/199617 objects misplaced (1.233%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:03:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:03 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/690142981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:03 smithi076 ceph-mon[110963]: pgmap v1420: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 350 op/s; 36503/199617 objects degraded (18.287%); 2462/199617 objects misplaced (1.233%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:03:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:03 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/690142981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:04.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:04.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:04.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:04.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:04.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:04.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:04.472 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:04.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:04.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:04.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:04.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:04.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:04.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:04.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:04.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:03:04.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:04.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:04.943 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:04.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:04.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:04.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:05.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:05.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/199617 objects degraded (18.287%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/199617 objects degraded (18.287%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/199617 objects degraded (18.287%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:05.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.245 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:05.272 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:05.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.375 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:05.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:05.497 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:05.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:05.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:05.624 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:05.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:05.686 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:05 smithi076 ceph-mon[106482]: pgmap v1421: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 345 op/s; 36503/197841 objects degraded (18.451%); 2360/197841 objects misplaced (1.193%); 1.2 MiB/s, 18 objects/s recovering 2024-09-21T10:03:05.687 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:05 smithi076 ceph-mon[110963]: pgmap v1421: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 345 op/s; 36503/197841 objects degraded (18.451%); 2360/197841 objects misplaced (1.193%); 1.2 MiB/s, 18 objects/s recovering 2024-09-21T10:03:05.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:05.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:05.820 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:05.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:05.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:05.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:05.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:06.010 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:06.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:05 smithi191 ceph-mon[102206]: pgmap v1421: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 345 op/s; 36503/197841 objects degraded (18.451%); 2360/197841 objects misplaced (1.193%); 1.2 MiB/s, 18 objects/s recovering 2024-09-21T10:03:06.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:06.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:06.237 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:06.261 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:06.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:06.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:06.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:06.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:06.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:06.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:06.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:06.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:06.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:06.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:06.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:06.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:06.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:06.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:07.002 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:07.024 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:07.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:07.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:07.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:03:07.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:07.271 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:07.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:07.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.396 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:07.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:07.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:07.523 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:07.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:07.624 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:07.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:07.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:07.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:07.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:07.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:07.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:07 smithi076 ceph-mon[106482]: pgmap v1422: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 36503/195774 objects degraded (18.645%); 2360/195774 objects misplaced (1.205%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:03:07.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:07 smithi076 ceph-mon[110963]: pgmap v1422: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 36503/195774 objects degraded (18.645%); 2360/195774 objects misplaced (1.205%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:03:07.894 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:07.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:07.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:08.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:07 smithi191 ceph-mon[102206]: pgmap v1422: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 357 op/s; 36503/195774 objects degraded (18.645%); 2360/195774 objects misplaced (1.205%); 1.0 MiB/s, 16 objects/s recovering 2024-09-21T10:03:08.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:08.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:08.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:08.266 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:08.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:08.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:08.389 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:08.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:08.511 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:08.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:08.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:08.618 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:08.650 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:08.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:08.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:08.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:08.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:08.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:08.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:08.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:08.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:09.007 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:09.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:09.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:09.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:09.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:09.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:09.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:09.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:09.406 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:09.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:09.530 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:09.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:09.631 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:09.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:09.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:09.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:09.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:09.896 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:09 smithi076 ceph-mon[110963]: pgmap v1423: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 36503/195165 objects degraded (18.704%); 2360/195165 objects misplaced (1.209%); 997 KiB/s, 15 objects/s recovering 2024-09-21T10:03:09.896 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/195165 objects degraded (18.704%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:09.897 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:09 smithi076 ceph-mon[106482]: pgmap v1423: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 36503/195165 objects degraded (18.704%); 2360/195165 objects misplaced (1.209%); 997 KiB/s, 15 objects/s recovering 2024-09-21T10:03:09.897 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/195165 objects degraded (18.704%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:09.915 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:09 smithi191 ceph-mon[102206]: pgmap v1423: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 283 op/s; 36503/195165 objects degraded (18.704%); 2360/195165 objects misplaced (1.209%); 997 KiB/s, 15 objects/s recovering 2024-09-21T10:03:09.915 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/195165 objects degraded (18.704%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T10:03:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:09.916 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:09.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:10.017 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:10.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:10.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.176 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:10.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:10.277 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:10.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.311 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:10.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:10.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:10.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:10.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:10.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:10.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:10.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:10.677 INFO:teuthology.orchestra.run.smithi076.stderr:set pool 7 pgp_num to 16 2024-09-21T10:03:10.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:10.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:10.795 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:10.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:10.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/729660275' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:03:10.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:03:10.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:10.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:10.945 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:10.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:10.955 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/729660275' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:03:10.956 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:03:10.956 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:10.956 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/729660275' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:03:10.956 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-21T10:03:10.956 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:11.048 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:11.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:11.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:11.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:11.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:11.328 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:11.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:11.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:11.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:11.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:11.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:11.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:11.682 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:03:11.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:11.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:11.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:11.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:11 smithi191 ceph-mon[102206]: pgmap v1424: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 36503/193575 objects degraded (18.857%); 2256/193575 objects misplaced (1.165%); 1.5 MiB/s, 24 objects/s recovering 2024-09-21T10:03:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:03:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:11 smithi191 ceph-mon[102206]: osdmap e1166: 8 total, 8 up, 6 in 2024-09-21T10:03:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3769431458' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:11.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:11.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:11.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:12.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:12.095 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[106482]: pgmap v1424: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 36503/193575 objects degraded (18.857%); 2256/193575 objects misplaced (1.165%); 1.5 MiB/s, 24 objects/s recovering 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[106482]: osdmap e1166: 8 total, 8 up, 6 in 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3769431458' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[110963]: pgmap v1424: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 36503/193575 objects degraded (18.857%); 2256/193575 objects misplaced (1.165%); 1.5 MiB/s, 24 objects/s recovering 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[110963]: osdmap e1166: 8 total, 8 up, 6 in 2024-09-21T10:03:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3769431458' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:12.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:03:12.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:12.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:12.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:12.345 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:12.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:12.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:12.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:12.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:12.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:12.621 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:12.723 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:12.745 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:12.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:12.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:12.869 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:12.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:12 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:12 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:03:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:12 smithi076 ceph-mon[110963]: osdmap e1167: 8 total, 8 up, 6 in 2024-09-21T10:03:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:12 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:12 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:03:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:12 smithi076 ceph-mon[106482]: osdmap e1167: 8 total, 8 up, 6 in 2024-09-21T10:03:12.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:12.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:12.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:13.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:13.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:12 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:12 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:03:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:12 smithi191 ceph-mon[102206]: osdmap e1167: 8 total, 8 up, 6 in 2024-09-21T10:03:13.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:13.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:13.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:13.240 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:13.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:13.342 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:13.372 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:13.378 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:03:13.378 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:03:13.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:13.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:13.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:13.496 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:03:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:03:13] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-21T10:03:13.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:13.625 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:13.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:13.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:13.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:13.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:13.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:13 smithi076 ceph-mon[106482]: pgmap v1427: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 294 op/s; 36503/190761 objects degraded (19.135%); 2256/190761 objects misplaced (1.183%); 840 KiB/s, 13 objects/s recovering 2024-09-21T10:03:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:13 smithi076 ceph-mon[110963]: pgmap v1427: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 294 op/s; 36503/190761 objects degraded (19.135%); 2256/190761 objects misplaced (1.183%); 840 KiB/s, 13 objects/s recovering 2024-09-21T10:03:13.973 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:14.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:14.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:13 smithi191 ceph-mon[102206]: pgmap v1427: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 294 op/s; 36503/190761 objects degraded (19.135%); 2256/190761 objects misplaced (1.183%); 840 KiB/s, 13 objects/s recovering 2024-09-21T10:03:14.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:14.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:14.253 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:14.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.377 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:14.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:14.478 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:14.503 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:14.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:14.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:14.633 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:14.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:14.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:14.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:14.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:14.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/190761 objects degraded (19.135%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:14.797 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:14.797 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/190761 objects degraded (19.135%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:14.797 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:14.831 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:03:14.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:14.880 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:14.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:14.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.008 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:15.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:15.109 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/190761 objects degraded (19.135%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:15.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:15.143 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:15.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:15.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:15.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:15.367 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:15.392 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:15.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:15.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:15.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:15.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:15.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:15.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:15.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:15.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:15.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:15.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:15.907 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:15.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:15.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[106482]: pgmap v1428: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 291 op/s; 36503/188769 objects degraded (19.337%); 2154/188769 objects misplaced (1.141%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[110963]: pgmap v1428: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 291 op/s; 36503/188769 objects degraded (19.337%); 2154/188769 objects misplaced (1.141%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.965 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.966 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:15.995 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:03:15.995 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:16.009 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:16.026 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:03:16.026 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:16.036 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:16.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:03:16.061 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:16.095 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:03:16.096 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:15 smithi191 ceph-mon[102206]: pgmap v1428: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 291 op/s; 36503/188769 objects degraded (19.337%); 2154/188769 objects misplaced (1.141%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:03:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:16.128 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:03:16.128 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:16.138 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:16.162 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:03:16.162 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:16.170 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:16.189 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:03:16.190 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:16.217 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:03:16.217 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:16.272 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:16.305 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:16.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:16.428 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:16.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:16.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:16.551 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:16.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:16.674 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:16.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:16.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:16.801 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/375855635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/375855635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:16.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:16.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:16.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:17.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:17.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/375855635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:17.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:17.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:17.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:17.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:17.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:17.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:17.430 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:17.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:17.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:17.494 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:17.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:17.559 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:17.661 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.691 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:17.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:17.792 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:17.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:17.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:17.849 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:17 smithi076 ceph-mon[106482]: pgmap v1429: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 36503/186177 objects degraded (19.607%); 2154/186177 objects misplaced (1.157%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:03:17.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:17 smithi076 ceph-mon[110963]: pgmap v1429: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 36503/186177 objects degraded (19.607%); 2154/186177 objects misplaced (1.157%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:03:17.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:17.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:17.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:17.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:18.041 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:17 smithi191 ceph-mon[102206]: pgmap v1429: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 36503/186177 objects degraded (19.607%); 2154/186177 objects misplaced (1.157%); 1.6 MiB/s, 25 objects/s recovering 2024-09-21T10:03:18.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:18.076 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:18.165 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 3, 6] out_osds: [7, 5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:03:18.165 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:03:18.165 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 4, 3, 6] 2024-09-21T10:03:18.166 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 0 2024-09-21T10:03:18.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:18.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:18.345 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:18.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:18.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:18.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:18.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:18.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:18.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:18.721 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:18.751 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:18.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:18.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:18.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:18.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:03:18.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:19.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:19.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:19.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:19.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:19.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:19.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:19.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:19.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:19.415 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:19.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:19.545 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:19.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:19.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:19.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:19.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:19.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:19.791 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:19.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:19.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:19.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:20.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:20.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:19 smithi191 ceph-mon[102206]: pgmap v1430: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 36503/185583 objects degraded (19.669%); 2154/185583 objects misplaced (1.161%); 816 KiB/s, 12 objects/s recovering 2024-09-21T10:03:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/185583 objects degraded (19.669%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[106482]: pgmap v1430: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 36503/185583 objects degraded (19.669%); 2154/185583 objects misplaced (1.161%); 816 KiB/s, 12 objects/s recovering 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/185583 objects degraded (19.669%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[110963]: pgmap v1430: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 36503/185583 objects degraded (19.669%); 2154/185583 objects misplaced (1.161%); 816 KiB/s, 12 objects/s recovering 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/185583 objects degraded (19.669%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:20.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:20.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:20.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:20.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:20.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:20.283 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:20.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:20.385 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:20.413 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:20.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:20.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:20.536 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:20.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:20.659 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:20.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:20.782 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:20.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:20.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:20.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:20.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:20.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:20.935 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:03:20.935 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:03:20.935 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:03:20.935 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:03:21.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:21.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:21.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:21.137 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:21.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:21.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:21.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:21.382 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:21.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:21.389 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:21.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:21.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:21.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:21.646 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:21.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:21.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:21.788 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:03:21.806 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:21 smithi191 ceph-mon[102206]: pgmap v1431: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 330 op/s; 36503/183747 objects degraded (19.866%); 2051/183747 objects misplaced (1.116%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:03:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2666349119' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:21.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:03:21.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:21.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:21.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:22.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.082 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:21 smithi076 ceph-mon[106482]: pgmap v1431: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 330 op/s; 36503/183747 objects degraded (19.866%); 2051/183747 objects misplaced (1.116%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:03:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2666349119' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:21 smithi076 ceph-mon[110963]: pgmap v1431: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 330 op/s; 36503/183747 objects degraded (19.866%); 2051/183747 objects misplaced (1.116%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:03:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2666349119' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:03:22.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:22.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:22.328 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:22.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:22.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:22.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:22.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:22.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:22.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:22.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:22.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:22.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:22.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:22.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:22.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:23.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:23.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:22 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:22 smithi191 ceph-mon[102206]: osdmap e1168: 8 total, 8 up, 6 in 2024-09-21T10:03:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:22 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:22 smithi076 ceph-mon[106482]: osdmap e1168: 8 total, 8 up, 6 in 2024-09-21T10:03:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:22 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:03:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:22 smithi076 ceph-mon[110963]: osdmap e1168: 8 total, 8 up, 6 in 2024-09-21T10:03:23.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.215 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:23.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:23.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:23.350 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:23.428 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:03:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:03:23] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-21T10:03:23.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:23.486 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:23.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:23.587 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:23.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:23.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:23.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:23.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:23.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:23.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:23.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:23.982 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:23.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:24.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:24.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:24.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:23 smithi191 ceph-mon[102206]: pgmap v1433: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 36503/180942 objects degraded (20.174%); 2051/180942 objects misplaced (1.134%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:03:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:23 smithi076 ceph-mon[106482]: pgmap v1433: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 36503/180942 objects degraded (20.174%); 2051/180942 objects misplaced (1.134%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:03:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:23 smithi076 ceph-mon[110963]: pgmap v1433: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 327 op/s; 36503/180942 objects degraded (20.174%); 2051/180942 objects misplaced (1.134%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:03:24.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:24.232 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:24.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:24.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:24.355 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:24.457 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:24.486 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:24.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:24.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:24.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:24.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:24.742 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:24.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:24.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:24.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:24.946 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:03:24.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:24.973 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/180942 objects degraded (20.174%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:24.973 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/180942 objects degraded (20.174%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:24.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:25.101 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/180942 objects degraded (20.174%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:25.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:25.164 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:25.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:25.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:25.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:25.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:25.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:25.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:25.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:25.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:25.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:25.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:25.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:25.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:25.784 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:25.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:25.885 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:25.908 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:25.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:25.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:26.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:26.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:25 smithi076 ceph-mon[106482]: pgmap v1434: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 36503/179085 objects degraded (20.383%); 1937/179085 objects misplaced (1.082%); 1.4 MiB/s, 21 objects/s recovering 2024-09-21T10:03:26.031 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:26.031 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:26.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:25 smithi076 ceph-mon[110963]: pgmap v1434: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 36503/179085 objects degraded (20.383%); 1937/179085 objects misplaced (1.082%); 1.4 MiB/s, 21 objects/s recovering 2024-09-21T10:03:26.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:26.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:26.064 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:25 smithi191 ceph-mon[102206]: pgmap v1434: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 36503/179085 objects degraded (20.383%); 1937/179085 objects misplaced (1.082%); 1.4 MiB/s, 21 objects/s recovering 2024-09-21T10:03:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:26.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:26.188 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:26.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:26.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:26.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:26.434 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:26.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:26.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:26.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:26.687 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:26.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:26.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:26.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:26.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:26.938 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:26.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:26.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:27.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:27.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:27.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:26 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:03:26.818+0000 7f240edc2640 -1 mon.a@0(leader).osd e1168 definitely_dead 0 2024-09-21T10:03:27.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:27.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:27.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:27.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:27.394 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:03:27.395 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:27.426 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:03:27.426 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:27.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:27.444 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:03:27.444 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:27.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:27.466 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:03:27.466 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:27.486 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:03:27.487 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:27.511 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:03:27.511 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:27.533 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:03:27.533 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:27.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:27.556 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:03:27.556 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:27.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:27.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:27.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:27.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:27.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:27.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:27.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:27.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:27.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:28.065 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:28.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:27 smithi191 ceph-mon[102206]: pgmap v1435: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 36503/176517 objects degraded (20.680%); 1937/176517 objects misplaced (1.097%); 1.4 MiB/s, 21 objects/s recovering 2024-09-21T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2490712796' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:27 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:27 smithi191 ceph-mon[102206]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-21T10:03:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/863406769' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[106482]: pgmap v1435: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 36503/176517 objects degraded (20.680%); 1937/176517 objects misplaced (1.097%); 1.4 MiB/s, 21 objects/s recovering 2024-09-21T10:03:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2490712796' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[106482]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/863406769' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[110963]: pgmap v1435: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 321 op/s; 36503/176517 objects degraded (20.680%); 1937/176517 objects misplaced (1.097%); 1.4 MiB/s, 21 objects/s recovering 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2490712796' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[110963]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-21T10:03:28.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/863406769' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:28.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.214 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:28.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:28.294 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.0. 2024-09-21T10:03:28.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:28.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.341 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:28.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:28.442 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:28.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:28.509 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:28.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:28.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:28.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:28.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:28.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:28.834 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:28.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:28 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:03:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:28 smithi076 ceph-mon[106482]: osdmap e1169: 8 total, 8 up, 5 in 2024-09-21T10:03:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:28 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:03:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:28 smithi076 ceph-mon[110963]: osdmap e1169: 8 total, 8 up, 5 in 2024-09-21T10:03:28.936 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:28.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:28.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:29.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:29.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:28 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-21T10:03:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:28 smithi191 ceph-mon[102206]: osdmap e1169: 8 total, 8 up, 5 in 2024-09-21T10:03:29.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:29.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:29.304 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:29.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:29.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:29.448 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:29.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:29.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:29.570 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:29.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:29.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:29.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:29.820 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:29.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:29.965 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:29.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:30.067 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:30.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:29 smithi191 ceph-mon[102206]: pgmap v1437: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 36503/175848 objects degraded (20.758%); 1937/175848 objects misplaced (1.102%); 888 KiB/s, 13 objects/s recovering 2024-09-21T10:03:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:29 smithi191 ceph-mon[102206]: osdmap e1170: 8 total, 8 up, 5 in 2024-09-21T10:03:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 36503/175848 objects degraded (20.758%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:29 smithi076 ceph-mon[106482]: pgmap v1437: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 36503/175848 objects degraded (20.758%); 1937/175848 objects misplaced (1.102%); 888 KiB/s, 13 objects/s recovering 2024-09-21T10:03:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:29 smithi076 ceph-mon[106482]: osdmap e1170: 8 total, 8 up, 5 in 2024-09-21T10:03:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 36503/175848 objects degraded (20.758%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:29 smithi076 ceph-mon[110963]: pgmap v1437: 89 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 77 active+clean; 3.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 36503/175848 objects degraded (20.758%); 1937/175848 objects misplaced (1.102%); 888 KiB/s, 13 objects/s recovering 2024-09-21T10:03:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:29 smithi076 ceph-mon[110963]: osdmap e1170: 8 total, 8 up, 5 in 2024-09-21T10:03:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 36503/175848 objects degraded (20.758%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:30.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:30.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:30.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:30.342 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:30.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:30.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:30.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:30.588 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:30.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:30.713 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:30.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:30.815 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:30.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:30.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:30.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:31.004 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:31.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:30 smithi191 ceph-mon[102206]: osdmap e1171: 8 total, 8 up, 5 in 2024-09-21T10:03:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:30 smithi076 ceph-mon[106482]: osdmap e1171: 8 total, 8 up, 5 in 2024-09-21T10:03:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:30 smithi076 ceph-mon[110963]: osdmap e1171: 8 total, 8 up, 5 in 2024-09-21T10:03:31.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:31.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:31.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:31.358 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:31.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:31.483 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:31.511 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:31.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:31.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:31.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:31.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:31.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:31.759 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:31.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:31.888 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:31.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:31.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:31.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:32.012 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:32.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:32.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:31 smithi076 ceph-mon[106482]: pgmap v1440: 89 pgs: 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 28504/174366 objects degraded (16.347%); 2790/174366 objects misplaced (1.600%); 1.9 MiB/s, 31 objects/s recovering 2024-09-21T10:03:32.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:31 smithi076 ceph-mon[110963]: pgmap v1440: 89 pgs: 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 28504/174366 objects degraded (16.347%); 2790/174366 objects misplaced (1.600%); 1.9 MiB/s, 31 objects/s recovering 2024-09-21T10:03:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:31 smithi191 ceph-mon[102206]: pgmap v1440: 89 pgs: 3 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 262 op/s; 28504/174366 objects degraded (16.347%); 2790/174366 objects misplaced (1.600%); 1.9 MiB/s, 31 objects/s recovering 2024-09-21T10:03:32.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:32.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:32.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:32.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:32.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:32.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:32.362 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:32.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:32.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:32.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:32.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:32.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:32.633 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:32.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:32.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:32.759 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:32.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:32.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:32.884 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:32.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:32.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:33.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:33.110 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:33.133 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:33.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:33.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:33.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:33.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:33.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:03:33.541 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:03:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:03:33] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-09-21T10:03:33.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:33.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:33.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:33.713 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:33.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:33.815 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:33.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:33.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:33.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:33.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:33.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:34.065 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:34.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:34.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:33 smithi191 ceph-mon[102206]: pgmap v1441: 89 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 132 op/s; 38997/174132 objects degraded (22.395%); 5361/174132 objects misplaced (3.079%); 5.3 MiB/s, 86 objects/s recovering 2024-09-21T10:03:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:33 smithi076 ceph-mon[106482]: pgmap v1441: 89 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 132 op/s; 38997/174132 objects degraded (22.395%); 5361/174132 objects misplaced (3.079%); 5.3 MiB/s, 86 objects/s recovering 2024-09-21T10:03:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:33 smithi076 ceph-mon[110963]: pgmap v1441: 89 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 132 op/s; 38997/174132 objects degraded (22.395%); 5361/174132 objects misplaced (3.079%); 5.3 MiB/s, 86 objects/s recovering 2024-09-21T10:03:34.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:34.214 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:34.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:34.338 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:34.440 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:34.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:34.564 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:34.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:34.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:34.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:34.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:34.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:34.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:34.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:34.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:34.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:34.960 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:34.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:34.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:35.062 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:35.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:35.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 38997/174132 objects degraded (22.395%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:35.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 38997/174132 objects degraded (22.395%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:35.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 38997/174132 objects degraded (22.395%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:35.189 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:35.215 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:35.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:35.340 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:35.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:35.441 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:35.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:35.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:35.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:35.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:35.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:35.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:35.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:35.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:35.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:35.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:35.943 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 5, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:03:35.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:03:35.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:35.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:36.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:36.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:35 smithi191 ceph-mon[102206]: pgmap v1442: 89 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 84 op/s; 38997/174132 objects degraded (22.395%); 5361/174132 objects misplaced (3.079%); 4.7 MiB/s, 76 objects/s recovering 2024-09-21T10:03:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:35 smithi076 ceph-mon[106482]: pgmap v1442: 89 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 84 op/s; 38997/174132 objects degraded (22.395%); 5361/174132 objects misplaced (3.079%); 4.7 MiB/s, 76 objects/s recovering 2024-09-21T10:03:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:35 smithi076 ceph-mon[110963]: pgmap v1442: 89 pgs: 1 activating+undersized+degraded+remapped, 3 remapped+peering, 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 84 op/s; 38997/174132 objects degraded (22.395%); 5361/174132 objects misplaced (3.079%); 4.7 MiB/s, 76 objects/s recovering 2024-09-21T10:03:36.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:36.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:36.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:36.320 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:36.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:36.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:36.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:36.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:36.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:36.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:36.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:36.729 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:36.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:36.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:36.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:36.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:36.980 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:36.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:36.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:37.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:37.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:37.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:37.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:37.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:37.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:37.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:37.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:37.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:37.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:37.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:37.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:37.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:37.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:37.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:37.940 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:37 smithi076 ceph-mon[110963]: pgmap v1443: 89 pgs: 1 activating+undersized+remapped, 5 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 76 op/s; 61362/174006 objects degraded (35.264%); 4766/174006 objects misplaced (2.739%); 8.6 MiB/s, 139 objects/s recovering 2024-09-21T10:03:37.940 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:37 smithi076 ceph-mon[106482]: pgmap v1443: 89 pgs: 1 activating+undersized+remapped, 5 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 76 op/s; 61362/174006 objects degraded (35.264%); 4766/174006 objects misplaced (2.739%); 8.6 MiB/s, 139 objects/s recovering 2024-09-21T10:03:37.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:37.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:37.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:38.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:38.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:38.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:37 smithi191 ceph-mon[102206]: pgmap v1443: 89 pgs: 1 activating+undersized+remapped, 5 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 76 op/s; 61362/174006 objects degraded (35.264%); 4766/174006 objects misplaced (2.739%); 8.6 MiB/s, 139 objects/s recovering 2024-09-21T10:03:38.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:38.233 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:38.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:38.334 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:38.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:38.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.489 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:38.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:38.590 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.613 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:38.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:38.714 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:38.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.753 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:38.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:38.854 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:38.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:38.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:38.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:39.000 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:39.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:39.102 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:39.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:39.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:39.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:39.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:39.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:39.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:39.501 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:39.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:39.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:39.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:39.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:39.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:39.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:39.885 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:03:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:39.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:39.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:39.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:40.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:40.119 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[106482]: pgmap v1444: 89 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 65 op/s; 61362/173928 objects degraded (35.280%); 4232/173928 objects misplaced (2.433%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/203749285' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 61362/173928 objects degraded (35.280%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[110963]: pgmap v1444: 89 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 65 op/s; 61362/173928 objects degraded (35.280%); 4232/173928 objects misplaced (2.433%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/203749285' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 61362/173928 objects degraded (35.280%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:40.145 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:40.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:40.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:40.329 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:39 smithi191 ceph-mon[102206]: pgmap v1444: 89 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 65 op/s; 61362/173928 objects degraded (35.280%); 4232/173928 objects misplaced (2.433%); 10 MiB/s, 169 objects/s recovering 2024-09-21T10:03:40.329 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/203749285' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.329 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:39 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:03:40.329 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 61362/173928 objects degraded (35.280%), 15 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:40.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:40.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:40.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:40.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:40.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:40.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:40.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:40.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:40.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:40.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:40.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:40.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:41.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:41.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:03:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[110963]: osdmap e1172: 8 total, 8 up, 5 in 2024-09-21T10:03:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:03:41.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[106482]: osdmap e1172: 8 total, 8 up, 5 in 2024-09-21T10:03:41.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:41.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:41.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:41.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:41.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:41.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:40 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:03:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:40 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:03:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:40 smithi191 ceph-mon[102206]: osdmap e1172: 8 total, 8 up, 5 in 2024-09-21T10:03:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:41.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:41.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:41.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:41.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:41.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:41.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:41.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:41.789 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:41.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:41.891 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:41.921 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:41.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:41.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:42.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:42.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:41 smithi076 ceph-mon[106482]: pgmap v1446: 89 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 14 op/s; 61362/173928 objects degraded (35.280%); 4232/173928 objects misplaced (2.433%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:03:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:41 smithi076 ceph-mon[110963]: pgmap v1446: 89 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 14 op/s; 61362/173928 objects degraded (35.280%); 4232/173928 objects misplaced (2.433%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:03:42.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:42.179 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:42.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:42.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:42.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:42.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:41 smithi191 ceph-mon[102206]: pgmap v1446: 89 pgs: 1 activating+undersized+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 14 op/s; 61362/173928 objects degraded (35.280%); 4232/173928 objects misplaced (2.433%); 9.1 MiB/s, 145 objects/s recovering 2024-09-21T10:03:42.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:42.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:42.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:42.565 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:42.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:42.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:42.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:42.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:42.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:42.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:42.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:42.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:43.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:43.064 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:43.166 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:43.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:43.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:43.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:43.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:43.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:43.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:43.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:43.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:43.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:43.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:43.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:43.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:03:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:03:43] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-09-21T10:03:43.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:43.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:43.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:43.803 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:43.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:43.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:43.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:43.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:44.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:44.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:44.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:44.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:44.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:44.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:44.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:44.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:44.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:43 smithi191 ceph-mon[102206]: pgmap v1447: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 61177/172428 objects degraded (35.480%); 3363/172428 objects misplaced (1.950%); 13 MiB/s, 212 objects/s recovering 2024-09-21T10:03:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:43 smithi076 ceph-mon[106482]: pgmap v1447: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 61177/172428 objects degraded (35.480%); 3363/172428 objects misplaced (1.950%); 13 MiB/s, 212 objects/s recovering 2024-09-21T10:03:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:43 smithi076 ceph-mon[110963]: pgmap v1447: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 61177/172428 objects degraded (35.480%); 3363/172428 objects misplaced (1.950%); 13 MiB/s, 212 objects/s recovering 2024-09-21T10:03:44.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:44.466 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:44.567 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:44.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:44.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:44.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:44.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:44.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:44.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:44.846 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:44.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:44.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:44.970 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:44.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:45.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:45.095 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:45.197 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:45.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:45.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:45.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 61177/172428 objects degraded (35.480%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 61177/172428 objects degraded (35.480%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 61177/172428 objects degraded (35.480%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:45.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:03:45.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:45.471 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:45.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:45.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:45.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:45.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:45.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:45.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:45.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:45.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:45.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:45.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:46.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:46.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:46.188 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:46.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:46.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:46.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:46.345 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:45 smithi191 ceph-mon[102206]: pgmap v1448: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 61177/172428 objects degraded (35.480%); 3363/172428 objects misplaced (1.950%); 13 MiB/s, 212 objects/s recovering 2024-09-21T10:03:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:45 smithi076 ceph-mon[106482]: pgmap v1448: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 61177/172428 objects degraded (35.480%); 3363/172428 objects misplaced (1.950%); 13 MiB/s, 212 objects/s recovering 2024-09-21T10:03:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:45 smithi076 ceph-mon[110963]: pgmap v1448: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 56 op/s; 61177/172428 objects degraded (35.480%); 3363/172428 objects misplaced (1.950%); 13 MiB/s, 212 objects/s recovering 2024-09-21T10:03:46.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:46.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:46.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:46.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:46.602 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:46.704 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:46.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:46.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:46.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:46.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:46.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:46.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:47.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:47.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:47.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:47.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:47.236 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:47.338 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:47.365 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:47.466 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:47.496 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:47.598 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:47.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:47.684 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:47.685 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:47.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:47.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:47.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:47.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:47.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:47.979 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:47 smithi076 ceph-mon[106482]: pgmap v1449: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 97 op/s; 60860/171081 objects degraded (35.574%); 3117/171081 objects misplaced (1.822%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:03:47.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:47 smithi076 ceph-mon[110963]: pgmap v1449: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 97 op/s; 60860/171081 objects degraded (35.574%); 3117/171081 objects misplaced (1.822%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:03:47.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:48.009 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:48.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:48.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:48.244 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:48.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:48.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:47 smithi191 ceph-mon[102206]: pgmap v1449: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 97 op/s; 60860/171081 objects degraded (35.574%); 3117/171081 objects misplaced (1.822%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:03:48.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:48.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:48.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:48.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:48.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:48.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:48.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:48.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:48.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:48.871 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:48.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:48.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:48.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:48.950 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:03:48.950 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:03:49.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:49.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:49.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:49.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:49.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:49.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:49.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:49.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:49.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:49.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:49.629 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:49.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:49.716 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:03:49.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:49.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:03:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:49.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:49.905 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:49.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:49.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:50.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:50.050 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:50.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:49 smithi076 ceph-mon[106482]: pgmap v1450: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 114 op/s; 60860/170505 objects degraded (35.694%); 2516/170505 objects misplaced (1.476%); 13 MiB/s, 206 objects/s recovering 2024-09-21T10:03:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 60860/170505 objects degraded (35.694%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:49 smithi076 ceph-mon[110963]: pgmap v1450: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 114 op/s; 60860/170505 objects degraded (35.694%); 2516/170505 objects misplaced (1.476%); 13 MiB/s, 206 objects/s recovering 2024-09-21T10:03:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 60860/170505 objects degraded (35.694%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:50.152 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:50.179 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:50.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:50.313 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:50.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:49 smithi191 ceph-mon[102206]: pgmap v1450: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 114 op/s; 60860/170505 objects degraded (35.694%); 2516/170505 objects misplaced (1.476%); 13 MiB/s, 206 objects/s recovering 2024-09-21T10:03:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 60860/170505 objects degraded (35.694%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:50.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:50.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:50.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:50.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:50.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:50.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:50.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:50.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:50.830 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:50.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:50.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:50.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:50.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:51.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:51.086 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:51.187 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:51.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:51.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:51.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:51.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:51.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:51.460 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:51.562 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:51.588 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:51.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.716 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:51.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:51.817 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:51.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:51.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:51.943 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:51.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:51.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:51.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:51.984 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:52.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:52.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:52.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:52.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:51 smithi076 ceph-mon[106482]: pgmap v1451: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 106 op/s; 60860/170505 objects degraded (35.694%); 2516/170505 objects misplaced (1.476%); 12 MiB/s, 192 objects/s recovering 2024-09-21T10:03:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:51 smithi076 ceph-mon[110963]: pgmap v1451: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 106 op/s; 60860/170505 objects degraded (35.694%); 2516/170505 objects misplaced (1.476%); 12 MiB/s, 192 objects/s recovering 2024-09-21T10:03:52.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:52.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:52.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:52.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:52.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:51 smithi191 ceph-mon[102206]: pgmap v1451: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.5 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 106 op/s; 60860/170505 objects degraded (35.694%); 2516/170505 objects misplaced (1.476%); 12 MiB/s, 192 objects/s recovering 2024-09-21T10:03:52.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:52.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:52.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:52.581 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:03:52.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:52.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:52.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:52.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:52.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:52.861 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:52.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:52.921 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:03:52.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:52.984 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:52.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:53.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:53.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:53.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2185074027' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:03:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:52 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:03:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2185074027' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:03:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:52 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:03:53.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:53.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:53.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:53.362 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:53.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2185074027' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:03:53.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:52 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:03:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:53.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:03:53] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-09-21T10:03:53.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:53.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:53.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:53.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:53.614 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:53.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:53.742 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:53.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:53.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:53.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:53.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:53.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:53.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:53.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:54.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:54.120 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:54.146 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-mon[110963]: pgmap v1452: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 159 op/s; 60509/168183 objects degraded (35.978%); 1618/168183 objects misplaced (0.962%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:03:54.146 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:03:54.147 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-mon[110963]: osdmap e1173: 8 total, 8 up, 5 in 2024-09-21T10:03:54.147 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-mon[106482]: pgmap v1452: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 159 op/s; 60509/168183 objects degraded (35.978%); 1618/168183 objects misplaced (0.962%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:03:54.147 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:03:54.147 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:53 smithi076 ceph-mon[106482]: osdmap e1173: 8 total, 8 up, 5 in 2024-09-21T10:03:54.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:54.250 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:54.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:53 smithi191 ceph-mon[102206]: pgmap v1452: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 159 op/s; 60509/168183 objects degraded (35.978%); 1618/168183 objects misplaced (0.962%); 17 MiB/s, 266 objects/s recovering 2024-09-21T10:03:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:53 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:03:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:53 smithi191 ceph-mon[102206]: osdmap e1173: 8 total, 8 up, 5 in 2024-09-21T10:03:54.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:54.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:54.510 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:54.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:03:54.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:54.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:54.737 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:54.765 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:54.860 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:03:54.861 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:03:54.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:54.890 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:03:54.891 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:03:54.905 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:54.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:54.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:54.915 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:03:54.916 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:03:54.939 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:03:54.939 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:03:54.969 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:03:54.970 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:03:54.996 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:03:54.996 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:03:55.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:55.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:03:55.020 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:03:55.029 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:55.044 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:03:55.044 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:03:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 60509/168183 objects degraded (35.978%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/307162153' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 60509/168183 objects degraded (35.978%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/307162153' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:55.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:55.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:55.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:55.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.283 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:55.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 60509/168183 objects degraded (35.978%), 16 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:03:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/307162153' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:03:55.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:55.408 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:55.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:55.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:55.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:55.671 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:55.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:55.773 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:55.799 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:55.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:55.924 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:55.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:56.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:56.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:56.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:56.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:56.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:56.313 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:55 smithi191 ceph-mon[102206]: pgmap v1454: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 60509/168183 objects degraded (35.978%); 1618/168183 objects misplaced (0.962%); 14 MiB/s, 220 objects/s recovering 2024-09-21T10:03:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:55 smithi076 ceph-mon[106482]: pgmap v1454: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 60509/168183 objects degraded (35.978%); 1618/168183 objects misplaced (0.962%); 14 MiB/s, 220 objects/s recovering 2024-09-21T10:03:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:55 smithi076 ceph-mon[110963]: pgmap v1454: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 60509/168183 objects degraded (35.978%); 1618/168183 objects misplaced (0.962%); 14 MiB/s, 220 objects/s recovering 2024-09-21T10:03:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:56.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:56.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:56.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:56.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:56.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:56.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:56.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:56.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:56.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:56.911 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:56.934 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:56.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:57.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:57.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:57.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:57.189 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:57.207 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:56 smithi076 ceph-mon[106482]: osdmap e1174: 8 total, 8 up, 5 in 2024-09-21T10:03:57.208 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:56 smithi076 ceph-mon[110963]: osdmap e1174: 8 total, 8 up, 5 in 2024-09-21T10:03:57.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:57.313 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:57.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:56 smithi191 ceph-mon[102206]: osdmap e1174: 8 total, 8 up, 5 in 2024-09-21T10:03:57.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:03:57.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:03:57.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:57.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:57.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:57.662 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:57.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:57.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:57.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:57.820 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:57.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:57.952 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:57.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:58.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:03:58.076 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:03:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:57 smithi076 ceph-mon[106482]: pgmap v1456: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 157 op/s; 60189/166734 objects degraded (36.099%); 1587/166734 objects misplaced (0.952%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:03:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:57 smithi076 ceph-mon[106482]: osdmap e1175: 8 total, 8 up, 5 in 2024-09-21T10:03:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:57 smithi076 ceph-mon[110963]: pgmap v1456: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 157 op/s; 60189/166734 objects degraded (36.099%); 1587/166734 objects misplaced (0.952%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:03:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:57 smithi076 ceph-mon[110963]: osdmap e1175: 8 total, 8 up, 5 in 2024-09-21T10:03:58.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:58.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:58.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:58.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:58.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:58.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:57 smithi191 ceph-mon[102206]: pgmap v1456: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 157 op/s; 60189/166734 objects degraded (36.099%); 1587/166734 objects misplaced (0.952%); 11 MiB/s, 175 objects/s recovering 2024-09-21T10:03:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:57 smithi191 ceph-mon[102206]: osdmap e1175: 8 total, 8 up, 5 in 2024-09-21T10:03:58.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:58.450 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:58.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:58.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:58.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:58.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:58.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:58.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:58.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:58.824 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:58.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:58.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:58.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:03:59.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:59.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:59.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:59.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:03:59.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:03:59.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:03:59.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:03:59.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:59.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:03:59.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:03:59.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:03:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:03:59.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:59.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:59.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:03:59.710 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:59.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:03:59.812 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:03:59.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:03:59.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:03:59.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:03:59.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:03:59.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:00.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:00.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:00.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:00.217 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:00.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:00.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:00.352 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:59 smithi076 ceph-mon[106482]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 117 op/s; 59346/166074 objects degraded (35.735%); 1473/166074 objects misplaced (0.887%); 3.9 MiB/s, 202 objects/s recovering 2024-09-21T10:04:00.352 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:03:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 59346/166074 objects degraded (35.735%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:00.353 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:59 smithi076 ceph-mon[110963]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 117 op/s; 59346/166074 objects degraded (35.735%); 1473/166074 objects misplaced (0.887%); 3.9 MiB/s, 202 objects/s recovering 2024-09-21T10:04:00.353 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:03:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 59346/166074 objects degraded (35.735%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:59 smithi191 ceph-mon[102206]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 117 op/s; 59346/166074 objects degraded (35.735%); 1473/166074 objects misplaced (0.887%); 3.9 MiB/s, 202 objects/s recovering 2024-09-21T10:04:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:03:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 59346/166074 objects degraded (35.735%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:00.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:00.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:00.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:00.556 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:04:00.556 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:04:00.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:04:00.556 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:04:00.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.556 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159005 6.f 6:f6a4ad1c:::benchmark_data_smithi076_154645_object45099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.851352+0000", 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.81841376599999, 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0199435720000001, 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159005 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.557 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851352+0000", 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851352+0000", 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851354+0000", 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.327e-06 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851359+0000", 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9660000000000004e-06 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851368+0000", 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6369999999999994e-06 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.851375+0000", 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1409999999999999e-06 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.870075+0000", 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018699335000000001 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.870086+0000", 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1508999999999999e-05 2024-09-21T10:04:00.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.862921+0000", 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.99283502099999998 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.863032+0000", 2024-09-21T10:04:00.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110525 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.863134+0000", 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010220100000000001 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871068+0000", 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079340639999999994 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871112+0000", 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3816000000000002e-05 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871230+0000", 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118262 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871266+0000", 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6143000000000003e-05 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871296+0000", 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9624999999999998e-05 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159036 6.f 6:fa619a2f:::benchmark_data_smithi076_154645_object45130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.909283+0000", 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.76048334799998, 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96219932799999996, 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.564 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159036 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909283+0000", 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909283+0000", 2024-09-21T10:04:00.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909284+0000", 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4100000000000001e-06 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909287+0000", 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2339999999999999e-06 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909300+0000", 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2184999999999999e-05 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.909305+0000", 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6790000000000002e-06 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.911068+0000", 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017625690000000001 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.911077+0000", 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6190000000000006e-06 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.867281+0000", 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95620355999999995 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.867405+0000", 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012382699999999999 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.867537+0000", 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013225199999999999 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871344+0000", 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038068329999999999 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.569 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871353+0000", 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9789999999999999e-06 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871390+0000", 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6910999999999999e-05 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871443+0000", 2024-09-21T10:04:00.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3619000000000003e-05 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871482+0000", 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8651e-05 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159051 6.f 6:fab518e4:::benchmark_data_smithi076_154645_object45145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.571 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.926074+0000", 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.743692022, 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96485169299999995, 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159051 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.572 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926074+0000", 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926074+0000", 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926076+0000", 2024-09-21T10:04:00.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.578e-06 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926082+0000", 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5220000000000004e-06 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926099+0000", 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7258999999999999e-05 2024-09-21T10:04:00.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926107+0000", 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.148e-06 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926290+0000", 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018331599999999999 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.926314+0000", 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3791e-05 2024-09-21T10:04:00.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871532+0000", 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.94521803500000001 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871646+0000", 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000114288 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.871744+0000", 2024-09-21T10:04:00.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8425999999999996e-05 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881091+0000", 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.009346732 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881104+0000", 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.294e-05 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890814+0000", 2024-09-21T10:04:00.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097100260000000001 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890872+0000", 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7636999999999997e-05 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890926+0000", 2024-09-21T10:04:00.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3995000000000001e-05 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159079 6.f 6:fabef842:::benchmark_data_smithi076_154645_object45173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:02.961015+0000", 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.70875152399998, 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.93010314400000005, 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.579 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159079 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961015+0000", 2024-09-21T10:04:00.580 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961015+0000", 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961016+0000", 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7099999999999999e-06 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961022+0000", 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6710000000000004e-06 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961041+0000", 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8695e-05 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961049+0000", 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0709999999999995e-06 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.583 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961120+0000", 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1664999999999999e-05 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:02.961133+0000", 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2649999999999999e-05 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.876285+0000", 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91515186100000001 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.584 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.876390+0000", 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010521700000000001 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.876517+0000", 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012682500000000001 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881191+0000", 2024-09-21T10:04:00.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046744860000000003 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890750+0000", 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0095585719999999996 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891032+0000", 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00028154299999999998 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891081+0000", 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9280000000000003e-05 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891118+0000", 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6897999999999998e-05 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159136 6.f 6:f59b07bd:::benchmark_data_smithi076_154645_object45230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.028209+0000", 2024-09-21T10:04:00.587 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.641557487, 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88298491899999998, 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159136 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028209+0000", 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.588 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028209+0000", 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028210+0000", 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.437e-06 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028214+0000", 2024-09-21T10:04:00.589 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4989999999999999e-06 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028223+0000", 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6080000000000002e-06 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028229+0000", 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8420000000000001e-06 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030110+0000", 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018811570000000001 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030121+0000", 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1229e-05 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881239+0000", 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85111755600000005 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881336+0000", 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7343999999999995e-05 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.881437+0000", 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000100758 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.890973+0000", 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0095361449999999993 2024-09-21T10:04:00.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891198+0000", 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000224405 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911101+0000", 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.019903677000000002 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911146+0000", 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5164000000000003e-05 2024-09-21T10:04:00.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911193+0000", 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7098000000000002e-05 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159137 6.f 6:f9e69ffb:::benchmark_data_smithi076_154645_object45231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.64113043600003, 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88267890299999996, 2024-09-21T10:04:00.594 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159137 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.595 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028636+0000", 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5300000000000003e-07 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.596 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028640+0000", 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6780000000000002e-06 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028651+0000", 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0752e-05 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.028657+0000", 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4989999999999999e-06 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030209+0000", 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001551796 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.030219+0000", 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5149999999999995e-06 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.886339+0000", 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85612032900000001 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.886444+0000", 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.886554+0000", 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011025 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891158+0000", 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046040780000000002 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911230+0000", 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020071848 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911240+0000", 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7829999999999993e-06 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911282+0000", 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2216999999999999e-05 2024-09-21T10:04:00.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911315+0000", 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2404999999999999e-05 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159138 6.f 6:ff6bf720:::benchmark_data_smithi076_154645_object45232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.032925+0000", 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.63684113599999, 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87857126900000004, 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.601 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159138 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032925+0000", 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032925+0000", 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032927+0000", 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9690000000000001e-06 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032932+0000", 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5470000000000004e-06 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.603 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032964+0000", 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1260000000000002e-05 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.032972+0000", 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1499999999999999e-06 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.033026+0000", 2024-09-21T10:04:00.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4296999999999999e-05 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.033036+0000", 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9280000000000005e-06 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891238+0000", 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85820230900000005 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891379+0000", 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000140586 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.891507+0000", 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012846900000000001 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901276+0000", 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097683360000000007 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911320+0000", 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010044136 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911407+0000", 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6948999999999997e-05 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911462+0000", 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5572999999999998e-05 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911496+0000", 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3760000000000002e-05 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.608 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159153 6.f 6:fc7c58f9:::benchmark_data_smithi076_154645_object45247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.047418+0000", 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.62234855399998, 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86424884499999999, 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159153 2024-09-21T10:04:00.609 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047418+0000", 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047418+0000", 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047420+0000", 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1449999999999998e-06 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047425+0000", 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5879999999999997e-06 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047441+0000", 2024-09-21T10:04:00.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5801999999999999e-05 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.047452+0000", 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0648999999999999e-05 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.049292+0000", 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018407250000000001 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.049304+0000", 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2035000000000001e-05 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.896156+0000", 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84685153999999996 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.896260+0000", 2024-09-21T10:04:00.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103546 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.896370+0000", 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011090399999999999 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911089+0000", 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.014718491 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911325+0000", 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000235691 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911585+0000", 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000260277 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911618+0000", 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3068999999999998e-05 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911666+0000", 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8383000000000001e-05 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159165 6.f 6:fe126f68:::benchmark_data_smithi076_154645_object45259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.063179+0000", 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.60658704299999, 2024-09-21T10:04:00.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85356281499999997, 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159165 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063179+0000", 2024-09-21T10:04:00.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063179+0000", 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063181+0000", 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.691e-06 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063187+0000", 2024-09-21T10:04:00.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3559999999999998e-06 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063204+0000", 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6668e-05 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063212+0000", 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9000000000000006e-06 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063477+0000", 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026537700000000002 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.063496+0000", 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8651000000000002e-05 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901305+0000", 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83780932399999997 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901415+0000", 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110479 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.901531+0000", 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011521799999999999 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911329+0000", 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097980210000000005 2024-09-21T10:04:00.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911358+0000", 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8867999999999999e-05 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916649+0000", 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052918260000000003 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916695+0000", 2024-09-21T10:04:00.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5355999999999997e-05 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916742+0000", 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7080000000000003e-05 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159170 6.f 6:f90e3041:::benchmark_data_smithi076_154645_object45264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.070623+0000", 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.59914310400001, 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84630223000000004, 2024-09-21T10:04:00.624 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159170 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070623+0000", 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070623+0000", 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070625+0000", 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9920000000000002e-06 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070630+0000", 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4179999999999996e-06 2024-09-21T10:04:00.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070647+0000", 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6229e-05 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070654+0000", 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8639999999999996e-06 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070720+0000", 2024-09-21T10:04:00.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5827000000000005e-05 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.070730+0000", 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9650000000000003e-06 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.906055+0000", 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83532494199999996 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.906178+0000", 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012309300000000001 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.906307+0000", 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012857600000000001 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911367+0000", 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050601049999999996 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911532+0000", 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000165315 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916803+0000", 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052708980000000004 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916882+0000", 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8366000000000002e-05 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916925+0000", 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3640000000000002e-05 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159194 6.f 6:f0fd59f5:::benchmark_data_smithi076_154645_object45288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.102922+0000", 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.56684394500002, 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84748727099999999, 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.631 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159194 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102922+0000", 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102922+0000", 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102924+0000", 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9649999999999998e-06 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102930+0000", 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2199999999999997e-06 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102949+0000", 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8638999999999999e-05 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.102957+0000", 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0860000000000002e-06 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.103021+0000", 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3607000000000003e-05 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.103031+0000", 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0549e-05 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911710+0000", 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.80867835600000004 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911782+0000", 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2279000000000004e-05 2024-09-21T10:04:00.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.911899+0000", 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116721 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.926988+0000", 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015089111000000001 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.926999+0000", 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1079e-05 2024-09-21T10:04:00.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950320+0000", 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023321278000000001 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950370+0000", 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9959000000000001e-05 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950409+0000", 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9422000000000003e-05 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159224 6.f 6:f634f675:::benchmark_data_smithi076_154645_object45318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.158844+0000", 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.51092254600002, 2024-09-21T10:04:00.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.79173849500000004, 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159224 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158844+0000", 2024-09-21T10:04:00.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158844+0000", 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158845+0000", 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7689999999999999e-06 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158851+0000", 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.998e-06 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158869+0000", 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7356000000000001e-05 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158876+0000", 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6229999999999997e-06 2024-09-21T10:04:00.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158948+0000", 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1767000000000006e-05 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.158958+0000", 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9879999999999999e-06 2024-09-21T10:04:00.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.916959+0000", 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75800074799999995 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.917095+0000", 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013579900000000001 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.917204+0000", 2024-09-21T10:04:00.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109182 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927003+0000", 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097995760000000008 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927174+0000", 2024-09-21T10:04:00.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000170786 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950512+0000", 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023337994000000001 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950551+0000", 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9227999999999998e-05 2024-09-21T10:04:00.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950582+0000", 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0681e-05 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159255 6.f 6:fe2e5df9:::benchmark_data_smithi076_154645_object45349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.342273+0000", 2024-09-21T10:04:00.646 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.32749256599999, 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.60849325200000004, 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159255 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342273+0000", 2024-09-21T10:04:00.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342273+0000", 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342275+0000", 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.575e-06 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342280+0000", 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5299999999999998e-06 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342292+0000", 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2115e-05 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342297+0000", 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3800000000000002e-06 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342353+0000", 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6195e-05 2024-09-21T10:04:00.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.342360+0000", 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5629999999999997e-06 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.922043+0000", 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.57968352300000003 2024-09-21T10:04:00.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.922229+0000", 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018597700000000001 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.922339+0000", 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010936499999999999 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937451+0000", 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015112038 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.943758+0000", 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006307511 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950664+0000", 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069058239999999996 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950704+0000", 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9446999999999997e-05 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950767+0000", 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3208999999999997e-05 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.654 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159258 6.f 6:f50607f5:::benchmark_data_smithi076_154645_object45352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.345695+0000", 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.32407099900001, 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.60567683400000005, 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159258 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345695+0000", 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345695+0000", 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345696+0000", 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3289999999999999e-06 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345702+0000", 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1379999999999998e-06 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345714+0000", 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2772000000000001e-05 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345721+0000", 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7379999999999997e-06 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345776+0000", 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4976999999999999e-05 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.345783+0000", 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4949999999999996e-06 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927211+0000", 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.58142859400000002 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927328+0000", 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011688699999999999 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.927452+0000", 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012365299999999999 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.943763+0000", 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016311811999999998 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950452+0000", 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066881589999999999 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951082+0000", 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00063082299999999997 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951227+0000", 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014471699999999999 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951372+0000", 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014474 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159302 6.f 6:f3d805b1:::benchmark_data_smithi076_154645_object45396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.661 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.450435+0000", 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.21933074600003, 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.50150189700000003, 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159302 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.662 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450435+0000", 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450435+0000", 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450437+0000", 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.443e-06 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450442+0000", 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3820000000000003e-06 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450455+0000", 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2689e-05 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450462+0000", 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9739999999999996e-06 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450527+0000", 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5129999999999995e-05 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.450539+0000", 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2199e-05 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.932309+0000", 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.48176940899999998 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.665 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.932411+0000", 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010220799999999999 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.932534+0000", 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012294300000000001 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.943768+0000", 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011234177999999999 2024-09-21T10:04:00.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950630+0000", 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0068617390000000004 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951676+0000", 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010461310000000001 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951819+0000", 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014361300000000001 2024-09-21T10:04:00.667 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.951937+0000", 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000117859 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34674.0:159325 6.f 6:f4e968b7:::benchmark_data_smithi076_154645_object45419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T09:59:03.552683+0000", 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 296.11708342399999, 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.39968836299999999, 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.668 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34674", 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 159325 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552683+0000", 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.669 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552683+0000", 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552684+0000", 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.39e-06 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552689+0000", 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.583e-06 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552700+0000", 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1615e-05 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552706+0000", 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3879999999999999e-06 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552758+0000", 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.189e-05 2024-09-21T10:04:00.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.552766+0000", 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9090000000000006e-06 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937484+0000", 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.38471757299999998 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937595+0000", 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110933 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.937707+0000", 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011215200000000001 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950419+0000", 2024-09-21T10:04:00.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012711524 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.950927+0000", 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00050878800000000004 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.952090+0000", 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011627440000000001 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.952240+0000", 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000149479 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T09:59:03.952371+0000", 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000131395 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.675 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:26453 7.b e1143/1137 7:d9a335b8:::benchmark_data_smithi076_190198_object26452:head v 1143'1680, mlcod=1143'1678)", 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.033992+0000", 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 152.63577365800001, 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22480366199999999, 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033992+0000", 2024-09-21T10:04:00.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033992+0000", 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033993+0000", 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4e-07 2024-09-21T10:04:00.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.034070+0000", 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6508999999999995e-05 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.034081+0000", 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1022999999999999e-05 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.034087+0000", 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5560000000000004e-06 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035516+0000", 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001428756 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035535+0000", 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9188999999999999e-05 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.258743+0000", 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22320741599999999 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.258796+0000", 2024-09-21T10:04:00.680 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3273000000000003e-05 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:26456 7.c e1143/1137 7:3f550937:::benchmark_data_smithi076_190198_object26455:head v 1143'1736, mlcod=1143'1734)", 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.035558+0000", 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 152.63420798300001, 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22499829199999999, 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.681 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035558+0000", 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035558+0000", 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035559+0000", 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8199999999999998e-07 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035588+0000", 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9096000000000001e-05 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035597+0000", 2024-09-21T10:04:00.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6139999999999992e-06 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035603+0000", 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5969999999999997e-06 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035654+0000", 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0708000000000003e-05 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035673+0000", 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9083e-05 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.260439+0000", 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22476589499999999 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.260556+0000", 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000117417 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:26458 7.e e1143/1142 7:7d2003d9:::benchmark_data_smithi076_190198_object26457:head v 1143'1632, mlcod=1143'1630)", 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.035730+0000", 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 152.634036528, 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22607488100000001, 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035730+0000", 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.686 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035730+0000", 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035730+0000", 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9299999999999997e-07 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035787+0000", 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6552999999999999e-05 2024-09-21T10:04:00.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035795+0000", 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5609999999999999e-06 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.035801+0000", 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4609999999999999e-06 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036328+0000", 2024-09-21T10:04:00.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00052715599999999998 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036347+0000", 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8632000000000001e-05 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261761+0000", 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22541404500000001 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261804+0000", 2024-09-21T10:04:00.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3779999999999998e-05 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:26460 7.a e1143/1137 7:5c18703f:::benchmark_data_smithi076_190198_object26459:head v 1143'1709, mlcod=1143'1707)", 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.036629+0000", 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 152.63313752100001, 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22511524099999999, 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:00.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036629+0000", 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036629+0000", 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036629+0000", 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1800000000000005e-07 2024-09-21T10:04:00.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036652+0000", 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2427000000000001e-05 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.258745+0000", 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22209356599999999 2024-09-21T10:04:00.692 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.258761+0000", 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5546e-05 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.258895+0000", 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013452099999999999 2024-09-21T10:04:00.693 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.258925+0000", 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9818000000000001e-05 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261705+0000", 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002780327 2024-09-21T10:04:00.694 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261744+0000", 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8318000000000001e-05 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:00.695 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:04:00.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:00.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:00.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:00.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:00.846 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:00.906 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:00.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:00.959 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:04:00.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:00.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:01.074 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:01.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:01.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:01.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:01.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:01.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:01.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:01.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1810403858' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:01.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:00 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1810403858' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:00 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1810403858' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:00 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:01.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:01.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:01.486 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:04:01.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.601 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:01.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:01.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:01.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.726 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:01.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:01.827 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:01.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:01.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:01.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:01.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[110963]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.4 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 91 op/s; 59346/166074 objects degraded (35.735%); 1473/166074 objects misplaced (0.887%); 3.1 MiB/s, 158 objects/s recovering 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[110963]: osdmap e1176: 8 total, 8 up, 5 in 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[106482]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.4 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 91 op/s; 59346/166074 objects degraded (35.735%); 1473/166074 objects misplaced (0.887%); 3.1 MiB/s, 158 objects/s recovering 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:02.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:01 smithi076 ceph-mon[106482]: osdmap e1176: 8 total, 8 up, 5 in 2024-09-21T10:04:02.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:02.096 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:02.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.198 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:02.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:02.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:02.349 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:01 smithi191 ceph-mon[102206]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.4 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 91 op/s; 59346/166074 objects degraded (35.735%); 1473/166074 objects misplaced (0.887%); 3.1 MiB/s, 158 objects/s recovering 2024-09-21T10:04:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:01 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:01 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:01 smithi191 ceph-mon[102206]: osdmap e1176: 8 total, 8 up, 5 in 2024-09-21T10:04:02.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:02.473 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:02.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:02.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:02.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:02.698 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:02.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:02.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:02.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:02.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:02.945 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:02.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:02.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:02.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:03.069 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:03.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:03.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:03.198 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:03.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:03.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.350 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:03.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:04:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:04:03] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-09-21T10:04:03.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:03.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:03.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:03.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:03.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:03.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:03.783 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:03.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:03.841 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:03.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:03.964 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:03.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:04.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:04.087 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:04.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:04.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:03 smithi076 ceph-mon[106482]: pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 211 op/s; 59027/163959 objects degraded (36.001%); 1210/163959 objects misplaced (0.738%); 8.1 MiB/s, 256 objects/s recovering 2024-09-21T10:04:04.101 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:03 smithi076 ceph-mon[110963]: pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 211 op/s; 59027/163959 objects degraded (36.001%); 1210/163959 objects misplaced (0.738%); 8.1 MiB/s, 256 objects/s recovering 2024-09-21T10:04:04.189 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:04.214 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:04.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:04.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:04.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:03 smithi191 ceph-mon[102206]: pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 211 op/s; 59027/163959 objects degraded (36.001%); 1210/163959 objects misplaced (0.738%); 8.1 MiB/s, 256 objects/s recovering 2024-09-21T10:04:04.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:04.465 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:04.652 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:04.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:04.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:04.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:04.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:04.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:04.935 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:04.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:04.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:05.037 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:05.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:05.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 59027/163959 objects degraded (36.001%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:05.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 59027/163959 objects degraded (36.001%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:05.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:04:05.161 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:05.213 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:05.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:05.337 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:05.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 59027/163959 objects degraded (36.001%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:05.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:05.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:05.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:05.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:05.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:05.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:05.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:05.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:05.834 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:05.936 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:05.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:05.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:06.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:06.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:06.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:06.189 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:06.213 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:06.297 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:05 smithi076 ceph-mon[106482]: pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 115 op/s; 59027/163959 objects degraded (36.001%); 1210/163959 objects misplaced (0.738%); 4.7 MiB/s, 180 objects/s recovering 2024-09-21T10:04:06.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:05 smithi076 ceph-mon[110963]: pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 115 op/s; 59027/163959 objects degraded (36.001%); 1210/163959 objects misplaced (0.738%); 4.7 MiB/s, 180 objects/s recovering 2024-09-21T10:04:06.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:06.337 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:06.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:05 smithi191 ceph-mon[102206]: pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 115 op/s; 59027/163959 objects degraded (36.001%); 1210/163959 objects misplaced (0.738%); 4.7 MiB/s, 180 objects/s recovering 2024-09-21T10:04:06.415 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:04:06.415 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:04:06.415 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.415 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89889 7.e 7:7b12db03:::benchmark_data_smithi076_190198_object17900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.415 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.525014+0000", 2024-09-21T10:04:06.415 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.76895011000000002, 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76898328599999999, 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89889 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.525014+0000", 2024-09-21T10:04:06.416 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.525014+0000", 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.525016+0000", 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.601e-06 2024-09-21T10:04:06.417 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.525022+0000", 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1399999999999997e-06 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.525035+0000", 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3451e-05 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.418 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.525043+0000", 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5719999999999997e-06 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.652672+0000", 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.127629091 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.658015+0000", 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053425130000000001 2024-09-21T10:04:06.419 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.681687+0000", 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023672011999999999 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.685878+0000", 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0041915709999999998 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.420 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89972 7.e 7:787f889b:::benchmark_data_smithi076_190198_object17983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.936240+0000", 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.35772458299999998, 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.35782980800000003, 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89972 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.421 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.936240+0000", 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.936240+0000", 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.936241+0000", 2024-09-21T10:04:06.422 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3659999999999999e-06 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.936246+0000", 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.203e-06 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.936260+0000", 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3750999999999999e-05 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.423 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.936268+0000", 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8460000000000008e-06 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.029907+0000", 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.093638694999999994 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.034785+0000", 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00487807 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.424 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.067475+0000", 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.032690618999999997 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.072444+0000", 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004968727 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:90019 7.6 7:637e5ed1:::benchmark_data_smithi076_190198_object18030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.425 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.269578+0000", 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.024386333999999999, 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.024555819999999999, 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for degraded object", 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 90019 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.426 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.269578+0000", 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.269578+0000", 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.269580+0000", 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.483e-06 2024-09-21T10:04:06.427 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.269584+0000", 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7879999999999997e-06 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.269596+0000", 2024-09-21T10:04:06.428 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2051e-05 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.269603+0000", 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8560000000000001e-06 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.271955+0000", 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023518129999999999 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.429 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for degraded object", 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.272701+0000", 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00074609299999999997 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89974 7.e 7:780b0171:::benchmark_data_smithi076_190198_object17985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.430 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.943003+0000", 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.350961776, 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.35118757900000003, 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89974 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.431 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.943003+0000", 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.943003+0000", 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.432 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.943004+0000", 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.516e-06 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.943010+0000", 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4709999999999998e-06 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.943021+0000", 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0983e-05 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.433 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.943029+0000", 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0600000000000008e-06 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.035849+0000", 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.092820250000000007 2024-09-21T10:04:06.434 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.040988+0000", 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051389820000000003 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.091355+0000", 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.050367473000000003 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.435 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.095455+0000", 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040991350000000003 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89893 7.e 7:79cd7745:::benchmark_data_smithi076_190198_object17904:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.542278+0000", 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.75168648500000002, 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75197443200000003, 2024-09-21T10:04:06.436 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89893 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.437 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.542278+0000", 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.542278+0000", 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.438 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.542279+0000", 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5009999999999999e-06 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.542285+0000", 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7610000000000004e-06 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.439 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.542300+0000", 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4989e-05 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.542308+0000", 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7470000000000005e-06 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.658581+0000", 2024-09-21T10:04:06.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.11627317299999999 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.664359+0000", 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0057781200000000003 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.702989+0000", 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.038629284999999999 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.707533+0000", 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045446289999999997 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89906 7.e 7:7469aaa5:::benchmark_data_smithi076_190198_object17917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.611478+0000", 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.68248672399999999, 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.68283592299999996, 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.442 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89906 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.611478+0000", 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.443 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.611478+0000", 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.611479+0000", 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.61e-06 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.611485+0000", 2024-09-21T10:04:06.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3630000000000002e-06 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.611496+0000", 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1039999999999999e-05 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.611502+0000", 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4459999999999998e-06 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.445 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.709941+0000", 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.098439281000000003 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.715149+0000", 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00520727 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.734434+0000", 2024-09-21T10:04:06.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.019285098000000001 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.738669+0000", 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042352609999999997 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "MOSDPGPushReply(7.6 1176/1170 [PushReplyOp(7:637e5ed1:::benchmark_data_smithi076_190198_object18030:head)])", 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.278097+0000", 2024-09-21T10:04:06.447 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.015867665999999999, 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016277969, 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "reached pg", 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.278097+0000", 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.278097+0000", 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.278097+0000", 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2199999999999999e-07 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.278101+0000", 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4759999999999998e-06 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.449 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.278108+0000", 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1110000000000002e-06 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.278113+0000", 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9699999999999998e-06 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.450 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293163+0000", 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015049849000000001 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "MOSDPGPushReply(7.6 1176/1170 [PushReplyOp(7:6773768c:::benchmark_data_smithi076_190198_object18024:head)])", 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.279205+0000", 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.014759415999999999, 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015210660000000001, 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.451 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279205+0000", 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279205+0000", 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279206+0000", 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0100000000000001e-06 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279212+0000", 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1190000000000002e-06 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279227+0000", 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4657e-05 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.453 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279236+0000", 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5440000000000004e-06 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "MOSDPGPushReply(7.6 1176/1170 [PushReplyOp(7:637e5ed1:::benchmark_data_smithi076_190198_object18030:head)])", 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.279350+0000", 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.01461399, 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015102081999999999, 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.454 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279350+0000", 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279350+0000", 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279351+0000", 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3900000000000004e-07 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279355+0000", 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4209999999999997e-06 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279367+0000", 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1738e-05 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.279374+0000", 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2359999999999998e-06 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:90006 7.d e1176/1170 7:be533352:::benchmark_data_smithi076_190198_object18017:head v 1176'5719, mlcod=1141'1140)", 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.291965+0000", 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.001998952, 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025271569999999999, 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.457 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.291965+0000", 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.291965+0000", 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.291967+0000", 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2890000000000001e-06 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.291976+0000", 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1449999999999997e-06 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.291995+0000", 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9235000000000001e-05 2024-09-21T10:04:06.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.292004+0000", 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9039999999999998e-06 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.460 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.292088+0000", 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4004000000000004e-05 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.292116+0000", 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8373000000000001e-05 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.461 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:90012 7.6 7:616b5c23:::benchmark_data_smithi076_190198_object18023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.199544+0000", 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.094420583000000002, 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.094996118000000004, 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 90012 2024-09-21T10:04:06.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.199544+0000", 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.199544+0000", 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.199546+0000", 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.776e-06 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.199553+0000", 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1899999999999998e-06 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.199569+0000", 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.668e-05 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.199579+0000", 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5130000000000002e-06 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.270343+0000", 2024-09-21T10:04:06.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.070763533000000003 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for degraded object", 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.271122+0000", 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00077923399999999996 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.287876+0000", 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016754346 2024-09-21T10:04:06.466 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.287954+0000", 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.818e-05 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.288045+0000", 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0706999999999997e-05 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.467 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop_reply(client.34992.0:90012 7.6 e1176/1170)", 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.293816+0000", 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.00014870000000000001, 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00078568900000000001, 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293816+0000", 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293816+0000", 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293817+0000", 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2389999999999999e-06 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293825+0000", 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1450000000000008e-06 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293836+0000", 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0956e-05 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.293844+0000", 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9310000000000008e-06 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89978 7.e 7:7a50b87a:::benchmark_data_smithi076_190198_object17989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.964026+0000", 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.32993864099999998, 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.33061641600000002, 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.472 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89978 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.964026+0000", 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.964026+0000", 2024-09-21T10:04:06.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.964027+0000", 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5430000000000001e-06 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.964035+0000", 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4379999999999998e-06 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.964049+0000", 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4188e-05 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.964058+0000", 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5830000000000007e-06 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.041465+0000", 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.077407130000000005 2024-09-21T10:04:06.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.046649+0000", 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051844450000000002 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.114342+0000", 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.067692402999999998 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.118988+0000", 2024-09-21T10:04:06.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046468739999999996 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89959 7.e 7:7d82c0ff:::benchmark_data_smithi076_190198_object17970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.860183+0000", 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.43378132800000002, 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.43452683199999997, 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.477 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89959 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.860183+0000", 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.478 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.860183+0000", 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.860184+0000", 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.347e-06 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.860190+0000", 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4829999999999999e-06 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.860202+0000", 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2312e-05 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.860209+0000", 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9779999999999999e-06 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.882982+0000", 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.022773003 2024-09-21T10:04:06.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.888878+0000", 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0058962659999999998 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.930263+0000", 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.041384675000000003 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.935482+0000", 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052188039999999996 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:90013 7.6 7:6773768c:::benchmark_data_smithi076_190198_object18024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:06.240467+0000", 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.053497014000000002, 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.054311068999999997, 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.482 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for degraded object", 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 90013 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.240467+0000", 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.483 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.240467+0000", 2024-09-21T10:04:06.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.240469+0000", 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.852e-06 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.240475+0000", 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5330000000000002e-06 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.240489+0000", 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3831e-05 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.240496+0000", 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7039999999999994e-06 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.271165+0000", 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.030669096999999999 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for degraded object", 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.271907+0000", 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00074153800000000003 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89969 7.e 7:7a33afc8:::benchmark_data_smithi076_190198_object17980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:05.926985+0000", 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.36697894599999997, 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.36784873000000001, 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89969 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:06.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.926985+0000", 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.926985+0000", 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.926988+0000", 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2189999999999998e-06 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.926994+0000", 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2759999999999997e-06 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.927006+0000", 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2279999999999999e-05 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.927014+0000", 2024-09-21T10:04:06.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0099999999999995e-06 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.947071+0000", 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020057209999999999 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.953235+0000", 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0061631580000000002 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:06.491 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.007212+0000", 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.053976930999999999 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.012046+0000", 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0048347340000000003 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 16 2024-09-21T10:04:06.492 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:04:06.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:06.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:06.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:06.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:06.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:06.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:06.772 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:06.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:06.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:06.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:06.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:06.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:06.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:07.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:07.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:07.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:07.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:07.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:07.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:07.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:07.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:07.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:07.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:07.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:07.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:07.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:07.643 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:07.745 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:07.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:07.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:07.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:07.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:07.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:08.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:08.027 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:08.105 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:07 smithi076 ceph-mon[106482]: pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 137 op/s; 58762/162735 objects degraded (36.109%); 1189/162735 objects misplaced (0.731%); 5.1 MiB/s, 168 objects/s recovering 2024-09-21T10:04:08.105 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:07 smithi076 ceph-mon[110963]: pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 137 op/s; 58762/162735 objects degraded (36.109%); 1189/162735 objects misplaced (0.731%); 5.1 MiB/s, 168 objects/s recovering 2024-09-21T10:04:08.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:08.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:08.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:08.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:08.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:07 smithi191 ceph-mon[102206]: pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 137 op/s; 58762/162735 objects degraded (36.109%); 1189/162735 objects misplaced (0.731%); 5.1 MiB/s, 168 objects/s recovering 2024-09-21T10:04:08.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:08.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:08.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:08.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:08.526 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:08.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:08.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:08.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:08.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:08.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:08.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:08.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:08.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:09.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:09.030 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:09.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:09.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:09.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:09.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:09.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:09.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:09.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:09.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:09.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:09.669 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:09.771 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:09.794 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:09.896 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:09.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:09.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:09.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:10.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:10.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:10.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:10.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:10.187 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:10.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:10.316 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:09 smithi191 ceph-mon[102206]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 58675/161832 objects degraded (36.257%); 1189/161832 objects misplaced (0.735%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:04:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 58675/161832 objects degraded (36.257%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:09 smithi076 ceph-mon[106482]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 58675/161832 objects degraded (36.257%); 1189/161832 objects misplaced (0.735%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:04:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 58675/161832 objects degraded (36.257%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:09 smithi076 ceph-mon[110963]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 58675/161832 objects degraded (36.257%); 1189/161832 objects misplaced (0.735%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:04:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 58675/161832 objects degraded (36.257%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:10.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:10.440 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:10.542 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:10.571 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:10.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:10.694 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:10.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:10.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:10.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:10.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:10.954 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:10.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:11.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:11.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:11.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:11.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:04:11.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:11.315 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:11.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:11.451 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:11.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:11.559 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:04:11.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:11.622 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:11.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:11.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:11.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:11.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:11.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[106482]: pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 58675/161832 objects degraded (36.257%); 1189/161832 objects misplaced (0.735%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:04:11.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/901460017' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:04:11.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:11.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:11.737 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[110963]: pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 58675/161832 objects degraded (36.257%); 1189/161832 objects misplaced (0.735%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:04:11.737 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/901460017' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:04:11.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:11.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:11.856 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:11 smithi191 ceph-mon[102206]: pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 141 op/s; 58675/161832 objects degraded (36.257%); 1189/161832 objects misplaced (0.735%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:04:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/901460017' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:04:11.868 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:04:11.869 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:11.884 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:11.894 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:04:11.894 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:11.916 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:04:11.916 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:11.944 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:04:11.944 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:11.961 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:04:11.961 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:11.983 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:04:11.984 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:11.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:12.003 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:04:12.003 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:12.010 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:12.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:12.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:04:12.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:12.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:12.135 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:12.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:12.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:12.370 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:12.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:12.473 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:12.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:12.524 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:12.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:12.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:12.753 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:12.775 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/901460017' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:04:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:12 smithi191 ceph-mon[102206]: osdmap e1177: 8 total, 8 up, 5 in 2024-09-21T10:04:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1457600668' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/901460017' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:04:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:12 smithi076 ceph-mon[106482]: osdmap e1177: 8 total, 8 up, 5 in 2024-09-21T10:04:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1457600668' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/901460017' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:04:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:12 smithi076 ceph-mon[110963]: osdmap e1177: 8 total, 8 up, 5 in 2024-09-21T10:04:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1457600668' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:12.878 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:12.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:12.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:12.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:13.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:13.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:13.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:13.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:13.154 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:13.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:13.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:13.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:13.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:04:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:04:13] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:04:13.380 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:13.402 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:13.504 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:13.528 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:13.630 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:13.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:13.657 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:13.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:04:13.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:13.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:13.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:13 smithi191 ceph-mon[102206]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 58290/158934 objects degraded (36.676%); 632/158934 objects misplaced (0.398%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:04:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:13 smithi076 ceph-mon[106482]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 58290/158934 objects degraded (36.676%); 632/158934 objects misplaced (0.398%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:04:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:13 smithi076 ceph-mon[110963]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 58290/158934 objects degraded (36.676%); 632/158934 objects misplaced (0.398%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:04:13.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:13.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:13.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:14.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:14.045 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:14.147 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:14.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:14.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:14.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:14.303 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:14.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:14.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:14.427 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:14.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:14.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:14.556 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:14.658 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:14.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:14.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:14.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:14.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 58290/158934 objects degraded (36.676%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 58290/158934 objects degraded (36.676%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 58290/158934 objects degraded (36.676%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:14.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:14.937 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:14.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:15.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:15.066 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:04:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:15.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:15.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:15.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:15.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:15.298 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:15.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:15.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:15.453 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:15.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:15.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:15.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:15.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:15.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:15.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:15.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:15.840 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:15 smithi076 ceph-mon[106482]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 58290/158934 objects degraded (36.676%); 632/158934 objects misplaced (0.398%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:04:15.841 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:15 smithi076 ceph-mon[110963]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 58290/158934 objects degraded (36.676%); 632/158934 objects misplaced (0.398%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:04:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:15 smithi191 ceph-mon[102206]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 58290/158934 objects degraded (36.676%); 632/158934 objects misplaced (0.398%); 6.9 MiB/s, 109 objects/s recovering 2024-09-21T10:04:15.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:15.961 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:15.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:16.063 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:16.084 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:16.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:16.206 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:16.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:16.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:16.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:16.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:16.458 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:16.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:16.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:16.581 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:16.598 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:04:16.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:16.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:16.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:16.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2753750999' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:16.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:16 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:16.757 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2753750999' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:16.757 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:16 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:16.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:16.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:16.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2753750999' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:16 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:04:16.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:16.958 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:16.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:17.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:17.086 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:17.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:17.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:17.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:17.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:17.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:17.465 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:17.567 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:17.592 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:17.682 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:17.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:17.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:17.817 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:17.846 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:17 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:17 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:04:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:17 smithi191 ceph-mon[102206]: osdmap e1178: 8 total, 8 up, 5 in 2024-09-21T10:04:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:17 smithi191 ceph-mon[102206]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 187 op/s; 58004/157341 objects degraded (36.865%); 117/157341 objects misplaced (0.074%); 12 MiB/s, 190 objects/s recovering 2024-09-21T10:04:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:04:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[106482]: osdmap e1178: 8 total, 8 up, 5 in 2024-09-21T10:04:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[106482]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 187 op/s; 58004/157341 objects degraded (36.865%); 117/157341 objects misplaced (0.074%); 12 MiB/s, 190 objects/s recovering 2024-09-21T10:04:17.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:17.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:04:17.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[110963]: osdmap e1178: 8 total, 8 up, 5 in 2024-09-21T10:04:17.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:17 smithi076 ceph-mon[110963]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 187 op/s; 58004/157341 objects degraded (36.865%); 117/157341 objects misplaced (0.074%); 12 MiB/s, 190 objects/s recovering 2024-09-21T10:04:17.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:17.971 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:17.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:17.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:18.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:18.095 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:18.197 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:18.226 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:18.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:18.318 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:18.318 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:18.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:18.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:18.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:18.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:18.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:18.603 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:18.705 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:18.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:18.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:18.851 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:18.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:18.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:18.976 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:18.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:19.063 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:19.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:19.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:19.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:19.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:19.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:19.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:19.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:19.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:19.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:19.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:04:19.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:19.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:19.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:04:19.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:19.735 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:19.763 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:19 smithi076 ceph-mon[106482]: pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 57892/156126 objects degraded (37.080%); 117/156126 objects misplaced (0.075%); 13 MiB/s, 205 objects/s recovering 2024-09-21T10:04:19.763 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 57892/156126 objects degraded (37.080%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:19.763 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:19 smithi076 ceph-mon[110963]: pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 57892/156126 objects degraded (37.080%); 117/156126 objects misplaced (0.075%); 13 MiB/s, 205 objects/s recovering 2024-09-21T10:04:19.763 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 57892/156126 objects degraded (37.080%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:19.837 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:19.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:19.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:19.996 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:19.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:19.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:20.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:19 smithi191 ceph-mon[102206]: pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 237 op/s; 57892/156126 objects degraded (37.080%); 117/156126 objects misplaced (0.075%); 13 MiB/s, 205 objects/s recovering 2024-09-21T10:04:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 57892/156126 objects degraded (37.080%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:20.123 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:20.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:20.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:20.247 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:20.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:20.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:20.374 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:20.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:20.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:20.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:20.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:20.727 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:20.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:20.856 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:20.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:20.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:20.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:21.010 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:21.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:21.136 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:21.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:21.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:21.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:21.388 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:21.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:21.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:21.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:21.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:21.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:21.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:21.763 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:21.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:21.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:21.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:21.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:21.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:21.974 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:21 smithi076 ceph-mon[106482]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 151 op/s; 57892/156126 objects degraded (37.080%); 117/156126 objects misplaced (0.075%); 6.4 MiB/s, 101 objects/s recovering 2024-09-21T10:04:21.975 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:21 smithi076 ceph-mon[110963]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 151 op/s; 57892/156126 objects degraded (37.080%); 117/156126 objects misplaced (0.075%); 6.4 MiB/s, 101 objects/s recovering 2024-09-21T10:04:21.988 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:22.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:21 smithi191 ceph-mon[102206]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.2 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 151 op/s; 57892/156126 objects degraded (37.080%); 117/156126 objects misplaced (0.075%); 6.4 MiB/s, 101 objects/s recovering 2024-09-21T10:04:22.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:22.138 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:22.240 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:22.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:22.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:22.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:22.402 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:22.402 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:22.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:22.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:22.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:22.638 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:22.673 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:04:22.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:22.767 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:22.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3988005505' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:04:22.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:04:22.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3988005505' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:04:22.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:04:22.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:22.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:22.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:22.927 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:04:22.927 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:22.947 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:04:22.947 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:22.974 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:04:22.975 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:22.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:22.998 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:04:22.998 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:23.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:23.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:23.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:04:23.020 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:23.043 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:04:23.044 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:23.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:04:23.061 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:23.083 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:04:23.083 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3988005505' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:04:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:04:23.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.144 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:23.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:23.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:23.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:23.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:23.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:04:23] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:04:23.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:23.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:23.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:23.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:23.620 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:23.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:23.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:23.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:23.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:23.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:23.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:23.930 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[106482]: pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 56858/153453 objects degraded (37.052%); 7.6 MiB/s, 183 objects/s recovering 2024-09-21T10:04:23.930 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:04:23.930 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[106482]: osdmap e1179: 8 total, 8 up, 5 in 2024-09-21T10:04:23.931 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2362568466' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:23.931 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[110963]: pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 56858/153453 objects degraded (37.052%); 7.6 MiB/s, 183 objects/s recovering 2024-09-21T10:04:23.931 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:04:23.931 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[110963]: osdmap e1179: 8 total, 8 up, 5 in 2024-09-21T10:04:23.931 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:23 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2362568466' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:24.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:24.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:24.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:23 smithi191 ceph-mon[102206]: pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 56858/153453 objects degraded (37.052%); 7.6 MiB/s, 183 objects/s recovering 2024-09-21T10:04:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:23 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:04:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:23 smithi191 ceph-mon[102206]: osdmap e1179: 8 total, 8 up, 5 in 2024-09-21T10:04:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:23 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2362568466' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:24.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:24.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:24.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:24.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:24.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:24.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:24.511 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:24.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:24.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:24.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:24.663 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:24.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:24.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:24.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:24.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:24.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:24.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:25.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:25.039 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 56858/153453 objects degraded (37.052%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:25.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 56858/153453 objects degraded (37.052%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:25.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 56858/153453 objects degraded (37.052%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:25.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:25.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:25.270 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:25.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:25.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:25.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:25.417 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:25.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:04:25.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:25.541 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:25.643 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:25.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:25.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:25.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:25.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:25 smithi076 ceph-mon[106482]: pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 161 op/s; 56858/153453 objects degraded (37.052%); 4.1 MiB/s, 143 objects/s recovering 2024-09-21T10:04:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:25 smithi076 ceph-mon[110963]: pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 161 op/s; 56858/153453 objects degraded (37.052%); 4.1 MiB/s, 143 objects/s recovering 2024-09-21T10:04:25.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:25.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:25.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:25.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:26.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:26.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:26.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:25 smithi191 ceph-mon[102206]: pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 161 op/s; 56858/153453 objects degraded (37.052%); 4.1 MiB/s, 143 objects/s recovering 2024-09-21T10:04:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:26.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:26.180 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:26.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:26.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:26.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:26.399 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:04:26.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:26.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:26.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:26.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:26.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:26.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:26.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:26.817 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:26.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.846 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:26.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:26 smithi076 ceph-mon[106482]: osdmap e1180: 8 total, 8 up, 5 in 2024-09-21T10:04:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:26 smithi076 ceph-mon[110963]: osdmap e1180: 8 total, 8 up, 5 in 2024-09-21T10:04:26.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:26.973 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:26.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:27.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:27.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:26 smithi191 ceph-mon[102206]: osdmap e1180: 8 total, 8 up, 5 in 2024-09-21T10:04:27.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:27.239 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:27.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:27.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:27.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:27.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:27.499 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:27.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:27.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:27.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:27.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:27.706 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:04:27.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:27.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:27.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:27.871 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:27.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:27.972 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:27.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[106482]: pgmap v1477: 89 pgs: 1 peering, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 172 op/s; 56296/151995 objects degraded (37.038%); 4.4 MiB/s, 187 objects/s recovering 2024-09-21T10:04:27.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[106482]: osdmap e1181: 8 total, 8 up, 5 in 2024-09-21T10:04:27.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1531374926' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:27.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:27.999 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[110963]: pgmap v1477: 89 pgs: 1 peering, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 172 op/s; 56296/151995 objects degraded (37.038%); 4.4 MiB/s, 187 objects/s recovering 2024-09-21T10:04:27.999 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[110963]: osdmap e1181: 8 total, 8 up, 5 in 2024-09-21T10:04:27.999 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1531374926' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:27.999 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:27 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:28.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:28.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:28.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:27 smithi191 ceph-mon[102206]: pgmap v1477: 89 pgs: 1 peering, 3 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 172 op/s; 56296/151995 objects degraded (37.038%); 4.4 MiB/s, 187 objects/s recovering 2024-09-21T10:04:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:27 smithi191 ceph-mon[102206]: osdmap e1181: 8 total, 8 up, 5 in 2024-09-21T10:04:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1531374926' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:27 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:28.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:28.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:28.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:28.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:28.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:28.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:28.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:28.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:28.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:28.632 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:28.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:28.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:28.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:28.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:28.888 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:28.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:28.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:28.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.015 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:29.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:29.020 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:28 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:29.021 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:28 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:29.021 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:28 smithi076 ceph-mon[106482]: osdmap e1182: 8 total, 8 up, 5 in 2024-09-21T10:04:29.021 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:28 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:29.021 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:28 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:29.021 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:28 smithi076 ceph-mon[110963]: osdmap e1182: 8 total, 8 up, 5 in 2024-09-21T10:04:29.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:28 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:28 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:28 smithi191 ceph-mon[102206]: osdmap e1182: 8 total, 8 up, 5 in 2024-09-21T10:04:29.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:29.143 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:29.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:29.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:29.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:29.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:29.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:29.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:29.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:29.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:29.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:29.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:29.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:29.750 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:29.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:29.769 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:04:29.769 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:04:29.769 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:04:29.769 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52756 7.8 7:10c9f1bf:::benchmark_data_smithi076_190198_object52755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125262+0000", 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63758379800001, 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9378176539999998, 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52756 2024-09-21T10:04:29.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125262+0000", 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125262+0000", 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125263+0000", 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3790000000000001e-06 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125287+0000", 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4253e-05 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125298+0000", 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0248000000000001e-05 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125306+0000", 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5650000000000002e-06 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125505+0000", 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000199271 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125518+0000", 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2286000000000001e-05 2024-09-21T10:04:29.773 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:54.179180+0000", 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0536622609999999 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:54.182680+0000", 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034997119999999999 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:54.188088+0000", 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054078939999999999 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.051092+0000", 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86300423900000001 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.052612+0000", 2024-09-21T10:04:29.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001519699 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.053801+0000", 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011891689999999999 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.055577+0000", 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017763480000000001 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.776 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.063079+0000", 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075023299999999998 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52768 7.8 7:15148440:::benchmark_data_smithi076_190198_object52767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125318+0000", 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63752708199999, 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9777066379999999, 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.777 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52768 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125318+0000", 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125318+0000", 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125319+0000", 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.82e-07 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125340+0000", 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1426000000000001e-05 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125347+0000", 2024-09-21T10:04:29.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5359999999999998e-06 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125352+0000", 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1549999999999998e-06 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125545+0000", 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019330899999999999 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125553+0000", 2024-09-21T10:04:29.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9449999999999993e-06 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.063708+0000", 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.938154607 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.068305+0000", 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004596915 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.074704+0000", 2024-09-21T10:04:29.781 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063993690000000002 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.092571+0000", 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017866988 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.093768+0000", 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011968129999999999 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.094457+0000", 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00068903300000000003 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.095958+0000", 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001501346 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.103025+0000", 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.007066714 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.783 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52784 7.8 7:1ac06dd3:::benchmark_data_smithi076_190198_object52783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.637486462, 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9892106549999999, 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.784 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52784 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9400000000000001e-07 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125381+0000", 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1469999999999999e-05 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125387+0000", 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1739999999999997e-06 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125392+0000", 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7110000000000003e-06 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125577+0000", 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018529399999999999 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125584+0000", 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6619999999999999e-06 2024-09-21T10:04:29.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.078364+0000", 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.952780089 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.081931+0000", 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003566657 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.087347+0000", 2024-09-21T10:04:29.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054161340000000004 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.104186+0000", 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016838907 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.105762+0000", 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015761690000000001 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.789 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.106422+0000", 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00065996800000000003 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.107987+0000", 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001565676 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.114570+0000", 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065823499999999998 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.790 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52790 7.8 7:1cf5039a:::benchmark_data_smithi076_190198_object52789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63744762900001, 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0320771440000001, 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52790 2024-09-21T10:04:29.791 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:04:29.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0499999999999999e-07 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125420+0000", 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1310000000000001e-05 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125426+0000", 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0929999999999999e-06 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125430+0000", 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6879999999999998e-06 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125611+0000", 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018086400000000001 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125619+0000", 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2490000000000004e-06 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.794 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.114931+0000", 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.989312263 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.118768+0000", 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038371450000000001 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.124328+0000", 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0055604110000000003 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.145040+0000", 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020711415 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.146315+0000", 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012752219999999999 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.148584+0000", 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022692369999999999 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.150147+0000", 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001562475 2024-09-21T10:04:29.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.157475+0000", 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073283669999999997 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52807 7.8 7:175fc991:::benchmark_data_smithi076_190198_object52806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125436+0000", 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63740924999999, 2024-09-21T10:04:29.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.042878151, 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52807 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125436+0000", 2024-09-21T10:04:29.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125436+0000", 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125437+0000", 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7599999999999998e-07 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125499+0000", 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2487e-05 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125506+0000", 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4309999999999999e-06 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125511+0000", 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9300000000000002e-06 2024-09-21T10:04:29.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125643+0000", 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000132325 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125650+0000", 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3529999999999997e-06 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.128543+0000", 2024-09-21T10:04:29.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0028929029999998 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.133053+0000", 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045100019999999999 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.139776+0000", 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0067227229999999999 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.147802+0000", 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0080258459999999997 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.158680+0000", 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010877886 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.159319+0000", 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00063960700000000004 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.161032+0000", 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017127119999999999 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.168314+0000", 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072825700000000004 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52808 7.8 7:106b279b:::benchmark_data_smithi076_190198_object52807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.805 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63732872400001, 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0821514780000001, 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52808 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1800000000000001e-07 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125538+0000", 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0531999999999999e-05 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125545+0000", 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9879999999999998e-06 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125550+0000", 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1390000000000003e-06 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125674+0000", 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124404 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125681+0000", 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7730000000000002e-06 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.168728+0000", 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0430472690000001 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.172808+0000", 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040795359999999999 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.179117+0000", 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063089900000000004 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.197374+0000", 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018256989000000001 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.198951+0000", 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015774210000000001 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.199662+0000", 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00071113999999999997 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.201077+0000", 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014148800000000001 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.207668+0000", 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065909990000000002 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.812 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52818 7.8 7:179d769e:::benchmark_data_smithi076_190198_object52817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125556+0000", 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.637289369, 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1853145110000001, 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52818 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.813 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125556+0000", 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125556+0000", 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125557+0000", 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0900000000000002e-07 2024-09-21T10:04:29.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125576+0000", 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9735999999999999e-05 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125582+0000", 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0759999999999999e-06 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125587+0000", 2024-09-21T10:04:29.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7430000000000002e-06 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125874+0000", 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00028725700000000002 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125883+0000", 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3739999999999998e-06 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.183475+0000", 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0575924579999998 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.187473+0000", 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039973689999999997 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.192809+0000", 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053367349999999996 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.208062+0000", 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015252388 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.297826+0000", 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.089764627999999999 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.300098+0000", 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022719319999999999 2024-09-21T10:04:29.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.302169+0000", 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020704209999999998 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.310871+0000", 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0087019850000000006 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52827 7.8 7:10f0ddf6:::benchmark_data_smithi076_190198_object52826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125838+0000", 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63700766299999, 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2459487029999998, 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52827 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125838+0000", 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125838+0000", 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125839+0000", 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.68e-07 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.821 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125887+0000", 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8659000000000003e-05 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125897+0000", 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7210000000000006e-06 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125903+0000", 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4860000000000002e-06 2024-09-21T10:04:29.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125997+0000", 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3828999999999999e-05 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.126006+0000", 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.619e-06 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.311618+0000", 2024-09-21T10:04:29.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.185612619 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.316660+0000", 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005041149 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.323970+0000", 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.007310024 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.362884+0000", 2024-09-21T10:04:29.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.038914532000000002 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.363448+0000", 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056372300000000002 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.364299+0000", 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00085096100000000001 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.366052+0000", 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017532139999999999 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.371787+0000", 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0057344989999999997 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52845 7.8 7:1187a13e:::benchmark_data_smithi076_190198_object52844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125927+0000", 2024-09-21T10:04:29.826 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.63691836500001, 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2574380139999999, 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52845 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125927+0000", 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125927+0000", 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125928+0000", 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8999999999999996e-07 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.828 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125974+0000", 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6186999999999997e-05 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125982+0000", 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1950000000000003e-06 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.829 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125988+0000", 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9660000000000001e-06 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.126030+0000", 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1625999999999998e-05 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.126037+0000", 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.883e-06 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.830 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.328679+0000", 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2026427630000001 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.332779+0000", 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040996399999999999 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.340983+0000", 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0082042360000000002 2024-09-21T10:04:29.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.372677+0000", 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.031693547000000002 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.373231+0000", 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00055373499999999997 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.374472+0000", 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012418139999999999 2024-09-21T10:04:29.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.376369+0000", 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018962149999999999 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.383365+0000", 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069966170000000001 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52853 7.8 7:1f7ef906:::benchmark_data_smithi076_190198_object52852:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.159054+0000", 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.603791704, 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2362974119999999, 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52853 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.834 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159054+0000", 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159054+0000", 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159055+0000", 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4699999999999999e-06 2024-09-21T10:04:29.835 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159077+0000", 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1679e-05 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159088+0000", 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1447000000000001e-05 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159096+0000", 2024-09-21T10:04:29.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.374e-06 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159185+0000", 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9529000000000003e-05 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159197+0000", 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1537e-05 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.346249+0000", 2024-09-21T10:04:29.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1870516360000001 2024-09-21T10:04:29.838 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.838 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:29.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:29.840 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:29.840 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.840 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.351584+0000", 2024-09-21T10:04:29.840 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053351199999999996 2024-09-21T10:04:29.840 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.357971+0000", 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063877669999999999 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.373507+0000", 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015535739999999999 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.384672+0000", 2024-09-21T10:04:29.841 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011165059999999999 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.386260+0000", 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015877350000000001 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.388538+0000", 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022780880000000002 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.395351+0000", 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00681323 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52867 7.8 7:1ff0d522:::benchmark_data_smithi076_190198_object52866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.171941+0000", 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.59090432900001, 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2690444109999999, 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.843 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52867 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171941+0000", 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.844 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171941+0000", 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171942+0000", 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4099999999999999e-07 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171975+0000", 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3419e-05 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.845 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171981+0000", 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9519999999999999e-06 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171985+0000", 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5090000000000002e-06 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.172045+0000", 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0010999999999999e-05 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.846 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.172055+0000", 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0739e-05 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.395713+0000", 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.223657384 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.400221+0000", 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045087979999999996 2024-09-21T10:04:29.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.407140+0000", 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069187329999999998 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.429665+0000", 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.022524371000000001 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.848 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.430411+0000", 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00074638000000000005 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.431429+0000", 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001017712 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.433802+0000", 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023734369999999999 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.440986+0000", 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0071835249999999996 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52869 7.8 7:179415fd:::benchmark_data_smithi076_190198_object52868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.175548+0000", 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.587297568, 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2876107210000001, 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.850 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52869 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175548+0000", 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175548+0000", 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175549+0000", 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3799999999999999e-06 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175596+0000", 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6592999999999999e-05 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175604+0000", 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3829999999999992e-06 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175611+0000", 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2729999999999999e-06 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175673+0000", 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2162999999999994e-05 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.853 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175682+0000", 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2429999999999994e-06 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.412358+0000", 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2366758820000001 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.416943+0000", 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045848190000000004 2024-09-21T10:04:29.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.424233+0000", 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072906760000000003 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.442258+0000", 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018024212000000001 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.443888+0000", 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0016305110000000001 2024-09-21T10:04:29.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.446498+0000", 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026097339999999998 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.450291+0000", 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003793088 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.463159+0000", 2024-09-21T10:04:29.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012867764 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52884 7.8 7:106bbe8d:::benchmark_data_smithi076_190198_object52883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.196889+0000", 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.56595678900001, 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3147955009999999, 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.857 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52884 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196889+0000", 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196889+0000", 2024-09-21T10:04:29.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196890+0000", 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.325e-06 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196949+0000", 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8884000000000001e-05 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196960+0000", 2024-09-21T10:04:29.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0795000000000001e-05 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196967+0000", 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1110000000000002e-06 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.197033+0000", 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6381e-05 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.860 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.197044+0000", 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0789e-05 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.463653+0000", 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2666088929999999 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.470290+0000", 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066372159999999996 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.477840+0000", 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075500630000000001 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.499602+0000", 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021761857999999999 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.500852+0000", 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012494400000000001 2024-09-21T10:04:29.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.502175+0000", 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001323418 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.504437+0000", 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022620190000000001 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.511684+0000", 2024-09-21T10:04:29.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072473090000000004 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52887 7.8 7:127cfd92:::benchmark_data_smithi076_190198_object52886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.199967+0000", 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.56287888400001, 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3250906430000002, 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.864 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52887 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.199967+0000", 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.199967+0000", 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.199968+0000", 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4670000000000001e-06 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200013+0000", 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4697999999999998e-05 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200023+0000", 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0026e-05 2024-09-21T10:04:29.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200028+0000", 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6339999999999998e-06 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200082+0000", 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3136000000000001e-05 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200090+0000", 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6659999999999997e-06 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.483007+0000", 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2829167049999999 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.487146+0000", 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004138821 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.494516+0000", 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073705519999999998 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.512278+0000", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017761328999999999 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.513504+0000", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012263020000000001 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.514981+0000", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014765399999999999 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.517827+0000", 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028469369999999999 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.525057+0000", 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072298299999999996 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52908 7.8 7:19324e83:::benchmark_data_smithi076_190198_object52907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.249069+0000", 2024-09-21T10:04:29.870 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.513776702, 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3212883099999999, 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52908 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249069+0000", 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249069+0000", 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249070+0000", 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0210000000000001e-06 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249090+0000", 2024-09-21T10:04:29.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0531e-05 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249102+0000", 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1420000000000001e-05 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249112+0000", 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7829999999999993e-06 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249195+0000", 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3673999999999999e-05 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249207+0000", 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1392e-05 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.526267+0000", 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2770600129999998 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.874 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.530238+0000", 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039713839999999997 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.537764+0000", 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075257170000000003 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.559351+0000", 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021587614000000001 2024-09-21T10:04:29.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.560460+0000", 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001108567 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.561858+0000", 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001398299 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.564078+0000", 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022197470000000002 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.570357+0000", 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006279148 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52921 7.8 7:1be3829c:::benchmark_data_smithi076_190198_object52920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.284229+0000", 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 156.47861630099999, 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3019164519999999, 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.877 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52921 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284229+0000", 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284229+0000", 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284231+0000", 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2640000000000001e-06 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284251+0000", 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0055999999999999e-05 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284262+0000", 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1423000000000001e-05 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284268+0000", 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.913e-06 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284333+0000", 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5591000000000003e-05 2024-09-21T10:04:29.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284345+0000", 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1977999999999999e-05 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.541864+0000", 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2575185150000001 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.546200+0000", 2024-09-21T10:04:29.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043359399999999999 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.554034+0000", 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0078339180000000005 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.571643+0000", 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017609265999999998 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.573665+0000", 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020218509999999999 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.574919+0000", 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012535739999999999 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.577250+0000", 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023317590000000001 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.586146+0000", 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0088954040000000009 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:74029 7.8 7:1790c434:::benchmark_data_smithi076_190198_object2040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:50.799065+0000", 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 98.963780990999993, 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3692967949999999, 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.884 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 74029 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.799065+0000", 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.799065+0000", 2024-09-21T10:04:29.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.799066+0000", 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3120000000000001e-06 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.799071+0000", 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.135e-06 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.799081+0000", 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0463e-05 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.799087+0000", 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.446e-06 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.822266+0000", 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.02317866 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:29.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:50.823671+0000", 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00140566 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:51.483665+0000", 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.65999370800000001 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:51.487865+0000", 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004199726 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.888 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.111478+0000", 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.62361351099999995 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.114660+0000", 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031822180000000001 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.119025+0000", 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043650219999999997 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.154228+0000", 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.035203017000000003 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.159101+0000", 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0048729799999999998 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.160475+0000", 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0013736779999999999 2024-09-21T10:04:29.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.162681+0000", 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002205701 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:52.168361+0000", 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056805579999999996 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:75410 7.8 7:1ce9f334:::benchmark_data_smithi076_190198_object3421:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:54.918150+0000", 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 94.844695607000006, 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.383227207, 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.892 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 75410 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.918150+0000", 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.918150+0000", 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.918151+0000", 2024-09-21T10:04:29.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9999999999999996e-07 2024-09-21T10:04:29.894 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.894 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.918155+0000", 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1999999999999996e-06 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.918165+0000", 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6509999999999996e-06 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.918171+0000", 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.93e-06 2024-09-21T10:04:29.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.952514+0000", 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.034343246000000001 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:54.953999+0000", 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014850760000000001 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:55.649545+0000", 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.69554600200000005 2024-09-21T10:04:29.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:55.652789+0000", 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0032437350000000002 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.263511+0000", 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.61072272699999997 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.266614+0000", 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031023919999999998 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.272241+0000", 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056267030000000003 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.290860+0000", 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018619688999999998 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.291414+0000", 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00055376199999999996 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.292303+0000", 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00088862299999999995 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.294341+0000", 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020386660000000002 2024-09-21T10:04:29.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:56.301377+0000", 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0070359050000000003 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89565 7.e 7:7981dcad:::benchmark_data_smithi076_190198_object17576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:03.061373+0000", 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 26.701472975000001, 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.264562805, 2024-09-21T10:04:29.900 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89565 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.061373+0000", 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.061373+0000", 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.061374+0000", 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2920000000000001e-06 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.061379+0000", 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8770000000000001e-06 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.061392+0000", 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3505e-05 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.061400+0000", 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5859999999999999e-06 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.124764+0000", 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.063364466999999994 2024-09-21T10:04:29.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.129095+0000", 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043311690000000002 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.150297+0000", 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021201378999999999 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:03.154956+0000", 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046591369999999998 2024-09-21T10:04:29.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.286852+0000", 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1318956550000001 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.291129+0000", 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042777700000000002 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.298459+0000", 2024-09-21T10:04:29.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073295069999999999 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.309472+0000", 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011013347 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.310844+0000", 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0013715909999999999 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.312816+0000", 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019721719999999999 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.315715+0000", 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00289873 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.325935+0000", 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010220620999999999 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.907 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:89814 7.e 7:73bc031b:::benchmark_data_smithi076_190198_object17825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:04.956518+0000", 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 24.806327495000001, 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2962438119999999, 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 89814 2024-09-21T10:04:29.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.956518+0000", 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.956518+0000", 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.956519+0000", 2024-09-21T10:04:29.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.004e-06 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.956524+0000", 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6659999999999999e-06 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.956534+0000", 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9569999999999997e-06 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:04:29.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.956542+0000", 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2409999999999995e-06 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.978777+0000", 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.022235141999999999 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for missing object", 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:04.982973+0000", 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004196371 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.035249+0000", 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.052275375999999998 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:05.039644+0000", 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043946899999999997 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.149152+0000", 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1095086999999999 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:04:29.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.153797+0000", 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004644909 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.160358+0000", 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006561051 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.200261+0000", 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.039902396999999999 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.239314+0000", 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.039053500999999997 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.241198+0000", 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001884295 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.243813+0000", 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002614239 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:04:29.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:06.252762+0000", 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0089492730000000006 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:04:29.915 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:04:29.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:29.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:29.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:30.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:29 smithi076 ceph-mon[106482]: pgmap v1480: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 148 op/s; 53211/150804 objects degraded (35.285%); 2826/150804 objects misplaced (1.874%); 4.3 MiB/s, 121 objects/s recovering 2024-09-21T10:04:30.051 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 53211/150804 objects degraded (35.285%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:30.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:29 smithi076 ceph-mon[110963]: pgmap v1480: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 148 op/s; 53211/150804 objects degraded (35.285%); 2826/150804 objects misplaced (1.874%); 4.3 MiB/s, 121 objects/s recovering 2024-09-21T10:04:30.051 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 53211/150804 objects degraded (35.285%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:30.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:30.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:29 smithi191 ceph-mon[102206]: pgmap v1480: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 148 op/s; 53211/150804 objects degraded (35.285%); 2826/150804 objects misplaced (1.874%); 4.3 MiB/s, 121 objects/s recovering 2024-09-21T10:04:30.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 53211/150804 objects degraded (35.285%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:30.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:30.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:30.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:30.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:30.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:30.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:30.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:30.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:30.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:30.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:30.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:30.837 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:30.865 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:30.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:30.993 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:30.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:30.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:31.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:31.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:31.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:31.247 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:31.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:31.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:31.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:31.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:31.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:04:31.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:31.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:31.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:31.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:31.625 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:31.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:31.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:31.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:31.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:31 smithi076 ceph-mon[106482]: pgmap v1481: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 147 op/s; 53211/150804 objects degraded (35.285%); 2826/150804 objects misplaced (1.874%); 4.3 MiB/s, 120 objects/s recovering 2024-09-21T10:04:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:31 smithi076 ceph-mon[110963]: pgmap v1481: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 147 op/s; 53211/150804 objects degraded (35.285%); 2826/150804 objects misplaced (1.874%); 4.3 MiB/s, 120 objects/s recovering 2024-09-21T10:04:31.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:31.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:31.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:32.004 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:32.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:32.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:31 smithi191 ceph-mon[102206]: pgmap v1481: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 74 active+clean; 3.1 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 147 op/s; 53211/150804 objects degraded (35.285%); 2826/150804 objects misplaced (1.874%); 4.3 MiB/s, 120 objects/s recovering 2024-09-21T10:04:32.139 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:32.241 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:32.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:32.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:32.388 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:32.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:32.516 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:32.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:32.562 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:04:32.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:32.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:32.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:32.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:32.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:32.807 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:32.874 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:32.900 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:32.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:33.002 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:33.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:33.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:33.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:33.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:33.283 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:33.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:33.409 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:33.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:33.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:33.535 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:04:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:04:33] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.51.0" 2024-09-21T10:04:33.536 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:33.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:33.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.671 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:33.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:33.772 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:33.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:33.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:33 smithi076 ceph-mon[106482]: pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 264 op/s; 52818/147969 objects degraded (35.695%); 2481/147969 objects misplaced (1.677%); 9.4 MiB/s, 195 objects/s recovering 2024-09-21T10:04:33.808 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:33 smithi076 ceph-mon[110963]: pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 264 op/s; 52818/147969 objects degraded (35.695%); 2481/147969 objects misplaced (1.677%); 9.4 MiB/s, 195 objects/s recovering 2024-09-21T10:04:33.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:33.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:33.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:34.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:34.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:33 smithi191 ceph-mon[102206]: pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 264 op/s; 52818/147969 objects degraded (35.695%); 2481/147969 objects misplaced (1.677%); 9.4 MiB/s, 195 objects/s recovering 2024-09-21T10:04:34.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:34.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:34.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:34.317 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:34.419 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:34.442 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:34.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:34.567 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:34.668 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:04:34.669 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:34.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:34.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:34.698 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:04:34.699 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:34.728 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:04:34.728 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:34.759 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:34.762 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:04:34.763 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:34.792 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:34.793 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:04:34.793 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:34.794 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 52818/147969 objects degraded (35.695%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:34.794 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:34 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1994534729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:34.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 52818/147969 objects degraded (35.695%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:34.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:34 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1994534729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:34.821 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:04:34.822 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:34.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:34.844 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:04:34.845 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:34.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 52818/147969 objects degraded (35.695%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:34 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1994534729' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:34.871 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:04:34.871 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:34.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:34.955 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:34.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:35.000 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:35.000 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:35.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:35.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:35.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:35.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:35.365 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:35.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:35.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:35.491 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:35.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:35.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:35.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:35.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:04:35.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:35.796 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:35.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:35.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:35.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:35.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:35.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:36.001 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:36.028 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:36.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:35 smithi076 ceph-mon[106482]: pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 52818/147969 objects degraded (35.695%); 2481/147969 objects misplaced (1.677%); 6.9 MiB/s, 111 objects/s recovering 2024-09-21T10:04:36.049 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:35 smithi076 ceph-mon[110963]: pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 52818/147969 objects degraded (35.695%); 2481/147969 objects misplaced (1.677%); 6.9 MiB/s, 111 objects/s recovering 2024-09-21T10:04:36.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:36.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:35 smithi191 ceph-mon[102206]: pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 167 op/s; 52818/147969 objects degraded (35.695%); 2481/147969 objects misplaced (1.677%); 6.9 MiB/s, 111 objects/s recovering 2024-09-21T10:04:36.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:36.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:36.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:36.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:36.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:36.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:36.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:36.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:36.564 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:36.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:36.691 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:36.744 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:04:36.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:36.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:36 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/604766136' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:04:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:36 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:04:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:36 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/604766136' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:04:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:36 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:04:36.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:36.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:36.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:36.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:37.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:36 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/604766136' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:04:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:36 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:04:37.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:37.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:37.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:37.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:37.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:37.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:37.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:37.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:37.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:37.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:37.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:37.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:37.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:04:37.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:37 smithi191 ceph-mon[102206]: pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 196 op/s; 52519/146145 objects degraded (35.936%); 2481/146145 objects misplaced (1.698%); 6.8 MiB/s, 108 objects/s recovering 2024-09-21T10:04:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:37 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:04:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:37 smithi191 ceph-mon[102206]: osdmap e1183: 8 total, 8 up, 5 in 2024-09-21T10:04:37.888 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:37.888 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:37.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:37.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:37.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:37.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:38.065 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:38.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:38.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:37 smithi076 ceph-mon[106482]: pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 196 op/s; 52519/146145 objects degraded (35.936%); 2481/146145 objects misplaced (1.698%); 6.8 MiB/s, 108 objects/s recovering 2024-09-21T10:04:38.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:37 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:04:38.101 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:37 smithi076 ceph-mon[106482]: osdmap e1183: 8 total, 8 up, 5 in 2024-09-21T10:04:38.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:37 smithi076 ceph-mon[110963]: pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.0 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 196 op/s; 52519/146145 objects degraded (35.936%); 2481/146145 objects misplaced (1.698%); 6.8 MiB/s, 108 objects/s recovering 2024-09-21T10:04:38.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:37 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:04:38.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:37 smithi076 ceph-mon[110963]: osdmap e1183: 8 total, 8 up, 5 in 2024-09-21T10:04:38.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:38.223 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:38.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:38.350 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:38.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:38.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:38.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:38.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:38.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:38.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:04:38.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:38.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:38.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:38.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:38.869 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:38.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:38.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:38.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:39.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:39.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:39.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:39.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:39.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:39.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:39.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:39.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:39.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:39.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:39.626 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:39.728 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:39.750 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:39.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:39.876 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:39.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:39.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:39.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:40.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:40.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:40.103 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:40.110 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:39 smithi191 ceph-mon[102206]: pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 52387/144873 objects degraded (36.161%); 2104/144873 objects misplaced (1.452%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:04:40.111 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 52387/144873 objects degraded (36.161%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:39 smithi076 ceph-mon[106482]: pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 52387/144873 objects degraded (36.161%); 2104/144873 objects misplaced (1.452%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:04:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 52387/144873 objects degraded (36.161%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:39 smithi076 ceph-mon[110963]: pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 52387/144873 objects degraded (36.161%); 2104/144873 objects misplaced (1.452%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:04:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 52387/144873 objects degraded (36.161%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:40.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:40.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:40.255 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:40.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:40.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:40.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:40.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:40.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:40.637 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:40.739 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:40.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:40.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:40.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:40.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:40.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:41.018 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:41.020 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:41.039 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:41.039 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:41.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:41.086 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:41.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:04:41.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:41.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:41.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:41.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.274 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:41.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:41.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:41.399 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:41.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:41.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:41.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:41.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:41.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:41.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:41.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:04:41.782 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:41.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:41.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:41.917 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:41.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:41.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:41.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:42.044 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:42.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:42.063 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:41 smithi076 ceph-mon[106482]: pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 52387/144873 objects degraded (36.161%); 2104/144873 objects misplaced (1.452%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:04:42.064 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:41 smithi076 ceph-mon[110963]: pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 52387/144873 objects degraded (36.161%); 2104/144873 objects misplaced (1.452%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:04:42.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:42.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:41 smithi191 ceph-mon[102206]: pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 197 op/s; 52387/144873 objects degraded (36.161%); 2104/144873 objects misplaced (1.452%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:04:42.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:42.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:42.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:42.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:42.328 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:42.364 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:42.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:42.458 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:42.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:42.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:42.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:42.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:42.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:42.762 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:04:42.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:42.835 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4260147817' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4260147817' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4260147817' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:04:42.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:42.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:42.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:43.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:43.092 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:43.194 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:43.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:43.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:43.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:43.400 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:04:43] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-21T10:04:43.446 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:43.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:43.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:43.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:43.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:43.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:43.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:43.863 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:43.965 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:43.975 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:43.975 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:43.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:43 smithi191 ceph-mon[102206]: pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 189 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T10:04:43.982 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:43 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:43.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4260147817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:04:43.983 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:43 smithi191 ceph-mon[102206]: osdmap e1184: 8 total, 8 up, 5 in 2024-09-21T10:04:43.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:43.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:44.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:44.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[110963]: pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 189 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4260147817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[110963]: osdmap e1184: 8 total, 8 up, 5 in 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[106482]: pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 189 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4260147817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:04:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:43 smithi076 ceph-mon[106482]: osdmap e1184: 8 total, 8 up, 5 in 2024-09-21T10:04:44.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:44.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:44.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:44.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:44.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:44.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:44.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:44.613 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:44.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:44.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:44.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:44.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:44.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:44.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:04:44.966 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:44.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:44.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:45.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 52056/142293 objects degraded (36.584%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:45.118 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:45.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 52056/142293 objects degraded (36.584%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 52056/142293 objects degraded (36.584%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:45.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:45.255 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:45.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:45.356 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:45.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:45.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:45.511 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:45.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:45.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:45.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:45.767 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:45.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:45.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:45.894 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:45.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:45.935 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:45.946 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:04:45.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:46.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:46.063 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:45 smithi191 ceph-mon[102206]: pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 160 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:04:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:45 smithi076 ceph-mon[106482]: pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 160 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:04:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:45 smithi076 ceph-mon[110963]: pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 160 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:04:46.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:46.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:46.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:46.251 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:46.274 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:46.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:46.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:46.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:46.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:46.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:46.530 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:46.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:46.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:46.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:46.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:46.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:46.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:46.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:46.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:46.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:47.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:47.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:47.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:47.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:47.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:47.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:47.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:47.290 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:47.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:47.416 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:47.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:47.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:47.549 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:47.650 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:47.676 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:47.777 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:47.802 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:47 smithi191 ceph-mon[102206]: pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 130 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:04:47.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:47.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:47.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:47.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:48.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:48.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:48.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:47 smithi076 ceph-mon[106482]: pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 130 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:04:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:47 smithi076 ceph-mon[110963]: pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 130 op/s; 52056/142293 objects degraded (36.584%); 1673/142293 objects misplaced (1.176%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:04:48.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:48.183 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:48.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:48.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:48.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:48.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:48.442 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:48.521 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:04:48.521 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:04:48.544 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:48.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:48.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:48.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:48.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:48.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:48.819 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:48.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3407450944' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3407450944' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:48.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:48.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:48.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:48.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:49.045 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:49.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3407450944' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:04:49.172 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:49.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:04:49.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:49.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:49.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:49.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:49.354 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:04:49.354 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:04:49.354 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:04:49.354 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:04:49.355 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:04:49.356 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:04:49.356 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:04:49.356 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:04:49.356 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:04:49.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:49.453 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:49.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:49.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:49.585 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:49.687 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:49.709 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:49.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:49.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:49.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:49 smithi191 ceph-mon[102206]: pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 51680/139719 objects degraded (36.989%); 1209/139719 objects misplaced (0.865%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:04:49.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 51680/139719 objects degraded (36.989%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:49.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:49.964 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:49.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:49.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:50.064 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:50.066 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:50.091 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:49 smithi076 ceph-mon[106482]: pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 51680/139719 objects degraded (36.989%); 1209/139719 objects misplaced (0.865%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:04:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 51680/139719 objects degraded (36.989%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:49 smithi076 ceph-mon[110963]: pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 51680/139719 objects degraded (36.989%); 1209/139719 objects misplaced (0.865%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:04:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 51680/139719 objects degraded (36.989%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:50.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:50.216 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:50.317 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:50.343 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:50.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:50.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:50.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:50.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:50.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:50.697 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:50.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:50.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:50.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:50.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:50.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:50.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:50.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:51.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:51.103 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:51.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:51.205 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:51.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:51.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:51.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:51.355 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:51.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:51.479 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:51.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:51.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:51.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:51.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:51.740 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:51.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:51.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:51.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:51.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:51.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:51.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:51.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:52.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:51 smithi191 ceph-mon[102206]: pgmap v1493: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 51680/139719 objects degraded (36.989%); 1209/139719 objects misplaced (0.865%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:04:52.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:51 smithi076 ceph-mon[106482]: pgmap v1493: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 51680/139719 objects degraded (36.989%); 1209/139719 objects misplaced (0.865%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:04:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:51 smithi076 ceph-mon[110963]: pgmap v1493: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 171 op/s; 51680/139719 objects degraded (36.989%); 1209/139719 objects misplaced (0.865%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:04:52.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:52.249 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:52.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:52.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:52.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:52.501 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:52.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:52.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:52.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:52.757 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:52.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:52.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:52.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:52.984 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:53.008 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:53.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:53.071 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:04:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:04:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:04:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:04:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.071 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132669 6.7 6:ee3236d4:::benchmark_data_smithi076_154645_object18763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.701663+0000", 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.36464091900001, 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0345916079999999, 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132669 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701663+0000", 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701663+0000", 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701662+0000", 2024-09-21T10:04:53.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701668+0000", 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5389999999999998e-06 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701685+0000", 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6798000000000001e-05 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701693+0000", 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2679999999999994e-06 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701770+0000", 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7224000000000005e-05 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.701782+0000", 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2174000000000001e-05 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723044+0000", 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0212620189999999 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723136+0000", 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1699000000000004e-05 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.723211+0000", 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5343e-05 2024-09-21T10:04:53.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729587+0000", 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063752499999999998 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732288+0000", 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027012820000000002 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736194+0000", 2024-09-21T10:04:53.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003905974 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736229+0000", 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5385000000000001e-05 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736255+0000", 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5789e-05 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.078 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132674 6.7 6:e6149bab:::benchmark_data_smithi076_154645_object18768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.709017+0000", 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.35728697899998, 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.027360359, 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132674 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.079 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709017+0000", 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709017+0000", 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709016+0000", 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709023+0000", 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1690000000000003e-06 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709040+0000", 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7295999999999998e-05 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709048+0000", 2024-09-21T10:04:53.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0309999999999998e-06 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709157+0000", 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109302 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.709170+0000", 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3006999999999999e-05 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726637+0000", 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.017466998 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726724+0000", 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6982000000000002e-05 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.726801+0000", 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6508999999999995e-05 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736264+0000", 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094625979999999991 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736288+0000", 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4314000000000001e-05 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736328+0000", 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0627000000000003e-05 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736358+0000", 2024-09-21T10:04:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9179999999999998e-05 2024-09-21T10:04:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736378+0000", 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9995000000000001e-05 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132678 6.7 6:e83a6585:::benchmark_data_smithi076_154645_object18772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.716074+0000", 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.35023035799998, 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0204681069999999, 2024-09-21T10:04:53.086 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132678 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716074+0000", 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716074+0000", 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716072+0000", 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716080+0000", 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2679999999999997e-06 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716092+0000", 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2102e-05 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716100+0000", 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9829999999999996e-06 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716143+0000", 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2979999999999998e-05 2024-09-21T10:04:53.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.716151+0000", 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9750000000000007e-06 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729620+0000", 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.013469213 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729708+0000", 2024-09-21T10:04:53.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7615999999999996e-05 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.729777+0000", 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9509000000000001e-05 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736385+0000", 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.006608198 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736425+0000", 2024-09-21T10:04:53.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0061999999999997e-05 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736481+0000", 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5494000000000001e-05 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736512+0000", 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1439000000000002e-05 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736542+0000", 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9791e-05 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132696 6.7 6:e3a68bfc:::benchmark_data_smithi076_154645_object18790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.735329+0000", 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.33097480599997, 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.001302906, 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.093 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132696 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735329+0000", 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735329+0000", 2024-09-21T10:04:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735328+0000", 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735335+0000", 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4699999999999999e-06 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735349+0000", 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4493e-05 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735357+0000", 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5259999999999996e-06 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735402+0000", 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5572999999999999e-05 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.735420+0000", 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7569999999999999e-05 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732376+0000", 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99695626999999998 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732509+0000", 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013274200000000001 2024-09-21T10:04:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.732636+0000", 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000127346 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736458+0000", 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038217440000000002 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736462+0000", 2024-09-21T10:04:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5299999999999998e-06 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736583+0000", 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120709 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736612+0000", 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9308000000000001e-05 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736632+0000", 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.001e-05 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132704 6.7 6:ee4531e9:::benchmark_data_smithi076_154645_object18798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.744678+0000", 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.32162609, 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.004240655, 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132704 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744678+0000", 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744678+0000", 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744676+0000", 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744685+0000", 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5040000000000002e-06 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744702+0000", 2024-09-21T10:04:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7399000000000001e-05 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744711+0000", 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4200000000000007e-06 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744768+0000", 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6725999999999999e-05 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.744776+0000", 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5329999999999994e-06 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736669+0000", 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99189255799999998 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736774+0000", 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105721 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.736877+0000", 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010238699999999999 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748781+0000", 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011904068 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748790+0000", 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.679e-06 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748855+0000", 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4892000000000001e-05 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748889+0000", 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3206e-05 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748919+0000", 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0226000000000002e-05 2024-09-21T10:04:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132719 6.7 6:ecb28ace:::benchmark_data_smithi076_154645_object18813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.758432+0000", 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.30787222399999, 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99870091400000005, 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132719 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758432+0000", 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758432+0000", 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758430+0000", 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758438+0000", 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7360000000000001e-06 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758453+0000", 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4759999999999999e-05 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758460+0000", 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6350000000000006e-06 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758522+0000", 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2273000000000005e-05 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.758533+0000", 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0237e-05 2024-09-21T10:04:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741029+0000", 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98249609900000001 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741177+0000", 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014844599999999999 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.741300+0000", 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122748 2024-09-21T10:04:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748825+0000", 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0075247990000000004 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748931+0000", 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010597799999999999 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757064+0000", 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008132841 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757105+0000", 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1198000000000001e-05 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757133+0000", 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8116999999999999e-05 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132729 6.7 6:ec0c304a:::benchmark_data_smithi076_154645_object18823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.776958+0000", 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.289346427, 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98031006499999995, 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132729 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776958+0000", 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776958+0000", 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776957+0000", 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776962+0000", 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4190000000000001e-06 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776974+0000", 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2303999999999999e-05 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.776982+0000", 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.486e-06 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.777032+0000", 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9852999999999999e-05 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.777039+0000", 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4900000000000003e-06 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745236+0000", 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96819714999999995 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745349+0000", 2024-09-21T10:04:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112273 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.745455+0000", 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106005 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757033+0000", 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011578564 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757190+0000", 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000156661 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757223+0000", 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3413e-05 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757249+0000", 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5522000000000001e-05 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757268+0000", 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9046999999999999e-05 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132747 6.7 6:ef3db1a5:::benchmark_data_smithi076_154645_object18841:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.807098+0000", 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.25920619099998, 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95029417199999999, 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132747 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807098+0000", 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807098+0000", 2024-09-21T10:04:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807097+0000", 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807101+0000", 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4649999999999999e-06 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807107+0000", 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.207e-06 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807111+0000", 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7620000000000001e-06 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807144+0000", 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2768999999999997e-05 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.807150+0000", 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9540000000000001e-06 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.748957+0000", 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94180735199999999 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.749051+0000", 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3607e-05 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.749151+0000", 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100497 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757182+0000", 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080311150000000001 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757197+0000", 2024-09-21T10:04:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4562000000000001e-05 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757311+0000", 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011408000000000001 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:53.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757358+0000", 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7392999999999998e-05 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757392+0000", 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4044000000000001e-05 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132752 6.7 6:e393e829:::benchmark_data_smithi076_154645_object18846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.808248+0000", 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.25805635299997, 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96938841399999998, 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132752 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808248+0000", 2024-09-21T10:04:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808248+0000", 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808247+0000", 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808251+0000", 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6559999999999998e-06 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808259+0000", 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8590000000000005e-06 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808263+0000", 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4039999999999997e-06 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808297+0000", 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3346000000000002e-05 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.808302+0000", 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.964e-06 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753045+0000", 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94474228999999998 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753148+0000", 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103346 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.753251+0000", 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010297 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757404+0000", 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041525709999999999 2024-09-21T10:04:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757440+0000", 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6028999999999997e-05 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777567+0000", 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020127794000000001 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777606+0000", 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8336e-05 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777636+0000", 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0482000000000001e-05 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132759 6.7 6:ec6776a2:::benchmark_data_smithi076_154645_object18853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.822499+0000", 2024-09-21T10:04:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.24380535099999, 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95524379100000001, 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132759 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822499+0000", 2024-09-21T10:04:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822499+0000", 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822498+0000", 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822504+0000", 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2910000000000004e-06 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822515+0000", 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1122e-05 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822521+0000", 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8850000000000004e-06 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822576+0000", 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5012000000000001e-05 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.822586+0000", 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0198e-05 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757477+0000", 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93489060499999999 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757603+0000", 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000125699 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.757727+0000", 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012464799999999999 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.769569+0000", 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011841539 2024-09-21T10:04:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777648+0000", 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080794749999999992 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777682+0000", 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3782000000000003e-05 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777710+0000", 2024-09-21T10:04:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8453e-05 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777743+0000", 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2187000000000002e-05 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132762 6.7 6:e3ba75f7:::benchmark_data_smithi076_154645_object18856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:04:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.24025897199999, 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97741979800000001, 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132762 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826045+0000", 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826049+0000", 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7199999999999997e-06 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826058+0000", 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8810000000000002e-06 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826063+0000", 2024-09-21T10:04:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5870000000000003e-06 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826098+0000", 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5222999999999998e-05 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.826104+0000", 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1140000000000002e-06 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761541+0000", 2024-09-21T10:04:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93543742500000004 2024-09-21T10:04:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761637+0000", 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5283000000000002e-05 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.761731+0000", 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3795999999999994e-05 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777779+0000", 2024-09-21T10:04:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016048020999999999 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777784+0000", 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1710000000000001e-06 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803357+0000", 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025573056 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803440+0000", 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3496000000000002e-05 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.803465+0000", 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4794999999999999e-05 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:132777 6.7 6:e0ee62be:::benchmark_data_smithi076_154645_object18871:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1113)", 2024-09-21T10:04:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:28.868119+0000", 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 384.19818547400001, 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94012129, 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 132777 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868119+0000", 2024-09-21T10:04:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868119+0000", 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868117+0000", 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868125+0000", 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4200000000000001e-06 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868139+0000", 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4076999999999999e-05 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868146+0000", 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7950000000000002e-06 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868192+0000", 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.693e-05 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:28.868201+0000", 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6580000000000008e-06 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765747+0000", 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89754568400000001 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765878+0000", 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013122700000000001 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.765999+0000", 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120541 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777788+0000", 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011788997000000001 2024-09-21T10:04:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.777826+0000", 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8155999999999999e-05 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808156+0000", 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.030329964000000001 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808203+0000", 2024-09-21T10:04:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7398000000000003e-05 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:29.808240+0000", 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6976e-05 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18337 7.e 7:790a4a7c:::benchmark_data_smithi076_190198_object18336:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661783+0000", 2024-09-21T10:04:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.404520975, 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0020676100000001, 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18337 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661783+0000", 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661783+0000", 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661782+0000", 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661814+0000", 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2070999999999998e-05 2024-09-21T10:04:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661825+0000", 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0596e-05 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661833+0000", 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3480000000000004e-06 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661880+0000", 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6983000000000001e-05 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op must wait for map", 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661891+0000", 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0540000000000001e-05 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663450+0000", 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015596760000000001 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663456+0000", 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4750000000000001e-06 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663740+0000", 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0002838890000001 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663851+0000", 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011123799999999999 2024-09-21T10:04:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18338 7.e 7:7ab9ca86:::benchmark_data_smithi076_190198_object18337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.40445915999999, 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0022534350000001, 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18338 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661845+0000", 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661868+0000", 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2954000000000001e-05 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661873+0000", 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4399999999999996e-06 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661876+0000", 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4039999999999999e-06 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661943+0000", 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6816000000000005e-05 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661949+0000", 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5720000000000002e-06 2024-09-21T10:04:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663476+0000", 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015270419999999999 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663486+0000", 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9089999999999995e-06 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664045+0000", 2024-09-21T10:04:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.000559376 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664099+0000", 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3440999999999999e-05 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18343 7.e 7:715bc4a5:::benchmark_data_smithi076_190198_object18342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:04:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.40442328, 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0044391960000001, 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18343 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:04:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661881+0000", 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661905+0000", 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4547000000000001e-05 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661912+0000", 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3829999999999997e-06 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661916+0000", 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6299999999999997e-06 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661985+0000", 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8645000000000004e-05 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661989+0000", 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3619999999999999e-06 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663500+0000", 2024-09-21T10:04:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001511075 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.663504+0000", 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0199999999999996e-06 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664142+0000", 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0006375890000001 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664250+0000", 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010797 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666217+0000", 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019669169999999999 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666320+0000", 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010336 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18376 7.e 7:7e9a0281:::benchmark_data_smithi076_190198_object18375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.686399+0000", 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.379905286, 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98100495300000001, 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18376 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686399+0000", 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686399+0000", 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686398+0000", 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686477+0000", 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9285999999999999e-05 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686488+0000", 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0686999999999999e-05 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686493+0000", 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3340000000000001e-06 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686542+0000", 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8446000000000003e-05 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.686549+0000", 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9330000000000004e-06 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664284+0000", 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97773493499999997 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664406+0000", 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122318 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664521+0000", 2024-09-21T10:04:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114911 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667148+0000", 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026273590000000001 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667234+0000", 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5628000000000001e-05 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667278+0000", 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4450999999999998e-05 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667313+0000", 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4839000000000003e-05 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667404+0000", 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0903999999999996e-05 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18381 7.e 7:744d80e9:::benchmark_data_smithi076_190198_object18380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.692250+0000", 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.37405396299999, 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97926354900000001, 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18381 2024-09-21T10:04:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692250+0000", 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692250+0000", 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692249+0000", 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692293+0000", 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3548999999999999e-05 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692302+0000", 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.516e-06 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692308+0000", 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.203e-06 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692342+0000", 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4851999999999998e-05 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.692349+0000", 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6209999999999998e-06 2024-09-21T10:04:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666352+0000", 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97400270799999999 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666462+0000", 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110207 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:04:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666578+0000", 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115673 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669202+0000", 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026243019999999998 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669317+0000", 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011520599999999999 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671370+0000", 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020528909999999998 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671430+0000", 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9469999999999998e-05 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671514+0000", 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4266999999999998e-05 2024-09-21T10:04:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18429 7.e 7:714a56fb:::benchmark_data_smithi076_190198_object18428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.725346+0000", 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.34095836200001, 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94707176999999998, 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18429 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725346+0000", 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725346+0000", 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725345+0000", 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725445+0000", 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9802000000000004e-05 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725456+0000", 2024-09-21T10:04:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1443999999999999e-05 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725463+0000", 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3149999999999997e-06 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725502+0000", 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9968000000000001e-05 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.725509+0000", 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3489999999999997e-06 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667477+0000", 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94196792100000004 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667605+0000", 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000127985 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667718+0000", 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113751 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669324+0000", 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001605277 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671567+0000", 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002242881 2024-09-21T10:04:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.672272+0000", 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00070508799999999998 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.672318+0000", 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6419999999999999e-05 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.672418+0000", 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9507999999999997e-05 2024-09-21T10:04:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18445 7.e 7:7dbc857c:::benchmark_data_smithi076_190198_object18444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.734496+0000", 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.33180813499999, 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93906263300000004, 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18445 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734496+0000", 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734496+0000", 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734495+0000", 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734564+0000", 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8957999999999999e-05 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734573+0000", 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9080000000000001e-06 2024-09-21T10:04:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734578+0000", 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8049999999999997e-06 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734622+0000", 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4554000000000001e-05 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.734629+0000", 2024-09-21T10:04:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3539999999999996e-06 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.668318+0000", 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93368945000000003 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.668449+0000", 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013118200000000001 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:04:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.668550+0000", 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100407 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671162+0000", 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026126690000000002 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671617+0000", 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000454504 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673479+0000", 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018617390000000001 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673524+0000", 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5828000000000002e-05 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673559+0000", 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4276999999999997e-05 2024-09-21T10:04:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18460 7.e 7:785ec73b:::benchmark_data_smithi076_190198_object18459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.743709+0000", 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 211.322594955, 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92995887700000002, 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18460 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743709+0000", 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743709+0000", 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743708+0000", 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743754+0000", 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6079999999999999e-05 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743764+0000", 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0465000000000001e-05 2024-09-21T10:04:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743770+0000", 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2340000000000002e-06 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743825+0000", 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5464999999999998e-05 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.743840+0000", 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4936999999999999e-05 2024-09-21T10:04:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669358+0000", 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92551827799999997 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669491+0000", 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013267300000000001 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 0,2", 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.669636+0000", 2024-09-21T10:04:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014477000000000001 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671527+0000", 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018911889999999999 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.671656+0000", 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129417 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:04:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673600+0000", 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019433020000000001 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673640+0000", 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9900000000000001e-05 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.673668+0000", 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8639000000000001e-05 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:53.212 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:04:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:53.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:53.252 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:53.320 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.355 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:53.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:04:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:04:53] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.51.0" 2024-09-21T10:04:53.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:53.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:53.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:53.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:53.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:53.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:53.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:53.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:53.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:53.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:53.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:53.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:53.994 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:53 smithi076 ceph-mon[106482]: pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:04:53.994 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:53 smithi076 ceph-mon[110963]: pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:04:54.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:53 smithi191 ceph-mon[102206]: pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:04:54.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:54.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:54.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:54.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:04:54.245 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:54.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.377 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:54.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:54.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:54.501 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:54.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:54.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:54.611 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.633 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:54.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:54.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:54.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:54.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:54.783 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:04:54.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:54.893 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:54.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:54.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:55.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:55.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 51266/136827 objects degraded (37.468%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:55.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/425709655' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:04:55.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:55.061 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:55.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 51266/136827 objects degraded (37.468%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/425709655' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:04:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:55.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 51266/136827 objects degraded (37.468%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/425709655' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:04:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:55.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:04:55.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:04:55.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:55.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:55.249 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:55.279 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:55.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:55.409 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:55.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:55.511 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:55.536 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:55.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:55.665 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:55.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:55.794 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:55.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:55.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:55.921 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:55.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:56.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:56.054 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:56.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:56.075 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:55 smithi191 ceph-mon[102206]: pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 153 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T10:04:56.075 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/425709655' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:04:56.075 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:55 smithi191 ceph-mon[102206]: osdmap e1185: 8 total, 8 up, 5 in 2024-09-21T10:04:56.075 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[106482]: pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 153 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T10:04:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/425709655' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:04:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[106482]: osdmap e1185: 8 total, 8 up, 5 in 2024-09-21T10:04:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[110963]: pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 153 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 8.3 MiB/s, 132 objects/s recovering 2024-09-21T10:04:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/425709655' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:04:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[110963]: osdmap e1185: 8 total, 8 up, 5 in 2024-09-21T10:04:56.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:56.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:56.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:56.282 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:56.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:56.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:56.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:56.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:56.570 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:56.635 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:04:56.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:56.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:56.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:56.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99345 7.d 7:b8821bda:::benchmark_data_smithi076_190198_object27356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.799042+0000", 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.022220889000000001, 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022251724000000001, 2024-09-21T10:04:56.824 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for missing object", 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99345 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.799042+0000", 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.825 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.799042+0000", 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.799040+0000", 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.799050+0000", 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9259999999999995e-06 2024-09-21T10:04:56.826 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.799066+0000", 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.658e-05 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.799075+0000", 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5590000000000006e-06 2024-09-21T10:04:56.827 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806573+0000", 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074981859999999996 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.810509+0000", 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039361099999999996 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.828 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99328 7.d 7:b82ccc95:::benchmark_data_smithi076_190198_object27339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.679785+0000", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.14147808100000001, 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14156738099999999, 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99328 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.679785+0000", 2024-09-21T10:04:56.829 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.679785+0000", 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.679783+0000", 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.830 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.679790+0000", 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2110000000000003e-06 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.679802+0000", 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.278e-05 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.679808+0000", 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5269999999999998e-06 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.831 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.719655+0000", 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.039847012000000001 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.723816+0000", 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041615030000000004 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.787437+0000", 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063620593000000003 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.832 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.791283+0000", 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038459269999999999 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGPush(7.d 1185/1170 [PushOp(7:b493e54c:::benchmark_data_smithi076_190198_object27354:head, version: 1143'1741, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:b493e54c:::benchmark_data_smithi076_190198_object27354:head@1143'1741, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.806835+0000", 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.014427799, 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014579171, 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.833 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806835+0000", 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806835+0000", 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806834+0000", 2024-09-21T10:04:56.834 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806885+0000", 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1174000000000002e-05 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806905+0000", 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0409000000000001e-05 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.835 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806924+0000", 2024-09-21T10:04:56.836 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8997000000000001e-05 2024-09-21T10:04:56.836 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.836 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.836 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.836 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.836 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "MOSDPGPush(7.d 1185/1170 [PushOp(7:b8821bda:::benchmark_data_smithi076_190198_object27356:head, version: 1143'1742, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:b8821bda:::benchmark_data_smithi076_190198_object27356:head@1143'1742, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.811287+0000", 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0099758599999999996, 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010165528, 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.811287+0000", 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.837 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.811287+0000", 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.811286+0000", 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.811394+0000", 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107991 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.838 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.811411+0000", 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7127999999999999e-05 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.811423+0000", 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1387e-05 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.839 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99333 7.d 7:b930280b:::benchmark_data_smithi076_190198_object27344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.695245+0000", 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.12601794099999999, 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12623547299999999, 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99333 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.840 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.695245+0000", 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.695245+0000", 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.695243+0000", 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.841 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.842 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.842 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.695252+0000", 2024-09-21T10:04:56.842 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8440000000000004e-06 2024-09-21T10:04:56.842 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.695268+0000", 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6112000000000001e-05 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.695275+0000", 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3769999999999998e-06 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.843 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.724261+0000", 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028985935000000001 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.728243+0000", 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039816840000000001 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.766954+0000", 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038711284999999998 2024-09-21T10:04:56.844 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.771022+0000", 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040676009999999997 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99291 7.d 7:b3b40fd9:::benchmark_data_smithi076_190198_object27302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.495597+0000", 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.32566630699999999, 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.32593620800000001, 2024-09-21T10:04:56.845 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99291 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.495597+0000", 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.846 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.495597+0000", 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.495595+0000", 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.495603+0000", 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3520000000000007e-06 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.847 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.495619+0000", 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5888000000000001e-05 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.495628+0000", 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8990000000000007e-06 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.558186+0000", 2024-09-21T10:04:56.848 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.062557704000000006 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.564808+0000", 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066219970000000001 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.590425+0000", 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.025617705000000001 2024-09-21T10:04:56.849 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.594259+0000", 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003833749 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99292 7.d 7:b9dc142a:::benchmark_data_smithi076_190198_object27303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.499311+0000", 2024-09-21T10:04:56.850 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.321951773, 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.32226348399999999, 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99292 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.499311+0000", 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.851 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.499311+0000", 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.499310+0000", 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.499317+0000", 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1509999999999998e-06 2024-09-21T10:04:56.852 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.499330+0000", 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3091999999999999e-05 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.499335+0000", 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3929999999999999e-06 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.566263+0000", 2024-09-21T10:04:56.853 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.066928087999999997 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.570234+0000", 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039709130000000004 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.611074+0000", 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.040839908000000001 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:56.854 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.616105+0000", 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005030512 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99277 7.d 7:b5a32c61:::benchmark_data_smithi076_190198_object27288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.431051+0000", 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.39021186099999999, 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.39056655800000001, 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.855 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99277 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.431051+0000", 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.431051+0000", 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.856 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.431050+0000", 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.431056+0000", 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7710000000000001e-06 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.431067+0000", 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0397000000000001e-05 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.857 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.431072+0000", 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2340000000000002e-06 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.480637+0000", 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.049564618999999997 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.484430+0000", 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0037937790000000002 2024-09-21T10:04:56.858 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.548256+0000", 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.063825466999999997 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.552108+0000", 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038521900000000001 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.820790+0000", 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.268681962 2024-09-21T10:04:56.859 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99317 7.d 7:b7095d20:::benchmark_data_smithi076_190198_object27328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.623695+0000", 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.197567412, 2024-09-21T10:04:56.860 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.19796613099999999, 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for rw locks", 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99317 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.623695+0000", 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.861 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.623695+0000", 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.623694+0000", 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.623702+0000", 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9699999999999999e-06 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.862 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.623714+0000", 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.255e-05 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.623722+0000", 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6499999999999996e-06 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.645177+0000", 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.021455086000000002 2024-09-21T10:04:56.863 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.649024+0000", 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038468270000000001 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.676131+0000", 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027107214000000001 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for rw locks", 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.680613+0000", 2024-09-21T10:04:56.864 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0044814570000000003 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:99343 7.d 7:b493e54c:::benchmark_data_smithi076_190198_object27354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:04:56.769172+0000", 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.052090696999999998, 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.052530332999999999, 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for missing object", 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:04:56.865 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 99343 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.769172+0000", 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.769172+0000", 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:04:56.866 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.769170+0000", 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.769179+0000", 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0389999999999993e-06 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.769193+0000", 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4049000000000001e-05 2024-09-21T10:04:56.867 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.769200+0000", 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6599999999999998e-06 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.801854+0000", 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032653663999999999 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:04:56.806056+0000", 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0042022470000000001 2024-09-21T10:04:56.868 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.869 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:04:56.869 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.869 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:04:56.869 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2024-09-21T10:04:56.869 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 10 2024-09-21T10:04:56.869 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:04:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.870 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:56.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:56.971 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:56.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:56.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:56.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:57.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.123 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.157 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:57.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:57.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:57.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.385 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:57.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:04:57.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:04:57.736 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:57.768 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:57.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:57.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:57.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:57.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:57.976 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:04:58.000 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:58.029 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:58.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:58.099 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:57 smithi191 ceph-mon[102206]: pgmap v1497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:04:58.106 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:57 smithi076 ceph-mon[106482]: pgmap v1497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:04:58.107 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:57 smithi076 ceph-mon[110963]: pgmap v1497: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.8 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 182 op/s; 51266/136827 objects degraded (37.468%); 670/136827 objects misplaced (0.490%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:04:58.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:04:58.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:04:58.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:04:58.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:04:58.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:58.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:58.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:58.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:58.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:58.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:58.668 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:58.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:58.794 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:58.813 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:04:58.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:58.921 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:58.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:59.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:04:59.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:04:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3305851993' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:58 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3305851993' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:58 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3305851993' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:58 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:04:59.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:59.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:59.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:59.313 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:59.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:04:59.438 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:04:59.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:04:59.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:04:59.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:04:59.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:04:59.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:04:59.821 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:04:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:04:59.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:59 smithi191 ceph-mon[102206]: pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 50845/133920 objects degraded (37.967%); 26/133920 objects misplaced (0.019%); 11 MiB/s, 178 objects/s recovering 2024-09-21T10:04:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:59 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:04:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:59 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:04:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:59 smithi191 ceph-mon[102206]: osdmap e1186: 8 total, 8 up, 5 in 2024-09-21T10:04:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:04:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 50845/133920 objects degraded (37.967%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:04:59.892 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:04:59.893 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:04:59.893 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:04:59.893 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:04:59.893 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:04:59.923 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:04:59.935 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:59.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:04:59.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:59.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:00.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:00.060 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[106482]: pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 50845/133920 objects degraded (37.967%); 26/133920 objects misplaced (0.019%); 11 MiB/s, 178 objects/s recovering 2024-09-21T10:05:00.060 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:00.060 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:00.060 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[106482]: osdmap e1186: 8 total, 8 up, 5 in 2024-09-21T10:05:00.060 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 50845/133920 objects degraded (37.967%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:00.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[110963]: pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 50845/133920 objects degraded (37.967%); 26/133920 objects misplaced (0.019%); 11 MiB/s, 178 objects/s recovering 2024-09-21T10:05:00.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:05:00.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:00.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[110963]: osdmap e1186: 8 total, 8 up, 5 in 2024-09-21T10:05:00.061 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:04:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 50845/133920 objects degraded (37.967%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:00.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:00.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:00.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:00.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:00.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:00.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:00.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:00.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:00.462 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:00.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:00.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:00.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:00.591 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:05:00.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:00.712 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:00.775 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:00.778 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:05:00.778 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:00.800 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:05:00.800 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:00.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:00.827 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:05:00.828 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:00.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:00.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:00.886 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:00.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:00.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:00.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:00.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:01.070 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2441079621' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:01.070 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2441079621' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:01.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:01.103 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:01.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:01.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2441079621' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:01.204 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:01.227 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:01.329 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:01.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:01.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:01.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:01.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:01.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:01.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:01.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:01.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:01.817 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:01 smithi191 ceph-mon[102206]: pgmap v1500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 120 op/s; 50845/133920 objects degraded (37.967%); 26/133920 objects misplaced (0.019%); 7.4 MiB/s, 117 objects/s recovering 2024-09-21T10:05:01.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:01.860 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:01.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:01.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:05:01.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:01.988 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:01.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:02.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:02.115 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:02.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:01 smithi076 ceph-mon[110963]: pgmap v1500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 120 op/s; 50845/133920 objects degraded (37.967%); 26/133920 objects misplaced (0.019%); 7.4 MiB/s, 117 objects/s recovering 2024-09-21T10:05:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:01 smithi076 ceph-mon[106482]: pgmap v1500: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 120 op/s; 50845/133920 objects degraded (37.967%); 26/133920 objects misplaced (0.019%); 7.4 MiB/s, 117 objects/s recovering 2024-09-21T10:05:02.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:02.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:02.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:02.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:02.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:02.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:02.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:02.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:02.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:02.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:02.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:02.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:02.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:02.787 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:02.787 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:02.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:02.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:02.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:02.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:03.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:03.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:03.138 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:03.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:03.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:03.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:03.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:03.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:03.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:03.518 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:03.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:05:03.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:05:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:05:03] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-09-21T10:05:03.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:03.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:03.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:03.781 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:03.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:03.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:03.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:03.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:03.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:04.021 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:04.085 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:03 smithi191 ceph-mon[102206]: pgmap v1501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 49363/131028 objects degraded (37.674%); 9.9 MiB/s, 290 objects/s recovering 2024-09-21T10:05:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:03 smithi076 ceph-mon[106482]: pgmap v1501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 49363/131028 objects degraded (37.674%); 9.9 MiB/s, 290 objects/s recovering 2024-09-21T10:05:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:03 smithi076 ceph-mon[110963]: pgmap v1501: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 49363/131028 objects degraded (37.674%); 9.9 MiB/s, 290 objects/s recovering 2024-09-21T10:05:04.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:04.145 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:04.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:04.268 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:04.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:04.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:04.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:04.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:04.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:04.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:04.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:04.768 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:04.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:04.896 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:05:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:04.900 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:04.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:04.996 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:04 smithi191 ceph-mon[102206]: osdmap e1187: 8 total, 8 up, 5 in 2024-09-21T10:05:04.996 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 49363/131028 objects degraded (37.674%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:05.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:05.026 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:05.087 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:04 smithi076 ceph-mon[106482]: osdmap e1187: 8 total, 8 up, 5 in 2024-09-21T10:05:05.087 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 49363/131028 objects degraded (37.674%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:05.087 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:04 smithi076 ceph-mon[110963]: osdmap e1187: 8 total, 8 up, 5 in 2024-09-21T10:05:05.087 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 49363/131028 objects degraded (37.674%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:05.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:05.153 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:05.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:05.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:05.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:05.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:05.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:05.530 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:05.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:05.604 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:05.604 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:05.632 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:05.666 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:05.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:05.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:05.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:05.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:05 smithi191 ceph-mon[102206]: pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 49363/131028 objects degraded (37.674%); 9.9 MiB/s, 291 objects/s recovering 2024-09-21T10:05:05.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:05 smithi191 ceph-mon[102206]: osdmap e1188: 8 total, 8 up, 5 in 2024-09-21T10:05:05.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:05.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:05.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:06.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:06.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:05 smithi076 ceph-mon[106482]: pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 49363/131028 objects degraded (37.674%); 9.9 MiB/s, 291 objects/s recovering 2024-09-21T10:05:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:05 smithi076 ceph-mon[106482]: osdmap e1188: 8 total, 8 up, 5 in 2024-09-21T10:05:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:05 smithi076 ceph-mon[110963]: pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 49363/131028 objects degraded (37.674%); 9.9 MiB/s, 291 objects/s recovering 2024-09-21T10:05:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:05 smithi076 ceph-mon[110963]: osdmap e1188: 8 total, 8 up, 5 in 2024-09-21T10:05:06.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:06.180 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:06.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:06.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:06.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:05:06.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:06.432 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:06.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:06.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:06.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:06.604 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:06.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:06.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:06.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:06.799 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:06.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:06.927 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:06.950 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:06.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:07.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:07.078 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:07.179 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:07.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:07.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:07.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:07.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:07.440 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:07.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:07.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:07.592 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:07.693 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:07.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:07.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:07.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:07.844 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:07.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:07.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:07.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:07.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:08.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:08.098 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:07 smithi191 ceph-mon[102206]: pgmap v1505: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 123 op/s; 49363/131028 objects degraded (37.674%); 2.6 MiB/s, 177 objects/s recovering 2024-09-21T10:05:08.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:08.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:08.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:07 smithi076 ceph-mon[106482]: pgmap v1505: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 123 op/s; 49363/131028 objects degraded (37.674%); 2.6 MiB/s, 177 objects/s recovering 2024-09-21T10:05:08.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:07 smithi076 ceph-mon[110963]: pgmap v1505: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 2.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 123 op/s; 49363/131028 objects degraded (37.674%); 2.6 MiB/s, 177 objects/s recovering 2024-09-21T10:05:08.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:08.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:08.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:08.353 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:08.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:08.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:08.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:08.509 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:08.509 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:08.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:08.607 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:08.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:08.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:08.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:08.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:08.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:08.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:08.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:08.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:08.987 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:08.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:08.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:09.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:09.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:09.166 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:05:09.169 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:05:09.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:05:09.214 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:09.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:09.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:09.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:09.416 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:05:09.447 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:09.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:09.490 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:09.591 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:09.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:09.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:09.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:09.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:09.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:09.876 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:09.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:09.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.013 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:10.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:10.014 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1506,"stamp":"2024-09-21T10:05:08.626744+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2792298493,"num_objects":42625,"num_object_clones":0,"num_object_copies":127875,"num_objects_missing_on_primary":10475,"num_objects_missing":10475,"num_objects_degraded":40691,"num_objects_misplaced":5014,"num_objects_unfound":0,"num_objects_dirty":42625,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":101654,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26100,"num_bytes_recovered":1502164486,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77349,"ondisk_log_size":77349,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8263192,"kb_used_data":6698668,"kb_used_omap":119,"kb_used_meta":1564296,"kb_avail":460462568,"statfs":{"total":479975178240,"available":471513669632,"internally_reserved":0,"allocated":6859436032,"data_stored":6847173505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1601839304},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-132055040,"num_objects":-2015,"num_object_clones":0,"num_object_copies":-6045,"num_objects_missing_on_primary":-511,"num_objects_missing":-511,"num_objects_degraded":-10154,"num_objects_misplaced":4988,"num_objects_unfound":0,"num_objects_dirty":-2015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2015,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2371,"num_bytes_recovered":65274400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002513"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841488+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:05:04.841488+0000","last_peered":"2024-09-21T10:05:04.841488+0000","last_clean":"2024-09-21T10:05:04.841488+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:05:04.841488+0000","last_undegraded":"2024-09-21T10:05:04.841488+0000","last_fullsized":"2024-09-21T10:05:04.841488+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":44,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833446+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:05:03.833446+0000","last_peered":"2024-09-21T10:05:03.833446+0000","last_clean":"2024-09-21T10:05:03.833446+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:05:03.833446+0000","last_undegraded":"2024-09-21T10:05:03.833446+0000","last_fullsized":"2024-09-21T10:05:03.833446+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1197,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841904+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:05:04.841904+0000","last_peered":"2024-09-21T10:05:04.841904+0000","last_clean":"2024-09-21T10:05:04.841904+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:05:04.841904+0000","last_undegraded":"2024-09-21T10:05:04.841904+0000","last_fullsized":"2024-09-21T10:05:04.841904+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1528,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840857+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:05:04.840857+0000","last_peered":"2024-09-21T10:05:04.840857+0000","last_clean":"2024-09-21T10:05:04.840857+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:05:04.840857+0000","last_undegraded":"2024-09-21T10:05:04.840857+0000","last_fullsized":"2024-09-21T10:05:04.840857+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1528,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.873682+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:05:04.873682+0000","last_peered":"2024-09-21T10:05:04.873682+0000","last_clean":"2024-09-21T10:05:04.873682+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:05:04.873682+0000","last_undegraded":"2024-09-21T10:05:04.873682+0000","last_fullsized":"2024-09-21T10:05:04.873682+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.836731+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:05:03.836731+0000","last_peered":"2024-09-21T10:05:03.836731+0000","last_clean":"2024-09-21T10:05:03.836731+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:05:03.836731+0000","last_undegraded":"2024-09-21T10:05:03.836731+0000","last_fullsized":"2024-09-21T10:05:03.836731+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.846429+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:05:04.846429+0000","last_peered":"2024-09-21T10:05:04.846429+0000","last_clean":"2024-09-21T10:05:04.846429+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:05:04.846429+0000","last_undegraded":"2024-09-21T10:05:04.846429+0000","last_fullsized":"2024-09-21T10:05:04.846429+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":67,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.832847+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:05:03.832847+0000","last_peered":"2024-09-21T10:05:03.832847+0000","last_clean":"2024-09-21T10:05:03.832847+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:05:03.832847+0000","last_undegraded":"2024-09-21T10:05:03.832847+0000","last_fullsized":"2024-09-21T10:05:03.832847+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1188'6378","reported_seq":15223,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:07.265618+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:05:07.265618+0000","last_peered":"2024-09-21T10:05:07.265618+0000","last_clean":"2024-09-21T10:05:07.265618+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:05:07.265618+0000","last_undegraded":"2024-09-21T10:05:07.265618+0000","last_fullsized":"2024-09-21T10:05:07.265618+0000","mapping_epoch":1187,"log_start":"1143'1631","ondisk_log_start":"1143'1631","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4747,"log_dups_size":0,"ondisk_log_size":4747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174850048,"num_objects":2668,"num_object_clones":0,"num_object_copies":8004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6378,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":67,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825567+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:05:04.825567+0000","last_peered":"2024-09-21T10:05:04.825567+0000","last_clean":"2024-09-21T10:05:04.825567+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:05:04.825567+0000","last_undegraded":"2024-09-21T10:05:04.825567+0000","last_fullsized":"2024-09-21T10:05:04.825567+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833009+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:05:03.833009+0000","last_peered":"2024-09-21T10:05:03.833009+0000","last_clean":"2024-09-21T10:05:03.833009+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:05:03.833009+0000","last_undegraded":"2024-09-21T10:05:03.833009+0000","last_fullsized":"2024-09-21T10:05:03.833009+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1188'6390","reported_seq":14089,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:06.803958+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:05:06.803958+0000","last_peered":"2024-09-21T10:05:06.803958+0000","last_clean":"2024-09-21T10:05:06.803958+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:05:06.803958+0000","last_undegraded":"2024-09-21T10:05:06.803958+0000","last_fullsized":"2024-09-21T10:05:06.803958+0000","mapping_epoch":1180,"log_start":"1143'1803","ondisk_log_start":"1143'1803","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4587,"log_dups_size":0,"ondisk_log_size":4587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172883968,"num_objects":2638,"num_object_clones":0,"num_object_copies":7914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6390,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826517+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:05:04.826517+0000","last_peered":"2024-09-21T10:05:04.826517+0000","last_clean":"2024-09-21T10:05:04.826517+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:05:04.826517+0000","last_undegraded":"2024-09-21T10:05:04.826517+0000","last_fullsized":"2024-09-21T10:05:04.826517+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1152,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825169+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:05:04.825169+0000","last_peered":"2024-09-21T10:05:04.825169+0000","last_clean":"2024-09-21T10:05:04.825169+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:05:04.825169+0000","last_undegraded":"2024-09-21T10:05:04.825169+0000","last_fullsized":"2024-09-21T10:05:04.825169+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1188'6452","reported_seq":6570,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:07.084574+0000","last_change":"2024-09-21T10:03:30.139654+0000","last_active":"2024-09-21T10:05:07.084574+0000","last_peered":"2024-09-21T10:05:07.084574+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:05:07.084574+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1143'1682","ondisk_log_start":"1143'1682","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180617216,"num_objects":2756,"num_object_clones":0,"num_object_copies":8268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6452,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2756},{"shards":"1,4","objects":-983}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":42,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.850477+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:05:03.850477+0000","last_peered":"2024-09-21T10:05:03.850477+0000","last_clean":"2024-09-21T10:05:03.850477+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:05:03.850477+0000","last_undegraded":"2024-09-21T10:05:03.850477+0000","last_fullsized":"2024-09-21T10:05:03.850477+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833236+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:05:03.833236+0000","last_peered":"2024-09-21T10:05:03.833236+0000","last_clean":"2024-09-21T10:05:03.833236+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:05:03.833236+0000","last_undegraded":"2024-09-21T10:05:03.833236+0000","last_fullsized":"2024-09-21T10:05:03.833236+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1188'6308","reported_seq":10065,"reported_epoch":1188,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:05:06.642580+0000","last_change":"2024-09-21T10:03:28.893028+0000","last_active":"2024-09-21T10:05:06.642580+0000","last_peered":"2024-09-21T10:05:06.642580+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:05:06.642580+0000","last_undegraded":"2024-09-21T10:03:28.884675+0000","last_fullsized":"2024-09-21T10:05:06.642580+0000","mapping_epoch":1151,"log_start":"1143'1736","ondisk_log_start":"1143'1736","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4572,"log_dups_size":0,"ondisk_log_size":4572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505408,"num_objects":2678,"num_object_clones":0,"num_object_copies":8034,"num_objects_missing_on_primary":940,"num_objects_missing":940,"num_objects_degraded":940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6308,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1815,"num_bytes_recovered":118947840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":1738},{"shards":"2,4,7","objects":940}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1182,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841213+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:05:04.841213+0000","last_peered":"2024-09-21T10:05:04.841213+0000","last_clean":"2024-09-21T10:05:04.841213+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:05:04.841213+0000","last_undegraded":"2024-09-21T10:05:04.841213+0000","last_fullsized":"2024-09-21T10:05:04.841213+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1538,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826867+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:05:04.826867+0000","last_peered":"2024-09-21T10:05:04.826867+0000","last_clean":"2024-09-21T10:05:04.826867+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:05:04.826867+0000","last_undegraded":"2024-09-21T10:05:04.826867+0000","last_fullsized":"2024-09-21T10:05:04.826867+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1188'6339","reported_seq":12086,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:06.653301+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:05:06.653301+0000","last_peered":"2024-09-21T10:05:06.653301+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:05:06.653301+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1143'1748","ondisk_log_start":"1143'1748","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4591,"log_dups_size":0,"ondisk_log_size":4591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175308800,"num_objects":2675,"num_object_clones":0,"num_object_copies":8025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6339,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"0,3,4","objects":2675},{"shards":"3,4","objects":-946}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":220,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841337+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:05:04.841337+0000","last_peered":"2024-09-21T10:05:04.841337+0000","last_clean":"2024-09-21T10:05:04.841337+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:05:04.841337+0000","last_undegraded":"2024-09-21T10:05:04.841337+0000","last_fullsized":"2024-09-21T10:05:04.841337+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1197,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841131+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:05:04.841131+0000","last_peered":"2024-09-21T10:05:04.841131+0000","last_clean":"2024-09-21T10:05:04.841131+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:05:04.841131+0000","last_undegraded":"2024-09-21T10:05:04.841131+0000","last_fullsized":"2024-09-21T10:05:04.841131+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1188'6240","reported_seq":9179,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:06.819521+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:05:06.819521+0000","last_peered":"2024-09-21T10:05:06.819521+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:05:06.819521+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1143'1653","ondisk_log_start":"1143'1653","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4587,"log_dups_size":0,"ondisk_log_size":4587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172490752,"num_objects":2632,"num_object_clones":0,"num_object_copies":7896,"num_objects_missing_on_primary":2744,"num_objects_missing":2744,"num_objects_degraded":6238,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":944,"num_bytes_recovered":61865984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,3","objects":749},{"shards":"0,2,3,5","objects":1},{"shards":"0,2,5","objects":1882}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833276+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:05:03.833276+0000","last_peered":"2024-09-21T10:05:03.833276+0000","last_clean":"2024-09-21T10:05:03.833276+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:05:03.833276+0000","last_undegraded":"2024-09-21T10:05:03.833276+0000","last_fullsized":"2024-09-21T10:05:03.833276+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":840,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827693+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:05:04.827693+0000","last_peered":"2024-09-21T10:05:04.827693+0000","last_clean":"2024-09-21T10:05:04.827693+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:05:04.827693+0000","last_undegraded":"2024-09-21T10:05:04.827693+0000","last_fullsized":"2024-09-21T10:05:04.827693+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1188'6452","reported_seq":8616,"reported_epoch":1188,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:05:07.557596+0000","last_change":"2024-09-21T10:05:03.425944+0000","last_active":"2024-09-21T10:05:07.557596+0000","last_peered":"2024-09-21T10:05:07.557596+0000","last_clean":"2024-09-21T10:02:38.228948+0000","last_became_active":"2024-09-21T10:03:36.276110+0000","last_became_peered":"2024-09-21T10:03:36.276110+0000","last_unstale":"2024-09-21T10:05:07.557596+0000","last_undegraded":"2024-09-21T10:05:07.557596+0000","last_fullsized":"2024-09-21T10:03:29.833661+0000","mapping_epoch":1170,"log_start":"1141'400","ondisk_log_start":"1141'400","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:04.270084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274904,"num_objects":2706,"num_object_clones":0,"num_object_copies":8118,"num_objects_missing_on_primary":3376,"num_objects_missing":3376,"num_objects_degraded":0,"num_objects_misplaced":2676,"num_objects_unfound":0,"num_objects_dirty":2706,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6452,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1030,"num_bytes_recovered":67502080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1,3","objects":2503},{"shards":"0,1,6","objects":173},{"shards":"1,6","objects":30}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1183,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825005+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:05:04.825005+0000","last_peered":"2024-09-21T10:05:04.825005+0000","last_clean":"2024-09-21T10:05:04.825005+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:05:04.825005+0000","last_undegraded":"2024-09-21T10:05:04.825005+0000","last_fullsized":"2024-09-21T10:05:04.825005+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833565+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:05:03.833565+0000","last_peered":"2024-09-21T10:05:03.833565+0000","last_clean":"2024-09-21T10:05:03.833565+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:05:03.833565+0000","last_undegraded":"2024-09-21T10:05:03.833565+0000","last_fullsized":"2024-09-21T10:05:03.833565+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1188'6578","reported_seq":10221,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:06.674094+0000","last_change":"2024-09-21T10:02:39.200547+0000","last_active":"2024-09-21T10:05:06.674094+0000","last_peered":"2024-09-21T10:05:06.674094+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:05:06.674094+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"1143'2025","ondisk_log_start":"1143'2025","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178126848,"num_objects":2718,"num_object_clones":0,"num_object_copies":8154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4497,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6578,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"0,2,3","objects":1584},{"shards":"2,3","objects":-1779},{"shards":"2,3,7","objects":983}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840601+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:05:04.840601+0000","last_peered":"2024-09-21T10:05:04.840601+0000","last_clean":"2024-09-21T10:05:04.840601+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:05:04.840601+0000","last_undegraded":"2024-09-21T10:05:04.840601+0000","last_fullsized":"2024-09-21T10:05:04.840601+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1542,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826845+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:05:04.826845+0000","last_peered":"2024-09-21T10:05:04.826845+0000","last_clean":"2024-09-21T10:05:04.826845+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:05:04.826845+0000","last_undegraded":"2024-09-21T10:05:04.826845+0000","last_fullsized":"2024-09-21T10:05:04.826845+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1188'6231","reported_seq":10827,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:06.567252+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:05:06.567252+0000","last_peered":"2024-09-21T10:05:06.567252+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:05:06.567252+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1143'1647","ondisk_log_start":"1143'1647","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4584,"log_dups_size":0,"ondisk_log_size":4584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170983424,"num_objects":2609,"num_object_clones":0,"num_object_copies":7827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5828,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6231,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1765,"num_bytes_recovered":115671040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,2,3","objects":732},{"shards":"0,3,5","objects":957},{"shards":"0,3,7","objects":920}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":710,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.832785+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:05:03.832785+0000","last_peered":"2024-09-21T10:05:03.832785+0000","last_clean":"2024-09-21T10:05:03.832785+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:05:03.832785+0000","last_undegraded":"2024-09-21T10:05:03.832785+0000","last_fullsized":"2024-09-21T10:05:03.832785+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1196,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841086+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:05:04.841086+0000","last_peered":"2024-09-21T10:05:04.841086+0000","last_clean":"2024-09-21T10:05:04.841086+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:05:04.841086+0000","last_undegraded":"2024-09-21T10:05:04.841086+0000","last_fullsized":"2024-09-21T10:05:04.841086+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840985+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:05:04.840985+0000","last_peered":"2024-09-21T10:05:04.840985+0000","last_clean":"2024-09-21T10:05:04.840985+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:05:04.840985+0000","last_undegraded":"2024-09-21T10:05:04.840985+0000","last_fullsized":"2024-09-21T10:05:04.840985+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.850406+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:05:03.850406+0000","last_peered":"2024-09-21T10:05:03.850406+0000","last_clean":"2024-09-21T10:05:03.850406+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:05:03.850406+0000","last_undegraded":"2024-09-21T10:05:03.850406+0000","last_fullsized":"2024-09-21T10:05:03.850406+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1188'6294","reported_seq":11916,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:07.332962+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:05:07.332962+0000","last_peered":"2024-09-21T10:05:07.332962+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:05:07.332962+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1143'1554","ondisk_log_start":"1143'1554","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4740,"log_dups_size":0,"ondisk_log_size":4740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173801472,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6294,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"0,1,2","objects":2652},{"shards":"1,2","objects":-953}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":826,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825438+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:05:04.825438+0000","last_peered":"2024-09-21T10:05:04.825438+0000","last_clean":"2024-09-21T10:05:04.825438+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:05:04.825438+0000","last_undegraded":"2024-09-21T10:05:04.825438+0000","last_fullsized":"2024-09-21T10:05:04.825438+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1415,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840363+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:05:04.840363+0000","last_peered":"2024-09-21T10:05:04.840363+0000","last_clean":"2024-09-21T10:05:04.840363+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:05:04.840363+0000","last_undegraded":"2024-09-21T10:05:04.840363+0000","last_fullsized":"2024-09-21T10:05:04.840363+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":824,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825268+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:05:04.825268+0000","last_peered":"2024-09-21T10:05:04.825268+0000","last_clean":"2024-09-21T10:05:04.825268+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:05:04.825268+0000","last_undegraded":"2024-09-21T10:05:04.825268+0000","last_fullsized":"2024-09-21T10:05:04.825268+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1188'6007","reported_seq":6768,"reported_epoch":1188,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:07.340239+0000","last_change":"2024-09-21T10:05:03.300674+0000","last_active":"2024-09-21T10:05:07.340239+0000","last_peered":"2024-09-21T10:05:07.340239+0000","last_clean":"2024-09-21T10:03:27.561377+0000","last_became_active":"2024-09-21T10:03:30.061640+0000","last_became_peered":"2024-09-21T10:03:30.061640+0000","last_unstale":"2024-09-21T10:05:07.340239+0000","last_undegraded":"2024-09-21T10:03:29.959075+0000","last_fullsized":"2024-09-21T10:03:29.836831+0000","mapping_epoch":1170,"log_start":"1143'1237","ondisk_log_start":"1143'1237","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171376640,"num_objects":2615,"num_object_clones":0,"num_object_copies":7845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":839,"num_objects_misplaced":2338,"num_objects_unfound":0,"num_objects_dirty":2615,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6007,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":321,"num_bytes_recovered":21037056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2338},{"shards":"1,4","objects":-562}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1456,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.842267+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:05:04.842267+0000","last_peered":"2024-09-21T10:05:04.842267+0000","last_clean":"2024-09-21T10:05:04.842267+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:05:04.842267+0000","last_undegraded":"2024-09-21T10:05:04.842267+0000","last_fullsized":"2024-09-21T10:05:04.842267+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1543,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825607+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:05:04.825607+0000","last_peered":"2024-09-21T10:05:04.825607+0000","last_clean":"2024-09-21T10:05:04.825607+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:05:04.825607+0000","last_undegraded":"2024-09-21T10:05:04.825607+0000","last_fullsized":"2024-09-21T10:05:04.825607+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1551,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826359+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:05:04.826359+0000","last_peered":"2024-09-21T10:05:04.826359+0000","last_clean":"2024-09-21T10:05:04.826359+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:05:04.826359+0000","last_undegraded":"2024-09-21T10:05:04.826359+0000","last_fullsized":"2024-09-21T10:05:04.826359+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1188'6323","reported_seq":8350,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:07.545799+0000","last_change":"2024-09-21T10:03:36.351943+0000","last_active":"2024-09-21T10:05:07.545799+0000","last_peered":"2024-09-21T10:05:07.545799+0000","last_clean":"2024-09-21T10:02:38.308681+0000","last_became_active":"2024-09-21T10:03:36.318082+0000","last_became_peered":"2024-09-21T10:03:36.318082+0000","last_unstale":"2024-09-21T10:05:07.545799+0000","last_undegraded":"2024-09-21T10:03:36.183243+0000","last_fullsized":"2024-09-21T10:03:29.833927+0000","mapping_epoch":1170,"log_start":"1141'300","ondisk_log_start":"1141'300","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6023,"log_dups_size":0,"ondisk_log_size":6023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:52.296275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176619520,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":3415,"num_objects_missing":3415,"num_objects_degraded":4399,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6323,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":962,"num_bytes_recovered":63045632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,1,3","objects":2563},{"shards":"0,3,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":120,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826221+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:05:04.826221+0000","last_peered":"2024-09-21T10:05:04.826221+0000","last_clean":"2024-09-21T10:05:04.826221+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:05:04.826221+0000","last_undegraded":"2024-09-21T10:05:04.826221+0000","last_fullsized":"2024-09-21T10:05:04.826221+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1148,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833299+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:05:03.833299+0000","last_peered":"2024-09-21T10:05:03.833299+0000","last_clean":"2024-09-21T10:05:03.833299+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:05:03.833299+0000","last_undegraded":"2024-09-21T10:05:03.833299+0000","last_fullsized":"2024-09-21T10:05:03.833299+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1554,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825146+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:05:04.825146+0000","last_peered":"2024-09-21T10:05:04.825146+0000","last_clean":"2024-09-21T10:05:04.825146+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:05:04.825146+0000","last_undegraded":"2024-09-21T10:05:04.825146+0000","last_fullsized":"2024-09-21T10:05:04.825146+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1188'6434","reported_seq":6544,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:06.618781+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:05:06.618781+0000","last_peered":"2024-09-21T10:05:06.618781+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:05:06.618781+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1143'1885","ondisk_log_start":"1143'1885","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4549,"log_dups_size":0,"ondisk_log_size":4549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171048960,"num_objects":2610,"num_object_clones":0,"num_object_copies":7830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6434,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":2455},{"shards":"3,4","objects":-1757}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1144,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841767+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:05:04.841767+0000","last_peered":"2024-09-21T10:05:04.841767+0000","last_clean":"2024-09-21T10:05:04.841767+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:05:04.841767+0000","last_undegraded":"2024-09-21T10:05:04.841767+0000","last_fullsized":"2024-09-21T10:05:04.841767+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1565,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825230+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:05:04.825230+0000","last_peered":"2024-09-21T10:05:04.825230+0000","last_clean":"2024-09-21T10:05:04.825230+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:05:04.825230+0000","last_undegraded":"2024-09-21T10:05:04.825230+0000","last_fullsized":"2024-09-21T10:05:04.825230+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":84,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.824870+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:05:04.824870+0000","last_peered":"2024-09-21T10:05:04.824870+0000","last_clean":"2024-09-21T10:05:04.824870+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:05:04.824870+0000","last_undegraded":"2024-09-21T10:05:04.824870+0000","last_fullsized":"2024-09-21T10:05:04.824870+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1188'6277","reported_seq":13861,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:07.311415+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:05:07.311415+0000","last_peered":"2024-09-21T10:05:07.311415+0000","last_clean":"2024-09-21T10:05:07.311415+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:05:07.311415+0000","last_undegraded":"2024-09-21T10:05:07.311415+0000","last_fullsized":"2024-09-21T10:05:07.311415+0000","mapping_epoch":1174,"log_start":"1143'1494","ondisk_log_start":"1143'1494","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4783,"log_dups_size":0,"ondisk_log_size":4783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170328064,"num_objects":2599,"num_object_clones":0,"num_object_copies":7797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6277,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1542,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826268+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:05:04.826268+0000","last_peered":"2024-09-21T10:05:04.826268+0000","last_clean":"2024-09-21T10:05:04.826268+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:05:04.826268+0000","last_undegraded":"2024-09-21T10:05:04.826268+0000","last_fullsized":"2024-09-21T10:05:04.826268+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":722,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827374+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:05:04.827374+0000","last_peered":"2024-09-21T10:05:04.827374+0000","last_clean":"2024-09-21T10:05:04.827374+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:05:04.827374+0000","last_undegraded":"2024-09-21T10:05:04.827374+0000","last_fullsized":"2024-09-21T10:05:04.827374+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1534,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826909+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:05:04.826909+0000","last_peered":"2024-09-21T10:05:04.826909+0000","last_clean":"2024-09-21T10:05:04.826909+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:05:04.826909+0000","last_undegraded":"2024-09-21T10:05:04.826909+0000","last_fullsized":"2024-09-21T10:05:04.826909+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1188'6462","reported_seq":6577,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:06.255055+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:05:06.255055+0000","last_peered":"2024-09-21T10:05:06.255055+0000","last_clean":"2024-09-21T10:05:06.255055+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:05:06.255055+0000","last_undegraded":"2024-09-21T10:05:06.255055+0000","last_fullsized":"2024-09-21T10:05:06.255055+0000","mapping_epoch":1137,"log_start":"1143'1826","ondisk_log_start":"1143'1826","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177602560,"num_objects":2710,"num_object_clones":0,"num_object_copies":8130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6462,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":242,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.846505+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:05:04.846505+0000","last_peered":"2024-09-21T10:05:04.846505+0000","last_clean":"2024-09-21T10:05:04.846505+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:05:04.846505+0000","last_undegraded":"2024-09-21T10:05:04.846505+0000","last_fullsized":"2024-09-21T10:05:04.846505+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1543,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826616+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:05:04.826616+0000","last_peered":"2024-09-21T10:05:04.826616+0000","last_clean":"2024-09-21T10:05:04.826616+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:05:04.826616+0000","last_undegraded":"2024-09-21T10:05:04.826616+0000","last_fullsized":"2024-09-21T10:05:04.826616+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":220,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840511+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:05:04.840511+0000","last_peered":"2024-09-21T10:05:04.840511+0000","last_clean":"2024-09-21T10:05:04.840511+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:05:04.840511+0000","last_undegraded":"2024-09-21T10:05:04.840511+0000","last_fullsized":"2024-09-21T10:05:04.840511+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1188'6210","reported_seq":6325,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:06.608824+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:05:06.608824+0000","last_peered":"2024-09-21T10:05:06.608824+0000","last_clean":"2024-09-21T10:05:06.608824+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:05:06.608824+0000","last_undegraded":"2024-09-21T10:05:06.608824+0000","last_fullsized":"2024-09-21T10:05:06.608824+0000","mapping_epoch":1137,"log_start":"1143'1613","ondisk_log_start":"1143'1613","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4597,"log_dups_size":0,"ondisk_log_size":4597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172883968,"num_objects":2638,"num_object_clones":0,"num_object_copies":7914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6210,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":83,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825538+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:05:04.825538+0000","last_peered":"2024-09-21T10:05:04.825538+0000","last_clean":"2024-09-21T10:05:04.825538+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:05:04.825538+0000","last_undegraded":"2024-09-21T10:05:04.825538+0000","last_fullsized":"2024-09-21T10:05:04.825538+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.840278+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:05:03.840278+0000","last_peered":"2024-09-21T10:05:03.840278+0000","last_clean":"2024-09-21T10:05:03.840278+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:05:03.840278+0000","last_undegraded":"2024-09-21T10:05:03.840278+0000","last_fullsized":"2024-09-21T10:05:03.840278+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.836671+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:05:03.836671+0000","last_peered":"2024-09-21T10:05:03.836671+0000","last_clean":"2024-09-21T10:05:03.836671+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:05:03.836671+0000","last_undegraded":"2024-09-21T10:05:03.836671+0000","last_fullsized":"2024-09-21T10:05:03.836671+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833368+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:05:03.833368+0000","last_peered":"2024-09-21T10:05:03.833368+0000","last_clean":"2024-09-21T10:05:03.833368+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:05:03.833368+0000","last_undegraded":"2024-09-21T10:05:03.833368+0000","last_fullsized":"2024-09-21T10:05:03.833368+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":48,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.834724+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:05:03.834724+0000","last_peered":"2024-09-21T10:05:03.834724+0000","last_clean":"2024-09-21T10:05:03.834724+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:05:03.834724+0000","last_undegraded":"2024-09-21T10:05:03.834724+0000","last_fullsized":"2024-09-21T10:05:03.834724+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1542,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826308+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:05:04.826308+0000","last_peered":"2024-09-21T10:05:04.826308+0000","last_clean":"2024-09-21T10:05:04.826308+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:05:04.826308+0000","last_undegraded":"2024-09-21T10:05:04.826308+0000","last_fullsized":"2024-09-21T10:05:04.826308+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1155,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841995+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:05:04.841995+0000","last_peered":"2024-09-21T10:05:04.841995+0000","last_clean":"2024-09-21T10:05:04.841995+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:05:04.841995+0000","last_undegraded":"2024-09-21T10:05:04.841995+0000","last_fullsized":"2024-09-21T10:05:04.841995+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1166,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826695+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:05:04.826695+0000","last_peered":"2024-09-21T10:05:04.826695+0000","last_clean":"2024-09-21T10:05:04.826695+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:05:04.826695+0000","last_undegraded":"2024-09-21T10:05:04.826695+0000","last_fullsized":"2024-09-21T10:05:04.826695+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":240,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840574+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:05:04.840574+0000","last_peered":"2024-09-21T10:05:04.840574+0000","last_clean":"2024-09-21T10:05:04.840574+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:05:04.840574+0000","last_undegraded":"2024-09-21T10:05:04.840574+0000","last_fullsized":"2024-09-21T10:05:04.840574+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1568,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.846295+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:05:04.846295+0000","last_peered":"2024-09-21T10:05:04.846295+0000","last_clean":"2024-09-21T10:05:04.846295+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:05:04.846295+0000","last_undegraded":"2024-09-21T10:05:04.846295+0000","last_fullsized":"2024-09-21T10:05:04.846295+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":219,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840761+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:05:04.840761+0000","last_peered":"2024-09-21T10:05:04.840761+0000","last_clean":"2024-09-21T10:05:04.840761+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:05:04.840761+0000","last_undegraded":"2024-09-21T10:05:04.840761+0000","last_fullsized":"2024-09-21T10:05:04.840761+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.840191+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:05:03.840191+0000","last_peered":"2024-09-21T10:05:03.840191+0000","last_clean":"2024-09-21T10:05:03.840191+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:05:03.840191+0000","last_undegraded":"2024-09-21T10:05:03.840191+0000","last_fullsized":"2024-09-21T10:05:03.840191+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840405+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:05:04.840405+0000","last_peered":"2024-09-21T10:05:04.840405+0000","last_clean":"2024-09-21T10:05:04.840405+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:05:04.840405+0000","last_undegraded":"2024-09-21T10:05:04.840405+0000","last_fullsized":"2024-09-21T10:05:04.840405+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1530,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.873943+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:05:04.873943+0000","last_peered":"2024-09-21T10:05:04.873943+0000","last_clean":"2024-09-21T10:05:04.873943+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:05:04.873943+0000","last_undegraded":"2024-09-21T10:05:04.873943+0000","last_fullsized":"2024-09-21T10:05:04.873943+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1541,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825254+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:05:04.825254+0000","last_peered":"2024-09-21T10:05:04.825254+0000","last_clean":"2024-09-21T10:05:04.825254+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:05:04.825254+0000","last_undegraded":"2024-09-21T10:05:04.825254+0000","last_fullsized":"2024-09-21T10:05:04.825254+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":205,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.842079+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:05:04.842079+0000","last_peered":"2024-09-21T10:05:04.842079+0000","last_clean":"2024-09-21T10:05:04.842079+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:05:04.842079+0000","last_undegraded":"2024-09-21T10:05:04.842079+0000","last_fullsized":"2024-09-21T10:05:04.842079+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":89,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827210+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:05:04.827210+0000","last_peered":"2024-09-21T10:05:04.827210+0000","last_clean":"2024-09-21T10:05:04.827210+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:05:04.827210+0000","last_undegraded":"2024-09-21T10:05:04.827210+0000","last_fullsized":"2024-09-21T10:05:04.827210+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1425,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826050+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:05:04.826050+0000","last_peered":"2024-09-21T10:05:04.826050+0000","last_clean":"2024-09-21T10:05:04.826050+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:05:04.826050+0000","last_undegraded":"2024-09-21T10:05:04.826050+0000","last_fullsized":"2024-09-21T10:05:04.826050+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1506,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826795+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:05:04.826795+0000","last_peered":"2024-09-21T10:05:04.826795+0000","last_clean":"2024-09-21T10:05:04.826795+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:05:04.826795+0000","last_undegraded":"2024-09-21T10:05:04.826795+0000","last_fullsized":"2024-09-21T10:05:04.826795+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1151,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841257+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:05:04.841257+0000","last_peered":"2024-09-21T10:05:04.841257+0000","last_clean":"2024-09-21T10:05:04.841257+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:05:04.841257+0000","last_undegraded":"2024-09-21T10:05:04.841257+0000","last_fullsized":"2024-09-21T10:05:04.841257+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":105,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826366+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:05:04.826366+0000","last_peered":"2024-09-21T10:05:04.826366+0000","last_clean":"2024-09-21T10:05:04.826366+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:05:04.826366+0000","last_undegraded":"2024-09-21T10:05:04.826366+0000","last_fullsized":"2024-09-21T10:05:04.826366+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1158,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840828+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:05:04.840828+0000","last_peered":"2024-09-21T10:05:04.840828+0000","last_clean":"2024-09-21T10:05:04.840828+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:05:04.840828+0000","last_undegraded":"2024-09-21T10:05:04.840828+0000","last_fullsized":"2024-09-21T10:05:04.840828+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":41,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.850356+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:05:03.850356+0000","last_peered":"2024-09-21T10:05:03.850356+0000","last_clean":"2024-09-21T10:05:03.850356+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:05:03.850356+0000","last_undegraded":"2024-09-21T10:05:03.850356+0000","last_fullsized":"2024-09-21T10:05:03.850356+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.842338+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:05:04.842338+0000","last_peered":"2024-09-21T10:05:04.842338+0000","last_clean":"2024-09-21T10:05:04.842338+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:05:04.842338+0000","last_undegraded":"2024-09-21T10:05:04.842338+0000","last_fullsized":"2024-09-21T10:05:04.842338+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":220,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.873769+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:05:04.873769+0000","last_peered":"2024-09-21T10:05:04.873769+0000","last_clean":"2024-09-21T10:05:04.873769+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:05:04.873769+0000","last_undegraded":"2024-09-21T10:05:04.873769+0000","last_fullsized":"2024-09-21T10:05:04.873769+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827939+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:05:04.827939+0000","last_peered":"2024-09-21T10:05:04.827939+0000","last_clean":"2024-09-21T10:05:04.827939+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:05:04.827939+0000","last_undegraded":"2024-09-21T10:05:04.827939+0000","last_fullsized":"2024-09-21T10:05:04.827939+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1556,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833653+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:05:03.833653+0000","last_peered":"2024-09-21T10:05:03.833653+0000","last_clean":"2024-09-21T10:05:03.833653+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:05:03.833653+0000","last_undegraded":"2024-09-21T10:05:03.833653+0000","last_fullsized":"2024-09-21T10:05:03.833653+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2791702552,"num_objects":42599,"num_object_clones":0,"num_object_copies":127797,"num_objects_missing_on_primary":10475,"num_objects_missing":10475,"num_objects_degraded":40691,"num_objects_misplaced":5014,"num_objects_unfound":0,"num_objects_dirty":42599,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101375,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26035,"num_bytes_recovered":1499201536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9910628352,"data_stored":9910616136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77141,"ondisk_log_size":77141,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234351,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495987,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":465212,"kb_used_data":235636,"kb_used_omap":1,"kb_used_meta":229566,"kb_avail":93279940,"statfs":{"total":95995035648,"available":95518658560,"internally_reserved":0,"allocated":241291264,"data_stored":238824874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":235076039},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757626,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019262,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332980,"kb_used_data":1073004,"kb_used_omap":1,"kb_used_meta":259966,"kb_avail":92412172,"statfs":{"total":95995035648,"available":94630064128,"internally_reserved":0,"allocated":1098756096,"data_stored":1096297333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":266205638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888558,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2367720,"kb_used_data":1916444,"kb_used_omap":34,"kb_used_meta":451229,"kb_avail":91377432,"statfs":{"total":95995035648,"available":93570490368,"internally_reserved":0,"allocated":1962438656,"data_stored":1959990936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":462059507},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967797,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1945116,"kb_used_data":1606228,"kb_used_omap":41,"kb_used_meta":338838,"kb_avail":91800036,"statfs":{"total":95995035648,"available":94003236864,"internally_reserved":0,"allocated":1644777472,"data_stored":1642335130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":346970527},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047036,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2152164,"kb_used_data":1867356,"kb_used_omap":41,"kb_used_meta":284694,"kb_avail":91592988,"statfs":{"total":95995035648,"available":93791219712,"internally_reserved":0,"allocated":1912172544,"data_stored":1909725232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":291527593},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126274,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2116096000,"data_stored":2116091928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1903300608,"data_stored":1903296536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1631453184,"data_stored":1631453184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1953824768,"data_stored":1953824768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1090584576,"data_stored":1090584576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":235802624,"data_stored":235798552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":647364608,"data_stored":647364608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:05:10.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:05:10.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:09 smithi076 ceph-mon[106482]: pgmap v1506: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 251 op/s; 40691/127875 objects degraded (31.821%); 5014/127875 objects misplaced (3.921%); 7.8 MiB/s, 296 objects/s recovering 2024-09-21T10:05:10.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 40691/127875 objects degraded (31.821%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:10.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:09 smithi076 ceph-mon[106482]: from='client.35238 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:10.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:09 smithi076 ceph-mon[110963]: pgmap v1506: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 251 op/s; 40691/127875 objects degraded (31.821%); 5014/127875 objects misplaced (3.921%); 7.8 MiB/s, 296 objects/s recovering 2024-09-21T10:05:10.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 40691/127875 objects degraded (31.821%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:10.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:09 smithi076 ceph-mon[110963]: from='client.35238 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:09 smithi191 ceph-mon[102206]: pgmap v1506: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 251 op/s; 40691/127875 objects degraded (31.821%); 5014/127875 objects misplaced (3.921%); 7.8 MiB/s, 296 objects/s recovering 2024-09-21T10:05:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 40691/127875 objects degraded (31.821%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:09 smithi191 ceph-mon[102206]: from='client.35238 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:10.114 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:10.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:10.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:10.281 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:10.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:10.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:10.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:10.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:10.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:10.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:10.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:10.660 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:10.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:10.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:10.771 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:10.815 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:10.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:10.952 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:10.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:10.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:11.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:11.084 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:11.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:11.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:11.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:11.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:11.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:11.347 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:11.347 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:11.372 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:11.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:11.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:11.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:11.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:11.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:11.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:11.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:11 smithi191 ceph-mon[102206]: pgmap v1507: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 40691/127875 objects degraded (31.821%); 5014/127875 objects misplaced (3.921%); 5.3 MiB/s, 123 objects/s recovering 2024-09-21T10:05:11.617 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:11.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:11.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:11.743 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:11.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:11.872 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:11 smithi076 ceph-mon[106482]: pgmap v1507: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 40691/127875 objects degraded (31.821%); 5014/127875 objects misplaced (3.921%); 5.3 MiB/s, 123 objects/s recovering 2024-09-21T10:05:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:11 smithi076 ceph-mon[110963]: pgmap v1507: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.6 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 40691/127875 objects degraded (31.821%); 5014/127875 objects misplaced (3.921%); 5.3 MiB/s, 123 objects/s recovering 2024-09-21T10:05:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:11.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:11.975 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:11.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:11.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:12.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:05:12.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:12.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:12.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:12.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:12.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:12.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.389 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:12.523 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:12.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:12.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:12.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:12.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:12.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:12.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:12.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:13.007 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:13.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:13.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:13.169 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:13.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:13.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:13.298 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:13.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:13.400 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:13.439 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:13.534 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:05:13] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-21T10:05:13.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:13.565 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:13.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:13.574 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:05:13.667 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:13.693 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:13.795 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:13.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:13 smithi191 ceph-mon[102206]: pgmap v1508: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 10 MiB/s, 200 objects/s recovering 2024-09-21T10:05:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1620992352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:13 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[106482]: pgmap v1508: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 10 MiB/s, 200 objects/s recovering 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1620992352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[110963]: pgmap v1508: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 10 MiB/s, 200 objects/s recovering 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1620992352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:13.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:13 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:05:13.923 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:13.948 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:13.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:14.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:14.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:14.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:14.203 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:14.304 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:14.328 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:14.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:14.457 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:14.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:14.584 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:14.686 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:14.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:14.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:14.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:14.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:14.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:14.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: osdmap e1189: 8 total, 8 up, 5 in 2024-09-21T10:05:14.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 40307/124722 objects degraded (32.317%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:14.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: osdmap e1189: 8 total, 8 up, 5 in 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 40307/124722 objects degraded (32.317%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:14.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:14.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:05:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: osdmap e1189: 8 total, 8 up, 5 in 2024-09-21T10:05:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 40307/124722 objects degraded (32.317%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:14.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:14.970 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:14.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:14.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:15.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:15.097 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:15.198 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:15.221 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:15.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:15.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:15.448 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:15.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:15.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:15.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:15.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:15.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:15.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:15.746 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:15.759 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:05:15.759 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:15.793 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:05:15.793 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:15.818 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:05:15.819 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:15.819 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:15 smithi076 ceph-mon[106482]: pgmap v1510: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 214 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 9.3 MiB/s, 179 objects/s recovering 2024-09-21T10:05:15.819 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:15 smithi076 ceph-mon[110963]: pgmap v1510: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 214 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 9.3 MiB/s, 179 objects/s recovering 2024-09-21T10:05:15.843 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:05:15.843 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:15.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:15 smithi191 ceph-mon[102206]: pgmap v1510: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 214 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 9.3 MiB/s, 179 objects/s recovering 2024-09-21T10:05:15.870 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:05:15.870 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:15.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:15.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:15.894 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:05:15.895 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:15.917 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:05:15.918 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:15.942 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:05:15.942 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:15.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:15.996 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:15.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:16.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:16.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:16.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:16.247 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:16.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:16.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:16.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:16.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.515 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:16.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:16.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:16.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.649 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:16.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:16.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:16.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:16.837 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1814838855' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:16.838 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1814838855' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:16.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1814838855' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:16.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:16.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:16.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:17.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.024 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:17.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:17.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.273 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:17.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:17.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:17.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:17.522 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:17.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:17.662 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:17.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:17.699 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:17.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:17.787 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:17.828 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:17 smithi076 ceph-mon[106482]: pgmap v1511: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 9.1 MiB/s, 176 objects/s recovering 2024-09-21T10:05:17.828 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:17 smithi076 ceph-mon[110963]: pgmap v1511: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 9.1 MiB/s, 176 objects/s recovering 2024-09-21T10:05:17.835 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:05:17.836 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:05:17.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:17 smithi191 ceph-mon[102206]: pgmap v1511: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 210 op/s; 40307/124722 objects degraded (32.317%); 4481/124722 objects misplaced (3.593%); 9.1 MiB/s, 176 objects/s recovering 2024-09-21T10:05:17.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:17.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:17.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:18.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:18.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:18.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:18.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:18.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:18.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:18.296 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:18.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:18.421 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:18.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:18.549 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:18.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:18.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:18.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:18.800 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:18.864 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:05:18.866 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:18 smithi191 ceph-mon[102206]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:18 smithi076 ceph-mon[106482]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:18 smithi076 ceph-mon[110963]: from='client.54874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:05:18.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:18.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:18.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:18.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:05:19.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:19.053 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:19.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:19.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:19.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:19.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:19.305 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:19.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:19.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:19.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:19.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:19.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:19.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:19 smithi076 ceph-mon[106482]: pgmap v1512: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 39878/120846 objects degraded (32.999%); 3940/120846 objects misplaced (3.260%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:05:19.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:19 smithi076 ceph-mon[110963]: pgmap v1512: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 39878/120846 objects degraded (32.999%); 3940/120846 objects misplaced (3.260%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:05:19.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:19.705 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:19.807 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:19.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:19 smithi191 ceph-mon[102206]: pgmap v1512: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 39878/120846 objects degraded (32.999%); 3940/120846 objects misplaced (3.260%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:05:19.895 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1511,"stamp":"2024-09-21T10:05:16.629293+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2723420157,"num_objects":41574,"num_object_clones":0,"num_object_copies":124722,"num_objects_missing_on_primary":10177,"num_objects_missing":10177,"num_objects_degraded":40307,"num_objects_misplaced":4481,"num_objects_unfound":0,"num_objects_dirty":41574,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":102705,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26880,"num_bytes_recovered":1553282566,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77200,"ondisk_log_size":77200,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8092168,"kb_used_data":6487324,"kb_used_omap":119,"kb_used_meta":1604680,"kb_avail":460633592,"statfs":{"total":479975178240,"available":471688798208,"internally_reserved":0,"allocated":6643019776,"data_stored":6630733668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1643192520},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-137756672,"num_objects":-2102,"num_object_clones":0,"num_object_copies":-6306,"num_objects_missing_on_primary":-575,"num_objects_missing":-575,"num_objects_degraded":-9056,"num_objects_misplaced":4481,"num_objects_unfound":0,"num_objects_dirty":-2102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2102,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1765,"num_bytes_recovered":95355424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003088"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841488+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:05:04.841488+0000","last_peered":"2024-09-21T10:05:04.841488+0000","last_clean":"2024-09-21T10:05:04.841488+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:05:04.841488+0000","last_undegraded":"2024-09-21T10:05:04.841488+0000","last_fullsized":"2024-09-21T10:05:04.841488+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":44,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833446+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:05:03.833446+0000","last_peered":"2024-09-21T10:05:03.833446+0000","last_clean":"2024-09-21T10:05:03.833446+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:05:03.833446+0000","last_undegraded":"2024-09-21T10:05:03.833446+0000","last_fullsized":"2024-09-21T10:05:03.833446+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1197,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841904+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:05:04.841904+0000","last_peered":"2024-09-21T10:05:04.841904+0000","last_clean":"2024-09-21T10:05:04.841904+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:05:04.841904+0000","last_undegraded":"2024-09-21T10:05:04.841904+0000","last_fullsized":"2024-09-21T10:05:04.841904+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1528,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840857+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:05:04.840857+0000","last_peered":"2024-09-21T10:05:04.840857+0000","last_clean":"2024-09-21T10:05:04.840857+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:05:04.840857+0000","last_undegraded":"2024-09-21T10:05:04.840857+0000","last_fullsized":"2024-09-21T10:05:04.840857+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1528,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.873682+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:05:04.873682+0000","last_peered":"2024-09-21T10:05:04.873682+0000","last_clean":"2024-09-21T10:05:04.873682+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:05:04.873682+0000","last_undegraded":"2024-09-21T10:05:04.873682+0000","last_fullsized":"2024-09-21T10:05:04.873682+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.836731+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:05:03.836731+0000","last_peered":"2024-09-21T10:05:03.836731+0000","last_clean":"2024-09-21T10:05:03.836731+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:05:03.836731+0000","last_undegraded":"2024-09-21T10:05:03.836731+0000","last_fullsized":"2024-09-21T10:05:03.836731+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.846429+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:05:04.846429+0000","last_peered":"2024-09-21T10:05:04.846429+0000","last_clean":"2024-09-21T10:05:04.846429+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:05:04.846429+0000","last_undegraded":"2024-09-21T10:05:04.846429+0000","last_fullsized":"2024-09-21T10:05:04.846429+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":67,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.832847+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:05:03.832847+0000","last_peered":"2024-09-21T10:05:03.832847+0000","last_clean":"2024-09-21T10:05:03.832847+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:05:03.832847+0000","last_undegraded":"2024-09-21T10:05:03.832847+0000","last_fullsized":"2024-09-21T10:05:03.832847+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1188'6438","reported_seq":15283,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:12.246355+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:05:12.246355+0000","last_peered":"2024-09-21T10:05:12.246355+0000","last_clean":"2024-09-21T10:05:12.246355+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:05:12.246355+0000","last_undegraded":"2024-09-21T10:05:12.246355+0000","last_fullsized":"2024-09-21T10:05:12.246355+0000","mapping_epoch":1187,"log_start":"1143'1731","ondisk_log_start":"1143'1731","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170917888,"num_objects":2608,"num_object_clones":0,"num_object_copies":7824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6438,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":67,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825567+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:05:04.825567+0000","last_peered":"2024-09-21T10:05:04.825567+0000","last_clean":"2024-09-21T10:05:04.825567+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:05:04.825567+0000","last_undegraded":"2024-09-21T10:05:04.825567+0000","last_fullsized":"2024-09-21T10:05:04.825567+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833009+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:05:03.833009+0000","last_peered":"2024-09-21T10:05:03.833009+0000","last_clean":"2024-09-21T10:05:03.833009+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:05:03.833009+0000","last_undegraded":"2024-09-21T10:05:03.833009+0000","last_fullsized":"2024-09-21T10:05:03.833009+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1188'6459","reported_seq":14158,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:11.801990+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:05:11.801990+0000","last_peered":"2024-09-21T10:05:11.801990+0000","last_clean":"2024-09-21T10:05:11.801990+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:05:11.801990+0000","last_undegraded":"2024-09-21T10:05:11.801990+0000","last_fullsized":"2024-09-21T10:05:11.801990+0000","mapping_epoch":1180,"log_start":"1143'1903","ondisk_log_start":"1143'1903","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4556,"log_dups_size":0,"ondisk_log_size":4556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168361984,"num_objects":2569,"num_object_clones":0,"num_object_copies":7707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6459,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":111,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826517+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:05:04.826517+0000","last_peered":"2024-09-21T10:05:04.826517+0000","last_clean":"2024-09-21T10:05:04.826517+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:05:04.826517+0000","last_undegraded":"2024-09-21T10:05:04.826517+0000","last_fullsized":"2024-09-21T10:05:04.826517+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1152,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825169+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:05:04.825169+0000","last_peered":"2024-09-21T10:05:04.825169+0000","last_clean":"2024-09-21T10:05:04.825169+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:05:04.825169+0000","last_undegraded":"2024-09-21T10:05:04.825169+0000","last_fullsized":"2024-09-21T10:05:04.825169+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1188'6527","reported_seq":6645,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:12.290944+0000","last_change":"2024-09-21T10:03:30.139654+0000","last_active":"2024-09-21T10:05:12.290944+0000","last_peered":"2024-09-21T10:05:12.290944+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:05:12.290944+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1143'1782","ondisk_log_start":"1143'1782","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4745,"log_dups_size":0,"ondisk_log_size":4745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175702016,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6527,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2681},{"shards":"1,4","objects":-1058}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":42,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.850477+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:05:03.850477+0000","last_peered":"2024-09-21T10:05:03.850477+0000","last_clean":"2024-09-21T10:05:03.850477+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:05:03.850477+0000","last_undegraded":"2024-09-21T10:05:03.850477+0000","last_fullsized":"2024-09-21T10:05:03.850477+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833236+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:05:03.833236+0000","last_peered":"2024-09-21T10:05:03.833236+0000","last_clean":"2024-09-21T10:05:03.833236+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:05:03.833236+0000","last_undegraded":"2024-09-21T10:05:03.833236+0000","last_fullsized":"2024-09-21T10:05:03.833236+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1188'6369","reported_seq":10248,"reported_epoch":1188,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:05:11.798842+0000","last_change":"2024-09-21T10:03:28.893028+0000","last_active":"2024-09-21T10:05:11.798842+0000","last_peered":"2024-09-21T10:05:11.798842+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:05:11.798842+0000","last_undegraded":"2024-09-21T10:03:28.884675+0000","last_fullsized":"2024-09-21T10:05:11.798842+0000","mapping_epoch":1151,"log_start":"1143'1736","ondisk_log_start":"1143'1736","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4633,"log_dups_size":0,"ondisk_log_size":4633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171507712,"num_objects":2617,"num_object_clones":0,"num_object_copies":7851,"num_objects_missing_on_primary":879,"num_objects_missing":879,"num_objects_degraded":879,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6369,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1876,"num_bytes_recovered":122945536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":1738},{"shards":"2,4,7","objects":879}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1182,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841213+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:05:04.841213+0000","last_peered":"2024-09-21T10:05:04.841213+0000","last_clean":"2024-09-21T10:05:04.841213+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:05:04.841213+0000","last_undegraded":"2024-09-21T10:05:04.841213+0000","last_fullsized":"2024-09-21T10:05:04.841213+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1538,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826867+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:05:04.826867+0000","last_peered":"2024-09-21T10:05:04.826867+0000","last_clean":"2024-09-21T10:05:04.826867+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:05:04.826867+0000","last_undegraded":"2024-09-21T10:05:04.826867+0000","last_fullsized":"2024-09-21T10:05:04.826867+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1188'6409","reported_seq":12156,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:11.661232+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:05:11.661232+0000","last_peered":"2024-09-21T10:05:11.661232+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:05:11.661232+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1143'1848","ondisk_log_start":"1143'1848","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4561,"log_dups_size":0,"ondisk_log_size":4561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170721280,"num_objects":2605,"num_object_clones":0,"num_object_copies":7815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6409,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"0,3,4","objects":2605},{"shards":"3,4","objects":-1016}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":220,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841337+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:05:04.841337+0000","last_peered":"2024-09-21T10:05:04.841337+0000","last_clean":"2024-09-21T10:05:04.841337+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:05:04.841337+0000","last_undegraded":"2024-09-21T10:05:04.841337+0000","last_fullsized":"2024-09-21T10:05:04.841337+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1197,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841131+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:05:04.841131+0000","last_peered":"2024-09-21T10:05:04.841131+0000","last_clean":"2024-09-21T10:05:04.841131+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:05:04.841131+0000","last_undegraded":"2024-09-21T10:05:04.841131+0000","last_fullsized":"2024-09-21T10:05:04.841131+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1188'6300","reported_seq":9429,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:11.858155+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:05:11.858155+0000","last_peered":"2024-09-21T10:05:11.858155+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:05:11.858155+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1143'1753","ondisk_log_start":"1143'1753","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4547,"log_dups_size":0,"ondisk_log_size":4547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168558592,"num_objects":2572,"num_object_clones":0,"num_object_copies":7716,"num_objects_missing_on_primary":2680,"num_objects_missing":2680,"num_objects_degraded":6113,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6300,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1005,"num_bytes_recovered":65863680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,3","objects":749},{"shards":"0,2,3,5","objects":4},{"shards":"0,2,5","objects":1818},{"shards":"2,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833276+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:05:03.833276+0000","last_peered":"2024-09-21T10:05:03.833276+0000","last_clean":"2024-09-21T10:05:03.833276+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:05:03.833276+0000","last_undegraded":"2024-09-21T10:05:03.833276+0000","last_fullsized":"2024-09-21T10:05:03.833276+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":840,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827693+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:05:04.827693+0000","last_peered":"2024-09-21T10:05:04.827693+0000","last_clean":"2024-09-21T10:05:04.827693+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:05:04.827693+0000","last_undegraded":"2024-09-21T10:05:04.827693+0000","last_fullsized":"2024-09-21T10:05:04.827693+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1188'6509","reported_seq":8898,"reported_epoch":1188,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:05:12.574496+0000","last_change":"2024-09-21T10:05:03.425944+0000","last_active":"2024-09-21T10:05:12.574496+0000","last_peered":"2024-09-21T10:05:12.574496+0000","last_clean":"2024-09-21T10:02:38.228948+0000","last_became_active":"2024-09-21T10:03:36.276110+0000","last_became_peered":"2024-09-21T10:03:36.276110+0000","last_unstale":"2024-09-21T10:05:12.574496+0000","last_undegraded":"2024-09-21T10:05:12.574496+0000","last_fullsized":"2024-09-21T10:03:29.833661+0000","mapping_epoch":1170,"log_start":"1141'500","ondisk_log_start":"1141'500","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:04.270084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173539352,"num_objects":2649,"num_object_clones":0,"num_object_copies":7947,"num_objects_missing_on_primary":3263,"num_objects_missing":3263,"num_objects_degraded":0,"num_objects_misplaced":2563,"num_objects_unfound":0,"num_objects_dirty":2649,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6509,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1142,"num_bytes_recovered":74842112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1,3","objects":2390},{"shards":"0,1,3,6","objects":1},{"shards":"0,1,6","objects":173},{"shards":"1,6","objects":85}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1183,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825005+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:05:04.825005+0000","last_peered":"2024-09-21T10:05:04.825005+0000","last_clean":"2024-09-21T10:05:04.825005+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:05:04.825005+0000","last_undegraded":"2024-09-21T10:05:04.825005+0000","last_fullsized":"2024-09-21T10:05:04.825005+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833565+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:05:03.833565+0000","last_peered":"2024-09-21T10:05:03.833565+0000","last_clean":"2024-09-21T10:05:03.833565+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:05:03.833565+0000","last_undegraded":"2024-09-21T10:05:03.833565+0000","last_fullsized":"2024-09-21T10:05:03.833565+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1188'6646","reported_seq":10289,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:11.743245+0000","last_change":"2024-09-21T10:02:39.200547+0000","last_active":"2024-09-21T10:05:11.743245+0000","last_peered":"2024-09-21T10:05:11.743245+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:05:11.743245+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"1143'2025","ondisk_log_start":"1143'2025","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4621,"log_dups_size":0,"ondisk_log_size":4621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173670400,"num_objects":2650,"num_object_clones":0,"num_object_copies":7950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4497,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6646,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1759,"num_bytes_recovered":115277824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"0,2,3","objects":1584},{"shards":"2,3","objects":-1847},{"shards":"2,3,7","objects":915}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840601+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:05:04.840601+0000","last_peered":"2024-09-21T10:05:04.840601+0000","last_clean":"2024-09-21T10:05:04.840601+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:05:04.840601+0000","last_undegraded":"2024-09-21T10:05:04.840601+0000","last_fullsized":"2024-09-21T10:05:04.840601+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1542,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826845+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:05:04.826845+0000","last_peered":"2024-09-21T10:05:04.826845+0000","last_clean":"2024-09-21T10:05:04.826845+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:05:04.826845+0000","last_undegraded":"2024-09-21T10:05:04.826845+0000","last_fullsized":"2024-09-21T10:05:04.826845+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1188'6300","reported_seq":11103,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:11.825662+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:05:11.825662+0000","last_peered":"2024-09-21T10:05:11.825662+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:05:11.825662+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1143'1747","ondisk_log_start":"1143'1747","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166461440,"num_objects":2540,"num_object_clones":0,"num_object_copies":7620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5690,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6300,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1834,"num_bytes_recovered":120193024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,2,3","objects":732},{"shards":"0,3,5","objects":957},{"shards":"0,3,7","objects":851}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":710,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.832785+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:05:03.832785+0000","last_peered":"2024-09-21T10:05:03.832785+0000","last_clean":"2024-09-21T10:05:03.832785+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:05:03.832785+0000","last_undegraded":"2024-09-21T10:05:03.832785+0000","last_fullsized":"2024-09-21T10:05:03.832785+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1196,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841086+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:05:04.841086+0000","last_peered":"2024-09-21T10:05:04.841086+0000","last_clean":"2024-09-21T10:05:04.841086+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:05:04.841086+0000","last_undegraded":"2024-09-21T10:05:04.841086+0000","last_fullsized":"2024-09-21T10:05:04.841086+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":227,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840985+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:05:04.840985+0000","last_peered":"2024-09-21T10:05:04.840985+0000","last_clean":"2024-09-21T10:05:04.840985+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:05:04.840985+0000","last_undegraded":"2024-09-21T10:05:04.840985+0000","last_fullsized":"2024-09-21T10:05:04.840985+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.850406+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:05:03.850406+0000","last_peered":"2024-09-21T10:05:03.850406+0000","last_clean":"2024-09-21T10:05:03.850406+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:05:03.850406+0000","last_undegraded":"2024-09-21T10:05:03.850406+0000","last_fullsized":"2024-09-21T10:05:03.850406+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1188'6363","reported_seq":11985,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:12.182904+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:05:12.182904+0000","last_peered":"2024-09-21T10:05:12.182904+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:05:12.182904+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1143'1654","ondisk_log_start":"1143'1654","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169279488,"num_objects":2583,"num_object_clones":0,"num_object_copies":7749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6363,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"0,1,2","objects":2583},{"shards":"1,2","objects":-1022}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":826,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825438+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:05:04.825438+0000","last_peered":"2024-09-21T10:05:04.825438+0000","last_clean":"2024-09-21T10:05:04.825438+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:05:04.825438+0000","last_undegraded":"2024-09-21T10:05:04.825438+0000","last_fullsized":"2024-09-21T10:05:04.825438+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1415,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840363+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:05:04.840363+0000","last_peered":"2024-09-21T10:05:04.840363+0000","last_clean":"2024-09-21T10:05:04.840363+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:05:04.840363+0000","last_undegraded":"2024-09-21T10:05:04.840363+0000","last_fullsized":"2024-09-21T10:05:04.840363+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":824,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825268+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:05:04.825268+0000","last_peered":"2024-09-21T10:05:04.825268+0000","last_clean":"2024-09-21T10:05:04.825268+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:05:04.825268+0000","last_undegraded":"2024-09-21T10:05:04.825268+0000","last_fullsized":"2024-09-21T10:05:04.825268+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1188'6073","reported_seq":7674,"reported_epoch":1188,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:12.346350+0000","last_change":"2024-09-21T10:05:03.300674+0000","last_active":"2024-09-21T10:05:12.346350+0000","last_peered":"2024-09-21T10:05:12.346350+0000","last_clean":"2024-09-21T10:03:27.561377+0000","last_became_active":"2024-09-21T10:03:30.061640+0000","last_became_peered":"2024-09-21T10:03:30.061640+0000","last_unstale":"2024-09-21T10:05:12.346350+0000","last_undegraded":"2024-09-21T10:03:29.959075+0000","last_fullsized":"2024-09-21T10:03:29.836831+0000","mapping_epoch":1170,"log_start":"1143'1337","ondisk_log_start":"1143'1337","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4736,"log_dups_size":0,"ondisk_log_size":4736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167051264,"num_objects":2549,"num_object_clones":0,"num_object_copies":7647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":839,"num_objects_misplaced":1918,"num_objects_unfound":0,"num_objects_dirty":2549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6073,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":741,"num_bytes_recovered":48562176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1918},{"shards":"1,4","objects":-208}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1456,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.842267+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:05:04.842267+0000","last_peered":"2024-09-21T10:05:04.842267+0000","last_clean":"2024-09-21T10:05:04.842267+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:05:04.842267+0000","last_undegraded":"2024-09-21T10:05:04.842267+0000","last_fullsized":"2024-09-21T10:05:04.842267+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1543,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825607+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:05:04.825607+0000","last_peered":"2024-09-21T10:05:04.825607+0000","last_clean":"2024-09-21T10:05:04.825607+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:05:04.825607+0000","last_undegraded":"2024-09-21T10:05:04.825607+0000","last_fullsized":"2024-09-21T10:05:04.825607+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1551,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826359+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:05:04.826359+0000","last_peered":"2024-09-21T10:05:04.826359+0000","last_clean":"2024-09-21T10:05:04.826359+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:05:04.826359+0000","last_undegraded":"2024-09-21T10:05:04.826359+0000","last_fullsized":"2024-09-21T10:05:04.826359+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1188'6379","reported_seq":8523,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:12.569099+0000","last_change":"2024-09-21T10:03:36.351943+0000","last_active":"2024-09-21T10:05:12.569099+0000","last_peered":"2024-09-21T10:05:12.569099+0000","last_clean":"2024-09-21T10:02:38.308681+0000","last_became_active":"2024-09-21T10:03:36.318082+0000","last_became_peered":"2024-09-21T10:03:36.318082+0000","last_unstale":"2024-09-21T10:05:12.569099+0000","last_undegraded":"2024-09-21T10:03:36.183243+0000","last_fullsized":"2024-09-21T10:03:29.833927+0000","mapping_epoch":1170,"log_start":"1141'300","ondisk_log_start":"1141'300","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:52.296275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949504,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":3355,"num_objects_missing":3355,"num_objects_degraded":4339,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6379,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1019,"num_bytes_recovered":66781184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,1,3","objects":2503},{"shards":"0,1,3,6","objects":3},{"shards":"0,3,6","objects":132},{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":120,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826221+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:05:04.826221+0000","last_peered":"2024-09-21T10:05:04.826221+0000","last_clean":"2024-09-21T10:05:04.826221+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:05:04.826221+0000","last_undegraded":"2024-09-21T10:05:04.826221+0000","last_fullsized":"2024-09-21T10:05:04.826221+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1148,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833299+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:05:03.833299+0000","last_peered":"2024-09-21T10:05:03.833299+0000","last_clean":"2024-09-21T10:05:03.833299+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:05:03.833299+0000","last_undegraded":"2024-09-21T10:05:03.833299+0000","last_fullsized":"2024-09-21T10:05:03.833299+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1554,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825146+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:05:04.825146+0000","last_peered":"2024-09-21T10:05:04.825146+0000","last_clean":"2024-09-21T10:05:04.825146+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:05:04.825146+0000","last_undegraded":"2024-09-21T10:05:04.825146+0000","last_fullsized":"2024-09-21T10:05:04.825146+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1188'6508","reported_seq":6618,"reported_epoch":1188,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:11.864299+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:05:11.864299+0000","last_peered":"2024-09-21T10:05:11.864299+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:05:11.864299+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1143'1885","ondisk_log_start":"1143'1885","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4623,"log_dups_size":0,"ondisk_log_size":4623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166199296,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6508,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":2381},{"shards":"3,4","objects":-1831}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1144,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841767+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:05:04.841767+0000","last_peered":"2024-09-21T10:05:04.841767+0000","last_clean":"2024-09-21T10:05:04.841767+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:05:04.841767+0000","last_undegraded":"2024-09-21T10:05:04.841767+0000","last_fullsized":"2024-09-21T10:05:04.841767+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1565,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825230+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:05:04.825230+0000","last_peered":"2024-09-21T10:05:04.825230+0000","last_clean":"2024-09-21T10:05:04.825230+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:05:04.825230+0000","last_undegraded":"2024-09-21T10:05:04.825230+0000","last_fullsized":"2024-09-21T10:05:04.825230+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":84,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.824870+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:05:04.824870+0000","last_peered":"2024-09-21T10:05:04.824870+0000","last_clean":"2024-09-21T10:05:04.824870+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:05:04.824870+0000","last_undegraded":"2024-09-21T10:05:04.824870+0000","last_fullsized":"2024-09-21T10:05:04.824870+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1188'6345","reported_seq":13929,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:12.285955+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:05:12.285955+0000","last_peered":"2024-09-21T10:05:12.285955+0000","last_clean":"2024-09-21T10:05:12.285955+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:05:12.285955+0000","last_undegraded":"2024-09-21T10:05:12.285955+0000","last_fullsized":"2024-09-21T10:05:12.285955+0000","mapping_epoch":1174,"log_start":"1143'1594","ondisk_log_start":"1143'1594","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165871616,"num_objects":2531,"num_object_clones":0,"num_object_copies":7593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6345,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1542,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826268+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:05:04.826268+0000","last_peered":"2024-09-21T10:05:04.826268+0000","last_clean":"2024-09-21T10:05:04.826268+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:05:04.826268+0000","last_undegraded":"2024-09-21T10:05:04.826268+0000","last_fullsized":"2024-09-21T10:05:04.826268+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":722,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827374+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:05:04.827374+0000","last_peered":"2024-09-21T10:05:04.827374+0000","last_clean":"2024-09-21T10:05:04.827374+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:05:04.827374+0000","last_undegraded":"2024-09-21T10:05:04.827374+0000","last_fullsized":"2024-09-21T10:05:04.827374+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1534,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826909+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:05:04.826909+0000","last_peered":"2024-09-21T10:05:04.826909+0000","last_clean":"2024-09-21T10:05:04.826909+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:05:04.826909+0000","last_undegraded":"2024-09-21T10:05:04.826909+0000","last_fullsized":"2024-09-21T10:05:04.826909+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1188'6533","reported_seq":6648,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:11.599574+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:05:11.599574+0000","last_peered":"2024-09-21T10:05:11.599574+0000","last_clean":"2024-09-21T10:05:11.599574+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:05:11.599574+0000","last_undegraded":"2024-09-21T10:05:11.599574+0000","last_fullsized":"2024-09-21T10:05:11.599574+0000","mapping_epoch":1137,"log_start":"1143'1926","ondisk_log_start":"1143'1926","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4607,"log_dups_size":0,"ondisk_log_size":4607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949504,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6533,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":242,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.846505+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:05:04.846505+0000","last_peered":"2024-09-21T10:05:04.846505+0000","last_clean":"2024-09-21T10:05:04.846505+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:05:04.846505+0000","last_undegraded":"2024-09-21T10:05:04.846505+0000","last_fullsized":"2024-09-21T10:05:04.846505+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1543,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826616+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:05:04.826616+0000","last_peered":"2024-09-21T10:05:04.826616+0000","last_clean":"2024-09-21T10:05:04.826616+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:05:04.826616+0000","last_undegraded":"2024-09-21T10:05:04.826616+0000","last_fullsized":"2024-09-21T10:05:04.826616+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":220,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840511+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:05:04.840511+0000","last_peered":"2024-09-21T10:05:04.840511+0000","last_clean":"2024-09-21T10:05:04.840511+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:05:04.840511+0000","last_undegraded":"2024-09-21T10:05:04.840511+0000","last_fullsized":"2024-09-21T10:05:04.840511+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1188'6268","reported_seq":6383,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:11.746679+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:05:11.746679+0000","last_peered":"2024-09-21T10:05:11.746679+0000","last_clean":"2024-09-21T10:05:11.746679+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:05:11.746679+0000","last_undegraded":"2024-09-21T10:05:11.746679+0000","last_fullsized":"2024-09-21T10:05:11.746679+0000","mapping_epoch":1137,"log_start":"1143'1713","ondisk_log_start":"1143'1713","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4555,"log_dups_size":0,"ondisk_log_size":4555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169082880,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6268,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":83,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825538+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:05:04.825538+0000","last_peered":"2024-09-21T10:05:04.825538+0000","last_clean":"2024-09-21T10:05:04.825538+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:05:04.825538+0000","last_undegraded":"2024-09-21T10:05:04.825538+0000","last_fullsized":"2024-09-21T10:05:04.825538+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.840278+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:05:03.840278+0000","last_peered":"2024-09-21T10:05:03.840278+0000","last_clean":"2024-09-21T10:05:03.840278+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:05:03.840278+0000","last_undegraded":"2024-09-21T10:05:03.840278+0000","last_fullsized":"2024-09-21T10:05:03.840278+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.836671+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:05:03.836671+0000","last_peered":"2024-09-21T10:05:03.836671+0000","last_clean":"2024-09-21T10:05:03.836671+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:05:03.836671+0000","last_undegraded":"2024-09-21T10:05:03.836671+0000","last_fullsized":"2024-09-21T10:05:03.836671+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833368+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:05:03.833368+0000","last_peered":"2024-09-21T10:05:03.833368+0000","last_clean":"2024-09-21T10:05:03.833368+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:05:03.833368+0000","last_undegraded":"2024-09-21T10:05:03.833368+0000","last_fullsized":"2024-09-21T10:05:03.833368+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":48,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.834724+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:05:03.834724+0000","last_peered":"2024-09-21T10:05:03.834724+0000","last_clean":"2024-09-21T10:05:03.834724+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:05:03.834724+0000","last_undegraded":"2024-09-21T10:05:03.834724+0000","last_fullsized":"2024-09-21T10:05:03.834724+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1542,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826308+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:05:04.826308+0000","last_peered":"2024-09-21T10:05:04.826308+0000","last_clean":"2024-09-21T10:05:04.826308+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:05:04.826308+0000","last_undegraded":"2024-09-21T10:05:04.826308+0000","last_fullsized":"2024-09-21T10:05:04.826308+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1155,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841995+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:05:04.841995+0000","last_peered":"2024-09-21T10:05:04.841995+0000","last_clean":"2024-09-21T10:05:04.841995+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:05:04.841995+0000","last_undegraded":"2024-09-21T10:05:04.841995+0000","last_fullsized":"2024-09-21T10:05:04.841995+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1166,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826695+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:05:04.826695+0000","last_peered":"2024-09-21T10:05:04.826695+0000","last_clean":"2024-09-21T10:05:04.826695+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:05:04.826695+0000","last_undegraded":"2024-09-21T10:05:04.826695+0000","last_fullsized":"2024-09-21T10:05:04.826695+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":240,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840574+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:05:04.840574+0000","last_peered":"2024-09-21T10:05:04.840574+0000","last_clean":"2024-09-21T10:05:04.840574+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:05:04.840574+0000","last_undegraded":"2024-09-21T10:05:04.840574+0000","last_fullsized":"2024-09-21T10:05:04.840574+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1568,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.846295+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:05:04.846295+0000","last_peered":"2024-09-21T10:05:04.846295+0000","last_clean":"2024-09-21T10:05:04.846295+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:05:04.846295+0000","last_undegraded":"2024-09-21T10:05:04.846295+0000","last_fullsized":"2024-09-21T10:05:04.846295+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":219,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840761+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:05:04.840761+0000","last_peered":"2024-09-21T10:05:04.840761+0000","last_clean":"2024-09-21T10:05:04.840761+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:05:04.840761+0000","last_undegraded":"2024-09-21T10:05:04.840761+0000","last_fullsized":"2024-09-21T10:05:04.840761+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":47,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.840191+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:05:03.840191+0000","last_peered":"2024-09-21T10:05:03.840191+0000","last_clean":"2024-09-21T10:05:03.840191+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:05:03.840191+0000","last_undegraded":"2024-09-21T10:05:03.840191+0000","last_fullsized":"2024-09-21T10:05:03.840191+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840405+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:05:04.840405+0000","last_peered":"2024-09-21T10:05:04.840405+0000","last_clean":"2024-09-21T10:05:04.840405+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:05:04.840405+0000","last_undegraded":"2024-09-21T10:05:04.840405+0000","last_fullsized":"2024-09-21T10:05:04.840405+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1530,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.873943+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:05:04.873943+0000","last_peered":"2024-09-21T10:05:04.873943+0000","last_clean":"2024-09-21T10:05:04.873943+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:05:04.873943+0000","last_undegraded":"2024-09-21T10:05:04.873943+0000","last_fullsized":"2024-09-21T10:05:04.873943+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1541,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.825254+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:05:04.825254+0000","last_peered":"2024-09-21T10:05:04.825254+0000","last_clean":"2024-09-21T10:05:04.825254+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:05:04.825254+0000","last_undegraded":"2024-09-21T10:05:04.825254+0000","last_fullsized":"2024-09-21T10:05:04.825254+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":205,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.842079+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:05:04.842079+0000","last_peered":"2024-09-21T10:05:04.842079+0000","last_clean":"2024-09-21T10:05:04.842079+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:05:04.842079+0000","last_undegraded":"2024-09-21T10:05:04.842079+0000","last_fullsized":"2024-09-21T10:05:04.842079+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":89,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827210+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:05:04.827210+0000","last_peered":"2024-09-21T10:05:04.827210+0000","last_clean":"2024-09-21T10:05:04.827210+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:05:04.827210+0000","last_undegraded":"2024-09-21T10:05:04.827210+0000","last_fullsized":"2024-09-21T10:05:04.827210+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1425,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826050+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:05:04.826050+0000","last_peered":"2024-09-21T10:05:04.826050+0000","last_clean":"2024-09-21T10:05:04.826050+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:05:04.826050+0000","last_undegraded":"2024-09-21T10:05:04.826050+0000","last_fullsized":"2024-09-21T10:05:04.826050+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1506,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826795+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:05:04.826795+0000","last_peered":"2024-09-21T10:05:04.826795+0000","last_clean":"2024-09-21T10:05:04.826795+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:05:04.826795+0000","last_undegraded":"2024-09-21T10:05:04.826795+0000","last_fullsized":"2024-09-21T10:05:04.826795+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1151,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.841257+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:05:04.841257+0000","last_peered":"2024-09-21T10:05:04.841257+0000","last_clean":"2024-09-21T10:05:04.841257+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:05:04.841257+0000","last_undegraded":"2024-09-21T10:05:04.841257+0000","last_fullsized":"2024-09-21T10:05:04.841257+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":105,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.826366+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:05:04.826366+0000","last_peered":"2024-09-21T10:05:04.826366+0000","last_clean":"2024-09-21T10:05:04.826366+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:05:04.826366+0000","last_undegraded":"2024-09-21T10:05:04.826366+0000","last_fullsized":"2024-09-21T10:05:04.826366+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1158,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.840828+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:05:04.840828+0000","last_peered":"2024-09-21T10:05:04.840828+0000","last_clean":"2024-09-21T10:05:04.840828+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:05:04.840828+0000","last_undegraded":"2024-09-21T10:05:04.840828+0000","last_fullsized":"2024-09-21T10:05:04.840828+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":42,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.835506+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:05:04.835506+0000","last_peered":"2024-09-21T10:05:04.835506+0000","last_clean":"2024-09-21T10:05:04.835506+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:05:04.835506+0000","last_undegraded":"2024-09-21T10:05:04.835506+0000","last_fullsized":"2024-09-21T10:05:04.835506+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":212,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.842338+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:05:04.842338+0000","last_peered":"2024-09-21T10:05:04.842338+0000","last_clean":"2024-09-21T10:05:04.842338+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:05:04.842338+0000","last_undegraded":"2024-09-21T10:05:04.842338+0000","last_fullsized":"2024-09-21T10:05:04.842338+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":220,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.873769+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:05:04.873769+0000","last_peered":"2024-09-21T10:05:04.873769+0000","last_clean":"2024-09-21T10:05:04.873769+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:05:04.873769+0000","last_undegraded":"2024-09-21T10:05:04.873769+0000","last_fullsized":"2024-09-21T10:05:04.873769+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":38,"reported_epoch":1188,"state":"active+clean","last_fresh":"2024-09-21T10:05:04.827939+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:05:04.827939+0000","last_peered":"2024-09-21T10:05:04.827939+0000","last_clean":"2024-09-21T10:05:04.827939+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:05:04.827939+0000","last_undegraded":"2024-09-21T10:05:04.827939+0000","last_fullsized":"2024-09-21T10:05:04.827939+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1556,"reported_epoch":1187,"state":"active+clean","last_fresh":"2024-09-21T10:05:03.833653+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:05:03.833653+0000","last_peered":"2024-09-21T10:05:03.833653+0000","last_clean":"2024-09-21T10:05:03.833653+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:05:03.833653+0000","last_undegraded":"2024-09-21T10:05:03.833653+0000","last_fullsized":"2024-09-21T10:05:03.833653+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2722824216,"num_objects":41548,"num_object_clones":0,"num_object_copies":124644,"num_objects_missing_on_primary":10177,"num_objects_missing":10177,"num_objects_degraded":40307,"num_objects_misplaced":4481,"num_objects_unfound":0,"num_objects_dirty":41548,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":102426,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26815,"num_bytes_recovered":1550319616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9688395776,"data_stored":9688383560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76992,"ondisk_log_size":76992,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234353,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495988,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528060,"kb_used_data":258292,"kb_used_omap":1,"kb_used_meta":269758,"kb_avail":93217092,"statfs":{"total":95995035648,"available":95454302208,"internally_reserved":0,"allocated":264491008,"data_stored":262024618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":276232647},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757628,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019264,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1244420,"kb_used_data":996732,"kb_used_omap":1,"kb_used_meta":247678,"kb_avail":92500732,"statfs":{"total":95995035648,"available":94720749568,"internally_reserved":0,"allocated":1020653568,"data_stored":1018186942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":253622726},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888559,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2325924,"kb_used_data":1874652,"kb_used_omap":34,"kb_used_meta":451229,"kb_avail":91419228,"statfs":{"total":95995035648,"available":93613289472,"internally_reserved":0,"allocated":1919643648,"data_stored":1917195928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":462059507},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967799,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1874688,"kb_used_data":1527476,"kb_used_omap":41,"kb_used_meta":347158,"kb_avail":91870464,"statfs":{"total":95995035648,"available":94075355136,"internally_reserved":0,"allocated":1564135424,"data_stored":1561677364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":355490207},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047037,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2119076,"kb_used_data":1830172,"kb_used_omap":41,"kb_used_meta":288854,"kb_avail":91626076,"statfs":{"total":95995035648,"available":93825101824,"internally_reserved":0,"allocated":1874096128,"data_stored":1871648816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":295787433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126276,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2116096000,"data_stored":2116091928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1865420800,"data_stored":1865416728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1547501568,"data_stored":1547501568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1910308864,"data_stored":1910308864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1010892800,"data_stored":1010892800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":258609152,"data_stored":258605080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":647364608,"data_stored":647364608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:05:19.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:05:19.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:19.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:19.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:19.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:20.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:20.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:20.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:20.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:20.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:20.350 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:20.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:20.448 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:20.451 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:20.453 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:20.490 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:05:20.490 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:05:20.490 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:05:20.490 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:05:20.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.499 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:20.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:20.600 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:20.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:20.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:20.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:20.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:20.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:20 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 39878/120846 objects degraded (32.999%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:20.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:20 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 39878/120846 objects degraded (32.999%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:20.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:20 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 39878/120846 objects degraded (32.999%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:20.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:20.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:20.979 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:21.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:21.109 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:21.134 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:21.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:21.260 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:21.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:21.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:05:21.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:21.399 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:21.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:21.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:21.528 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:21.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:21.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:21.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:21.794 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:21.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:21 smithi191 ceph-mon[102206]: pgmap v1513: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 39878/120846 objects degraded (32.999%); 3940/120846 objects misplaced (3.260%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:05:21.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:21 smithi076 ceph-mon[106482]: pgmap v1513: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 39878/120846 objects degraded (32.999%); 3940/120846 objects misplaced (3.260%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:05:21.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:21 smithi076 ceph-mon[110963]: pgmap v1513: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.5 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 39878/120846 objects degraded (32.999%); 3940/120846 objects misplaced (3.260%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:05:21.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:21.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:21.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:21.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:22.054 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:22.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:22.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:22.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:22.303 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:22.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:22.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:22.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:22.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:22.556 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:22.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:22.658 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:22.683 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:22.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:22.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:22.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:22.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:22.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:22.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:22.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:23.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:23.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:23.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:23.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:23.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:23.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:05:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:05:23] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-21T10:05:23.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:23.450 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:23.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:23.577 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:23.615 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:23.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:23.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:23.803 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:23.828 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:23 smithi076 ceph-mon[106482]: pgmap v1514: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 39588/118251 objects degraded (33.478%); 3423/118251 objects misplaced (2.895%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:05:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:23 smithi076 ceph-mon[110963]: pgmap v1514: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 39588/118251 objects degraded (33.478%); 3423/118251 objects misplaced (2.895%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:05:23.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:23.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:23.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:24.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:24.092 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:24.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:23 smithi191 ceph-mon[102206]: pgmap v1514: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 39588/118251 objects degraded (33.478%); 3423/118251 objects misplaced (2.895%); 9.5 MiB/s, 152 objects/s recovering 2024-09-21T10:05:24.194 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:24.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:24.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:24.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:24.346 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:24.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:24.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.469 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:24.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:24.571 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:24.596 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:24.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:24.698 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:24.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:24.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.849 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:24.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:24.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:24.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:24.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:25.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:25.104 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 39511/117759 objects degraded (33.552%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:25.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 39511/117759 objects degraded (33.552%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:25.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 39511/117759 objects degraded (33.552%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:25.206 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:25.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.332 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:25.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:25.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:25.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:25.615 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:25.717 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:25.743 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:25.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:25.870 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:25.972 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:25.996 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:25.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:26.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:25 smithi076 ceph-mon[106482]: pgmap v1515: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 209 op/s; 39511/117759 objects degraded (33.552%); 3317/117759 objects misplaced (2.817%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:05:26.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:26.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:26.018 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:25 smithi076 ceph-mon[110963]: pgmap v1515: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 209 op/s; 39511/117759 objects degraded (33.552%); 3317/117759 objects misplaced (2.817%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:05:26.018 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:26.018 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:26.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:25 smithi191 ceph-mon[102206]: pgmap v1515: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 209 op/s; 39511/117759 objects degraded (33.552%); 3317/117759 objects misplaced (2.817%); 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:05:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:05:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:26.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:26.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:26.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:26.244 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:26.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:26.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:26.369 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:26.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:26.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:26.492 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:26.594 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:26.622 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:26.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:26.753 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:26.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:26.878 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:26.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:26.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:26.980 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:27.004 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:27.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:27.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.233 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:27.261 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:27.362 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:27.386 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:27.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:27.511 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:27.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:27.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:27.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:27.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:27.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:27.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:27.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:27.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:27.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:27.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:28.013 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:27 smithi191 ceph-mon[102206]: pgmap v1516: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 39511/117759 objects degraded (33.552%); 3317/117759 objects misplaced (2.817%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:05:28.114 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:28.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:27 smithi076 ceph-mon[106482]: pgmap v1516: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 39511/117759 objects degraded (33.552%); 3317/117759 objects misplaced (2.817%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:05:28.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:27 smithi076 ceph-mon[110963]: pgmap v1516: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.4 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 39511/117759 objects degraded (33.552%); 3317/117759 objects misplaced (2.817%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:05:28.143 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:28.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:28.244 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:28.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:28.372 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:28.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:28.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:28.503 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:28.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:28.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:28.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:28.758 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:28.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:28.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:28.889 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:28.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:28.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:29.015 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:29.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:29.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:29.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:29.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:29.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:29.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:29.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:29.420 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:29.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:29.522 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:29.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:29.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:29.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:29.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:29.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:29.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:29.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:29.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:29.931 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:29.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:30.006 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:29 smithi076 ceph-mon[106482]: pgmap v1517: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 283 op/s; 39112/114504 objects degraded (34.158%); 2568/114504 objects misplaced (2.243%); 14 MiB/s, 227 objects/s recovering 2024-09-21T10:05:30.006 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:29 smithi076 ceph-mon[110963]: pgmap v1517: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 283 op/s; 39112/114504 objects degraded (34.158%); 2568/114504 objects misplaced (2.243%); 14 MiB/s, 227 objects/s recovering 2024-09-21T10:05:30.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:30.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:29 smithi191 ceph-mon[102206]: pgmap v1517: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 283 op/s; 39112/114504 objects degraded (34.158%); 2568/114504 objects misplaced (2.243%); 14 MiB/s, 227 objects/s recovering 2024-09-21T10:05:30.157 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:30.184 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:30.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:30.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:30.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:30.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:30.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:30.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:30.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:30.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:30.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:30.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:30.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:30.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:30.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:30.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:30.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:31.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:31.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:31.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:31.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:31.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:31.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:31.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:31.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:31.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:31.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:31.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:31.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:31.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:31.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:31.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:31.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:31.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:31.964 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:31.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:32.066 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:32.091 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:32.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:32.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:31 smithi191 ceph-mon[102206]: pgmap v1518: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 176 op/s; 39112/114504 objects degraded (34.158%); 2568/114504 objects misplaced (2.243%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:05:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:31 smithi076 ceph-mon[106482]: pgmap v1518: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 176 op/s; 39112/114504 objects degraded (34.158%); 2568/114504 objects misplaced (2.243%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:05:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:31 smithi076 ceph-mon[110963]: pgmap v1518: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 176 op/s; 39112/114504 objects degraded (34.158%); 2568/114504 objects misplaced (2.243%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:05:32.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:32.215 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:32.316 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:32.345 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:32.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:32.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:32.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:32.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:32.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:32.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:32.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:32.728 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:32.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:32.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:32.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:32.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:32.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:32.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:32.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:33.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:33.109 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:32 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 38802/111696 objects degraded (34.739%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:33.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:32 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 38802/111696 objects degraded (34.739%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:33.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:32 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 38802/111696 objects degraded (34.739%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:33.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:33.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:33.335 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:33.358 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:33.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:33.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:33.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:33.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:33.622 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:05:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:05:33] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-09-21T10:05:33.714 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:33.738 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:33.840 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:33.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:33.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:33.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:33.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:33.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:33.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:34.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:33 smithi191 ceph-mon[102206]: pgmap v1519: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 254 op/s; 38802/111696 objects degraded (34.739%); 2241/111696 objects misplaced (2.006%); 13 MiB/s, 248 objects/s recovering 2024-09-21T10:05:34.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:33 smithi076 ceph-mon[106482]: pgmap v1519: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 254 op/s; 38802/111696 objects degraded (34.739%); 2241/111696 objects misplaced (2.006%); 13 MiB/s, 248 objects/s recovering 2024-09-21T10:05:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:33 smithi076 ceph-mon[110963]: pgmap v1519: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 254 op/s; 38802/111696 objects degraded (34.739%); 2241/111696 objects misplaced (2.006%); 13 MiB/s, 248 objects/s recovering 2024-09-21T10:05:34.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:34.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:34.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:34.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:34.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:34.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:34.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:34.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:34.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:34.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:34.737 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:34.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:34.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:34.891 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:34.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:34.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:34.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:35.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:35.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:35.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:35.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:35.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:35.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:35.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:35.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:35.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:35.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:35.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:35.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:35.656 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:35.758 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:35.781 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:35 smithi076 ceph-mon[106482]: pgmap v1520: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 38731/111270 objects degraded (34.808%); 2112/111270 objects misplaced (1.898%); 10 MiB/s, 207 objects/s recovering 2024-09-21T10:05:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:35 smithi076 ceph-mon[110963]: pgmap v1520: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 38731/111270 objects degraded (34.808%); 2112/111270 objects misplaced (1.898%); 10 MiB/s, 207 objects/s recovering 2024-09-21T10:05:35.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:35.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:35.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:36.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:36.035 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:35 smithi191 ceph-mon[102206]: pgmap v1520: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 193 op/s; 38731/111270 objects degraded (34.808%); 2112/111270 objects misplaced (1.898%); 10 MiB/s, 207 objects/s recovering 2024-09-21T10:05:36.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:36.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:36.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:36.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:36.387 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:36.415 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:36.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:36.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:36.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:36.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:36.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:36.690 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:36.763 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:36.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:36.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:36.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:36.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:37.012 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:37.044 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:37.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:37.146 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:37.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:37.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:37.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:37.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:37.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:37.460 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:37.562 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:37.585 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:37.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:37.687 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:37.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:37.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:37.812 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:05:37.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:37.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:37.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:37.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:37.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:37.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:37.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:38.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:38.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:38.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:38.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:37 smithi191 ceph-mon[102206]: pgmap v1521: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 180 op/s; 38731/111270 objects degraded (34.808%); 2112/111270 objects misplaced (1.898%); 9.0 MiB/s, 191 objects/s recovering 2024-09-21T10:05:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:37 smithi191 ceph-mon[102206]: osdmap e1190: 8 total, 8 up, 5 in 2024-09-21T10:05:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1053708084' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:37 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1420550187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[106482]: pgmap v1521: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 180 op/s; 38731/111270 objects degraded (34.808%); 2112/111270 objects misplaced (1.898%); 9.0 MiB/s, 191 objects/s recovering 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[106482]: osdmap e1190: 8 total, 8 up, 5 in 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1053708084' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1420550187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[110963]: pgmap v1521: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.3 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 180 op/s; 38731/111270 objects degraded (34.808%); 2112/111270 objects misplaced (1.898%); 9.0 MiB/s, 191 objects/s recovering 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[110963]: osdmap e1190: 8 total, 8 up, 5 in 2024-09-21T10:05:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1053708084' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:38.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:05:38.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1420550187' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:38.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:38.241 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:38.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:38.339 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:38.340 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:38.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:05:38.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:05:38.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:05:38.341 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:38.341 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:38.341 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:38.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:38.366 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:38.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:38.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:38.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:38.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:38.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:38.726 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:38.752 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:38.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:38.854 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:38.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:38.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:38.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:39.010 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:39.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:38 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:38 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:38 smithi191 ceph-mon[102206]: osdmap e1191: 8 total, 8 up, 5 in 2024-09-21T10:05:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:38 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:38 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:39.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:38 smithi076 ceph-mon[106482]: osdmap e1191: 8 total, 8 up, 5 in 2024-09-21T10:05:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:38 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:38 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:05:39.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:38 smithi076 ceph-mon[110963]: osdmap e1191: 8 total, 8 up, 5 in 2024-09-21T10:05:39.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.260 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:39.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:39.388 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:39.489 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:39.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:39.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:39.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:39.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:39.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:39.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:39.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:39.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:39.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:39.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:39.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:40.024 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:39 smithi191 ceph-mon[102206]: pgmap v1524: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 275 op/s; 35361/107895 objects degraded (32.774%); 4065/107895 objects misplaced (3.768%); 9.0 MiB/s, 350 objects/s recovering 2024-09-21T10:05:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 35361/107895 objects degraded (32.774%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:40.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:39 smithi076 ceph-mon[106482]: pgmap v1524: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 275 op/s; 35361/107895 objects degraded (32.774%); 4065/107895 objects misplaced (3.768%); 9.0 MiB/s, 350 objects/s recovering 2024-09-21T10:05:40.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 35361/107895 objects degraded (32.774%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:40.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:39 smithi076 ceph-mon[110963]: pgmap v1524: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 275 op/s; 35361/107895 objects degraded (32.774%); 4065/107895 objects misplaced (3.768%); 9.0 MiB/s, 350 objects/s recovering 2024-09-21T10:05:40.115 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 35361/107895 objects degraded (32.774%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:40.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:40.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:40.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:40.399 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:40.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:40.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:40.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:40.633 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:40.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:40.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:40.783 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:40.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:40.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:40.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:40.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:41.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:41.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:41.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:41.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:41.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:41.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:41.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:41.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:41.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:41.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:41.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:41.536 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:41.638 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:41.662 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:41.764 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:41.790 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:41.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:41.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:41.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:41.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:42.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:41 smithi076 ceph-mon[106482]: pgmap v1525: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 158 op/s; 35361/107895 objects degraded (32.774%); 4065/107895 objects misplaced (3.768%); 5.0 MiB/s, 215 objects/s recovering 2024-09-21T10:05:42.008 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:41 smithi076 ceph-mon[110963]: pgmap v1525: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 158 op/s; 35361/107895 objects degraded (32.774%); 4065/107895 objects misplaced (3.768%); 5.0 MiB/s, 215 objects/s recovering 2024-09-21T10:05:42.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:42.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:42.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:41 smithi191 ceph-mon[102206]: pgmap v1525: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.2 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 158 op/s; 35361/107895 objects degraded (32.774%); 4065/107895 objects misplaced (3.768%); 5.0 MiB/s, 215 objects/s recovering 2024-09-21T10:05:42.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:42.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:42.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:42.267 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:42.304 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:42.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:42.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:42.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:42.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:42.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:42.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:42.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:42.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:42.683 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:05:42.683 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35361/107895 objects degraded (32.774%), 9 pgs degraded, 9 pgs undersized","count":18},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1245,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1191,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":5,"osd_in_since":1726913007,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":78},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"peering","count":1},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":35965,"data_bytes":2355828733,"bytes_used":7635271680,"bytes_avail":472339906560,"bytes_total":479975178240,"inactive_pgs_ratio":0.01123595517128706,"degraded_objects":35361,"degraded_total":107895,"degraded_ratio":0.32773529820658981,"misplaced_objects":4065,"misplaced_total":107895,"misplaced_ratio":0.037675517864590571,"recovering_objects_per_sec":215,"recovering_bytes_per_sec":5216835,"recovering_keys_per_sec":0,"num_objects_recovered":1721,"num_bytes_recovered":41746432,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":158},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-21T10:05:04.625691+0000","services":{}},"progress_events":{}} 2024-09-21T10:05:42.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:42.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:42.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:42.927 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1870097953' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:05:42.927 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1870097953' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:05:42.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:05:42.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:42.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:43.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1870097953' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:05:43.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:43.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:43.268 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:05:43.269 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:05:43.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:43.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:43.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:43.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:43.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:43.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:43.590 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:05:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:05:43] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:05:43.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:43.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:43.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:05:43.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:43.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:43 smithi076 ceph-mon[106482]: pgmap v1526: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 270 op/s; 34982/104769 objects degraded (33.390%); 3974/104769 objects misplaced (3.793%); 5.9 MiB/s, 229 objects/s recovering 2024-09-21T10:05:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:43 smithi076 ceph-mon[110963]: pgmap v1526: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 270 op/s; 34982/104769 objects degraded (33.390%); 3974/104769 objects misplaced (3.793%); 5.9 MiB/s, 229 objects/s recovering 2024-09-21T10:05:43.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:43.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:43.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:43.962 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:05:44.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:44.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:44.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:43 smithi191 ceph-mon[102206]: pgmap v1526: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 270 op/s; 34982/104769 objects degraded (33.390%); 3974/104769 objects misplaced (3.793%); 5.9 MiB/s, 229 objects/s recovering 2024-09-21T10:05:44.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:44.198 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:44.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:44.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:44.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:44.453 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:44.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:44.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:44.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:44.583 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:05:44.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:05:44.598 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:44.631 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:44.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:44.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:44.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:44.825 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:44.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:44.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:44.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:44.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:44.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:45.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:45.076 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 34982/104769 objects degraded (33.390%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 34982/104769 objects degraded (33.390%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 34982/104769 objects degraded (33.390%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:45.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:45.201 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:45.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:45.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:45.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:45.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:45.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:45.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:45.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:45.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:45.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:45.803 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:45.827 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:45.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:45.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:45.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:45.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:46.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:46.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:45 smithi191 ceph-mon[102206]: pgmap v1527: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 288 op/s; 34914/104346 objects degraded (33.460%); 3814/104346 objects misplaced (3.655%); 7.6 MiB/s, 257 objects/s recovering 2024-09-21T10:05:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:45 smithi076 ceph-mon[106482]: pgmap v1527: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 288 op/s; 34914/104346 objects degraded (33.460%); 3814/104346 objects misplaced (3.655%); 7.6 MiB/s, 257 objects/s recovering 2024-09-21T10:05:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:45 smithi076 ceph-mon[110963]: pgmap v1527: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 288 op/s; 34914/104346 objects degraded (33.460%); 3814/104346 objects misplaced (3.655%); 7.6 MiB/s, 257 objects/s recovering 2024-09-21T10:05:46.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:46.194 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:46.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:46.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:46.328 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:46.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:46.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:46.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:46.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:46.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:46.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:46.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:46.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:46.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:46.832 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:46.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:46.958 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:46.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:46.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:47.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:47.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:47.185 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:47.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:47.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:47.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:47.333 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:47.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:47.459 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:47.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:47.588 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:47.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:47.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:47.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:47.849 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:47.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:47.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:47.973 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:47.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:48.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.096 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:47 smithi076 ceph-mon[106482]: pgmap v1528: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 34914/104346 objects degraded (33.460%); 3814/104346 objects misplaced (3.655%); 6.2 MiB/s, 209 objects/s recovering 2024-09-21T10:05:48.102 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:47 smithi076 ceph-mon[110963]: pgmap v1528: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 34914/104346 objects degraded (33.460%); 3814/104346 objects misplaced (3.655%); 6.2 MiB/s, 209 objects/s recovering 2024-09-21T10:05:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:47 smithi191 ceph-mon[102206]: pgmap v1528: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 34914/104346 objects degraded (33.460%); 3814/104346 objects misplaced (3.655%); 6.2 MiB/s, 209 objects/s recovering 2024-09-21T10:05:48.199 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:48.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:48.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:48.323 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:48.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:48.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:48.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:48.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:48.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:48.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:48.705 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:48.729 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:48.831 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:48.856 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:48.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:48.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:48.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:48.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:49.084 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:49.109 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:49.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:49.361 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:49.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.491 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:49.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:49.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:49.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:49.741 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:49.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:49.842 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:49.870 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:05:49.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:49.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[106482]: pgmap v1529: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 220 op/s; 34462/101067 objects degraded (34.098%); 3577/101067 objects misplaced (3.539%); 7.5 MiB/s, 119 objects/s recovering 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 34462/101067 objects degraded (34.098%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2882142681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/535682648' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[110963]: pgmap v1529: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 220 op/s; 34462/101067 objects degraded (34.098%); 3577/101067 objects misplaced (3.539%); 7.5 MiB/s, 119 objects/s recovering 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 34462/101067 objects degraded (34.098%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2882142681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/535682648' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:49.892 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:49 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:49.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:50.000 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:50.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:49 smithi191 ceph-mon[102206]: pgmap v1529: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 220 op/s; 34462/101067 objects degraded (34.098%); 3577/101067 objects misplaced (3.539%); 7.5 MiB/s, 119 objects/s recovering 2024-09-21T10:05:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 34462/101067 objects degraded (34.098%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:49 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2882142681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:05:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:49 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/535682648' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:49 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:05:50.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:50.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:50.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:50.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:50.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:50.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:50.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:50.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:50.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:50.625 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:50.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:50.727 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:50.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:50.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:50.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:50.873 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:05:50.873 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44752 7.5 7:aa71fbe7:::benchmark_data_smithi076_190198_object44751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.236980+0000", 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.85301551699999, 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84928779700000001, 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.874 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44752 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.236980+0000", 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.875 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.236980+0000", 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.236981+0000", 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1099999999999996e-07 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237022+0000", 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0970000000000002e-05 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237029+0000", 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.328e-06 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237034+0000", 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4299999999999997e-06 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.070323+0000", 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83328886099999999 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.070479+0000", 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015550199999999999 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.070648+0000", 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016900199999999999 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.074462+0000", 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038139340000000002 2024-09-21T10:05:50.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078194+0000", 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00373216 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084829+0000", 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066349779999999997 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084884+0000", 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4749e-05 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086268+0000", 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001384372 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44755 7.5 7:a24569dd:::benchmark_data_smithi076_190198_object44754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.237890+0000", 2024-09-21T10:05:50.880 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.85210544399999, 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86201483000000001, 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44755 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237890+0000", 2024-09-21T10:05:50.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237890+0000", 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237891+0000", 2024-09-21T10:05:50.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.24e-06 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237939+0000", 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8021e-05 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237949+0000", 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6819999999999998e-06 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237954+0000", 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8670000000000002e-06 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.072413+0000", 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83445930700000004 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.072570+0000", 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015627800000000001 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.072781+0000", 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000211463 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078271+0000", 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054898880000000001 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.082585+0000", 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043141070000000002 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.098267+0000", 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015681725000000001 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.098316+0000", 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9376000000000003e-05 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.099905+0000", 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015888759999999999 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44759 7.5 7:a88bd5e4:::benchmark_data_smithi076_190198_object44758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:50.887 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.243453+0000", 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.84654237699999, 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86711577200000001, 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44759 2024-09-21T10:05:50.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243453+0000", 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243453+0000", 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243455+0000", 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5430000000000001e-06 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243508+0000", 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.312e-05 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.890 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243525+0000", 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6671e-05 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243533+0000", 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078310+0000", 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.834777769 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078495+0000", 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018510799999999999 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078691+0000", 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000195801 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084303+0000", 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056118959999999999 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086330+0000", 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002027067 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.108649+0000", 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.022319039999999998 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.108709+0000", 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9670000000000003e-05 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.894 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.110569+0000", 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001860149 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44760 7.5 7:a5bffe95:::benchmark_data_smithi076_190198_object44759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.243549+0000", 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.84644627700001, 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86892832200000003, 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.895 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44760 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243549+0000", 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243549+0000", 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243550+0000", 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3799999999999997e-07 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243592+0000", 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1882000000000003e-05 2024-09-21T10:05:50.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243599+0000", 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4549999999999998e-06 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243604+0000", 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1950000000000002e-06 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.080531+0000", 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83692627200000003 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.080721+0000", 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000190693 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.080875+0000", 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015366499999999999 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.899 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084309+0000", 2024-09-21T10:05:50.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034344050000000002 2024-09-21T10:05:50.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.090160+0000", 2024-09-21T10:05:50.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0058501669999999999 2024-09-21T10:05:50.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.110627+0000", 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020467793000000001 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.110975+0000", 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00034761299999999998 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.112478+0000", 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001502644 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.902 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:44762 7.d e1148/1137 7:b780d3c9:::benchmark_data_smithi076_190198_object44761:head v 1148'2804, mlcod=1146'2802)", 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.247040+0000", 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.84295556800001, 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77214165199999996, 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247040+0000", 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247040+0000", 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247041+0000", 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3700000000000001e-07 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247063+0000", 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2895e-05 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247071+0000", 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.055e-06 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247076+0000", 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7679999999999999e-06 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.996010+0000", 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74893399599999999 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.996035+0000", 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5103e-05 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019154+0000", 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023118825999999999 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019182+0000", 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7572e-05 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44766 7.5 7:a0cdd0d8:::benchmark_data_smithi076_190198_object44765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.249062+0000", 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.84093385899999, 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86492537199999997, 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.907 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44766 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249062+0000", 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.908 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249062+0000", 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249063+0000", 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5149999999999999e-06 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249093+0000", 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9734e-05 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249106+0000", 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3171e-05 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249114+0000", 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7270000000000007e-06 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086370+0000", 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83725620000000001 2024-09-21T10:05:50.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086493+0000", 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012246499999999999 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086659+0000", 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000166398 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.093835+0000", 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0071764580000000001 2024-09-21T10:05:50.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.106600+0000", 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012764559999999999 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.112529+0000", 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0059289649999999996 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.112584+0000", 2024-09-21T10:05:50.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4654000000000002e-05 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.113987+0000", 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014035250000000001 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44768 7.5 7:a3682ca2:::benchmark_data_smithi076_190198_object44767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.250180+0000", 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.83981556000001, 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87050010200000005, 2024-09-21T10:05:50.913 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44768 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250180+0000", 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250180+0000", 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250182+0000", 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4219999999999999e-06 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.915 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250205+0000", 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3748999999999999e-05 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250218+0000", 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2619000000000001e-05 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250225+0000", 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0079999999999996e-06 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.088311+0000", 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83808586100000004 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.088443+0000", 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013174000000000001 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.088606+0000", 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016336500000000001 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.100609+0000", 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012003565000000001 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.106606+0000", 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0059969589999999996 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.118424+0000", 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011817246 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.118512+0000", 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8101000000000002e-05 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.120680+0000", 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021684669999999999 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:44770 7.d e1148/1137 7:bab46a7f:::benchmark_data_smithi076_190198_object44769:head v 1148'2805, mlcod=1146'2802)", 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.254313+0000", 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.835682811, 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76489147199999996, 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.920 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254313+0000", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254313+0000", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254314+0000", 2024-09-21T10:05:50.921 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1379999999999999e-06 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254343+0000", 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.851e-05 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.922 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254356+0000", 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3869000000000001e-05 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254364+0000", 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4900000000000003e-06 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.997360+0000", 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.742996141 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.923 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.997380+0000", 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0123999999999999e-05 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019187+0000", 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021806447999999999 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.924 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019204+0000", 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7751999999999999e-05 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:44779 7.d e1148/1137 7:b6c25ae2:::benchmark_data_smithi076_190198_object44778:head v 1148'2806, mlcod=1146'2802)", 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.274461+0000", 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 246.81553447499999, 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74475925399999998, 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.925 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274461+0000", 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.926 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274461+0000", 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274462+0000", 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.92e-07 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274486+0000", 2024-09-21T10:05:50.927 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4417999999999999e-05 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274496+0000", 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0036e-05 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274502+0000", 2024-09-21T10:05:50.928 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1319999999999999e-06 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.998093+0000", 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.72359012099999998 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.929 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.998111+0000", 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8046999999999998e-05 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019206+0000", 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021095137999999999 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019220+0000", 2024-09-21T10:05:50.930 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.467e-05 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67019 7.5 7:addb2ccf:::benchmark_data_smithi076_190198_object67018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.328853+0000", 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.761142355, 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.98297844300000004, 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.931 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67019 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328853+0000", 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.932 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328853+0000", 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328854+0000", 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4499999999999995e-07 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328872+0000", 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7958e-05 2024-09-21T10:05:50.933 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328881+0000", 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8470000000000002e-06 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328887+0000", 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3230000000000003e-06 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328956+0000", 2024-09-21T10:05:50.934 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8629999999999999e-05 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328963+0000", 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4710000000000001e-06 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.303405+0000", 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.97444187100000002 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.935 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.303556+0000", 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015055900000000001 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.303683+0000", 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000126947 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311598+0000", 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079149809999999998 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.936 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311659+0000", 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1092000000000004e-05 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311733+0000", 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3565999999999993e-05 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311780+0000", 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7896999999999999e-05 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.937 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311832+0000", 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1356000000000001e-05 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67073 7.5 7:a0e58874:::benchmark_data_smithi076_190198_object67072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.468813+0000", 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.62118252499999, 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84322499799999995, 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.938 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67073 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468813+0000", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468813+0000", 2024-09-21T10:05:50.939 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468815+0000", 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6449999999999999e-06 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468837+0000", 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1844999999999998e-05 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468850+0000", 2024-09-21T10:05:50.940 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3105000000000001e-05 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468857+0000", 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3189999999999997e-06 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468937+0000", 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0020999999999998e-05 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468951+0000", 2024-09-21T10:05:50.941 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3529999999999999e-05 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.306149+0000", 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83719846499999995 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.306280+0000", 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013107900000000001 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.942 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.306452+0000", 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000171428 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311714+0000", 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005262353 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311868+0000", 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015407499999999999 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.943 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311959+0000", 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0593999999999994e-05 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312002+0000", 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.375e-05 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312038+0000", 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5788999999999999e-05 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.944 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67097 7.5 7:abaa65eb:::benchmark_data_smithi076_190198_object67096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.528415+0000", 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.56158075499999, 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.78379750800000003, 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67097 2024-09-21T10:05:50.945 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528415+0000", 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528415+0000", 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.946 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528416+0000", 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3e-06 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528438+0000", 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1664000000000001e-05 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528450+0000", 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2489e-05 2024-09-21T10:05:50.947 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528458+0000", 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2099999999999996e-06 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528544+0000", 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6278000000000003e-05 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528555+0000", 2024-09-21T10:05:50.948 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1453e-05 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.309199+0000", 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.78064412000000005 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.309329+0000", 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012979500000000001 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.949 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.309462+0000", 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013248399999999999 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312093+0000", 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002630945 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312132+0000", 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9480999999999997e-05 2024-09-21T10:05:50.950 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312143+0000", 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0529e-05 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312180+0000", 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7468000000000001e-05 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.951 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312212+0000", 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2292000000000002e-05 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67099 7.5 7:ac6bfae4:::benchmark_data_smithi076_190198_object67098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.535830+0000", 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.55416566599999, 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.791905674, 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.952 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67099 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.953 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535830+0000", 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535830+0000", 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535831+0000", 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2309999999999999e-06 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.954 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535853+0000", 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2084999999999999e-05 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535863+0000", 2024-09-21T10:05:50.955 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9250000000000007e-06 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535870+0000", 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1890000000000001e-06 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535932+0000", 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2063999999999994e-05 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.956 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535941+0000", 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3280000000000006e-06 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312251+0000", 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77631040100000004 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.957 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312383+0000", 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013156600000000001 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312508+0000", 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000125036 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327510+0000", 2024-09-21T10:05:50.958 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015002547999999999 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327562+0000", 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1341999999999997e-05 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327639+0000", 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7724000000000003e-05 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.959 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327690+0000", 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0467000000000003e-05 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327736+0000", 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5768e-05 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.960 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67106 7.5 7:a679e0fc:::benchmark_data_smithi076_190198_object67105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.603699+0000", 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.48629716299999, 2024-09-21T10:05:50.961 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.73135408599999996, 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67106 2024-09-21T10:05:50.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603699+0000", 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603699+0000", 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603700+0000", 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4190000000000001e-06 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603721+0000", 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1535999999999998e-05 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603734+0000", 2024-09-21T10:05:50.964 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2483999999999999e-05 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603740+0000", 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0979999999999999e-06 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603795+0000", 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5031000000000002e-05 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603803+0000", 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8450000000000003e-06 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.314640+0000", 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.71083746299999995 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.314936+0000", 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00029561700000000003 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.966 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.316359+0000", 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001423047 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327515+0000", 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011156142000000001 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327603+0000", 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8134999999999996e-05 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.334958+0000", 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073542360000000001 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335009+0000", 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1499999999999998e-05 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335053+0000", 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3532999999999998e-05 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.968 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67112 7.5 7:a6c7e4ba:::benchmark_data_smithi076_190198_object67111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.665613+0000", 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.424382834, 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.66975227199999998, 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.969 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67112 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665613+0000", 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665613+0000", 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665614+0000", 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.124e-06 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665633+0000", 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8783e-05 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.971 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665642+0000", 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1039999999999996e-06 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665646+0000", 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5820000000000003e-06 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665707+0000", 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0228e-05 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665719+0000", 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2174000000000001e-05 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.318684+0000", 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.65296505900000001 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.318806+0000", 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000121636 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.318945+0000", 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000139517 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327520+0000", 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0085746559999999999 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335115+0000", 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075948889999999996 2024-09-21T10:05:50.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335284+0000", 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016905800000000001 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335331+0000", 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7706e-05 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335365+0000", 2024-09-21T10:05:50.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3756e-05 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67138 7.5 7:a50cf1cf:::benchmark_data_smithi076_190198_object67137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.743306+0000", 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.346689309, 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.59216225700000003, 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.976 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67138 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743306+0000", 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743306+0000", 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743308+0000", 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4670000000000001e-06 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743329+0000", 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1608999999999999e-05 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743340+0000", 2024-09-21T10:05:50.978 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.079e-05 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743347+0000", 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4359999999999999e-06 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743406+0000", 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9503000000000003e-05 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.979 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743415+0000", 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3380000000000005e-06 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.321215+0000", 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.57780047899999998 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.321355+0000", 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000139791 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.980 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.321499+0000", 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000144516 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327572+0000", 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0060731170000000003 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335153+0000", 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075807390000000004 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.981 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335398+0000", 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000244966 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335438+0000", 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0074999999999998e-05 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335469+0000", 2024-09-21T10:05:50.982 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0431000000000001e-05 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67159 7.5 7:afaaa7d0:::benchmark_data_smithi076_190198_object67158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.793795+0000", 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.29620068200001, 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.54180674299999998, 2024-09-21T10:05:50.983 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67159 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793795+0000", 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.984 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793795+0000", 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793796+0000", 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1650000000000001e-06 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.985 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793816+0000", 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0100999999999999e-05 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793829+0000", 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2833000000000001e-05 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793836+0000", 2024-09-21T10:05:50.986 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1019999999999999e-06 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793896+0000", 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9976999999999999e-05 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793907+0000", 2024-09-21T10:05:50.987 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0929e-05 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.324145+0000", 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.53023783800000002 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.324253+0000", 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000107738 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.988 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.324405+0000", 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000152099 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.334917+0000", 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010512199999999999 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335182+0000", 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026484499999999999 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.989 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335533+0000", 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00035097999999999998 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335572+0000", 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9326999999999998e-05 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335602+0000", 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9609000000000001e-05 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.990 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67196 7.5 7:a910f998:::benchmark_data_smithi076_190198_object67195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.932286+0000", 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.157709251, 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.40341120400000002, 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67196 2024-09-21T10:05:50.991 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932286+0000", 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:50.992 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932286+0000", 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932288+0000", 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.747e-06 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932315+0000", 2024-09-21T10:05:50.993 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6750999999999998e-05 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932330+0000", 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4773999999999999e-05 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932338+0000", 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1289999999999996e-06 2024-09-21T10:05:50.994 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932400+0000", 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2397000000000006e-05 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932408+0000", 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2600000000000005e-06 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.995 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327766+0000", 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.39535715599999999 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327869+0000", 2024-09-21T10:05:50.996 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0001037 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327998+0000", 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012844000000000001 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335063+0000", 2024-09-21T10:05:50.997 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0070656149999999999 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335251+0000", 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018784199999999999 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335631+0000", 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00037950400000000002 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.998 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335669+0000", 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8624999999999997e-05 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335698+0000", 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8263999999999999e-05 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67209 7.5 7:a1f6f0be:::benchmark_data_smithi076_190198_object67208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:50.999 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.964507+0000", 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.12548838999999, 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.37128402999999999, 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67209 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:51.000 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964507+0000", 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964507+0000", 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964509+0000", 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4500000000000001e-06 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:51.001 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964531+0000", 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2345999999999999e-05 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964543+0000", 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2074e-05 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964551+0000", 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4340000000000003e-06 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.002 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964614+0000", 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3429000000000005e-05 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964623+0000", 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0990000000000005e-06 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.330022+0000", 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.36539935200000001 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.003 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.330167+0000", 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000144437 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.330315+0000", 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014777900000000001 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335499+0000", 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051839149999999999 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.004 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335508+0000", 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6139999999999998e-06 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335727+0000", 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021863899999999999 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335764+0000", 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7558999999999997e-05 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.005 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335791+0000", 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6903e-05 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67217 7.5 7:aa3c2a52:::benchmark_data_smithi076_190198_object67216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.991131+0000", 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 197.0988648, 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.356302434, 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:05:51.006 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67217 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991131+0000", 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:05:51.007 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991131+0000", 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991133+0000", 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.796e-06 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991181+0000", 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8643000000000002e-05 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:05:51.008 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991193+0000", 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.13e-05 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991200+0000", 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1149999999999996e-06 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991281+0000", 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1487000000000001e-05 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:05:51.009 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991292+0000", 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0635999999999999e-05 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335813+0000", 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.34452144299999998 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335943+0000", 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012933199999999999 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.010 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.336072+0000", 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012897399999999999 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347209+0000", 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011137651 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347307+0000", 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7627000000000006e-05 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.011 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347344+0000", 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7228000000000003e-05 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347390+0000", 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5884000000000003e-05 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347433+0000", 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3318000000000001e-05 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:51.012 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:51.013 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:05:51.013 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:05:51.013 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:05:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:51.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:50 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:50 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:50 smithi191 ceph-mon[102206]: osdmap e1192: 8 total, 8 up, 5 in 2024-09-21T10:05:51.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:50 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:50 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:50 smithi076 ceph-mon[106482]: osdmap e1192: 8 total, 8 up, 5 in 2024-09-21T10:05:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:50 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:05:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:50 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:05:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:50 smithi076 ceph-mon[110963]: osdmap e1192: 8 total, 8 up, 5 in 2024-09-21T10:05:51.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:51.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:51.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:51.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:51.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:51.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:51.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:51.523 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:51.625 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:51.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:51.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:51.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:51.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:51.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:51.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:51.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:52.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:52.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:52.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:05:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:51 smithi076 ceph-mon[106482]: pgmap v1531: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 227 op/s; 34462/101067 objects degraded (34.098%); 3577/101067 objects misplaced (3.539%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:05:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:51 smithi076 ceph-mon[110963]: pgmap v1531: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 227 op/s; 34462/101067 objects degraded (34.098%); 3577/101067 objects misplaced (3.539%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:05:52.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:52.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:05:52.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:05:52.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:05:52.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:52.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:52.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:51 smithi191 ceph-mon[102206]: pgmap v1531: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.1 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 227 op/s; 34462/101067 objects degraded (34.098%); 3577/101067 objects misplaced (3.539%); 6.7 MiB/s, 107 objects/s recovering 2024-09-21T10:05:52.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:52.421 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:52.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:52.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:52.550 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:52.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:52.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:52.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:52.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:52.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:52.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:52.907 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:52.932 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:52.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:52.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:53.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:53.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:53.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.187 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:53.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:53.288 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:53.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:53.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:53.331 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:05:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:05:53] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:05:53.397 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:53.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:53.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:53.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:53.585 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:53.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:53.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:53.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:53.843 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:53.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:53.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:53.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:53.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:54.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:54.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:54.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:53 smithi076 ceph-mon[106482]: pgmap v1532: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 239 op/s; 34027/97590 objects degraded (34.867%); 3423/97590 objects misplaced (3.508%); 7.3 MiB/s, 116 objects/s recovering 2024-09-21T10:05:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:53 smithi076 ceph-mon[110963]: pgmap v1532: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 239 op/s; 34027/97590 objects degraded (34.867%); 3423/97590 objects misplaced (3.508%); 7.3 MiB/s, 116 objects/s recovering 2024-09-21T10:05:54.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:54.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:54.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:54.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:54.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:54.356 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:05:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:53 smithi191 ceph-mon[102206]: pgmap v1532: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 239 op/s; 34027/97590 objects degraded (34.867%); 3423/97590 objects misplaced (3.508%); 7.3 MiB/s, 116 objects/s recovering 2024-09-21T10:05:54.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:54.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:54.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:54.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:54.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:54.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:54.741 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:54.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:54.843 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:54.867 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:54.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:54.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:54.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:55.053 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 34027/97590 objects degraded (34.867%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:55.054 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 34027/97590 objects degraded (34.867%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:55.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:55.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:55.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:55.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:55.253 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:55.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:55.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:55.359 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:05:55.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 34027/97590 objects degraded (34.867%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:55.378 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:55.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:55.480 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:55.506 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:55.591 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:05:55.608 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:55.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:55.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:55.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:55.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:55.907 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:55.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:56.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:56.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:56.134 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:56.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:56.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:56.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:56.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:56.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:56.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:56 smithi191 ceph-mon[102206]: pgmap v1533: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 33942/97101 objects degraded (34.955%); 3231/97101 objects misplaced (3.327%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:05:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:56 smithi076 ceph-mon[106482]: pgmap v1533: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 33942/97101 objects degraded (34.955%); 3231/97101 objects misplaced (3.327%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:05:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:56 smithi076 ceph-mon[110963]: pgmap v1533: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.7 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 33942/97101 objects degraded (34.955%); 3231/97101 objects misplaced (3.327%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:05:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:56.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:56.408 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:56.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.537 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:56.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:56.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:56.664 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:56.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:56.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:56.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:56.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:56.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:56.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:57.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:57.039 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:57.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:57.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:57.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:57.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:57.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:57.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:57.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:57.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:57.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:57.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:57.681 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:57.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:57.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:57.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:57.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:57.932 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:57.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:57.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:58.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:58.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:58.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:58.188 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:58.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:58.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:58.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:05:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:58 smithi191 ceph-mon[102206]: pgmap v1534: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 33942/97101 objects degraded (34.955%); 3231/97101 objects misplaced (3.327%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:05:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:58 smithi076 ceph-mon[106482]: pgmap v1534: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 33942/97101 objects degraded (34.955%); 3231/97101 objects misplaced (3.327%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:05:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:58 smithi076 ceph-mon[110963]: pgmap v1534: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 2.0 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 241 op/s; 33942/97101 objects degraded (34.955%); 3231/97101 objects misplaced (3.327%); 7.6 MiB/s, 121 objects/s recovering 2024-09-21T10:05:58.419 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:58.445 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:58.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:58.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:58.671 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:58.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:58.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:05:58.827 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:58.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:05:58.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:05:58.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:58.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:58.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:05:59.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:59.076 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:59.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:59.201 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:59.303 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:05:59.328 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:05:59.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:05:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 33942/97101 objects degraded (34.955%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:59.346 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:05:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 33942/97101 objects degraded (34.955%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:59.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:05:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 33942/97101 objects degraded (34.955%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:05:59.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:05:59.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:59.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:05:59.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:05:59.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:59.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:05:59.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:05:59.716 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:05:59.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:05:59.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:05:59.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:05:59.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:05:59.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:05:59.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:05:59.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:00.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:00.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:00.152 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:06:00.153 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:06:00.171 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:06:00.171 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:06:00.171 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:06:00.171 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:06:00.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:00.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:00.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:00.356 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:00 smithi191 ceph-mon[102206]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 33499/93537 objects degraded (35.814%); 2915/93537 objects misplaced (3.116%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:06:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:00 smithi076 ceph-mon[106482]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 33499/93537 objects degraded (35.814%); 2915/93537 objects misplaced (3.116%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:06:00.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:00 smithi076 ceph-mon[110963]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 250 op/s; 33499/93537 objects degraded (35.814%); 2915/93537 objects misplaced (3.116%); 8.1 MiB/s, 129 objects/s recovering 2024-09-21T10:06:00.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:00.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:00.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:00.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:00.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:00.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:00.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:00.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:00.863 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:00.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:00.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:00.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:01.093 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:01.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:01.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:01.244 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:01.345 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:01.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:01 smithi191 ceph-mon[102206]: from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:01.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:01 smithi076 ceph-mon[106482]: from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:01.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:01 smithi076 ceph-mon[110963]: from='client.45179 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:01.375 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:01.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:01.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:01.528 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1535,"stamp":"2024-09-21T10:05:58.641164+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2042173437,"num_objects":31179,"num_object_clones":0,"num_object_copies":93537,"num_objects_missing_on_primary":6826,"num_objects_missing":6826,"num_objects_degraded":33499,"num_objects_misplaced":2915,"num_objects_unfound":0,"num_objects_dirty":31179,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":113100,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34783,"num_bytes_recovered":1962948102,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77295,"ondisk_log_size":77295,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":6844900,"kb_used_data":5226172,"kb_used_omap":119,"kb_used_meta":1618568,"kb_avail":461880860,"statfs":{"total":479975178240,"available":472966000640,"internally_reserved":0,"allocated":5351600128,"data_stored":5339172236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1657413832},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2,0,3,0,3,6,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-164495360,"num_objects":-2510,"num_object_clones":0,"num_object_copies":-7530,"num_objects_missing_on_primary":-911,"num_objects_missing":-911,"num_objects_degraded":-963,"num_objects_misplaced":-662,"num_objects_unfound":0,"num_objects_dirty":-2510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2510,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1299,"num_bytes_recovered":85131264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002908"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":220,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.879381+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:05:49.879381+0000","last_peered":"2024-09-21T10:05:49.879381+0000","last_clean":"2024-09-21T10:05:49.879381+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:05:49.879381+0000","last_undegraded":"2024-09-21T10:05:49.879381+0000","last_fullsized":"2024-09-21T10:05:49.879381+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":48,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.830017+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:05:37.830017+0000","last_peered":"2024-09-21T10:05:37.830017+0000","last_clean":"2024-09-21T10:05:37.830017+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:05:37.830017+0000","last_undegraded":"2024-09-21T10:05:37.830017+0000","last_fullsized":"2024-09-21T10:05:37.830017+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1205,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.880694+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:05:49.880694+0000","last_peered":"2024-09-21T10:05:49.880694+0000","last_clean":"2024-09-21T10:05:49.880694+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:05:49.880694+0000","last_undegraded":"2024-09-21T10:05:49.880694+0000","last_fullsized":"2024-09-21T10:05:49.880694+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1537,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878782+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:05:49.878782+0000","last_peered":"2024-09-21T10:05:49.878782+0000","last_clean":"2024-09-21T10:05:49.878782+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:05:49.878782+0000","last_undegraded":"2024-09-21T10:05:49.878782+0000","last_fullsized":"2024-09-21T10:05:49.878782+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1537,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.914417+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:05:49.914417+0000","last_peered":"2024-09-21T10:05:49.914417+0000","last_clean":"2024-09-21T10:05:49.914417+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:05:49.914417+0000","last_undegraded":"2024-09-21T10:05:49.914417+0000","last_fullsized":"2024-09-21T10:05:49.914417+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829647+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:05:37.829647+0000","last_peered":"2024-09-21T10:05:37.829647+0000","last_clean":"2024-09-21T10:05:37.829647+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:05:37.829647+0000","last_undegraded":"2024-09-21T10:05:37.829647+0000","last_fullsized":"2024-09-21T10:05:37.829647+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":220,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.881682+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:05:49.881682+0000","last_peered":"2024-09-21T10:05:49.881682+0000","last_clean":"2024-09-21T10:05:49.881682+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:05:49.881682+0000","last_undegraded":"2024-09-21T10:05:49.881682+0000","last_fullsized":"2024-09-21T10:05:49.881682+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":71,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829624+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:05:37.829624+0000","last_peered":"2024-09-21T10:05:37.829624+0000","last_clean":"2024-09-21T10:05:37.829624+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:05:37.829624+0000","last_undegraded":"2024-09-21T10:05:37.829624+0000","last_fullsized":"2024-09-21T10:05:37.829624+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1192'7087","reported_seq":15941,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:57.228962+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:05:57.228962+0000","last_peered":"2024-09-21T10:05:57.228962+0000","last_clean":"2024-09-21T10:05:57.228962+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:05:57.228962+0000","last_undegraded":"2024-09-21T10:05:57.228962+0000","last_fullsized":"2024-09-21T10:05:57.228962+0000","mapping_epoch":1187,"log_start":"1143'2331","ondisk_log_start":"1143'2331","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4756,"log_dups_size":0,"ondisk_log_size":4756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128385024,"num_objects":1959,"num_object_clones":0,"num_object_copies":5877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7087,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":76,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.871081+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:05:49.871081+0000","last_peered":"2024-09-21T10:05:49.871081+0000","last_clean":"2024-09-21T10:05:49.871081+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:05:49.871081+0000","last_undegraded":"2024-09-21T10:05:49.871081+0000","last_fullsized":"2024-09-21T10:05:49.871081+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829712+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:05:37.829712+0000","last_peered":"2024-09-21T10:05:37.829712+0000","last_clean":"2024-09-21T10:05:37.829712+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:05:37.829712+0000","last_undegraded":"2024-09-21T10:05:37.829712+0000","last_fullsized":"2024-09-21T10:05:37.829712+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1192'7119","reported_seq":14827,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:56.982152+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:05:56.982152+0000","last_peered":"2024-09-21T10:05:56.982152+0000","last_clean":"2024-09-21T10:05:56.982152+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:05:56.982152+0000","last_undegraded":"2024-09-21T10:05:56.982152+0000","last_fullsized":"2024-09-21T10:05:56.982152+0000","mapping_epoch":1180,"log_start":"1143'2503","ondisk_log_start":"1143'2503","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4616,"log_dups_size":0,"ondisk_log_size":4616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125108224,"num_objects":1909,"num_object_clones":0,"num_object_copies":5727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7119,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":119,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.869097+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:05:49.869097+0000","last_peered":"2024-09-21T10:05:49.869097+0000","last_clean":"2024-09-21T10:05:49.869097+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:05:49.869097+0000","last_undegraded":"2024-09-21T10:05:49.869097+0000","last_fullsized":"2024-09-21T10:05:49.869097+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1160,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868881+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:05:49.868881+0000","last_peered":"2024-09-21T10:05:49.868881+0000","last_clean":"2024-09-21T10:05:49.868881+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:05:49.868881+0000","last_undegraded":"2024-09-21T10:05:49.868881+0000","last_fullsized":"2024-09-21T10:05:49.868881+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1192'7191","reported_seq":7318,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.118541+0000","last_change":"2024-09-21T10:03:30.139654+0000","last_active":"2024-09-21T10:05:57.118541+0000","last_peered":"2024-09-21T10:05:57.118541+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:05:57.118541+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1143'2482","ondisk_log_start":"1143'2482","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7191,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2017},{"shards":"1,4","objects":-1722}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":46,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.835479+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:05:37.835479+0000","last_peered":"2024-09-21T10:05:37.835479+0000","last_clean":"2024-09-21T10:05:37.835479+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:05:37.835479+0000","last_undegraded":"2024-09-21T10:05:37.835479+0000","last_fullsized":"2024-09-21T10:05:37.835479+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829879+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:05:37.829879+0000","last_peered":"2024-09-21T10:05:37.829879+0000","last_clean":"2024-09-21T10:05:37.829879+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:05:37.829879+0000","last_undegraded":"2024-09-21T10:05:37.829879+0000","last_fullsized":"2024-09-21T10:05:37.829879+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1192'7006","reported_seq":12167,"reported_epoch":1192,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-21T10:05:57.036208+0000","last_change":"2024-09-21T10:03:28.893028+0000","last_active":"2024-09-21T10:05:57.036208+0000","last_peered":"2024-09-21T10:05:57.036208+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:05:57.036208+0000","last_undegraded":"2024-09-21T10:03:28.884675+0000","last_fullsized":"2024-09-21T10:05:57.036208+0000","mapping_epoch":1151,"log_start":"1143'2436","ondisk_log_start":"1143'2436","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129761280,"num_objects":1980,"num_object_clones":0,"num_object_copies":5940,"num_objects_missing_on_primary":242,"num_objects_missing":242,"num_objects_degraded":242,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7006,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2513,"num_bytes_recovered":164691968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":1738},{"shards":"2,4,7","objects":242}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1190,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.879160+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:05:49.879160+0000","last_peered":"2024-09-21T10:05:49.879160+0000","last_clean":"2024-09-21T10:05:49.879160+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:05:49.879160+0000","last_undegraded":"2024-09-21T10:05:49.879160+0000","last_fullsized":"2024-09-21T10:05:49.879160+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1547,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868755+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:05:49.868755+0000","last_peered":"2024-09-21T10:05:49.868755+0000","last_clean":"2024-09-21T10:05:49.868755+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:05:49.868755+0000","last_undegraded":"2024-09-21T10:05:49.868755+0000","last_fullsized":"2024-09-21T10:05:49.868755+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1192'7079","reported_seq":12835,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.046019+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:05:57.046019+0000","last_peered":"2024-09-21T10:05:57.046019+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:05:57.046019+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1143'2448","ondisk_log_start":"1143'2448","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4631,"log_dups_size":0,"ondisk_log_size":4631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126812160,"num_objects":1935,"num_object_clones":0,"num_object_copies":5805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7079,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"0,3,4","objects":1935},{"shards":"3,4","objects":-1686}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":229,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878816+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:05:49.878816+0000","last_peered":"2024-09-21T10:05:49.878816+0000","last_clean":"2024-09-21T10:05:49.878816+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:05:49.878816+0000","last_undegraded":"2024-09-21T10:05:49.878816+0000","last_fullsized":"2024-09-21T10:05:49.878816+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1205,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.879059+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:05:49.879059+0000","last_peered":"2024-09-21T10:05:49.879059+0000","last_clean":"2024-09-21T10:05:49.879059+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:05:49.879059+0000","last_undegraded":"2024-09-21T10:05:49.879059+0000","last_fullsized":"2024-09-21T10:05:49.879059+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1192'6948","reported_seq":12032,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.073024+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:05:57.073024+0000","last_peered":"2024-09-21T10:05:57.073024+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:05:57.073024+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1143'2353","ondisk_log_start":"1143'2353","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4595,"log_dups_size":0,"ondisk_log_size":4595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126091264,"num_objects":1924,"num_object_clones":0,"num_object_copies":5772,"num_objects_missing_on_primary":2029,"num_objects_missing":2029,"num_objects_degraded":4815,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6948,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1652,"num_bytes_recovered":108265472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,3","objects":749},{"shards":"0,2,3,5","objects":8},{"shards":"0,2,5","objects":1167}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829804+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:05:37.829804+0000","last_peered":"2024-09-21T10:05:37.829804+0000","last_clean":"2024-09-21T10:05:37.829804+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:05:37.829804+0000","last_undegraded":"2024-09-21T10:05:37.829804+0000","last_fullsized":"2024-09-21T10:05:37.829804+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":848,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.872201+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:05:49.872201+0000","last_peered":"2024-09-21T10:05:49.872201+0000","last_clean":"2024-09-21T10:05:49.872201+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:05:49.872201+0000","last_undegraded":"2024-09-21T10:05:49.872201+0000","last_fullsized":"2024-09-21T10:05:49.872201+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1192'7126","reported_seq":12324,"reported_epoch":1192,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:05:57.951905+0000","last_change":"2024-09-21T10:05:03.425944+0000","last_active":"2024-09-21T10:05:57.951905+0000","last_peered":"2024-09-21T10:05:57.951905+0000","last_clean":"2024-09-21T10:02:38.228948+0000","last_became_active":"2024-09-21T10:03:36.276110+0000","last_became_peered":"2024-09-21T10:03:36.276110+0000","last_unstale":"2024-09-21T10:05:57.951905+0000","last_undegraded":"2024-09-21T10:05:57.951905+0000","last_fullsized":"2024-09-21T10:03:29.833661+0000","mapping_epoch":1170,"log_start":"1141'1100","ondisk_log_start":"1141'1100","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:04.270084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133103640,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":1861,"num_objects_missing":1861,"num_objects_degraded":0,"num_objects_misplaced":1161,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7126,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2544,"num_bytes_recovered":166723584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1,3","objects":988},{"shards":"0,1,3,6","objects":1},{"shards":"0,1,6","objects":173},{"shards":"1,6","objects":870}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1191,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868565+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:05:49.868565+0000","last_peered":"2024-09-21T10:05:49.868565+0000","last_clean":"2024-09-21T10:05:49.868565+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:05:49.868565+0000","last_undegraded":"2024-09-21T10:05:49.868565+0000","last_fullsized":"2024-09-21T10:05:49.868565+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.830104+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:05:37.830104+0000","last_peered":"2024-09-21T10:05:37.830104+0000","last_clean":"2024-09-21T10:05:37.830104+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:05:37.830104+0000","last_undegraded":"2024-09-21T10:05:37.830104+0000","last_fullsized":"2024-09-21T10:05:37.830104+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1192'7314","reported_seq":11661,"reported_epoch":1192,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.080241+0000","last_change":"2024-09-21T10:05:36.090756+0000","last_active":"2024-09-21T10:05:57.080241+0000","last_peered":"2024-09-21T10:05:57.080241+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:05:57.080241+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"1143'2725","ondisk_log_start":"1143'2725","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4589,"log_dups_size":0,"ondisk_log_size":4589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129892352,"num_objects":1982,"num_object_clones":0,"num_object_copies":5946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2396,"num_objects_misplaced":1754,"num_objects_unfound":0,"num_objects_dirty":1982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7314,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2106,"num_bytes_recovered":138018816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"0,2,3","objects":1584},{"shards":"2,3","objects":-2168},{"shards":"2,3,7","objects":170}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":221,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878045+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:05:49.878045+0000","last_peered":"2024-09-21T10:05:49.878045+0000","last_clean":"2024-09-21T10:05:49.878045+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:05:49.878045+0000","last_undegraded":"2024-09-21T10:05:49.878045+0000","last_fullsized":"2024-09-21T10:05:49.878045+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1551,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.876567+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:05:49.876567+0000","last_peered":"2024-09-21T10:05:49.876567+0000","last_clean":"2024-09-21T10:05:49.876567+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:05:49.876567+0000","last_undegraded":"2024-09-21T10:05:49.876567+0000","last_fullsized":"2024-09-21T10:05:49.876567+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1192'6936","reported_seq":13656,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:56.845870+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:05:56.845870+0000","last_peered":"2024-09-21T10:05:56.845870+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:05:56.845870+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1143'2347","ondisk_log_start":"1143'2347","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4589,"log_dups_size":0,"ondisk_log_size":4589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124780544,"num_objects":1904,"num_object_clones":0,"num_object_copies":5712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4418,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6936,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2470,"num_bytes_recovered":161873920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,2,3","objects":732},{"shards":"0,3,5","objects":957},{"shards":"0,3,7","objects":215}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":715,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829528+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:05:37.829528+0000","last_peered":"2024-09-21T10:05:37.829528+0000","last_clean":"2024-09-21T10:05:37.829528+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:05:37.829528+0000","last_undegraded":"2024-09-21T10:05:37.829528+0000","last_fullsized":"2024-09-21T10:05:37.829528+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1204,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878499+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:05:49.878499+0000","last_peered":"2024-09-21T10:05:49.878499+0000","last_clean":"2024-09-21T10:05:49.878499+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:05:49.878499+0000","last_undegraded":"2024-09-21T10:05:49.878499+0000","last_fullsized":"2024-09-21T10:05:49.878499+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":235,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878884+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:05:49.878884+0000","last_peered":"2024-09-21T10:05:49.878884+0000","last_clean":"2024-09-21T10:05:49.878884+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:05:49.878884+0000","last_undegraded":"2024-09-21T10:05:49.878884+0000","last_fullsized":"2024-09-21T10:05:49.878884+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.835404+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:05:37.835404+0000","last_peered":"2024-09-21T10:05:37.835404+0000","last_clean":"2024-09-21T10:05:37.835404+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:05:37.835404+0000","last_undegraded":"2024-09-21T10:05:37.835404+0000","last_fullsized":"2024-09-21T10:05:37.835404+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1192'6970","reported_seq":12601,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.248147+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:05:57.248147+0000","last_peered":"2024-09-21T10:05:57.248147+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:05:57.248147+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1143'2254","ondisk_log_start":"1143'2254","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4716,"log_dups_size":0,"ondisk_log_size":4716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129499136,"num_objects":1976,"num_object_clones":0,"num_object_copies":5928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6970,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"0,1,2","objects":1976},{"shards":"1,2","objects":-1629}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":835,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868355+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:05:49.868355+0000","last_peered":"2024-09-21T10:05:49.868355+0000","last_clean":"2024-09-21T10:05:49.868355+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:05:49.868355+0000","last_undegraded":"2024-09-21T10:05:49.868355+0000","last_fullsized":"2024-09-21T10:05:49.868355+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1424,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878170+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:05:49.878170+0000","last_peered":"2024-09-21T10:05:49.878170+0000","last_clean":"2024-09-21T10:05:49.878170+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:05:49.878170+0000","last_undegraded":"2024-09-21T10:05:49.878170+0000","last_fullsized":"2024-09-21T10:05:49.878170+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":833,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868807+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:05:49.868807+0000","last_peered":"2024-09-21T10:05:49.868807+0000","last_clean":"2024-09-21T10:05:49.868807+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:05:49.868807+0000","last_undegraded":"2024-09-21T10:05:49.868807+0000","last_fullsized":"2024-09-21T10:05:49.868807+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1192'6765","reported_seq":15523,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:57.286789+0000","last_change":"2024-09-21T10:05:37.849809+0000","last_active":"2024-09-21T10:05:57.286789+0000","last_peered":"2024-09-21T10:05:57.286789+0000","last_clean":"2024-09-21T10:05:57.286789+0000","last_became_active":"2024-09-21T10:05:37.830539+0000","last_became_peered":"2024-09-21T10:05:37.830539+0000","last_unstale":"2024-09-21T10:05:57.286789+0000","last_undegraded":"2024-09-21T10:05:57.286789+0000","last_fullsized":"2024-09-21T10:05:57.286789+0000","mapping_epoch":1190,"log_start":"1143'2037","ondisk_log_start":"1143'2037","created":1137,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4728,"log_dups_size":0,"ondisk_log_size":4728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:35.172792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121700352,"num_objects":1857,"num_object_clones":0,"num_object_copies":5571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6765,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4311,"num_bytes_recovered":174260224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1465,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878522+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:05:49.878522+0000","last_peered":"2024-09-21T10:05:49.878522+0000","last_clean":"2024-09-21T10:05:49.878522+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:05:49.878522+0000","last_undegraded":"2024-09-21T10:05:49.878522+0000","last_fullsized":"2024-09-21T10:05:49.878522+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1552,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868520+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:05:49.868520+0000","last_peered":"2024-09-21T10:05:49.868520+0000","last_clean":"2024-09-21T10:05:49.868520+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:05:49.868520+0000","last_undegraded":"2024-09-21T10:05:49.868520+0000","last_fullsized":"2024-09-21T10:05:49.868520+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1559,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.876406+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:05:49.876406+0000","last_peered":"2024-09-21T10:05:49.876406+0000","last_clean":"2024-09-21T10:05:49.876406+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:05:49.876406+0000","last_undegraded":"2024-09-21T10:05:49.876406+0000","last_fullsized":"2024-09-21T10:05:49.876406+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1192'7043","reported_seq":10517,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.954139+0000","last_change":"2024-09-21T10:03:36.351943+0000","last_active":"2024-09-21T10:05:57.954139+0000","last_peered":"2024-09-21T10:05:57.954139+0000","last_clean":"2024-09-21T10:02:38.308681+0000","last_became_active":"2024-09-21T10:03:36.318082+0000","last_became_peered":"2024-09-21T10:03:36.318082+0000","last_unstale":"2024-09-21T10:05:57.954139+0000","last_undegraded":"2024-09-21T10:03:36.183243+0000","last_fullsized":"2024-09-21T10:03:29.833927+0000","mapping_epoch":1170,"log_start":"1141'1000","ondisk_log_start":"1141'1000","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6043,"log_dups_size":0,"ondisk_log_size":6043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:52.296275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129433600,"num_objects":1975,"num_object_clones":0,"num_object_copies":5925,"num_objects_missing_on_primary":2694,"num_objects_missing":2694,"num_objects_degraded":3678,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7043,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1683,"num_bytes_recovered":110297088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,1,3","objects":1842},{"shards":"0,3,6","objects":132},{"shards":"3,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":128,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.869660+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:05:49.869660+0000","last_peered":"2024-09-21T10:05:49.869660+0000","last_clean":"2024-09-21T10:05:49.869660+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:05:49.869660+0000","last_undegraded":"2024-09-21T10:05:49.869660+0000","last_fullsized":"2024-09-21T10:05:49.869660+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1153,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:42.844439+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:05:42.844439+0000","last_peered":"2024-09-21T10:05:42.844439+0000","last_clean":"2024-09-21T10:05:42.844439+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:05:42.844439+0000","last_undegraded":"2024-09-21T10:05:42.844439+0000","last_fullsized":"2024-09-21T10:05:42.844439+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1562,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870102+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:05:49.870102+0000","last_peered":"2024-09-21T10:05:49.870102+0000","last_clean":"2024-09-21T10:05:49.870102+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:05:49.870102+0000","last_undegraded":"2024-09-21T10:05:49.870102+0000","last_fullsized":"2024-09-21T10:05:49.870102+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1192'7136","reported_seq":7254,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:05:57.005688+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:05:57.005688+0000","last_peered":"2024-09-21T10:05:57.005688+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:05:57.005688+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1143'2585","ondisk_log_start":"1143'2585","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4551,"log_dups_size":0,"ondisk_log_size":4551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125042688,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7136,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1753},{"shards":"3,4","objects":-2459}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1152,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.880535+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:05:49.880535+0000","last_peered":"2024-09-21T10:05:49.880535+0000","last_clean":"2024-09-21T10:05:49.880535+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:05:49.880535+0000","last_undegraded":"2024-09-21T10:05:49.880535+0000","last_fullsized":"2024-09-21T10:05:49.880535+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1573,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870189+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:05:49.870189+0000","last_peered":"2024-09-21T10:05:49.870189+0000","last_clean":"2024-09-21T10:05:49.870189+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:05:49.870189+0000","last_undegraded":"2024-09-21T10:05:49.870189+0000","last_fullsized":"2024-09-21T10:05:49.870189+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":92,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.869350+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:05:49.869350+0000","last_peered":"2024-09-21T10:05:49.869350+0000","last_clean":"2024-09-21T10:05:49.869350+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:05:49.869350+0000","last_undegraded":"2024-09-21T10:05:49.869350+0000","last_fullsized":"2024-09-21T10:05:49.869350+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1192'6977","reported_seq":14569,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:57.301132+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:05:57.301132+0000","last_peered":"2024-09-21T10:05:57.301132+0000","last_clean":"2024-09-21T10:05:57.301132+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:05:57.301132+0000","last_undegraded":"2024-09-21T10:05:57.301132+0000","last_fullsized":"2024-09-21T10:05:57.301132+0000","mapping_epoch":1174,"log_start":"1143'2194","ondisk_log_start":"1143'2194","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4783,"log_dups_size":0,"ondisk_log_size":4783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124452864,"num_objects":1899,"num_object_clones":0,"num_object_copies":5697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6977,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1550,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870676+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:05:49.870676+0000","last_peered":"2024-09-21T10:05:49.870676+0000","last_clean":"2024-09-21T10:05:49.870676+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:05:49.870676+0000","last_undegraded":"2024-09-21T10:05:49.870676+0000","last_fullsized":"2024-09-21T10:05:49.870676+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":731,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870870+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:05:49.870870+0000","last_peered":"2024-09-21T10:05:49.870870+0000","last_clean":"2024-09-21T10:05:49.870870+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:05:49.870870+0000","last_undegraded":"2024-09-21T10:05:49.870870+0000","last_fullsized":"2024-09-21T10:05:49.870870+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1543,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868979+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:05:49.868979+0000","last_peered":"2024-09-21T10:05:49.868979+0000","last_clean":"2024-09-21T10:05:49.868979+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:05:49.868979+0000","last_undegraded":"2024-09-21T10:05:49.868979+0000","last_fullsized":"2024-09-21T10:05:49.868979+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1192'7205","reported_seq":7329,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:57.046893+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:05:57.046893+0000","last_peered":"2024-09-21T10:05:57.046893+0000","last_clean":"2024-09-21T10:05:57.046893+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:05:57.046893+0000","last_undegraded":"2024-09-21T10:05:57.046893+0000","last_fullsized":"2024-09-21T10:05:57.046893+0000","mapping_epoch":1137,"log_start":"1143'2626","ondisk_log_start":"1143'2626","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4579,"log_dups_size":0,"ondisk_log_size":4579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128909312,"num_objects":1967,"num_object_clones":0,"num_object_copies":5901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7205,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":251,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.881759+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:05:49.881759+0000","last_peered":"2024-09-21T10:05:49.881759+0000","last_clean":"2024-09-21T10:05:49.881759+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:05:49.881759+0000","last_undegraded":"2024-09-21T10:05:49.881759+0000","last_fullsized":"2024-09-21T10:05:49.881759+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1551,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.869957+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:05:49.869957+0000","last_peered":"2024-09-21T10:05:49.869957+0000","last_clean":"2024-09-21T10:05:49.869957+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:05:49.869957+0000","last_undegraded":"2024-09-21T10:05:49.869957+0000","last_fullsized":"2024-09-21T10:05:49.869957+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":228,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.877974+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:05:49.877974+0000","last_peered":"2024-09-21T10:05:49.877974+0000","last_clean":"2024-09-21T10:05:49.877974+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:05:49.877974+0000","last_undegraded":"2024-09-21T10:05:49.877974+0000","last_fullsized":"2024-09-21T10:05:49.877974+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1192'6919","reported_seq":7043,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:56.857729+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:05:56.857729+0000","last_peered":"2024-09-21T10:05:56.857729+0000","last_clean":"2024-09-21T10:05:56.857729+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:05:56.857729+0000","last_undegraded":"2024-09-21T10:05:56.857729+0000","last_fullsized":"2024-09-21T10:05:56.857729+0000","mapping_epoch":1137,"log_start":"1143'2313","ondisk_log_start":"1143'2313","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4606,"log_dups_size":0,"ondisk_log_size":4606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126418944,"num_objects":1929,"num_object_clones":0,"num_object_copies":5787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6919,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":92,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868888+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:05:49.868888+0000","last_peered":"2024-09-21T10:05:49.868888+0000","last_clean":"2024-09-21T10:05:49.868888+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:05:49.868888+0000","last_undegraded":"2024-09-21T10:05:49.868888+0000","last_fullsized":"2024-09-21T10:05:49.868888+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.857331+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:05:37.857331+0000","last_peered":"2024-09-21T10:05:37.857331+0000","last_clean":"2024-09-21T10:05:37.857331+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:05:37.857331+0000","last_undegraded":"2024-09-21T10:05:37.857331+0000","last_fullsized":"2024-09-21T10:05:37.857331+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829652+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:05:37.829652+0000","last_peered":"2024-09-21T10:05:37.829652+0000","last_clean":"2024-09-21T10:05:37.829652+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:05:37.829652+0000","last_undegraded":"2024-09-21T10:05:37.829652+0000","last_fullsized":"2024-09-21T10:05:37.829652+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829967+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:05:37.829967+0000","last_peered":"2024-09-21T10:05:37.829967+0000","last_clean":"2024-09-21T10:05:37.829967+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:05:37.829967+0000","last_undegraded":"2024-09-21T10:05:37.829967+0000","last_fullsized":"2024-09-21T10:05:37.829967+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":52,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.830127+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:05:37.830127+0000","last_peered":"2024-09-21T10:05:37.830127+0000","last_clean":"2024-09-21T10:05:37.830127+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:05:37.830127+0000","last_undegraded":"2024-09-21T10:05:37.830127+0000","last_fullsized":"2024-09-21T10:05:37.830127+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1551,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.869781+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:05:49.869781+0000","last_peered":"2024-09-21T10:05:49.869781+0000","last_clean":"2024-09-21T10:05:49.869781+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:05:49.869781+0000","last_undegraded":"2024-09-21T10:05:49.869781+0000","last_fullsized":"2024-09-21T10:05:49.869781+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1163,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.880774+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:05:49.880774+0000","last_peered":"2024-09-21T10:05:49.880774+0000","last_clean":"2024-09-21T10:05:49.880774+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:05:49.880774+0000","last_undegraded":"2024-09-21T10:05:49.880774+0000","last_fullsized":"2024-09-21T10:05:49.880774+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1174,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870052+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:05:49.870052+0000","last_peered":"2024-09-21T10:05:49.870052+0000","last_clean":"2024-09-21T10:05:49.870052+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:05:49.870052+0000","last_undegraded":"2024-09-21T10:05:49.870052+0000","last_fullsized":"2024-09-21T10:05:49.870052+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":248,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878438+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:05:49.878438+0000","last_peered":"2024-09-21T10:05:49.878438+0000","last_clean":"2024-09-21T10:05:49.878438+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:05:49.878438+0000","last_undegraded":"2024-09-21T10:05:49.878438+0000","last_fullsized":"2024-09-21T10:05:49.878438+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1577,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.881593+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:05:49.881593+0000","last_peered":"2024-09-21T10:05:49.881593+0000","last_clean":"2024-09-21T10:05:49.881593+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:05:49.881593+0000","last_undegraded":"2024-09-21T10:05:49.881593+0000","last_fullsized":"2024-09-21T10:05:49.881593+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":227,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878445+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:05:49.878445+0000","last_peered":"2024-09-21T10:05:49.878445+0000","last_clean":"2024-09-21T10:05:49.878445+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:05:49.878445+0000","last_undegraded":"2024-09-21T10:05:49.878445+0000","last_fullsized":"2024-09-21T10:05:49.878445+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.846198+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:05:37.846198+0000","last_peered":"2024-09-21T10:05:37.846198+0000","last_clean":"2024-09-21T10:05:37.846198+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:05:37.846198+0000","last_undegraded":"2024-09-21T10:05:37.846198+0000","last_fullsized":"2024-09-21T10:05:37.846198+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":220,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.877849+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:05:49.877849+0000","last_peered":"2024-09-21T10:05:49.877849+0000","last_clean":"2024-09-21T10:05:49.877849+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:05:49.877849+0000","last_undegraded":"2024-09-21T10:05:49.877849+0000","last_fullsized":"2024-09-21T10:05:49.877849+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1539,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.914653+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:05:49.914653+0000","last_peered":"2024-09-21T10:05:49.914653+0000","last_clean":"2024-09-21T10:05:49.914653+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:05:49.914653+0000","last_undegraded":"2024-09-21T10:05:49.914653+0000","last_fullsized":"2024-09-21T10:05:49.914653+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1549,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.871857+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:05:49.871857+0000","last_peered":"2024-09-21T10:05:49.871857+0000","last_clean":"2024-09-21T10:05:49.871857+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:05:49.871857+0000","last_undegraded":"2024-09-21T10:05:49.871857+0000","last_fullsized":"2024-09-21T10:05:49.871857+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":213,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.880859+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:05:49.880859+0000","last_peered":"2024-09-21T10:05:49.880859+0000","last_clean":"2024-09-21T10:05:49.880859+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:05:49.880859+0000","last_undegraded":"2024-09-21T10:05:49.880859+0000","last_fullsized":"2024-09-21T10:05:49.880859+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":97,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.869888+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:05:49.869888+0000","last_peered":"2024-09-21T10:05:49.869888+0000","last_clean":"2024-09-21T10:05:49.869888+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:05:49.869888+0000","last_undegraded":"2024-09-21T10:05:49.869888+0000","last_fullsized":"2024-09-21T10:05:49.869888+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1433,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870409+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:05:49.870409+0000","last_peered":"2024-09-21T10:05:49.870409+0000","last_clean":"2024-09-21T10:05:49.870409+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:05:49.870409+0000","last_undegraded":"2024-09-21T10:05:49.870409+0000","last_fullsized":"2024-09-21T10:05:49.870409+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1514,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.868850+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:05:49.868850+0000","last_peered":"2024-09-21T10:05:49.868850+0000","last_clean":"2024-09-21T10:05:49.868850+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:05:49.868850+0000","last_undegraded":"2024-09-21T10:05:49.868850+0000","last_fullsized":"2024-09-21T10:05:49.868850+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1159,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878691+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:05:49.878691+0000","last_peered":"2024-09-21T10:05:49.878691+0000","last_clean":"2024-09-21T10:05:49.878691+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:05:49.878691+0000","last_undegraded":"2024-09-21T10:05:49.878691+0000","last_fullsized":"2024-09-21T10:05:49.878691+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":113,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.870765+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:05:49.870765+0000","last_peered":"2024-09-21T10:05:49.870765+0000","last_clean":"2024-09-21T10:05:49.870765+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:05:49.870765+0000","last_undegraded":"2024-09-21T10:05:49.870765+0000","last_fullsized":"2024-09-21T10:05:49.870765+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1166,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878650+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:05:49.878650+0000","last_peered":"2024-09-21T10:05:49.878650+0000","last_clean":"2024-09-21T10:05:49.878650+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:05:49.878650+0000","last_undegraded":"2024-09-21T10:05:49.878650+0000","last_fullsized":"2024-09-21T10:05:49.878650+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":46,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.835354+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:05:37.835354+0000","last_peered":"2024-09-21T10:05:37.835354+0000","last_clean":"2024-09-21T10:05:37.835354+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:05:37.835354+0000","last_undegraded":"2024-09-21T10:05:37.835354+0000","last_fullsized":"2024-09-21T10:05:37.835354+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":220,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.878603+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:05:49.878603+0000","last_peered":"2024-09-21T10:05:49.878603+0000","last_clean":"2024-09-21T10:05:49.878603+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:05:49.878603+0000","last_undegraded":"2024-09-21T10:05:49.878603+0000","last_fullsized":"2024-09-21T10:05:49.878603+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":228,"reported_epoch":1192,"state":"active+clean","last_fresh":"2024-09-21T10:05:49.914469+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:05:49.914469+0000","last_peered":"2024-09-21T10:05:49.914469+0000","last_clean":"2024-09-21T10:05:49.914469+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:05:49.914469+0000","last_undegraded":"2024-09-21T10:05:49.914469+0000","last_fullsized":"2024-09-21T10:05:49.914469+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829696+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:05:37.829696+0000","last_peered":"2024-09-21T10:05:37.829696+0000","last_clean":"2024-09-21T10:05:37.829696+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:05:37.829696+0000","last_undegraded":"2024-09-21T10:05:37.829696+0000","last_fullsized":"2024-09-21T10:05:37.829696+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1561,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.845332+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:05:37.845332+0000","last_peered":"2024-09-21T10:05:37.845332+0000","last_clean":"2024-09-21T10:05:37.845332+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:05:37.845332+0000","last_undegraded":"2024-09-21T10:05:37.845332+0000","last_fullsized":"2024-09-21T10:05:37.845332+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2041577496,"num_objects":31153,"num_object_clones":0,"num_object_copies":93459,"num_objects_missing_on_primary":6826,"num_objects_missing":6826,"num_objects_degraded":33499,"num_objects_misplaced":2915,"num_objects_unfound":0,"num_objects_dirty":31153,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":112821,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34718,"num_bytes_recovered":1959985152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8163504128,"data_stored":8163491912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77087,"ondisk_log_size":77087,"up":48,"acting":39,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234361,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495997,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642236,"kb_used_data":350324,"kb_used_omap":1,"kb_used_meta":291902,"kb_avail":93102916,"statfs":{"total":95995035648,"available":95337385984,"internally_reserved":0,"allocated":358731776,"data_stored":356233884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":298908103},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757636,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019272,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000692,"kb_used_data":757100,"kb_used_omap":1,"kb_used_meta":243582,"kb_avail":92744460,"statfs":{"total":95995035648,"available":94970327040,"internally_reserved":0,"allocated":775270400,"data_stored":772780135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":249428422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888568,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917028,"kb_used_data":1461596,"kb_used_omap":34,"kb_used_meta":455389,"kb_avail":91828124,"statfs":{"total":95995035648,"available":94031998976,"internally_reserved":0,"allocated":1496674304,"data_stored":1494195082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":466319347},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2,0,3,0,3,6,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967807,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1537964,"kb_used_data":1199076,"kb_used_omap":41,"kb_used_meta":338838,"kb_avail":92207188,"statfs":{"total":95995035648,"available":94420160512,"internally_reserved":0,"allocated":1227853824,"data_stored":1225372125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":346970527},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047046,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1746980,"kb_used_data":1458076,"kb_used_omap":41,"kb_used_meta":288854,"kb_avail":91998172,"statfs":{"total":95995035648,"available":94206128128,"internally_reserved":0,"allocated":1493069824,"data_stored":1490591010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":295787433},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126284,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1886851072,"data_stored":1886847000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1484132352,"data_stored":1484128280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1209663488,"data_stored":1209663488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1485832192,"data_stored":1485832192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":766902272,"data_stored":766902272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":350556160,"data_stored":350552088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":647364608,"data_stored":647364608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:06:01.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:01.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:01.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:01.730 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:01.732 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:01.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:01.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:01.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:01.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:01.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:01.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:02.008 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:02.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:02.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:02 smithi076 ceph-mon[110963]: pgmap v1536: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 232 op/s; 33499/93537 objects degraded (35.814%); 2915/93537 objects misplaced (3.116%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:06:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:02 smithi076 ceph-mon[106482]: pgmap v1536: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 232 op/s; 33499/93537 objects degraded (35.814%); 2915/93537 objects misplaced (3.116%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:06:02.133 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:02.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:02.157 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:06:02.235 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:02.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:02 smithi191 ceph-mon[102206]: pgmap v1536: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.9 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 232 op/s; 33499/93537 objects degraded (35.814%); 2915/93537 objects misplaced (3.116%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:06:02.365 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:02.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:02.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:02.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:02.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:02.644 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:02.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:02.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:02.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:02.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:02.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:02.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:02.942 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:02.991 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:02.995 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:03.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:03.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:03.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:03.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:03.350 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:06:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:06:03] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-21T10:06:03.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:03.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:03.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:03.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:03.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:03.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:03.666 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:03.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:03.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:03.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:03.926 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:03.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:04.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:04.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:04.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:04.180 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:04.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.282 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:04.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:04.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:04 smithi076 ceph-mon[110963]: pgmap v1537: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 33072/89871 objects degraded (36.799%); 2761/89871 objects misplaced (3.072%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:06:04.319 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 33072/89871 objects degraded (36.799%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:04.319 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:04 smithi076 ceph-mon[106482]: pgmap v1537: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 33072/89871 objects degraded (36.799%); 2761/89871 objects misplaced (3.072%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:06:04.319 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 33072/89871 objects degraded (36.799%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:04 smithi191 ceph-mon[102206]: pgmap v1537: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 33072/89871 objects degraded (36.799%); 2761/89871 objects misplaced (3.072%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:06:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 33072/89871 objects degraded (36.799%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:04.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:04.435 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:04.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:04.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:04.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:06:04.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:04.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:04.788 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:04.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:04.915 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:04.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:04.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:04.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:05.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:05.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:05.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:05.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:05.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:05.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:05.317 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:05.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:05.418 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:05.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:05.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:05.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:05.673 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:05.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:05.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:05.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:05.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:05.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:05.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:05.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:06.059 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:06.085 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:06.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:06.187 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:06.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:06.284 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:06 smithi076 ceph-mon[106482]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 228 op/s; 32983/89352 objects degraded (36.914%); 2504/89352 objects misplaced (2.802%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:06:06.284 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:06 smithi076 ceph-mon[110963]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 228 op/s; 32983/89352 objects degraded (36.914%); 2504/89352 objects misplaced (2.802%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:06:06.313 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:06.339 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:06.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:06 smithi191 ceph-mon[102206]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 228 op/s; 32983/89352 objects degraded (36.914%); 2504/89352 objects misplaced (2.802%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:06:06.440 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:06.466 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:06.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:06.590 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:06.610 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:06:06.610 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:06:06.610 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:06.611 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:06.612 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:06:06.612 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:06:06.612 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:06:06.612 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:06.612 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:06.612 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:06.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:06.714 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:06.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:06.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:06.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:06.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:06.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:06.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:07.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:07.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:07.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:07.188 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:07.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:07.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:07.346 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:07 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2325794234' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:07.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:07 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2325794234' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:07 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2325794234' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:07.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:07.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:07.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:07.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:07.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:07.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:07.725 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:07.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:07.850 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:07.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:07.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:07.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:08.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.103 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:08.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:08.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:08.228 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:08.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:08.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:08 smithi191 ceph-mon[102206]: pgmap v1539: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 32983/89352 objects degraded (36.914%); 2504/89352 objects misplaced (2.802%); 7.1 MiB/s, 114 objects/s recovering 2024-09-21T10:06:08.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:08 smithi076 ceph-mon[106482]: pgmap v1539: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 32983/89352 objects degraded (36.914%); 2504/89352 objects misplaced (2.802%); 7.1 MiB/s, 114 objects/s recovering 2024-09-21T10:06:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:08 smithi076 ceph-mon[110963]: pgmap v1539: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.8 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 32983/89352 objects degraded (36.914%); 2504/89352 objects misplaced (2.802%); 7.1 MiB/s, 114 objects/s recovering 2024-09-21T10:06:08.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:08.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:08.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:08.773 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:08.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:08.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:08.903 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:08.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:08.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:09.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.027 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:09.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:09.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:09.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:09.254 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:09.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:09.337 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 32983/89352 objects degraded (36.914%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:09.338 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 32983/89352 objects degraded (36.914%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:09.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 32983/89352 objects degraded (36.914%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:09.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:09.404 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:09.456 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:09.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.529 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:09.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:09.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:09.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:09.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:09.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:09.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:09.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:09.878 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:09.905 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:09.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:10.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:10.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:10.059 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:06:10.132 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:10.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:10.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:10.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:10 smithi191 ceph-mon[102206]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 334 op/s; 32477/85059 objects degraded (38.182%); 2061/85059 objects misplaced (2.423%); 11 MiB/s, 179 objects/s recovering 2024-09-21T10:06:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3698526921' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:10 smithi076 ceph-mon[106482]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 334 op/s; 32477/85059 objects degraded (38.182%); 2061/85059 objects misplaced (2.423%); 11 MiB/s, 179 objects/s recovering 2024-09-21T10:06:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3698526921' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:10 smithi076 ceph-mon[110963]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 334 op/s; 32477/85059 objects degraded (38.182%); 2061/85059 objects misplaced (2.423%); 11 MiB/s, 179 objects/s recovering 2024-09-21T10:06:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3698526921' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:10.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:10.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:10.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:10.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:10.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:10.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:10.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:10.789 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:10.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:10.916 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:10.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:11.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:11.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:11.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:11.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:11.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:11.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:11.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3698526921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[106482]: osdmap e1193: 8 total, 8 up, 5 in 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3698526921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[110963]: osdmap e1193: 8 total, 8 up, 5 in 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:11.307 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:11 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:06:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3698526921' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:11 smithi191 ceph-mon[102206]: osdmap e1193: 8 total, 8 up, 5 in 2024-09-21T10:06:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:11.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:11.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:11.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:11.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:11.546 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:11.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:11.669 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:11.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:11.792 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:11.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:11.916 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:11.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:12.018 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:12.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:12.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:12.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:12.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:12.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:06:12.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:12.294 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:12.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:12.297 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:12 smithi076 ceph-mon[106482]: pgmap v1542: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 282 op/s; 32477/85059 objects degraded (38.182%); 2061/85059 objects misplaced (2.423%); 9.6 MiB/s, 154 objects/s recovering 2024-09-21T10:06:12.298 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:12 smithi076 ceph-mon[110963]: pgmap v1542: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 282 op/s; 32477/85059 objects degraded (38.182%); 2061/85059 objects misplaced (2.423%); 9.6 MiB/s, 154 objects/s recovering 2024-09-21T10:06:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:12 smithi191 ceph-mon[102206]: pgmap v1542: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 282 op/s; 32477/85059 objects degraded (38.182%); 2061/85059 objects misplaced (2.423%); 9.6 MiB/s, 154 objects/s recovering 2024-09-21T10:06:12.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:12.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:12.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:12.551 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:12.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:12.610 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:12.610 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:12.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:12.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.676 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:12.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:12.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:12.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:12.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:12.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:12.929 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:12.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:13.031 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:13.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:13.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:13.182 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:13.284 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:13.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:13.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:06:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:13.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:13.434 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:13.507 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:06:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:06:13] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.51.0" 2024-09-21T10:06:13.536 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:13.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:13.661 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:06:13.662 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:06:13.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:13.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:13.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:13.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:13.834 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:13.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:06:13.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:13.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:13.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:14.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:14.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:14.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:14 smithi076 ceph-mon[106482]: pgmap v1543: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32084/81732 objects degraded (39.255%); 1951/81732 objects misplaced (2.387%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:06:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 32084/81732 objects degraded (39.255%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:14 smithi076 ceph-mon[110963]: pgmap v1543: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32084/81732 objects degraded (39.255%); 1951/81732 objects misplaced (2.387%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:06:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 32084/81732 objects degraded (39.255%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:14.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:14.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:14.295 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:14.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:14 smithi191 ceph-mon[102206]: pgmap v1543: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32084/81732 objects degraded (39.255%); 1951/81732 objects misplaced (2.387%); 9.2 MiB/s, 146 objects/s recovering 2024-09-21T10:06:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 32084/81732 objects degraded (39.255%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:14.429 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:14.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:14.557 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:14.585 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:14.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:14.687 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.713 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:14.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.815 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:14.943 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:14.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:14.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:14.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:15.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:15.099 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:15 smithi191 ceph-mon[102206]: from='client.45197 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:15.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:15.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:15.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:15.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:15.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:15 smithi076 ceph-mon[110963]: from='client.45197 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:15 smithi076 ceph-mon[106482]: from='client.45197 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:15.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:15.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:15.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:15.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:15.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:15.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:15.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:15.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:15.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:15.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:15.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:15.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:15.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:15.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:16.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:16.113 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:16.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:16.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:16.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:16 smithi191 ceph-mon[102206]: pgmap v1544: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32005/81198 objects degraded (39.416%); 1554/81198 objects misplaced (1.914%); 10 MiB/s, 176 objects/s recovering 2024-09-21T10:06:16.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:16.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:16 smithi076 ceph-mon[106482]: pgmap v1544: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32005/81198 objects degraded (39.416%); 1554/81198 objects misplaced (1.914%); 10 MiB/s, 176 objects/s recovering 2024-09-21T10:06:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:16 smithi076 ceph-mon[110963]: pgmap v1544: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.1 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32005/81198 objects degraded (39.416%); 1554/81198 objects misplaced (1.914%); 10 MiB/s, 176 objects/s recovering 2024-09-21T10:06:16.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:16.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:16.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:16.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:16.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:16.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:16.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:16.840 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:16.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:16.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:16.963 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1543,"stamp":"2024-09-21T10:06:12.645418+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1784289277,"num_objects":27244,"num_object_clones":0,"num_object_copies":81732,"num_objects_missing_on_primary":5544,"num_objects_missing":5544,"num_objects_degraded":32084,"num_objects_misplaced":1951,"num_objects_unfound":0,"num_objects_dirty":27244,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":117035,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36663,"num_bytes_recovered":2086155782,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77130,"ondisk_log_size":77130,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":6356060,"kb_used_data":4687340,"kb_used_omap":119,"kb_used_meta":1668552,"kb_avail":462369700,"statfs":{"total":479975178240,"available":473466572800,"internally_reserved":0,"allocated":4799836160,"data_stored":4787384592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1708597448},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,0,3,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-177799168,"num_objects":-2713,"num_object_clones":0,"num_object_copies":-8139,"num_objects_missing_on_primary":-1104,"num_objects_missing":-1104,"num_objects_degraded":-988,"num_objects_misplaced":-810,"num_objects_unfound":0,"num_objects_dirty":-2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2713,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1467,"num_bytes_recovered":96141312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002971"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":223,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.068195+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:06:10.068195+0000","last_peered":"2024-09-21T10:06:10.068195+0000","last_clean":"2024-09-21T10:06:10.068195+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:06:10.068195+0000","last_undegraded":"2024-09-21T10:06:10.068195+0000","last_fullsized":"2024-09-21T10:06:10.068195+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":48,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.830017+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:05:37.830017+0000","last_peered":"2024-09-21T10:05:37.830017+0000","last_clean":"2024-09-21T10:05:37.830017+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:05:37.830017+0000","last_undegraded":"2024-09-21T10:05:37.830017+0000","last_fullsized":"2024-09-21T10:05:37.830017+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1207,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066381+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:06:10.066381+0000","last_peered":"2024-09-21T10:06:10.066381+0000","last_clean":"2024-09-21T10:06:10.066381+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:06:10.066381+0000","last_undegraded":"2024-09-21T10:06:10.066381+0000","last_fullsized":"2024-09-21T10:06:10.066381+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1539,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.064885+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:06:10.064885+0000","last_peered":"2024-09-21T10:06:10.064885+0000","last_clean":"2024-09-21T10:06:10.064885+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:06:10.064885+0000","last_undegraded":"2024-09-21T10:06:10.064885+0000","last_fullsized":"2024-09-21T10:06:10.064885+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1539,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.114940+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:06:10.114940+0000","last_peered":"2024-09-21T10:06:10.114940+0000","last_clean":"2024-09-21T10:06:10.114940+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:06:10.114940+0000","last_undegraded":"2024-09-21T10:06:10.114940+0000","last_fullsized":"2024-09-21T10:06:10.114940+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829647+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:05:37.829647+0000","last_peered":"2024-09-21T10:05:37.829647+0000","last_clean":"2024-09-21T10:05:37.829647+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:05:37.829647+0000","last_undegraded":"2024-09-21T10:05:37.829647+0000","last_fullsized":"2024-09-21T10:05:37.829647+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":222,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.069128+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:06:10.069128+0000","last_peered":"2024-09-21T10:06:10.069128+0000","last_clean":"2024-09-21T10:06:10.069128+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:06:10.069128+0000","last_undegraded":"2024-09-21T10:06:10.069128+0000","last_fullsized":"2024-09-21T10:06:10.069128+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":71,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829624+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:05:37.829624+0000","last_peered":"2024-09-21T10:05:37.829624+0000","last_clean":"2024-09-21T10:05:37.829624+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:05:37.829624+0000","last_undegraded":"2024-09-21T10:05:37.829624+0000","last_fullsized":"2024-09-21T10:05:37.829624+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1193'7336","reported_seq":16192,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:12.263434+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:06:12.263434+0000","last_peered":"2024-09-21T10:06:12.263434+0000","last_clean":"2024-09-21T10:06:12.263434+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:06:12.263434+0000","last_undegraded":"2024-09-21T10:06:12.263434+0000","last_fullsized":"2024-09-21T10:06:12.263434+0000","mapping_epoch":1187,"log_start":"1143'2631","ondisk_log_start":"1143'2631","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112066560,"num_objects":1710,"num_object_clones":0,"num_object_copies":5130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7336,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":78,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062862+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:06:10.062862+0000","last_peered":"2024-09-21T10:06:10.062862+0000","last_clean":"2024-09-21T10:06:10.062862+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:06:10.062862+0000","last_undegraded":"2024-09-21T10:06:10.062862+0000","last_fullsized":"2024-09-21T10:06:10.062862+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829712+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:05:37.829712+0000","last_peered":"2024-09-21T10:05:37.829712+0000","last_clean":"2024-09-21T10:05:37.829712+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:05:37.829712+0000","last_undegraded":"2024-09-21T10:05:37.829712+0000","last_fullsized":"2024-09-21T10:05:37.829712+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1193'7353","reported_seq":15063,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:11.966150+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:06:11.966150+0000","last_peered":"2024-09-21T10:06:11.966150+0000","last_clean":"2024-09-21T10:06:11.966150+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:06:11.966150+0000","last_undegraded":"2024-09-21T10:06:11.966150+0000","last_fullsized":"2024-09-21T10:06:11.966150+0000","mapping_epoch":1180,"log_start":"1143'2803","ondisk_log_start":"1143'2803","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109772800,"num_objects":1675,"num_object_clones":0,"num_object_copies":5025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7353,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":121,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063849+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:06:10.063849+0000","last_peered":"2024-09-21T10:06:10.063849+0000","last_clean":"2024-09-21T10:06:10.063849+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:06:10.063849+0000","last_undegraded":"2024-09-21T10:06:10.063849+0000","last_fullsized":"2024-09-21T10:06:10.063849+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1163,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063370+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:06:10.063370+0000","last_peered":"2024-09-21T10:06:10.063370+0000","last_clean":"2024-09-21T10:06:10.063370+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:06:10.063370+0000","last_undegraded":"2024-09-21T10:06:10.063370+0000","last_fullsized":"2024-09-21T10:06:10.063370+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1193'7447","reported_seq":7576,"reported_epoch":1193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:12.334335+0000","last_change":"2024-09-21T10:03:30.139654+0000","last_active":"2024-09-21T10:06:12.334335+0000","last_peered":"2024-09-21T10:06:12.334335+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:06:12.334335+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1143'2682","ondisk_log_start":"1143'2682","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115408896,"num_objects":1761,"num_object_clones":0,"num_object_copies":5283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7447,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1761},{"shards":"1,4","objects":-1978}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":46,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.835479+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:05:37.835479+0000","last_peered":"2024-09-21T10:05:37.835479+0000","last_clean":"2024-09-21T10:05:37.835479+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:05:37.835479+0000","last_undegraded":"2024-09-21T10:05:37.835479+0000","last_fullsized":"2024-09-21T10:05:37.835479+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829879+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:05:37.829879+0000","last_peered":"2024-09-21T10:05:37.829879+0000","last_clean":"2024-09-21T10:05:37.829879+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:05:37.829879+0000","last_undegraded":"2024-09-21T10:05:37.829879+0000","last_fullsized":"2024-09-21T10:05:37.829879+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1193'7272","reported_seq":12919,"reported_epoch":1193,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:06:11.992081+0000","last_change":"2024-09-21T10:06:10.765843+0000","last_active":"2024-09-21T10:06:11.992081+0000","last_peered":"2024-09-21T10:06:11.992081+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:06:11.992081+0000","last_undegraded":"2024-09-21T10:06:11.992081+0000","last_fullsized":"2024-09-21T10:06:11.992081+0000","mapping_epoch":1151,"log_start":"1143'2636","ondisk_log_start":"1143'2636","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112328704,"num_objects":1714,"num_object_clones":0,"num_object_copies":5142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7272,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2755,"num_bytes_recovered":180551680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":1714}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1192,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066158+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:06:10.066158+0000","last_peered":"2024-09-21T10:06:10.066158+0000","last_clean":"2024-09-21T10:06:10.066158+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:06:10.066158+0000","last_undegraded":"2024-09-21T10:06:10.066158+0000","last_fullsized":"2024-09-21T10:06:10.066158+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1549,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063075+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:06:10.063075+0000","last_peered":"2024-09-21T10:06:10.063075+0000","last_clean":"2024-09-21T10:06:10.063075+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:06:10.063075+0000","last_undegraded":"2024-09-21T10:06:10.063075+0000","last_fullsized":"2024-09-21T10:06:10.063075+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1193'7325","reported_seq":13083,"reported_epoch":1193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:11.922353+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:06:11.922353+0000","last_peered":"2024-09-21T10:06:11.922353+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:06:11.922353+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1143'2748","ondisk_log_start":"1143'2748","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4577,"log_dups_size":0,"ondisk_log_size":4577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110690304,"num_objects":1689,"num_object_clones":0,"num_object_copies":5067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7325,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"0,3,4","objects":1689},{"shards":"3,4","objects":-1932}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":231,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.064778+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:06:10.064778+0000","last_peered":"2024-09-21T10:06:10.064778+0000","last_clean":"2024-09-21T10:06:10.064778+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:06:10.064778+0000","last_undegraded":"2024-09-21T10:06:10.064778+0000","last_fullsized":"2024-09-21T10:06:10.064778+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1207,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066065+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:06:10.066065+0000","last_peered":"2024-09-21T10:06:10.066065+0000","last_clean":"2024-09-21T10:06:10.066065+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:06:10.066065+0000","last_undegraded":"2024-09-21T10:06:10.066065+0000","last_fullsized":"2024-09-21T10:06:10.066065+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1193'7206","reported_seq":13068,"reported_epoch":1193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:12.146474+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:06:12.146474+0000","last_peered":"2024-09-21T10:06:12.146474+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:06:12.146474+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1143'2653","ondisk_log_start":"1143'2653","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":1769,"num_objects_missing":1769,"num_objects_degraded":4297,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7206,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1910,"num_bytes_recovered":125173760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,3","objects":749},{"shards":"0,2,3,5","objects":10},{"shards":"0,2,5","objects":907}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829804+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:05:37.829804+0000","last_peered":"2024-09-21T10:05:37.829804+0000","last_clean":"2024-09-21T10:05:37.829804+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:05:37.829804+0000","last_undegraded":"2024-09-21T10:05:37.829804+0000","last_fullsized":"2024-09-21T10:05:37.829804+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":850,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.069191+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:06:10.069191+0000","last_peered":"2024-09-21T10:06:10.069191+0000","last_clean":"2024-09-21T10:06:10.069191+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:06:10.069191+0000","last_undegraded":"2024-09-21T10:06:10.069191+0000","last_fullsized":"2024-09-21T10:06:10.069191+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1192'7314","reported_seq":13694,"reported_epoch":1192,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:06:08.029892+0000","last_change":"2024-09-21T10:05:03.425944+0000","last_active":"2024-09-21T10:06:08.029892+0000","last_peered":"2024-09-21T10:06:08.029892+0000","last_clean":"2024-09-21T10:02:38.228948+0000","last_became_active":"2024-09-21T10:03:36.276110+0000","last_became_peered":"2024-09-21T10:03:36.276110+0000","last_unstale":"2024-09-21T10:06:08.029892+0000","last_undegraded":"2024-09-21T10:06:08.029892+0000","last_fullsized":"2024-09-21T10:03:29.833661+0000","mapping_epoch":1170,"log_start":"1143'1300","ondisk_log_start":"1143'1300","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6014,"log_dups_size":0,"ondisk_log_size":6014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:04.270084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120782872,"num_objects":1844,"num_object_clones":0,"num_object_copies":5532,"num_objects_missing_on_primary":1270,"num_objects_missing":1270,"num_objects_degraded":0,"num_objects_misplaced":570,"num_objects_unfound":0,"num_objects_dirty":1844,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7314,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3135,"num_bytes_recovered":205455360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1,3","objects":397},{"shards":"0,1,3,6","objects":1},{"shards":"0,1,6","objects":173},{"shards":"1,6","objects":1273}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1194,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063098+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:06:10.063098+0000","last_peered":"2024-09-21T10:06:10.063098+0000","last_clean":"2024-09-21T10:06:10.063098+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:06:10.063098+0000","last_undegraded":"2024-09-21T10:06:10.063098+0000","last_fullsized":"2024-09-21T10:06:10.063098+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.830104+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:05:37.830104+0000","last_peered":"2024-09-21T10:05:37.830104+0000","last_clean":"2024-09-21T10:05:37.830104+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:05:37.830104+0000","last_undegraded":"2024-09-21T10:05:37.830104+0000","last_fullsized":"2024-09-21T10:05:37.830104+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1193'7575","reported_seq":12670,"reported_epoch":1193,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:12.062895+0000","last_change":"2024-09-21T10:05:36.090756+0000","last_active":"2024-09-21T10:06:12.062895+0000","last_peered":"2024-09-21T10:06:12.062895+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:06:12.062895+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"1148'3025","ondisk_log_start":"1148'3025","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2396,"num_objects_misplaced":1381,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7575,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2479,"num_bytes_recovered":162463744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"0,2,3","objects":1381},{"shards":"2,3","objects":-2056}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":223,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063018+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:06:10.063018+0000","last_peered":"2024-09-21T10:06:10.063018+0000","last_clean":"2024-09-21T10:06:10.063018+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:06:10.063018+0000","last_undegraded":"2024-09-21T10:06:10.063018+0000","last_fullsized":"2024-09-21T10:06:10.063018+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1553,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063197+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:06:10.063197+0000","last_peered":"2024-09-21T10:06:10.063197+0000","last_clean":"2024-09-21T10:06:10.063197+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:06:10.063197+0000","last_undegraded":"2024-09-21T10:06:10.063197+0000","last_fullsized":"2024-09-21T10:06:10.063197+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1193'7168","reported_seq":14589,"reported_epoch":1193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:12.162364+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:06:12.162364+0000","last_peered":"2024-09-21T10:06:12.162364+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:06:12.162364+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1143'2547","ondisk_log_start":"1143'2547","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4621,"log_dups_size":0,"ondisk_log_size":4621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109576192,"num_objects":1672,"num_object_clones":0,"num_object_copies":5016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7168,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2703,"num_bytes_recovered":177143808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,2,3","objects":732},{"shards":"0,3,5","objects":939},{"shards":"2,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":715,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829528+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:05:37.829528+0000","last_peered":"2024-09-21T10:05:37.829528+0000","last_clean":"2024-09-21T10:05:37.829528+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:05:37.829528+0000","last_undegraded":"2024-09-21T10:05:37.829528+0000","last_fullsized":"2024-09-21T10:05:37.829528+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1206,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.064508+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:06:10.064508+0000","last_peered":"2024-09-21T10:06:10.064508+0000","last_clean":"2024-09-21T10:06:10.064508+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:06:10.064508+0000","last_undegraded":"2024-09-21T10:06:10.064508+0000","last_fullsized":"2024-09-21T10:06:10.064508+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":237,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.065030+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:06:10.065030+0000","last_peered":"2024-09-21T10:06:10.065030+0000","last_clean":"2024-09-21T10:06:10.065030+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:06:10.065030+0000","last_undegraded":"2024-09-21T10:06:10.065030+0000","last_fullsized":"2024-09-21T10:06:10.065030+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.835404+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:05:37.835404+0000","last_peered":"2024-09-21T10:05:37.835404+0000","last_clean":"2024-09-21T10:05:37.835404+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:05:37.835404+0000","last_undegraded":"2024-09-21T10:05:37.835404+0000","last_fullsized":"2024-09-21T10:05:37.835404+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1193'7245","reported_seq":12878,"reported_epoch":1193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:12.152802+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:06:12.152802+0000","last_peered":"2024-09-21T10:06:12.152802+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:06:12.152802+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1143'2554","ondisk_log_start":"1143'2554","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4691,"log_dups_size":0,"ondisk_log_size":4691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111476736,"num_objects":1701,"num_object_clones":0,"num_object_copies":5103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7245,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"0,1,2","objects":1701},{"shards":"1,2","objects":-1904}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":837,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062503+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:06:10.062503+0000","last_peered":"2024-09-21T10:06:10.062503+0000","last_clean":"2024-09-21T10:06:10.062503+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:06:10.062503+0000","last_undegraded":"2024-09-21T10:06:10.062503+0000","last_fullsized":"2024-09-21T10:06:10.062503+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1426,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.065628+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:06:10.065628+0000","last_peered":"2024-09-21T10:06:10.065628+0000","last_clean":"2024-09-21T10:06:10.065628+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:06:10.065628+0000","last_undegraded":"2024-09-21T10:06:10.065628+0000","last_fullsized":"2024-09-21T10:06:10.065628+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":835,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063598+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:06:10.063598+0000","last_peered":"2024-09-21T10:06:10.063598+0000","last_clean":"2024-09-21T10:06:10.063598+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:06:10.063598+0000","last_undegraded":"2024-09-21T10:06:10.063598+0000","last_fullsized":"2024-09-21T10:06:10.063598+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1193'7019","reported_seq":15779,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:12.339015+0000","last_change":"2024-09-21T10:05:37.849809+0000","last_active":"2024-09-21T10:06:12.339015+0000","last_peered":"2024-09-21T10:06:12.339015+0000","last_clean":"2024-09-21T10:06:12.339015+0000","last_became_active":"2024-09-21T10:05:37.830539+0000","last_became_peered":"2024-09-21T10:05:37.830539+0000","last_unstale":"2024-09-21T10:06:12.339015+0000","last_undegraded":"2024-09-21T10:06:12.339015+0000","last_fullsized":"2024-09-21T10:06:12.339015+0000","mapping_epoch":1190,"log_start":"1143'2237","ondisk_log_start":"1143'2237","created":1137,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:35.172792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7019,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4311,"num_bytes_recovered":174260224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1467,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066229+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:06:10.066229+0000","last_peered":"2024-09-21T10:06:10.066229+0000","last_clean":"2024-09-21T10:06:10.066229+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:06:10.066229+0000","last_undegraded":"2024-09-21T10:06:10.066229+0000","last_fullsized":"2024-09-21T10:06:10.066229+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1554,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062774+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:06:10.062774+0000","last_peered":"2024-09-21T10:06:10.062774+0000","last_clean":"2024-09-21T10:06:10.062774+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:06:10.062774+0000","last_undegraded":"2024-09-21T10:06:10.062774+0000","last_fullsized":"2024-09-21T10:06:10.062774+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1561,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063057+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:06:10.063057+0000","last_peered":"2024-09-21T10:06:10.063057+0000","last_clean":"2024-09-21T10:06:10.063057+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:06:10.063057+0000","last_undegraded":"2024-09-21T10:06:10.063057+0000","last_fullsized":"2024-09-21T10:06:10.063057+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1192'7227","reported_seq":11073,"reported_epoch":1192,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:08.008541+0000","last_change":"2024-09-21T10:03:36.351943+0000","last_active":"2024-09-21T10:06:08.008541+0000","last_peered":"2024-09-21T10:06:08.008541+0000","last_clean":"2024-09-21T10:02:38.308681+0000","last_became_active":"2024-09-21T10:03:36.318082+0000","last_became_peered":"2024-09-21T10:03:36.318082+0000","last_unstale":"2024-09-21T10:06:08.008541+0000","last_undegraded":"2024-09-21T10:03:36.183243+0000","last_fullsized":"2024-09-21T10:03:29.833927+0000","mapping_epoch":1170,"log_start":"1143'1200","ondisk_log_start":"1143'1200","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:52.296275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117374976,"num_objects":1791,"num_object_clones":0,"num_object_copies":5373,"num_objects_missing_on_primary":2505,"num_objects_missing":2505,"num_objects_degraded":3489,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7227,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1866,"num_bytes_recovered":122290176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,1,3","objects":1653},{"shards":"0,1,3,6","objects":6},{"shards":"0,3,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":130,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062688+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:06:10.062688+0000","last_peered":"2024-09-21T10:06:10.062688+0000","last_clean":"2024-09-21T10:06:10.062688+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:06:10.062688+0000","last_undegraded":"2024-09-21T10:06:10.062688+0000","last_fullsized":"2024-09-21T10:06:10.062688+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1153,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:42.844439+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:05:42.844439+0000","last_peered":"2024-09-21T10:05:42.844439+0000","last_clean":"2024-09-21T10:05:42.844439+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:05:42.844439+0000","last_undegraded":"2024-09-21T10:05:42.844439+0000","last_fullsized":"2024-09-21T10:05:42.844439+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1564,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062458+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:06:10.062458+0000","last_peered":"2024-09-21T10:06:10.062458+0000","last_clean":"2024-09-21T10:06:10.062458+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:06:10.062458+0000","last_undegraded":"2024-09-21T10:06:10.062458+0000","last_fullsized":"2024-09-21T10:06:10.062458+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1193'7398","reported_seq":7519,"reported_epoch":1193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:12.087125+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:06:12.087125+0000","last_peered":"2024-09-21T10:06:12.087125+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:06:12.087125+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1143'2785","ondisk_log_start":"1143'2785","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4613,"log_dups_size":0,"ondisk_log_size":4613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107872256,"num_objects":1646,"num_object_clones":0,"num_object_copies":4938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7398,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1491},{"shards":"3,4","objects":-2721}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1154,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066218+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:06:10.066218+0000","last_peered":"2024-09-21T10:06:10.066218+0000","last_clean":"2024-09-21T10:06:10.066218+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:06:10.066218+0000","last_undegraded":"2024-09-21T10:06:10.066218+0000","last_fullsized":"2024-09-21T10:06:10.066218+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1575,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063310+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:06:10.063310+0000","last_peered":"2024-09-21T10:06:10.063310+0000","last_clean":"2024-09-21T10:06:10.063310+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:06:10.063310+0000","last_undegraded":"2024-09-21T10:06:10.063310+0000","last_fullsized":"2024-09-21T10:06:10.063310+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":94,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062374+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:06:10.062374+0000","last_peered":"2024-09-21T10:06:10.062374+0000","last_clean":"2024-09-21T10:06:10.062374+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:06:10.062374+0000","last_undegraded":"2024-09-21T10:06:10.062374+0000","last_fullsized":"2024-09-21T10:06:10.062374+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1193'7229","reported_seq":14824,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:12.319652+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:06:12.319652+0000","last_peered":"2024-09-21T10:06:12.319652+0000","last_clean":"2024-09-21T10:06:12.319652+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:06:12.319652+0000","last_undegraded":"2024-09-21T10:06:12.319652+0000","last_fullsized":"2024-09-21T10:06:12.319652+0000","mapping_epoch":1174,"log_start":"1143'2494","ondisk_log_start":"1143'2494","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4735,"log_dups_size":0,"ondisk_log_size":4735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7229,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1552,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063660+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:06:10.063660+0000","last_peered":"2024-09-21T10:06:10.063660+0000","last_clean":"2024-09-21T10:06:10.063660+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:06:10.063660+0000","last_undegraded":"2024-09-21T10:06:10.063660+0000","last_fullsized":"2024-09-21T10:06:10.063660+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":733,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.068862+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:06:10.068862+0000","last_peered":"2024-09-21T10:06:10.068862+0000","last_clean":"2024-09-21T10:06:10.068862+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:06:10.068862+0000","last_undegraded":"2024-09-21T10:06:10.068862+0000","last_fullsized":"2024-09-21T10:06:10.068862+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1545,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063244+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:06:10.063244+0000","last_peered":"2024-09-21T10:06:10.063244+0000","last_clean":"2024-09-21T10:06:10.063244+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:06:10.063244+0000","last_undegraded":"2024-09-21T10:06:10.063244+0000","last_fullsized":"2024-09-21T10:06:10.063244+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1193'7476","reported_seq":7602,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:11.996594+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:06:11.996594+0000","last_peered":"2024-09-21T10:06:11.996594+0000","last_clean":"2024-09-21T10:06:11.996594+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:06:11.996594+0000","last_undegraded":"2024-09-21T10:06:11.996594+0000","last_fullsized":"2024-09-21T10:06:11.996594+0000","mapping_epoch":1137,"log_start":"1148'2926","ondisk_log_start":"1148'2926","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111149056,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7476,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":253,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.069215+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:06:10.069215+0000","last_peered":"2024-09-21T10:06:10.069215+0000","last_clean":"2024-09-21T10:06:10.069215+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:06:10.069215+0000","last_undegraded":"2024-09-21T10:06:10.069215+0000","last_fullsized":"2024-09-21T10:06:10.069215+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1553,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063952+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:06:10.063952+0000","last_peered":"2024-09-21T10:06:10.063952+0000","last_clean":"2024-09-21T10:06:10.063952+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:06:10.063952+0000","last_undegraded":"2024-09-21T10:06:10.063952+0000","last_fullsized":"2024-09-21T10:06:10.063952+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":230,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062942+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:06:10.062942+0000","last_peered":"2024-09-21T10:06:10.062942+0000","last_clean":"2024-09-21T10:06:10.062942+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:06:10.062942+0000","last_undegraded":"2024-09-21T10:06:10.062942+0000","last_fullsized":"2024-09-21T10:06:10.062942+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1193'7166","reported_seq":7292,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:11.807616+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:06:11.807616+0000","last_peered":"2024-09-21T10:06:11.807616+0000","last_clean":"2024-09-21T10:06:11.807616+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:06:11.807616+0000","last_undegraded":"2024-09-21T10:06:11.807616+0000","last_fullsized":"2024-09-21T10:06:11.807616+0000","mapping_epoch":1137,"log_start":"1143'2613","ondisk_log_start":"1143'2613","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110231552,"num_objects":1682,"num_object_clones":0,"num_object_copies":5046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7166,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":94,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063690+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:06:10.063690+0000","last_peered":"2024-09-21T10:06:10.063690+0000","last_clean":"2024-09-21T10:06:10.063690+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:06:10.063690+0000","last_undegraded":"2024-09-21T10:06:10.063690+0000","last_fullsized":"2024-09-21T10:06:10.063690+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.857331+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:05:37.857331+0000","last_peered":"2024-09-21T10:05:37.857331+0000","last_clean":"2024-09-21T10:05:37.857331+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:05:37.857331+0000","last_undegraded":"2024-09-21T10:05:37.857331+0000","last_fullsized":"2024-09-21T10:05:37.857331+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829652+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:05:37.829652+0000","last_peered":"2024-09-21T10:05:37.829652+0000","last_clean":"2024-09-21T10:05:37.829652+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:05:37.829652+0000","last_undegraded":"2024-09-21T10:05:37.829652+0000","last_fullsized":"2024-09-21T10:05:37.829652+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829967+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:05:37.829967+0000","last_peered":"2024-09-21T10:05:37.829967+0000","last_clean":"2024-09-21T10:05:37.829967+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:05:37.829967+0000","last_undegraded":"2024-09-21T10:05:37.829967+0000","last_fullsized":"2024-09-21T10:05:37.829967+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":52,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.830127+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:05:37.830127+0000","last_peered":"2024-09-21T10:05:37.830127+0000","last_clean":"2024-09-21T10:05:37.830127+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:05:37.830127+0000","last_undegraded":"2024-09-21T10:05:37.830127+0000","last_fullsized":"2024-09-21T10:05:37.830127+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1553,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062778+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:06:10.062778+0000","last_peered":"2024-09-21T10:06:10.062778+0000","last_clean":"2024-09-21T10:06:10.062778+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:06:10.062778+0000","last_undegraded":"2024-09-21T10:06:10.062778+0000","last_fullsized":"2024-09-21T10:06:10.062778+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1165,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066463+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:06:10.066463+0000","last_peered":"2024-09-21T10:06:10.066463+0000","last_clean":"2024-09-21T10:06:10.066463+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:06:10.066463+0000","last_undegraded":"2024-09-21T10:06:10.066463+0000","last_fullsized":"2024-09-21T10:06:10.066463+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1176,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.064038+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:06:10.064038+0000","last_peered":"2024-09-21T10:06:10.064038+0000","last_clean":"2024-09-21T10:06:10.064038+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:06:10.064038+0000","last_undegraded":"2024-09-21T10:06:10.064038+0000","last_fullsized":"2024-09-21T10:06:10.064038+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":251,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066132+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:06:10.066132+0000","last_peered":"2024-09-21T10:06:10.066132+0000","last_clean":"2024-09-21T10:06:10.066132+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:06:10.066132+0000","last_undegraded":"2024-09-21T10:06:10.066132+0000","last_fullsized":"2024-09-21T10:06:10.066132+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1579,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.068860+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:06:10.068860+0000","last_peered":"2024-09-21T10:06:10.068860+0000","last_clean":"2024-09-21T10:06:10.068860+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:06:10.068860+0000","last_undegraded":"2024-09-21T10:06:10.068860+0000","last_fullsized":"2024-09-21T10:06:10.068860+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":230,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063159+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:06:10.063159+0000","last_peered":"2024-09-21T10:06:10.063159+0000","last_clean":"2024-09-21T10:06:10.063159+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:06:10.063159+0000","last_undegraded":"2024-09-21T10:06:10.063159+0000","last_fullsized":"2024-09-21T10:06:10.063159+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":51,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.846198+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:05:37.846198+0000","last_peered":"2024-09-21T10:05:37.846198+0000","last_clean":"2024-09-21T10:05:37.846198+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:05:37.846198+0000","last_undegraded":"2024-09-21T10:05:37.846198+0000","last_fullsized":"2024-09-21T10:05:37.846198+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":222,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062847+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:06:10.062847+0000","last_peered":"2024-09-21T10:06:10.062847+0000","last_clean":"2024-09-21T10:06:10.062847+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:06:10.062847+0000","last_undegraded":"2024-09-21T10:06:10.062847+0000","last_fullsized":"2024-09-21T10:06:10.062847+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1541,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.115091+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:06:10.115091+0000","last_peered":"2024-09-21T10:06:10.115091+0000","last_clean":"2024-09-21T10:06:10.115091+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:06:10.115091+0000","last_undegraded":"2024-09-21T10:06:10.115091+0000","last_fullsized":"2024-09-21T10:06:10.115091+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1551,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063419+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:06:10.063419+0000","last_peered":"2024-09-21T10:06:10.063419+0000","last_clean":"2024-09-21T10:06:10.063419+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:06:10.063419+0000","last_undegraded":"2024-09-21T10:06:10.063419+0000","last_fullsized":"2024-09-21T10:06:10.063419+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":215,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066551+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:06:10.066551+0000","last_peered":"2024-09-21T10:06:10.066551+0000","last_clean":"2024-09-21T10:06:10.066551+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:06:10.066551+0000","last_undegraded":"2024-09-21T10:06:10.066551+0000","last_fullsized":"2024-09-21T10:06:10.066551+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":99,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.062908+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:06:10.062908+0000","last_peered":"2024-09-21T10:06:10.062908+0000","last_clean":"2024-09-21T10:06:10.062908+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:06:10.062908+0000","last_undegraded":"2024-09-21T10:06:10.062908+0000","last_fullsized":"2024-09-21T10:06:10.062908+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1435,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063592+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:06:10.063592+0000","last_peered":"2024-09-21T10:06:10.063592+0000","last_clean":"2024-09-21T10:06:10.063592+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:06:10.063592+0000","last_undegraded":"2024-09-21T10:06:10.063592+0000","last_fullsized":"2024-09-21T10:06:10.063592+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1517,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063155+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:06:10.063155+0000","last_peered":"2024-09-21T10:06:10.063155+0000","last_clean":"2024-09-21T10:06:10.063155+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:06:10.063155+0000","last_undegraded":"2024-09-21T10:06:10.063155+0000","last_fullsized":"2024-09-21T10:06:10.063155+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1161,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.064696+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:06:10.064696+0000","last_peered":"2024-09-21T10:06:10.064696+0000","last_clean":"2024-09-21T10:06:10.064696+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:06:10.064696+0000","last_undegraded":"2024-09-21T10:06:10.064696+0000","last_fullsized":"2024-09-21T10:06:10.064696+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":115,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.063731+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:06:10.063731+0000","last_peered":"2024-09-21T10:06:10.063731+0000","last_clean":"2024-09-21T10:06:10.063731+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:06:10.063731+0000","last_undegraded":"2024-09-21T10:06:10.063731+0000","last_fullsized":"2024-09-21T10:06:10.063731+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1168,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.064809+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:06:10.064809+0000","last_peered":"2024-09-21T10:06:10.064809+0000","last_clean":"2024-09-21T10:06:10.064809+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:06:10.064809+0000","last_undegraded":"2024-09-21T10:06:10.064809+0000","last_fullsized":"2024-09-21T10:06:10.064809+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":46,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.835354+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:05:37.835354+0000","last_peered":"2024-09-21T10:05:37.835354+0000","last_clean":"2024-09-21T10:05:37.835354+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:05:37.835354+0000","last_undegraded":"2024-09-21T10:05:37.835354+0000","last_fullsized":"2024-09-21T10:05:37.835354+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":222,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.066317+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:06:10.066317+0000","last_peered":"2024-09-21T10:06:10.066317+0000","last_clean":"2024-09-21T10:06:10.066317+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:06:10.066317+0000","last_undegraded":"2024-09-21T10:06:10.066317+0000","last_fullsized":"2024-09-21T10:06:10.066317+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":230,"reported_epoch":1193,"state":"active+clean","last_fresh":"2024-09-21T10:06:10.114993+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:06:10.114993+0000","last_peered":"2024-09-21T10:06:10.114993+0000","last_clean":"2024-09-21T10:06:10.114993+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:06:10.114993+0000","last_undegraded":"2024-09-21T10:06:10.114993+0000","last_fullsized":"2024-09-21T10:06:10.114993+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":42,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.829696+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:05:37.829696+0000","last_peered":"2024-09-21T10:05:37.829696+0000","last_clean":"2024-09-21T10:05:37.829696+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:05:37.829696+0000","last_undegraded":"2024-09-21T10:05:37.829696+0000","last_fullsized":"2024-09-21T10:05:37.829696+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1561,"reported_epoch":1191,"state":"active+clean","last_fresh":"2024-09-21T10:05:37.845332+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:05:37.845332+0000","last_peered":"2024-09-21T10:05:37.845332+0000","last_clean":"2024-09-21T10:05:37.845332+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:05:37.845332+0000","last_undegraded":"2024-09-21T10:05:37.845332+0000","last_fullsized":"2024-09-21T10:05:37.845332+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1783693336,"num_objects":27218,"num_object_clones":0,"num_object_copies":81654,"num_objects_missing_on_primary":5544,"num_objects_missing":5544,"num_objects_degraded":32084,"num_objects_misplaced":1951,"num_objects_unfound":0,"num_objects_dirty":27218,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":116756,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36598,"num_bytes_recovered":2083192832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7600353280,"data_stored":7600341064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76922,"ondisk_log_size":76922,"up":48,"acting":39,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234364,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495999,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648508,"kb_used_data":360692,"kb_used_omap":1,"kb_used_meta":287806,"kb_avail":93096644,"statfs":{"total":95995035648,"available":95330963456,"internally_reserved":0,"allocated":369348608,"data_stored":366850716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":294713799},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,0,3,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757639,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019275,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":924296,"kb_used_data":658556,"kb_used_omap":1,"kb_used_meta":265726,"kb_avail":92820856,"statfs":{"total":95995035648,"available":95048556544,"internally_reserved":0,"allocated":674361344,"data_stored":671863187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":272103878},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888571,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1771828,"kb_used_data":1298348,"kb_used_omap":34,"kb_used_meta":473437,"kb_avail":91973324,"statfs":{"total":95995035648,"available":94180683776,"internally_reserved":0,"allocated":1329508352,"data_stored":1327021238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":484800499},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967810,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388652,"kb_used_data":1053924,"kb_used_omap":41,"kb_used_meta":334678,"kb_avail":92356500,"statfs":{"total":95995035648,"available":94573056000,"internally_reserved":0,"allocated":1079218176,"data_stored":1076736477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":342710687},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047049,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622776,"kb_used_data":1315820,"kb_used_omap":41,"kb_used_meta":306902,"kb_avail":92122376,"statfs":{"total":95995035648,"available":94333313024,"internally_reserved":0,"allocated":1347399680,"data_stored":1344912974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":314268585},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126287,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1886851072,"data_stored":1886847000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1333071872,"data_stored":1333067800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1061879808,"data_stored":1061879808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1312358400,"data_stored":1312358400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":665321472,"data_stored":665321472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":361304064,"data_stored":361299992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":647364608,"data_stored":647364608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:06:16.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:06:16.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:17.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:17.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:17.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:17.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:17.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:17.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:17.328 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:17.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:17.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:17.457 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:17.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:17.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:17.584 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:17.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:17.686 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:17.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:17.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.834 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:17.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:17.838 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:17.838 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:17.842 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:17.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:17.961 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:17.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:17.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:18.063 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:18.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:18.188 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:18.213 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:18.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:18.314 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.338 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:18.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:18.339 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:18 smithi191 ceph-mon[102206]: pgmap v1545: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32005/81198 objects degraded (39.416%); 1554/81198 objects misplaced (1.914%); 10 MiB/s, 176 objects/s recovering 2024-09-21T10:06:18.363 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:18 smithi076 ceph-mon[106482]: pgmap v1545: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32005/81198 objects degraded (39.416%); 1554/81198 objects misplaced (1.914%); 10 MiB/s, 176 objects/s recovering 2024-09-21T10:06:18.364 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:18 smithi076 ceph-mon[110963]: pgmap v1545: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.7 GiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 271 op/s; 32005/81198 objects degraded (39.416%); 1554/81198 objects misplaced (1.914%); 10 MiB/s, 176 objects/s recovering 2024-09-21T10:06:18.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:18.471 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:18.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:18.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:18.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:18.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:18.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:18.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:18.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:18.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:18.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:18.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:18.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:18.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:19.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:19.102 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:19.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:19.204 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:19.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:19.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:19.359 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:19.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:19.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:19.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:19.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:19.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:19.750 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:19.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:19.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:19.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:19.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:19.979 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:20.005 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:20.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:20.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:20.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:20.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:20.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:20.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:19 smithi191 ceph-mon[102206]: pgmap v1546: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 31582/77157 objects degraded (40.932%); 1278/77157 objects misplaced (1.656%); 8.5 MiB/s, 185 objects/s recovering 2024-09-21T10:06:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 32005/81198 objects degraded (39.416%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[106482]: pgmap v1546: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 31582/77157 objects degraded (40.932%); 1278/77157 objects misplaced (1.656%); 8.5 MiB/s, 185 objects/s recovering 2024-09-21T10:06:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 32005/81198 objects degraded (39.416%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[110963]: pgmap v1546: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 31582/77157 objects degraded (40.932%); 1278/77157 objects misplaced (1.656%); 8.5 MiB/s, 185 objects/s recovering 2024-09-21T10:06:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 32005/81198 objects degraded (39.416%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:20.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:20.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:20.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:20.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:20.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:20.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:20.759 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:20.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:20.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:20.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:20.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:21.016 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:21.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:21.118 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:21.144 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:21.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:21.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:21.268 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:21.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:21.391 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:21.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:21.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:21.524 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:21.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:21.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:21.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:21.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:21.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:21.826 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:06:21.826 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:21.845 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:06:21.845 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:21.870 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:06:21.870 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:21.884 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:21.901 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:06:21.901 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:21.907 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:21.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:21.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:21.924 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:06:21.924 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:21.948 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:06:21.948 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:21.968 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:06:21.968 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:21.989 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:06:21.989 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:22.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:22.033 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:21 smithi076 ceph-mon[106482]: pgmap v1547: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.6 GiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 31582/77157 objects degraded (40.932%); 1278/77157 objects misplaced (1.656%); 8.0 MiB/s, 175 objects/s recovering 2024-09-21T10:06:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2371264625' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:21 smithi076 ceph-mon[110963]: pgmap v1547: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.6 GiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 31582/77157 objects degraded (40.932%); 1278/77157 objects misplaced (1.656%); 8.0 MiB/s, 175 objects/s recovering 2024-09-21T10:06:22.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2371264625' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:22.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:22.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:22.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:22.285 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:21 smithi191 ceph-mon[102206]: pgmap v1547: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.6 GiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 31582/77157 objects degraded (40.932%); 1278/77157 objects misplaced (1.656%); 8.0 MiB/s, 175 objects/s recovering 2024-09-21T10:06:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2371264625' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:22.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:22.408 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:22.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:22.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:22.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:22.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:22.663 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:22.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:22.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:22.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:22.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:22.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:22.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:22.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:22.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:23.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.035 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:23.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:23.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.281 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.404 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.535 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:23.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:06:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:06:23] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.51.0" 2024-09-21T10:06:23.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:23.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:23.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:23.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:23.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:23.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:24.007 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:24.044 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:24.146 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:24.169 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:24.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:24.294 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:24.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:23 smithi191 ceph-mon[102206]: pgmap v1548: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 322 op/s; 31206/73434 objects degraded (42.495%); 1164/73434 objects misplaced (1.585%); 8.6 MiB/s, 179 objects/s recovering 2024-09-21T10:06:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:23 smithi191 ceph-mon[102206]: osdmap e1194: 8 total, 8 up, 5 in 2024-09-21T10:06:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:23 smithi076 ceph-mon[106482]: pgmap v1548: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 322 op/s; 31206/73434 objects degraded (42.495%); 1164/73434 objects misplaced (1.585%); 8.6 MiB/s, 179 objects/s recovering 2024-09-21T10:06:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:23 smithi076 ceph-mon[106482]: osdmap e1194: 8 total, 8 up, 5 in 2024-09-21T10:06:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:23 smithi076 ceph-mon[110963]: pgmap v1548: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 322 op/s; 31206/73434 objects degraded (42.495%); 1164/73434 objects misplaced (1.585%); 8.6 MiB/s, 179 objects/s recovering 2024-09-21T10:06:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:23 smithi076 ceph-mon[110963]: osdmap e1194: 8 total, 8 up, 5 in 2024-09-21T10:06:24.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:24.422 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:24.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:24.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:24.549 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:24.651 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:24.677 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:24.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:24.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:24.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:24.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:24.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:25.035 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:25.056 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:25.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:25.188 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:25.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:25.275 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:24 smithi076 ceph-mon[106482]: osdmap e1195: 8 total, 8 up, 5 in 2024-09-21T10:06:25.275 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 31206/73434 objects degraded (42.495%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:25.275 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:24 smithi076 ceph-mon[110963]: osdmap e1195: 8 total, 8 up, 5 in 2024-09-21T10:06:25.275 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 31206/73434 objects degraded (42.495%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:25.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:25.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:25.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:24 smithi191 ceph-mon[102206]: osdmap e1195: 8 total, 8 up, 5 in 2024-09-21T10:06:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 31206/73434 objects degraded (42.495%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:25.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:25.438 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:25.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:25.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:25.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:25.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:25.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:25.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:25.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:25.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:25.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:25.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:25.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:25.955 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:06:26.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:26.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:26.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:26.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:26.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:26.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:26.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:25 smithi191 ceph-mon[102206]: pgmap v1551: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 347 op/s; 27901/72867 objects degraded (38.290%); 2482/72867 objects misplaced (3.406%); 8.8 MiB/s, 228 objects/s recovering 2024-09-21T10:06:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2573464351' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[106482]: pgmap v1551: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 347 op/s; 27901/72867 objects degraded (38.290%); 2482/72867 objects misplaced (3.406%); 8.8 MiB/s, 228 objects/s recovering 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2573464351' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[110963]: pgmap v1551: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 347 op/s; 27901/72867 objects degraded (38.290%); 2482/72867 objects misplaced (3.406%); 8.8 MiB/s, 228 objects/s recovering 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2573464351' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:06:26.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:26.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:26.459 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:26.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:26.561 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:26.583 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:26.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:26.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:26.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:26.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:26.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:26.945 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:26 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:26.954 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:26.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:26.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:27.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:27.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:27.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:27.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:27.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:27.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:27.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:27.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:26 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:27.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:26 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:27.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:26 smithi191 ceph-mon[102206]: osdmap e1196: 8 total, 8 up, 5 in 2024-09-21T10:06:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:26 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:26 smithi076 ceph-mon[106482]: osdmap e1196: 8 total, 8 up, 5 in 2024-09-21T10:06:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:26 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:26 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:06:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:26 smithi076 ceph-mon[110963]: osdmap e1196: 8 total, 8 up, 5 in 2024-09-21T10:06:27.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:27.465 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:27.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:27.590 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:27.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:27.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:27.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:27.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:27.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:27.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:27.862 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:06:27.862 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 27901/72867 objects degraded (38.290%), 7 pgs degraded, 8 pgs undersized","count":15},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1290,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1196,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":5,"osd_in_since":1726913007,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":79},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"peering","count":1}],"num_pgs":89,"num_pools":5,"num_objects":24289,"data_bytes":1590630397,"bytes_used":6009102336,"bytes_avail":473966075904,"bytes_total":479975178240,"inactive_pgs_ratio":0.01123595517128706,"degraded_objects":27901,"degraded_total":72867,"degraded_ratio":0.38290309742407402,"misplaced_objects":2482,"misplaced_total":72867,"misplaced_ratio":0.034062058270547708,"recovering_objects_per_sec":160,"recovering_bytes_per_sec":6366314,"recovering_keys_per_sec":0,"num_objects_recovered":963,"num_bytes_recovered":38207512,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":238},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-21T10:05:04.625691+0000","services":{}},"progress_events":{}} 2024-09-21T10:06:27.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:27.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:27.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:28.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:06:28.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.115 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:28.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:28.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:28.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:28.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:27 smithi191 ceph-mon[102206]: pgmap v1553: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 27901/72867 objects degraded (38.290%); 2482/72867 objects misplaced (3.406%); 6.1 MiB/s, 160 objects/s recovering 2024-09-21T10:06:28.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2980068962' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:06:28.367 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:28.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:27 smithi076 ceph-mon[106482]: pgmap v1553: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 27901/72867 objects degraded (38.290%); 2482/72867 objects misplaced (3.406%); 6.1 MiB/s, 160 objects/s recovering 2024-09-21T10:06:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2980068962' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:06:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:27 smithi076 ceph-mon[110963]: pgmap v1553: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 1.5 GiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 238 op/s; 27901/72867 objects degraded (38.290%); 2482/72867 objects misplaced (3.406%); 6.1 MiB/s, 160 objects/s recovering 2024-09-21T10:06:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2980068962' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:06:28.471 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.600 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:28.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:28.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:28.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:28.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:28.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:28.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:28.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:29.013 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:29.043 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:06:29.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:29.137 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:29.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:29.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:29.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:29.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:29.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.406 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:29.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:29.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:29.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:29.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:29.673 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:29.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:29.797 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:29.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:29.920 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:29.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:30.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:30.045 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:30.146 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:30.168 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:30.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:30.292 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:30.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:29 smithi191 ceph-mon[102206]: pgmap v1554: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 319 op/s; 27466/67677 objects degraded (40.584%); 2186/67677 objects misplaced (3.230%); 8.3 MiB/s, 195 objects/s recovering 2024-09-21T10:06:30.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 27466/67677 objects degraded (40.584%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:29 smithi076 ceph-mon[106482]: pgmap v1554: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 319 op/s; 27466/67677 objects degraded (40.584%); 2186/67677 objects misplaced (3.230%); 8.3 MiB/s, 195 objects/s recovering 2024-09-21T10:06:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 27466/67677 objects degraded (40.584%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:29 smithi076 ceph-mon[110963]: pgmap v1554: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 319 op/s; 27466/67677 objects degraded (40.584%); 2186/67677 objects misplaced (3.230%); 8.3 MiB/s, 195 objects/s recovering 2024-09-21T10:06:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 27466/67677 objects degraded (40.584%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:30.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:30.416 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:30.509 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:06:30.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:06:30.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:30.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:30.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:30.664 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:30.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:30.787 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:30.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:30.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:30.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:30.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:31.006 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:06:31.006 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:06:31.006 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:26453 7.b 7:d9a335b8:::benchmark_data_smithi076_190198_object26452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.033441+0000", 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 303.96878832700003, 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.225782855, 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.007 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 26453 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033441+0000", 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033441+0000", 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.008 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033443+0000", 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8920000000000001e-06 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033468+0000", 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4833000000000001e-05 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033480+0000", 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1908e-05 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.009 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033487+0000", 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4490000000000002e-06 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033630+0000", 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014250499999999999 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033755+0000", 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012543600000000001 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.010 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033895+0000", 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000140016 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.038163+0000", 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042682950000000001 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.038178+0000", 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4443e-05 2024-09-21T10:06:31.011 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.259119+0000", 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.220941156 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.259177+0000", 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8393999999999997e-05 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.259224+0000", 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6527999999999999e-05 2024-09-21T10:06:31.012 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:26455 7.b 7:da9cbb59:::benchmark_data_smithi076_190198_object26454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.033508+0000", 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 303.96872179399998, 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22796824099999999, 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.013 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 26455 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033508+0000", 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033508+0000", 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.014 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033508+0000", 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6599999999999996e-07 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033528+0000", 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9734000000000001e-05 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033536+0000", 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3999999999999992e-06 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.015 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.033542+0000", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.789e-06 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036745+0000", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003203385 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.036877+0000", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000131166 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:06:31.016 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.037033+0000", 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015611499999999999 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.039072+0000", 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002039178 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.039179+0000", 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106632 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.017 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261348+0000", 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22216969 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261406+0000", 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7364000000000002e-05 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261476+0000", 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0222000000000006e-05 2024-09-21T10:06:31.018 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:26470 7.b 7:d1dd6348:::benchmark_data_smithi076_190198_object26469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:27.040496+0000", 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 303.961733659, 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.221324621, 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.019 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 26470 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040496+0000", 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.021 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040496+0000", 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040497+0000", 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.601e-06 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040521+0000", 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.317e-05 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.022 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040534+0000", 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3814e-05 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040542+0000", 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9640000000000003e-06 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040600+0000", 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7707999999999997e-05 2024-09-21T10:06:31.023 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040735+0000", 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000134914 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,5", 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.040881+0000", 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014656900000000001 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.024 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.042064+0000", 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001182815 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.042285+0000", 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000220206 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.025 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261724+0000", 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.21943958399999999 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261780+0000", 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6332000000000002e-05 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:27.261820+0000", 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9944000000000002e-05 2024-09-21T10:06:31.026 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:27795 7.d 7:b1903dd9:::benchmark_data_smithi076_190198_object27794:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:28.041008+0000", 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 302.96122154699998, 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.14765737200000001, 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.027 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 27795 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041008+0000", 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041008+0000", 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.028 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041009+0000", 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6270000000000001e-06 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041054+0000", 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4307000000000001e-05 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041064+0000", 2024-09-21T10:06:31.029 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8339999999999994e-06 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041071+0000", 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3499999999999999e-06 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041137+0000", 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6025000000000006e-05 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.030 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041260+0000", 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012312099999999999 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.041350+0000", 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0013999999999996e-05 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.042352+0000", 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010018589999999999 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.031 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.042499+0000", 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014659699999999999 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.188488+0000", 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.145989007 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.188605+0000", 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000117519 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.032 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:28.188665+0000", 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0112000000000003e-05 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44762 7.d 7:b780d3c9:::benchmark_data_smithi076_190198_object44761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.245148+0000", 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 287.75708117800002, 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77432460599999997, 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.033 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44762 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245148+0000", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245148+0000", 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.034 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245150+0000", 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.705e-06 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245177+0000", 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7118999999999999e-05 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245192+0000", 2024-09-21T10:06:31.035 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5153e-05 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245200+0000", 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9780000000000005e-06 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245266+0000", 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5495999999999998e-05 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245405+0000", 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000138882 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.036 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.245543+0000", 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013875899999999999 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247624+0000", 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020807249999999998 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249171+0000", 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015465209999999999 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.037 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019360+0000", 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77018937499999995 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019423+0000", 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3479999999999995e-05 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019473+0000", 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9413000000000003e-05 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.038 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44770 7.d 7:bab46a7f:::benchmark_data_smithi076_190198_object44769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.253737+0000", 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 287.74849259299998, 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76588910300000002, 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44770 2024-09-21T10:06:31.039 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253737+0000", 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253737+0000", 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253738+0000", 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.466e-06 2024-09-21T10:06:31.040 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253783+0000", 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4527999999999998e-05 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253802+0000", 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8833000000000001e-05 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253811+0000", 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4399999999999994e-06 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.041 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.253881+0000", 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9752999999999994e-05 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254056+0000", 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017469 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254223+0000", 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000167119 2024-09-21T10:06:31.042 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.256038+0000", 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001815636 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.256211+0000", 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017306299999999999 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019530+0000", 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76331833900000001 2024-09-21T10:06:31.043 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019584+0000", 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4001e-05 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019626+0000", 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2234999999999998e-05 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44779 7.d 7:b6c25ae2:::benchmark_data_smithi076_190198_object44778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:06:31.044 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.274122+0000", 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 287.72810781099997, 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74567194299999995, 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44779 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.045 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274122+0000", 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274122+0000", 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274123+0000", 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4270000000000001e-06 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.046 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274144+0000", 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0678000000000001e-05 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274155+0000", 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1663e-05 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274161+0000", 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7969999999999997e-06 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.047 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274208+0000", 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6783000000000003e-05 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274307+0000", 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9207000000000001e-05 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 0,3", 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274396+0000", 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8747999999999998e-05 2024-09-21T10:06:31.048 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.275753+0000", 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001357404 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.276113+0000", 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00035989000000000003 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019698+0000", 2024-09-21T10:06:31.049 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74358468099999997 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019758+0000", 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9676000000000003e-05 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019794+0000", 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5988999999999997e-05 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.050 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:61347 7.0 7:08f37e25:::benchmark_data_smithi076_190198_object61346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:20.446280+0000", 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 250.55594925099999, 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.153137675, 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 61347 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.051 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446280+0000", 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446280+0000", 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446282+0000", 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4869999999999999e-06 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.052 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446330+0000", 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.884e-05 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446344+0000", 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3603e-05 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446351+0000", 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8360000000000003e-06 2024-09-21T10:06:31.053 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446418+0000", 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6767000000000006e-05 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446546+0000", 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000128715 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.446682+0000", 2024-09-21T10:06:31.054 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013605799999999999 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.449292+0000", 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026097350000000002 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.452811+0000", 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0035187579999999999 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.055 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.597531+0000", 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.14472015999999999 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.597626+0000", 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5210999999999994e-05 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:20.599418+0000", 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017915050000000001 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.056 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:64436 7.0 7:048e100a:::benchmark_data_smithi076_190198_object64435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1158)", 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:27.101680+0000", 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 243.90054980400001, 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.14962550099999999, 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 64436 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.057 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.101680+0000", 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.101680+0000", 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.101680+0000", 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1500000000000004e-07 2024-09-21T10:06:31.058 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.101702+0000", 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1353999999999999e-05 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.101712+0000", 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0677e-05 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.101719+0000", 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7129999999999999e-06 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.059 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.104663+0000", 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029440920000000001 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.104859+0000", 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000196152 2024-09-21T10:06:31.060 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.105052+0000", 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000192215 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.107577+0000", 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025253810000000002 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.061 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.110581+0000", 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030037499999999999 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.249190+0000", 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.138609555 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.249339+0000", 2024-09-21T10:06:31.062 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000148753 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:27.251305+0000", 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001966144 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88593 7.0 7:03e88ef7:::benchmark_data_smithi076_190198_object16604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.016133+0000", 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.98609673000001, 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96461651299999995, 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.063 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88593 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016133+0000", 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.064 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016133+0000", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016135+0000", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3209999999999999e-06 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016143+0000", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.259e-06 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.065 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016158+0000", 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4630000000000001e-05 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016167+0000", 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8270000000000004e-06 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016254+0000", 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7548999999999998e-05 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.066 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.016269+0000", 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4600000000000001e-05 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.965274+0000", 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.949005031 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.965377+0000", 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010261 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.067 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.965469+0000", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2465000000000003e-05 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.970517+0000", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050480550000000001 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.970554+0000", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6519000000000001e-05 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.980648+0000", 2024-09-21T10:06:31.068 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010094031 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.980700+0000", 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1928999999999997e-05 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.980749+0000", 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9687e-05 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.069 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88600 7.0 7:069a9e89:::benchmark_data_smithi076_190198_object16611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.061508+0000", 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.94072171400001, 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92479728400000005, 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88600 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.070 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061508+0000", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061508+0000", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061510+0000", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3070000000000001e-06 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.071 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061516+0000", 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4520000000000002e-06 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061533+0000", 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6816999999999998e-05 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061542+0000", 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5299999999999996e-06 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.072 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061619+0000", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7406999999999999e-05 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.061632+0000", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2850999999999999e-05 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.968292+0000", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90665999900000005 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.968412+0000", 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000119978 2024-09-21T10:06:31.073 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.968529+0000", 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116722 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.975832+0000", 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073028199999999998 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.074 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.975920+0000", 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8373000000000002e-05 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986203+0000", 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010282909999999999 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986259+0000", 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6270999999999997e-05 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.075 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986305+0000", 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5847000000000003e-05 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88606 7.0 7:0bff37ee:::benchmark_data_smithi076_190198_object16617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.084573+0000", 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.91765643100001, 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.901832313, 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.076 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88606 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084573+0000", 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084573+0000", 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.077 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084575+0000", 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.722e-06 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084580+0000", 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2730000000000002e-06 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084593+0000", 2024-09-21T10:06:31.078 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.309e-05 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084602+0000", 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6610000000000006e-06 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084673+0000", 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1100999999999996e-05 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.079 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.084684+0000", 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1674999999999999e-05 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.970581+0000", 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.885896236 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.970679+0000", 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8346999999999999e-05 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.080 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.970795+0000", 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011641699999999999 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.975868+0000", 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050728839999999997 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983115+0000", 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072466980000000002 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.081 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986314+0000", 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031993709999999999 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986364+0000", 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9974e-05 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986405+0000", 2024-09-21T10:06:31.082 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0864e-05 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88622 7.0 7:091d98e7:::benchmark_data_smithi076_190198_object16633:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.221371+0000", 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.78085856600001, 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76514521300000005, 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.083 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88622 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221371+0000", 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221371+0000", 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.084 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221372+0000", 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.466e-06 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221379+0000", 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5710000000000003e-06 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221395+0000", 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5807999999999998e-05 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.085 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221405+0000", 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0074e-05 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221469+0000", 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4633000000000003e-05 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.221478+0000", 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9109999999999999e-06 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.086 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.973289+0000", 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75181078599999995 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.973409+0000", 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000120129 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.973523+0000", 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011369 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.087 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.978499+0000", 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0049756419999999997 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983151+0000", 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046520279999999999 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986415+0000", 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003264234 2024-09-21T10:06:31.088 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986459+0000", 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4431000000000002e-05 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986516+0000", 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.681e-05 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88626 7.0 7:022e461f:::benchmark_data_smithi076_190198_object16637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.089 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.261628+0000", 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.74060110900001, 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.72503645699999997, 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88626 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261628+0000", 2024-09-21T10:06:31.090 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261628+0000", 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261630+0000", 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.31e-06 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.091 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261634+0000", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6469999999999997e-06 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261645+0000", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1107000000000001e-05 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261651+0000", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.339e-06 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261712+0000", 2024-09-21T10:06:31.092 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0986000000000002e-05 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.261723+0000", 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1058e-05 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.975946+0000", 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.71422372300000003 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.976052+0000", 2024-09-21T10:06:31.093 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105973 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.976171+0000", 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011893800000000001 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983076+0000", 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069049410000000004 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.094 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986572+0000", 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034955120000000001 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986584+0000", 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1697e-05 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986628+0000", 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4817999999999997e-05 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.095 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986665+0000", 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6408000000000001e-05 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88648 7.0 7:0bd34903:::benchmark_data_smithi076_190198_object16659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.454868+0000", 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.54736175100001, 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.53196155000000001, 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.096 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88648 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454868+0000", 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.097 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454868+0000", 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454870+0000", 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3e-06 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454877+0000", 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5089999999999996e-06 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.098 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454895+0000", 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7067000000000001e-05 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454904+0000", 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1090000000000004e-06 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454983+0000", 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9635999999999994e-05 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.099 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.100 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.100 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.454993+0000", 2024-09-21T10:06:31.100 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.009e-05 2024-09-21T10:06:31.100 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.100 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.978532+0000", 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.52353855900000001 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.978657+0000", 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012553 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.101 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.978789+0000", 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013192600000000001 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983187+0000", 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043975630000000002 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986676+0000", 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034886129999999998 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.102 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986744+0000", 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8326999999999998e-05 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986794+0000", 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0132e-05 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986829+0000", 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5188999999999998e-05 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.103 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88657 7.0 7:06e4a79c:::benchmark_data_smithi076_190198_object16668:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.533247+0000", 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.46898254999999, 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.45968220399999998, 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.104 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88657 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533247+0000", 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533247+0000", 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.105 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533249+0000", 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0329999999999998e-06 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533255+0000", 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3029999999999996e-06 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533270+0000", 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4371e-05 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.106 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533277+0000", 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8800000000000008e-06 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533353+0000", 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5481000000000005e-05 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.533366+0000", 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.311e-05 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.107 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.980786+0000", 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.44741956100000002 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.980920+0000", 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013425399999999999 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.981063+0000", 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014310000000000001 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.108 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986084+0000", 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0050214140000000001 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986680+0000", 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00059611099999999997 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.992716+0000", 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0060351270000000004 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.109 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.992766+0000", 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0183999999999999e-05 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.992929+0000", 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000163275 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88661 7.0 7:09f04804:::benchmark_data_smithi076_190198_object16672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.110 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.609869+0000", 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.39236042799999, 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.38339572100000002, 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88661 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.111 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609869+0000", 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609869+0000", 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609870+0000", 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.474e-06 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.112 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609877+0000", 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0549999999999996e-06 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609890+0000", 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3345e-05 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609897+0000", 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2520000000000002e-06 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.113 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609971+0000", 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3796999999999998e-05 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:31.114 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:31.115 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:31.115 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.609980+0000", 2024-09-21T10:06:31.115 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0629999999999995e-06 2024-09-21T10:06:31.115 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.115 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.115 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983214+0000", 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.37323426199999998 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983312+0000", 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7621000000000006e-05 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.983512+0000", 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000200504 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.116 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986685+0000", 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031724230000000002 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.986965+0000", 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00028067700000000001 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.993182+0000", 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0062164910000000002 2024-09-21T10:06:31.117 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.993225+0000", 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2895000000000002e-05 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.993265+0000", 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9861999999999999e-05 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.118 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:88667 7.0 7:085090a4:::benchmark_data_smithi076_190198_object16678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:03:56.761901+0000", 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 154.24032861699999, 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.23407349, 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 88667 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.119 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761901+0000", 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761901+0000", 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761902+0000", 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.59e-06 2024-09-21T10:06:31.120 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761909+0000", 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4930000000000003e-06 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761928+0000", 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9301e-05 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761937+0000", 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5229999999999995e-06 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.121 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.761996+0000", 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9035e-05 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.762004+0000", 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6349999999999995e-06 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.987004+0000", 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.22500007299999999 2024-09-21T10:06:31.122 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.987098+0000", 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3503000000000002e-05 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.987197+0000", 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9518000000000005e-05 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.992674+0000", 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054767820000000004 2024-09-21T10:06:31.123 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.992685+0000", 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0596e-05 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.995849+0000", 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031644170000000001 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.995903+0000", 2024-09-21T10:06:31.124 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4129000000000003e-05 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:03:56.995974+0000", 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0895000000000005e-05 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:90383 7.5 7:a2d136d5:::benchmark_data_smithi076_190198_object18394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:08.323424+0000", 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 142.67880565300001, 2024-09-21T10:06:31.125 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.15385015499999999, 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 90383 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323424+0000", 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.126 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323424+0000", 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323425+0000", 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6169999999999999e-06 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323432+0000", 2024-09-21T10:06:31.127 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2580000000000001e-06 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323447+0000", 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5149e-05 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323455+0000", 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6440000000000006e-06 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.128 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323521+0000", 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5592999999999994e-05 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323702+0000", 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000180509 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.323886+0000", 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018407300000000001 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.129 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.326086+0000", 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021999960000000001 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.326572+0000", 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00048669400000000003 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.475272+0000", 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.148699308 2024-09-21T10:06:31.130 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.475336+0000", 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4642999999999998e-05 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:08.477274+0000", 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001937671 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.131 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:90681 7.5 7:a9454d49:::benchmark_data_smithi076_190198_object18692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1176)", 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:04:09.680766+0000", 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 141.32146332600001, 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.16505747800000001, 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 90681 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.132 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680766+0000", 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680766+0000", 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680769+0000", 2024-09-21T10:06:31.133 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.497e-06 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680777+0000", 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4109999999999996e-06 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680798+0000", 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0556000000000001e-05 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:06:31.134 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680807+0000", 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6940000000000007e-06 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.680883+0000", 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5369000000000003e-05 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.681083+0000", 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020019100000000001 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.135 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.681265+0000", 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018173199999999999 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.684012+0000", 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027469949999999999 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.684207+0000", 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019573400000000001 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.136 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.844621+0000", 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.16041369899999999 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.844658+0000", 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6990999999999999e-05 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:04:09.845824+0000", 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001165609 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:06:31.137 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:06:31.138 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:06:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.139 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:31.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:31.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:31.269 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:31.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:31.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:31.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:31.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:31.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:31.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:31.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:31.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:31.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:31.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:31.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:31.895 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:31.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:31.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:32.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:32.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:32.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:31 smithi076 ceph-mon[106482]: pgmap v1555: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 27466/67677 objects degraded (40.584%); 2186/67677 objects misplaced (3.230%); 6.4 MiB/s, 152 objects/s recovering 2024-09-21T10:06:32.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:31 smithi076 ceph-mon[110963]: pgmap v1555: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 27466/67677 objects degraded (40.584%); 2186/67677 objects misplaced (3.230%); 6.4 MiB/s, 152 objects/s recovering 2024-09-21T10:06:32.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:32.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:32.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:32.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:31 smithi191 ceph-mon[102206]: pgmap v1555: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.4 GiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 248 op/s; 27466/67677 objects degraded (40.584%); 2186/67677 objects misplaced (3.230%); 6.4 MiB/s, 152 objects/s recovering 2024-09-21T10:06:32.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:32.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:32.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:06:32.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:32.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:32.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:32.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:32.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:32.775 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:32.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:32.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:32.905 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:32.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:33.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:33.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:33.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:33.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:33.254 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:33.281 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:33.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:33.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:33.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:33.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:06:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:06:33] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-09-21T10:06:33.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:33.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:33.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:33.775 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:33.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:33.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:33.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:34.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:34.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:34.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:33 smithi076 ceph-mon[106482]: pgmap v1556: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 27068/63564 objects degraded (42.584%); 2139/63564 objects misplaced (3.365%); 5.4 MiB/s, 87 objects/s recovering 2024-09-21T10:06:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:33 smithi076 ceph-mon[110963]: pgmap v1556: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 27068/63564 objects degraded (42.584%); 2139/63564 objects misplaced (3.365%); 5.4 MiB/s, 87 objects/s recovering 2024-09-21T10:06:34.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:34.150 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:34.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:34.273 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:34.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:33 smithi191 ceph-mon[102206]: pgmap v1556: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 27068/63564 objects degraded (42.584%); 2139/63564 objects misplaced (3.365%); 5.4 MiB/s, 87 objects/s recovering 2024-09-21T10:06:34.374 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:34.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:34.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:34.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:34.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:34.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:34.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:34.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:34.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:34.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:34.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:34.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:35.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:35.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:35.136 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:35.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:35.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:35.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:35.283 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:35.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:35.311 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 27068/63564 objects degraded (42.584%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:35.311 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 27068/63564 objects degraded (42.584%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:35.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 27068/63564 objects degraded (42.584%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:35.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:35.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:35.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:35.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:35.537 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:35.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:35.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:35.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:35.765 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:35.793 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:35.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:35.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:35.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:36.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:36.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:36.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:36.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:36.177 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:36.222 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:35 smithi076 ceph-mon[106482]: pgmap v1557: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 27068/62979 objects degraded (42.979%); 1928/62979 objects misplaced (3.061%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:06:36.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:35 smithi076 ceph-mon[110963]: pgmap v1557: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 27068/62979 objects degraded (42.979%); 1928/62979 objects misplaced (3.061%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:06:36.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:36.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:36.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:36.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:35 smithi191 ceph-mon[102206]: pgmap v1557: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 329 op/s; 27068/62979 objects degraded (42.979%); 1928/62979 objects misplaced (3.061%); 6.1 MiB/s, 97 objects/s recovering 2024-09-21T10:06:36.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:36.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:36.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:36.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:36.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:36.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:36.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:36.787 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:36.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:36.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:36.915 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:36.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:36.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:37.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:37.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:37.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:37.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:37.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:37.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:37.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:37.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:37.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:37.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:37.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:37.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:37.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:37.705 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:37.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:37.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:37.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:37.955 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:37.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:37.976 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:06:38.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:38.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:38.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:38.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:38.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:38.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:38.314 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:37 smithi076 ceph-mon[106482]: pgmap v1558: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 307 op/s; 27068/62979 objects degraded (42.979%); 1928/62979 objects misplaced (3.061%); 5.7 MiB/s, 90 objects/s recovering 2024-09-21T10:06:38.315 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3934542348' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:38.315 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:37 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:38.315 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:37 smithi076 ceph-mon[110963]: pgmap v1558: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 307 op/s; 27068/62979 objects degraded (42.979%); 1928/62979 objects misplaced (3.061%); 5.7 MiB/s, 90 objects/s recovering 2024-09-21T10:06:38.315 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3934542348' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:38.316 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:37 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:38.329 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:37 smithi191 ceph-mon[102206]: pgmap v1558: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.3 GiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 307 op/s; 27068/62979 objects degraded (42.979%); 1928/62979 objects misplaced (3.061%); 5.7 MiB/s, 90 objects/s recovering 2024-09-21T10:06:38.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3934542348' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:38.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:37 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:06:38.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:38.472 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:06:38.472 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.473 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:38.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:38.507 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:06:38.507 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:38.537 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:06:38.537 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:38.568 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:06:38.568 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:38.575 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:38.596 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:06:38.596 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:38.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:38.625 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:06:38.625 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:38.655 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:06:38.655 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:38.685 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:06:38.686 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:38.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:38.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:38.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:38.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:38.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:38.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:38.989 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:38.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:39.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:39.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:39.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:39.235 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:39.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:39.245 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:39.245 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:39.246 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[106482]: osdmap e1197: 8 total, 8 up, 5 in 2024-09-21T10:06:39.246 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3858294400' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:39.246 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:39.246 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:39.246 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[110963]: osdmap e1197: 8 total, 8 up, 5 in 2024-09-21T10:06:39.246 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3858294400' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:39.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:39.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:38 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:06:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:38 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:06:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:38 smithi191 ceph-mon[102206]: osdmap e1197: 8 total, 8 up, 5 in 2024-09-21T10:06:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3858294400' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:39.367 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:39.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:39.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:39.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:39.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:39.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:39.625 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:40.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:39 smithi076 ceph-mon[106482]: pgmap v1560: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 333 op/s; 26571/57681 objects degraded (46.065%); 1609/57681 objects misplaced (2.789%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:06:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 26571/57681 objects degraded (46.065%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:40 smithi076 ceph-mon[110963]: pgmap v1560: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 333 op/s; 26571/57681 objects degraded (46.065%); 1609/57681 objects misplaced (2.789%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:06:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:40 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 26571/57681 objects degraded (46.065%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:40.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:40.220 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:40.263 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:39 smithi191 ceph-mon[102206]: pgmap v1560: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 333 op/s; 26571/57681 objects degraded (46.065%); 1609/57681 objects misplaced (2.789%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:06:40.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 26571/57681 objects degraded (46.065%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:40.322 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:40.350 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:40.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:40.478 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:40.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:40.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:40.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:40.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:40.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:40.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:40.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:40.752 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:40.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:40.863 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:40.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:40.965 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:40.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:40.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:40.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:41.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:41.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:41.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:41.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:41.254 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:41.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:41.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:41.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:41.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:41.481 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:41.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:41.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:41.642 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:41.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:41.773 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:41.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:41.845 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:41.845 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:41.845 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:41 smithi076 ceph-mon[106482]: pgmap v1561: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 333 op/s; 26571/57681 objects degraded (46.065%); 1609/57681 objects misplaced (2.789%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:06:41.845 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:41.845 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:41.845 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:41 smithi076 ceph-mon[110963]: pgmap v1561: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 333 op/s; 26571/57681 objects degraded (46.065%); 1609/57681 objects misplaced (2.789%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:06:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:41 smithi191 ceph-mon[102206]: pgmap v1561: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.2 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 333 op/s; 26571/57681 objects degraded (46.065%); 1609/57681 objects misplaced (2.789%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:06:41.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:41.899 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:41.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:41.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:42.001 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:42.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:42.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:42.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:42.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:06:42.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:42.276 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:42.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:42.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:42.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:42.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:42.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:42.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:42.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:42.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:06:42.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:42.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:42.916 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:42.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:43.018 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:43.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:43.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:43.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:43.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:43.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:43.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:43.395 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:06:43.396 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:06:43.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:43.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:43.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:43.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:43.577 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:43.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:06:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:06:43] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:06:43.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:43.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:43.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:43.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:43.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:43.934 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:43.961 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:43.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:43.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:43 smithi076 ceph-mon[106482]: pgmap v1562: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.1 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 26097/52245 objects degraded (49.951%); 1492/52245 objects misplaced (2.856%); 7.1 MiB/s, 112 objects/s recovering 2024-09-21T10:06:43.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:43 smithi076 ceph-mon[106482]: from='client.35325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:43.978 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:43 smithi076 ceph-mon[110963]: pgmap v1562: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.1 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 26097/52245 objects degraded (49.951%); 1492/52245 objects misplaced (2.856%); 7.1 MiB/s, 112 objects/s recovering 2024-09-21T10:06:43.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:43 smithi076 ceph-mon[110963]: from='client.35325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:44.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:44.087 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:43 smithi191 ceph-mon[102206]: pgmap v1562: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.1 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 26097/52245 objects degraded (49.951%); 1492/52245 objects misplaced (2.856%); 7.1 MiB/s, 112 objects/s recovering 2024-09-21T10:06:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:43 smithi191 ceph-mon[102206]: from='client.35325 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:06:44.127 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:06:44.127 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:06:44.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:44.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:44.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.350 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:44.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.451 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:44.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:44.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:44.600 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:44.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:44.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:44.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:44.826 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 26097/52245 objects degraded (49.951%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:44.826 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 26097/52245 objects degraded (49.951%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:44.837 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:44.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:06:44.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:44.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:44.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:44.975 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:44.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:44.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:45.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 26097/52245 objects degraded (49.951%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:45.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:45.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:45.247 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:45.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:45.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:45.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:45.497 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:45.599 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:45.625 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:45.656 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1562,"stamp":"2024-09-21T10:06:42.654976+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1140135933,"num_objects":17415,"num_object_clones":0,"num_object_copies":52245,"num_objects_missing_on_primary":2726,"num_objects_missing":2726,"num_objects_degraded":26097,"num_objects_misplaced":1492,"num_objects_unfound":0,"num_objects_dirty":17415,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":126864,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41033,"num_bytes_recovered":2315269662,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77159,"ondisk_log_size":77159,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4922496,"kb_used_data":3239952,"kb_used_omap":119,"kb_used_meta":1682376,"kb_avail":463803264,"statfs":{"total":479975178240,"available":474934542336,"internally_reserved":0,"allocated":3317710848,"data_stored":3305081189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1722753224},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-247267328,"num_objects":-3773,"num_object_clones":0,"num_object_copies":-11319,"num_objects_missing_on_primary":-698,"num_objects_missing":-698,"num_objects_degraded":-971,"num_objects_misplaced":-647,"num_objects_unfound":0,"num_objects_dirty":-3773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3773,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1130,"num_bytes_recovered":74055680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002763"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":231,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.978577+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:06:37.978577+0000","last_peered":"2024-09-21T10:06:37.978577+0000","last_clean":"2024-09-21T10:06:37.978577+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:06:37.978577+0000","last_undegraded":"2024-09-21T10:06:37.978577+0000","last_fullsized":"2024-09-21T10:06:37.978577+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":53,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955376+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:06:25.955376+0000","last_peered":"2024-09-21T10:06:25.955376+0000","last_clean":"2024-09-21T10:06:25.955376+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:06:25.955376+0000","last_undegraded":"2024-09-21T10:06:25.955376+0000","last_fullsized":"2024-09-21T10:06:25.955376+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1215,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.981228+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:06:37.981228+0000","last_peered":"2024-09-21T10:06:37.981228+0000","last_clean":"2024-09-21T10:06:37.981228+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:06:37.981228+0000","last_undegraded":"2024-09-21T10:06:37.981228+0000","last_fullsized":"2024-09-21T10:06:37.981228+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1547,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.977837+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:06:37.977837+0000","last_peered":"2024-09-21T10:06:37.977837+0000","last_clean":"2024-09-21T10:06:37.977837+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:06:37.977837+0000","last_undegraded":"2024-09-21T10:06:37.977837+0000","last_fullsized":"2024-09-21T10:06:37.977837+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1547,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:38.001284+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:06:38.001284+0000","last_peered":"2024-09-21T10:06:38.001284+0000","last_clean":"2024-09-21T10:06:38.001284+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:06:38.001284+0000","last_undegraded":"2024-09-21T10:06:38.001284+0000","last_fullsized":"2024-09-21T10:06:38.001284+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.958806+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:06:25.958806+0000","last_peered":"2024-09-21T10:06:25.958806+0000","last_clean":"2024-09-21T10:06:25.958806+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:06:25.958806+0000","last_undegraded":"2024-09-21T10:06:25.958806+0000","last_fullsized":"2024-09-21T10:06:25.958806+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":231,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.983149+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:06:37.983149+0000","last_peered":"2024-09-21T10:06:37.983149+0000","last_clean":"2024-09-21T10:06:37.983149+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:06:37.983149+0000","last_undegraded":"2024-09-21T10:06:37.983149+0000","last_fullsized":"2024-09-21T10:06:37.983149+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":76,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.954813+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:06:25.954813+0000","last_peered":"2024-09-21T10:06:25.954813+0000","last_clean":"2024-09-21T10:06:25.954813+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:06:25.954813+0000","last_undegraded":"2024-09-21T10:06:25.954813+0000","last_fullsized":"2024-09-21T10:06:25.954813+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1197'7941","reported_seq":16805,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:42.238158+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:06:42.238158+0000","last_peered":"2024-09-21T10:06:42.238158+0000","last_clean":"2024-09-21T10:06:42.238158+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:06:42.238158+0000","last_undegraded":"2024-09-21T10:06:42.238158+0000","last_fullsized":"2024-09-21T10:06:42.238158+0000","mapping_epoch":1187,"log_start":"1148'3231","ondisk_log_start":"1148'3231","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4710,"log_dups_size":0,"ondisk_log_size":4710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72417280,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7941,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":86,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.973633+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:06:37.973633+0000","last_peered":"2024-09-21T10:06:37.973633+0000","last_clean":"2024-09-21T10:06:37.973633+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:06:37.973633+0000","last_undegraded":"2024-09-21T10:06:37.973633+0000","last_fullsized":"2024-09-21T10:06:37.973633+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":47,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955011+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:06:25.955011+0000","last_peered":"2024-09-21T10:06:25.955011+0000","last_clean":"2024-09-21T10:06:25.955011+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:06:25.955011+0000","last_undegraded":"2024-09-21T10:06:25.955011+0000","last_fullsized":"2024-09-21T10:06:25.955011+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1197'7973","reported_seq":15691,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:42.231002+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:06:42.231002+0000","last_peered":"2024-09-21T10:06:42.231002+0000","last_clean":"2024-09-21T10:06:42.231002+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:06:42.231002+0000","last_undegraded":"2024-09-21T10:06:42.231002+0000","last_fullsized":"2024-09-21T10:06:42.231002+0000","mapping_epoch":1180,"log_start":"1152'3403","ondisk_log_start":"1152'3403","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7973,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":130,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.971537+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:06:37.971537+0000","last_peered":"2024-09-21T10:06:37.971537+0000","last_clean":"2024-09-21T10:06:37.971537+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:06:37.971537+0000","last_undegraded":"2024-09-21T10:06:37.971537+0000","last_fullsized":"2024-09-21T10:06:37.971537+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1171,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970194+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:06:37.970194+0000","last_peered":"2024-09-21T10:06:37.970194+0000","last_clean":"2024-09-21T10:06:37.970194+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:06:37.970194+0000","last_undegraded":"2024-09-21T10:06:37.970194+0000","last_fullsized":"2024-09-21T10:06:37.970194+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1197'8089","reported_seq":8226,"reported_epoch":1197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.369816+0000","last_change":"2024-09-21T10:03:30.139654+0000","last_active":"2024-09-21T10:06:42.369816+0000","last_peered":"2024-09-21T10:06:42.369816+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:06:42.369816+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1152'3382","ondisk_log_start":"1152'3382","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73334784,"num_objects":1119,"num_object_clones":0,"num_object_copies":3357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8089,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1119},{"shards":"1,4","objects":-2620}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":51,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955890+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:06:25.955890+0000","last_peered":"2024-09-21T10:06:25.955890+0000","last_clean":"2024-09-21T10:06:25.955890+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:06:25.955890+0000","last_undegraded":"2024-09-21T10:06:25.955890+0000","last_fullsized":"2024-09-21T10:06:25.955890+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":56,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955190+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:06:25.955190+0000","last_peered":"2024-09-21T10:06:25.955190+0000","last_clean":"2024-09-21T10:06:25.955190+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:06:25.955190+0000","last_undegraded":"2024-09-21T10:06:25.955190+0000","last_fullsized":"2024-09-21T10:06:25.955190+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1197'7882","reported_seq":13538,"reported_epoch":1197,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:06:42.275604+0000","last_change":"2024-09-21T10:06:10.765843+0000","last_active":"2024-09-21T10:06:42.275604+0000","last_peered":"2024-09-21T10:06:42.275604+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:06:42.275604+0000","last_undegraded":"2024-09-21T10:06:42.275604+0000","last_fullsized":"2024-09-21T10:06:42.275604+0000","mapping_epoch":1151,"log_start":"1152'3336","ondisk_log_start":"1152'3336","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4546,"log_dups_size":0,"ondisk_log_size":4546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72351744,"num_objects":1104,"num_object_clones":0,"num_object_copies":3312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2755,"num_bytes_recovered":180551680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":1104}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1201,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.977958+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:06:37.977958+0000","last_peered":"2024-09-21T10:06:37.977958+0000","last_clean":"2024-09-21T10:06:37.977958+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:06:37.977958+0000","last_undegraded":"2024-09-21T10:06:37.977958+0000","last_fullsized":"2024-09-21T10:06:37.977958+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1557,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970639+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:06:37.970639+0000","last_peered":"2024-09-21T10:06:37.970639+0000","last_clean":"2024-09-21T10:06:37.970639+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:06:37.970639+0000","last_undegraded":"2024-09-21T10:06:37.970639+0000","last_fullsized":"2024-09-21T10:06:37.970639+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1197'7913","reported_seq":13679,"reported_epoch":1197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.144413+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:06:42.144413+0000","last_peered":"2024-09-21T10:06:42.144413+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:06:42.144413+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1152'3348","ondisk_log_start":"1152'3348","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4565,"log_dups_size":0,"ondisk_log_size":4565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72155136,"num_objects":1101,"num_object_clones":0,"num_object_copies":3303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7913,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"0,3,4","objects":1101},{"shards":"3,4","objects":-2520}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":239,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.980694+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:06:37.980694+0000","last_peered":"2024-09-21T10:06:37.980694+0000","last_clean":"2024-09-21T10:06:37.980694+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:06:37.980694+0000","last_undegraded":"2024-09-21T10:06:37.980694+0000","last_fullsized":"2024-09-21T10:06:37.980694+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1216,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.977828+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:06:37.977828+0000","last_peered":"2024-09-21T10:06:37.977828+0000","last_clean":"2024-09-21T10:06:37.977828+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:06:37.977828+0000","last_undegraded":"2024-09-21T10:06:37.977828+0000","last_fullsized":"2024-09-21T10:06:37.977828+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1197'7800","reported_seq":15458,"reported_epoch":1197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.296253+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:06:42.296253+0000","last_peered":"2024-09-21T10:06:42.296253+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:06:42.296253+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1149'3253","ondisk_log_start":"1149'3253","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4547,"log_dups_size":0,"ondisk_log_size":4547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70254592,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":1169,"num_objects_missing":1169,"num_objects_degraded":3103,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7800,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2504,"num_bytes_recovered":164102144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,3","objects":749},{"shards":"0,2,3,5","objects":16},{"shards":"0,2,5","objects":307}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":47,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955035+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:06:25.955035+0000","last_peered":"2024-09-21T10:06:25.955035+0000","last_clean":"2024-09-21T10:06:25.955035+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:06:25.955035+0000","last_undegraded":"2024-09-21T10:06:25.955035+0000","last_fullsized":"2024-09-21T10:06:25.955035+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":859,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.973445+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:06:37.973445+0000","last_peered":"2024-09-21T10:06:37.973445+0000","last_clean":"2024-09-21T10:06:37.973445+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:06:37.973445+0000","last_undegraded":"2024-09-21T10:06:37.973445+0000","last_fullsized":"2024-09-21T10:06:37.973445+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1197'7899","reported_seq":18054,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.976602+0000","last_change":"2024-09-21T10:06:23.957394+0000","last_active":"2024-09-21T10:06:37.976602+0000","last_peered":"2024-09-21T10:06:37.976602+0000","last_clean":"2024-09-21T10:06:37.976602+0000","last_became_active":"2024-09-21T10:06:23.937063+0000","last_became_peered":"2024-09-21T10:06:23.937063+0000","last_unstale":"2024-09-21T10:06:37.976602+0000","last_undegraded":"2024-09-21T10:06:37.976602+0000","last_fullsized":"2024-09-21T10:06:37.976602+0000","mapping_epoch":1194,"log_start":"1143'1800","ondisk_log_start":"1143'1800","created":1137,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6099,"log_dups_size":0,"ondisk_log_size":6099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:57.739037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82444312,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7899,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4579,"num_bytes_recovered":242810904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1202,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.969990+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:06:37.969990+0000","last_peered":"2024-09-21T10:06:37.969990+0000","last_clean":"2024-09-21T10:06:37.969990+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:06:37.969990+0000","last_undegraded":"2024-09-21T10:06:37.969990+0000","last_fullsized":"2024-09-21T10:06:37.969990+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":47,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955769+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:06:25.955769+0000","last_peered":"2024-09-21T10:06:25.955769+0000","last_clean":"2024-09-21T10:06:25.955769+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:06:25.955769+0000","last_undegraded":"2024-09-21T10:06:25.955769+0000","last_fullsized":"2024-09-21T10:06:25.955769+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1197'8199","reported_seq":14754,"reported_epoch":1197,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.235143+0000","last_change":"2024-09-21T10:05:36.090756+0000","last_active":"2024-09-21T10:06:42.235143+0000","last_peered":"2024-09-21T10:06:42.235143+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:06:42.235143+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"1152'3625","ondisk_log_start":"1152'3625","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4574,"log_dups_size":0,"ondisk_log_size":4574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71892992,"num_objects":1097,"num_object_clones":0,"num_object_copies":3291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2396,"num_objects_misplaced":655,"num_objects_unfound":0,"num_objects_dirty":1097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8199,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3205,"num_bytes_recovered":210042880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"0,2,3","objects":655},{"shards":"2,3","objects":-1954}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":231,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.978171+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:06:37.978171+0000","last_peered":"2024-09-21T10:06:37.978171+0000","last_clean":"2024-09-21T10:06:37.978171+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:06:37.978171+0000","last_undegraded":"2024-09-21T10:06:37.978171+0000","last_fullsized":"2024-09-21T10:06:37.978171+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1561,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.975067+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:06:37.975067+0000","last_peered":"2024-09-21T10:06:37.975067+0000","last_clean":"2024-09-21T10:06:37.975067+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:06:37.975067+0000","last_undegraded":"2024-09-21T10:06:37.975067+0000","last_fullsized":"2024-09-21T10:06:37.975067+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1197'7821","reported_seq":17206,"reported_epoch":1197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.224371+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:06:42.224371+0000","last_peered":"2024-09-21T10:06:42.224371+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:06:42.224371+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1149'3247","ondisk_log_start":"1149'3247","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4574,"log_dups_size":0,"ondisk_log_size":4574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2648,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7821,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3355,"num_bytes_recovered":219873280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,2,3","objects":732},{"shards":"0,3,5","objects":287}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":720,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.954726+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:06:25.954726+0000","last_peered":"2024-09-21T10:06:25.954726+0000","last_clean":"2024-09-21T10:06:25.954726+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:06:25.954726+0000","last_undegraded":"2024-09-21T10:06:25.954726+0000","last_fullsized":"2024-09-21T10:06:25.954726+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1215,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.980441+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:06:37.980441+0000","last_peered":"2024-09-21T10:06:37.980441+0000","last_clean":"2024-09-21T10:06:37.980441+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:06:37.980441+0000","last_undegraded":"2024-09-21T10:06:37.980441+0000","last_fullsized":"2024-09-21T10:06:37.980441+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":245,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.978010+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:06:37.978010+0000","last_peered":"2024-09-21T10:06:37.978010+0000","last_clean":"2024-09-21T10:06:37.978010+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:06:37.978010+0000","last_undegraded":"2024-09-21T10:06:37.978010+0000","last_fullsized":"2024-09-21T10:06:37.978010+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":56,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955836+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:06:25.955836+0000","last_peered":"2024-09-21T10:06:25.955836+0000","last_clean":"2024-09-21T10:06:25.955836+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:06:25.955836+0000","last_undegraded":"2024-09-21T10:06:25.955836+0000","last_fullsized":"2024-09-21T10:06:25.955836+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1197'7911","reported_seq":13552,"reported_epoch":1197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.172639+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:06:42.172639+0000","last_peered":"2024-09-21T10:06:42.172639+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:06:42.172639+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1148'3154","ondisk_log_start":"1148'3154","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4757,"log_dups_size":0,"ondisk_log_size":4757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7911,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"0,1,2","objects":1035},{"shards":"1,2","objects":-2570}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":845,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.969749+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:06:37.969749+0000","last_peered":"2024-09-21T10:06:37.969749+0000","last_clean":"2024-09-21T10:06:37.969749+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:06:37.969749+0000","last_undegraded":"2024-09-21T10:06:37.969749+0000","last_fullsized":"2024-09-21T10:06:37.969749+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1434,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.979473+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:06:37.979473+0000","last_peered":"2024-09-21T10:06:37.979473+0000","last_clean":"2024-09-21T10:06:37.979473+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:06:37.979473+0000","last_undegraded":"2024-09-21T10:06:37.979473+0000","last_fullsized":"2024-09-21T10:06:37.979473+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":843,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.971349+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:06:37.971349+0000","last_peered":"2024-09-21T10:06:37.971349+0000","last_clean":"2024-09-21T10:06:37.971349+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:06:37.971349+0000","last_undegraded":"2024-09-21T10:06:37.971349+0000","last_fullsized":"2024-09-21T10:06:37.971349+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1197'7588","reported_seq":16356,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:42.243816+0000","last_change":"2024-09-21T10:05:37.849809+0000","last_active":"2024-09-21T10:06:42.243816+0000","last_peered":"2024-09-21T10:06:42.243816+0000","last_clean":"2024-09-21T10:06:42.243816+0000","last_became_active":"2024-09-21T10:05:37.830539+0000","last_became_peered":"2024-09-21T10:05:37.830539+0000","last_unstale":"2024-09-21T10:06:42.243816+0000","last_undegraded":"2024-09-21T10:06:42.243816+0000","last_fullsized":"2024-09-21T10:06:42.243816+0000","mapping_epoch":1190,"log_start":"1148'2837","ondisk_log_start":"1148'2837","created":1137,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:35.172792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67764224,"num_objects":1034,"num_object_clones":0,"num_object_copies":3102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7588,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4311,"num_bytes_recovered":174260224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1475,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.979723+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:06:37.979723+0000","last_peered":"2024-09-21T10:06:37.979723+0000","last_clean":"2024-09-21T10:06:37.979723+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:06:37.979723+0000","last_undegraded":"2024-09-21T10:06:37.979723+0000","last_fullsized":"2024-09-21T10:06:37.979723+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1562,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970004+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:06:37.970004+0000","last_peered":"2024-09-21T10:06:37.970004+0000","last_clean":"2024-09-21T10:06:37.970004+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:06:37.970004+0000","last_undegraded":"2024-09-21T10:06:37.970004+0000","last_fullsized":"2024-09-21T10:06:37.970004+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1569,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.973790+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:06:37.973790+0000","last_peered":"2024-09-21T10:06:37.973790+0000","last_clean":"2024-09-21T10:06:37.973790+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:06:37.973790+0000","last_undegraded":"2024-09-21T10:06:37.973790+0000","last_fullsized":"2024-09-21T10:06:37.973790+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1197'7828","reported_seq":13581,"reported_epoch":1197,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-21T10:06:38.094873+0000","last_change":"2024-09-21T10:06:22.125588+0000","last_active":"2024-09-21T10:06:38.094873+0000","last_peered":"2024-09-21T10:06:38.094873+0000","last_clean":"2024-09-21T10:02:38.308681+0000","last_became_active":"2024-09-21T10:03:36.318082+0000","last_became_peered":"2024-09-21T10:03:36.318082+0000","last_unstale":"2024-09-21T10:06:38.094873+0000","last_undegraded":"2024-09-21T10:06:38.094873+0000","last_fullsized":"2024-09-21T10:03:29.833927+0000","mapping_epoch":1170,"log_start":"1143'1800","ondisk_log_start":"1143'1800","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6028,"log_dups_size":0,"ondisk_log_size":6028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:52.296275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77987840,"num_objects":1190,"num_object_clones":0,"num_object_copies":3570,"num_objects_missing_on_primary":1557,"num_objects_missing":1557,"num_objects_degraded":0,"num_objects_misplaced":837,"num_objects_unfound":0,"num_objects_dirty":1190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7828,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2820,"num_bytes_recovered":184811520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,1,3","objects":705},{"shards":"0,3,6","objects":132},{"shards":"3,6","objects":353}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":139,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970279+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:06:37.970279+0000","last_peered":"2024-09-21T10:06:37.970279+0000","last_clean":"2024-09-21T10:06:37.970279+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:06:37.970279+0000","last_undegraded":"2024-09-21T10:06:37.970279+0000","last_fullsized":"2024-09-21T10:06:37.970279+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1158,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955226+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:06:25.955226+0000","last_peered":"2024-09-21T10:06:25.955226+0000","last_clean":"2024-09-21T10:06:25.955226+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:06:25.955226+0000","last_undegraded":"2024-09-21T10:06:25.955226+0000","last_fullsized":"2024-09-21T10:06:25.955226+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1573,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.972552+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:06:37.972552+0000","last_peered":"2024-09-21T10:06:37.972552+0000","last_clean":"2024-09-21T10:06:37.972552+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:06:37.972552+0000","last_undegraded":"2024-09-21T10:06:37.972552+0000","last_fullsized":"2024-09-21T10:06:37.972552+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1197'7969","reported_seq":8098,"reported_epoch":1197,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:06:42.215756+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:06:42.215756+0000","last_peered":"2024-09-21T10:06:42.215756+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:06:42.215756+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1152'3385","ondisk_log_start":"1152'3385","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4584,"log_dups_size":0,"ondisk_log_size":4584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7969,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":920},{"shards":"3,4","objects":-3292}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1163,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.981362+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:06:37.981362+0000","last_peered":"2024-09-21T10:06:37.981362+0000","last_clean":"2024-09-21T10:06:37.981362+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:06:37.981362+0000","last_undegraded":"2024-09-21T10:06:37.981362+0000","last_fullsized":"2024-09-21T10:06:37.981362+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1584,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.972652+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:06:37.972652+0000","last_peered":"2024-09-21T10:06:37.972652+0000","last_clean":"2024-09-21T10:06:37.972652+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:06:37.972652+0000","last_undegraded":"2024-09-21T10:06:37.972652+0000","last_fullsized":"2024-09-21T10:06:37.972652+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":103,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.969949+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:06:37.969949+0000","last_peered":"2024-09-21T10:06:37.969949+0000","last_clean":"2024-09-21T10:06:37.969949+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:06:37.969949+0000","last_undegraded":"2024-09-21T10:06:37.969949+0000","last_fullsized":"2024-09-21T10:06:37.969949+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1197'7841","reported_seq":15444,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:42.148132+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:06:42.148132+0000","last_peered":"2024-09-21T10:06:42.148132+0000","last_clean":"2024-09-21T10:06:42.148132+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:06:42.148132+0000","last_undegraded":"2024-09-21T10:06:42.148132+0000","last_fullsized":"2024-09-21T10:06:42.148132+0000","mapping_epoch":1174,"log_start":"1148'3094","ondisk_log_start":"1148'3094","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4747,"log_dups_size":0,"ondisk_log_size":4747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67829760,"num_objects":1035,"num_object_clones":0,"num_object_copies":3105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7841,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1561,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.973002+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:06:37.973002+0000","last_peered":"2024-09-21T10:06:37.973002+0000","last_clean":"2024-09-21T10:06:37.973002+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:06:37.973002+0000","last_undegraded":"2024-09-21T10:06:37.973002+0000","last_fullsized":"2024-09-21T10:06:37.973002+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":741,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.973170+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:06:37.973170+0000","last_peered":"2024-09-21T10:06:37.973170+0000","last_clean":"2024-09-21T10:06:37.973170+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:06:37.973170+0000","last_undegraded":"2024-09-21T10:06:37.973170+0000","last_fullsized":"2024-09-21T10:06:37.973170+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1553,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970795+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:06:37.970795+0000","last_peered":"2024-09-21T10:06:37.970795+0000","last_clean":"2024-09-21T10:06:37.970795+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:06:37.970795+0000","last_undegraded":"2024-09-21T10:06:37.970795+0000","last_fullsized":"2024-09-21T10:06:37.970795+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1197'8128","reported_seq":8262,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:42.180011+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:06:42.180011+0000","last_peered":"2024-09-21T10:06:42.180011+0000","last_clean":"2024-09-21T10:06:42.180011+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:06:42.180011+0000","last_undegraded":"2024-09-21T10:06:42.180011+0000","last_fullsized":"2024-09-21T10:06:42.180011+0000","mapping_epoch":1137,"log_start":"1152'3526","ondisk_log_start":"1152'3526","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4602,"log_dups_size":0,"ondisk_log_size":4602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68419584,"num_objects":1044,"num_object_clones":0,"num_object_copies":3132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8128,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":261,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.983240+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:06:37.983240+0000","last_peered":"2024-09-21T10:06:37.983240+0000","last_clean":"2024-09-21T10:06:37.983240+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:06:37.983240+0000","last_undegraded":"2024-09-21T10:06:37.983240+0000","last_fullsized":"2024-09-21T10:06:37.983240+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1561,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.971564+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:06:37.971564+0000","last_peered":"2024-09-21T10:06:37.971564+0000","last_clean":"2024-09-21T10:06:37.971564+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:06:37.971564+0000","last_undegraded":"2024-09-21T10:06:37.971564+0000","last_fullsized":"2024-09-21T10:06:37.971564+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":238,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.978088+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:06:37.978088+0000","last_peered":"2024-09-21T10:06:37.978088+0000","last_clean":"2024-09-21T10:06:37.978088+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:06:37.978088+0000","last_undegraded":"2024-09-21T10:06:37.978088+0000","last_fullsized":"2024-09-21T10:06:37.978088+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1197'7803","reported_seq":7937,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:42.220429+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:06:42.220429+0000","last_peered":"2024-09-21T10:06:42.220429+0000","last_clean":"2024-09-21T10:06:42.220429+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:06:42.220429+0000","last_undegraded":"2024-09-21T10:06:42.220429+0000","last_fullsized":"2024-09-21T10:06:42.220429+0000","mapping_epoch":1137,"log_start":"1149'3213","ondisk_log_start":"1149'3213","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4590,"log_dups_size":0,"ondisk_log_size":4590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68485120,"num_objects":1045,"num_object_clones":0,"num_object_copies":3135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7803,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":102,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.971432+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:06:37.971432+0000","last_peered":"2024-09-21T10:06:37.971432+0000","last_clean":"2024-09-21T10:06:37.971432+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:06:37.971432+0000","last_undegraded":"2024-09-21T10:06:37.971432+0000","last_fullsized":"2024-09-21T10:06:37.971432+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":47,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.968604+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:06:25.968604+0000","last_peered":"2024-09-21T10:06:25.968604+0000","last_clean":"2024-09-21T10:06:25.968604+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:06:25.968604+0000","last_undegraded":"2024-09-21T10:06:25.968604+0000","last_fullsized":"2024-09-21T10:06:25.968604+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.958751+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:06:25.958751+0000","last_peered":"2024-09-21T10:06:25.958751+0000","last_clean":"2024-09-21T10:06:25.958751+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:06:25.958751+0000","last_undegraded":"2024-09-21T10:06:25.958751+0000","last_fullsized":"2024-09-21T10:06:25.958751+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":47,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955213+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:06:25.955213+0000","last_peered":"2024-09-21T10:06:25.955213+0000","last_clean":"2024-09-21T10:06:25.955213+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:06:25.955213+0000","last_undegraded":"2024-09-21T10:06:25.955213+0000","last_fullsized":"2024-09-21T10:06:25.955213+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":57,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955806+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:06:25.955806+0000","last_peered":"2024-09-21T10:06:25.955806+0000","last_clean":"2024-09-21T10:06:25.955806+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:06:25.955806+0000","last_undegraded":"2024-09-21T10:06:25.955806+0000","last_fullsized":"2024-09-21T10:06:25.955806+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1561,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970465+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:06:37.970465+0000","last_peered":"2024-09-21T10:06:37.970465+0000","last_clean":"2024-09-21T10:06:37.970465+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:06:37.970465+0000","last_undegraded":"2024-09-21T10:06:37.970465+0000","last_fullsized":"2024-09-21T10:06:37.970465+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1174,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.981309+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:06:37.981309+0000","last_peered":"2024-09-21T10:06:37.981309+0000","last_clean":"2024-09-21T10:06:37.981309+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:06:37.981309+0000","last_undegraded":"2024-09-21T10:06:37.981309+0000","last_fullsized":"2024-09-21T10:06:37.981309+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1184,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.971628+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:06:37.971628+0000","last_peered":"2024-09-21T10:06:37.971628+0000","last_clean":"2024-09-21T10:06:37.971628+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:06:37.971628+0000","last_undegraded":"2024-09-21T10:06:37.971628+0000","last_fullsized":"2024-09-21T10:06:37.971628+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":259,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.979643+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:06:37.979643+0000","last_peered":"2024-09-21T10:06:37.979643+0000","last_clean":"2024-09-21T10:06:37.979643+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:06:37.979643+0000","last_undegraded":"2024-09-21T10:06:37.979643+0000","last_fullsized":"2024-09-21T10:06:37.979643+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1587,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.982998+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:06:37.982998+0000","last_peered":"2024-09-21T10:06:37.982998+0000","last_clean":"2024-09-21T10:06:37.982998+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:06:37.982998+0000","last_undegraded":"2024-09-21T10:06:37.982998+0000","last_fullsized":"2024-09-21T10:06:37.982998+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":238,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.978326+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:06:37.978326+0000","last_peered":"2024-09-21T10:06:37.978326+0000","last_clean":"2024-09-21T10:06:37.978326+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:06:37.978326+0000","last_undegraded":"2024-09-21T10:06:37.978326+0000","last_fullsized":"2024-09-21T10:06:37.978326+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":56,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.962096+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:06:25.962096+0000","last_peered":"2024-09-21T10:06:25.962096+0000","last_clean":"2024-09-21T10:06:25.962096+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:06:25.962096+0000","last_undegraded":"2024-09-21T10:06:25.962096+0000","last_fullsized":"2024-09-21T10:06:25.962096+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":231,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.977994+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:06:37.977994+0000","last_peered":"2024-09-21T10:06:37.977994+0000","last_clean":"2024-09-21T10:06:37.977994+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:06:37.977994+0000","last_undegraded":"2024-09-21T10:06:37.977994+0000","last_fullsized":"2024-09-21T10:06:37.977994+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1549,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:38.001556+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:06:38.001556+0000","last_peered":"2024-09-21T10:06:38.001556+0000","last_clean":"2024-09-21T10:06:38.001556+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:06:38.001556+0000","last_undegraded":"2024-09-21T10:06:38.001556+0000","last_fullsized":"2024-09-21T10:06:38.001556+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1560,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970272+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:06:37.970272+0000","last_peered":"2024-09-21T10:06:37.970272+0000","last_clean":"2024-09-21T10:06:37.970272+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:06:37.970272+0000","last_undegraded":"2024-09-21T10:06:37.970272+0000","last_fullsized":"2024-09-21T10:06:37.970272+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.981400+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:06:37.981400+0000","last_peered":"2024-09-21T10:06:37.981400+0000","last_clean":"2024-09-21T10:06:37.981400+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:06:37.981400+0000","last_undegraded":"2024-09-21T10:06:37.981400+0000","last_fullsized":"2024-09-21T10:06:37.981400+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":107,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970615+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:06:37.970615+0000","last_peered":"2024-09-21T10:06:37.970615+0000","last_clean":"2024-09-21T10:06:37.970615+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:06:37.970615+0000","last_undegraded":"2024-09-21T10:06:37.970615+0000","last_fullsized":"2024-09-21T10:06:37.970615+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1444,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.972967+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:06:37.972967+0000","last_peered":"2024-09-21T10:06:37.972967+0000","last_clean":"2024-09-21T10:06:37.972967+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:06:37.972967+0000","last_undegraded":"2024-09-21T10:06:37.972967+0000","last_fullsized":"2024-09-21T10:06:37.972967+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1525,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.970714+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:06:37.970714+0000","last_peered":"2024-09-21T10:06:37.970714+0000","last_clean":"2024-09-21T10:06:37.970714+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:06:37.970714+0000","last_undegraded":"2024-09-21T10:06:37.970714+0000","last_fullsized":"2024-09-21T10:06:37.970714+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1169,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.980625+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:06:37.980625+0000","last_peered":"2024-09-21T10:06:37.980625+0000","last_clean":"2024-09-21T10:06:37.980625+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:06:37.980625+0000","last_undegraded":"2024-09-21T10:06:37.980625+0000","last_fullsized":"2024-09-21T10:06:37.980625+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":123,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.973065+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:06:37.973065+0000","last_peered":"2024-09-21T10:06:37.973065+0000","last_clean":"2024-09-21T10:06:37.973065+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:06:37.973065+0000","last_undegraded":"2024-09-21T10:06:37.973065+0000","last_fullsized":"2024-09-21T10:06:37.973065+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1176,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.977744+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:06:37.977744+0000","last_peered":"2024-09-21T10:06:37.977744+0000","last_clean":"2024-09-21T10:06:37.977744+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:06:37.977744+0000","last_undegraded":"2024-09-21T10:06:37.977744+0000","last_fullsized":"2024-09-21T10:06:37.977744+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":51,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.955851+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:06:25.955851+0000","last_peered":"2024-09-21T10:06:25.955851+0000","last_clean":"2024-09-21T10:06:25.955851+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:06:25.955851+0000","last_undegraded":"2024-09-21T10:06:25.955851+0000","last_fullsized":"2024-09-21T10:06:25.955851+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":231,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:37.979800+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:06:37.979800+0000","last_peered":"2024-09-21T10:06:37.979800+0000","last_clean":"2024-09-21T10:06:37.979800+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:06:37.979800+0000","last_undegraded":"2024-09-21T10:06:37.979800+0000","last_fullsized":"2024-09-21T10:06:37.979800+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":238,"reported_epoch":1197,"state":"active+clean","last_fresh":"2024-09-21T10:06:38.001377+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:06:38.001377+0000","last_peered":"2024-09-21T10:06:38.001377+0000","last_clean":"2024-09-21T10:06:38.001377+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:06:38.001377+0000","last_undegraded":"2024-09-21T10:06:38.001377+0000","last_fullsized":"2024-09-21T10:06:38.001377+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.954936+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:06:25.954936+0000","last_peered":"2024-09-21T10:06:25.954936+0000","last_clean":"2024-09-21T10:06:25.954936+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:06:25.954936+0000","last_undegraded":"2024-09-21T10:06:25.954936+0000","last_fullsized":"2024-09-21T10:06:25.954936+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1566,"reported_epoch":1196,"state":"active+clean","last_fresh":"2024-09-21T10:06:25.961320+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:06:25.961320+0000","last_peered":"2024-09-21T10:06:25.961320+0000","last_clean":"2024-09-21T10:06:25.961320+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:06:25.961320+0000","last_undegraded":"2024-09-21T10:06:25.961320+0000","last_fullsized":"2024-09-21T10:06:25.961320+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1139539992,"num_objects":17389,"num_object_clones":0,"num_object_copies":52167,"num_objects_missing_on_primary":2726,"num_objects_missing":2726,"num_objects_degraded":26097,"num_objects_misplaced":1492,"num_objects_unfound":0,"num_objects_dirty":17389,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126585,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40968,"num_bytes_recovered":2312306712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5857751040,"data_stored":5857738824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76951,"ondisk_log_size":76951,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234370,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496005,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":600592,"kb_used_data":308676,"kb_used_omap":1,"kb_used_meta":291902,"kb_avail":93144560,"statfs":{"total":95995035648,"available":95380029440,"internally_reserved":0,"allocated":316084224,"data_stored":313546793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":298908103},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757645,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019281,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700612,"kb_used_data":434876,"kb_used_omap":1,"kb_used_meta":265726,"kb_avail":93044540,"statfs":{"total":95995035648,"available":95277608960,"internally_reserved":0,"allocated":445313024,"data_stored":442783220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":272103878},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888577,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1284920,"kb_used_data":823920,"kb_used_omap":34,"kb_used_meta":460957,"kb_avail":92460232,"statfs":{"total":95995035648,"available":94679277568,"internally_reserved":0,"allocated":843694080,"data_stored":841171247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":472020979},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967816,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1074944,"kb_used_data":713844,"kb_used_omap":41,"kb_used_meta":361046,"kb_avail":92670208,"statfs":{"total":95995035648,"available":94894292992,"internally_reserved":0,"allocated":730976256,"data_stored":728455018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":369711519},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047055,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1261428,"kb_used_data":958636,"kb_used_omap":41,"kb_used_meta":302742,"kb_avail":92483724,"statfs":{"total":95995035648,"available":94703333376,"internally_reserved":0,"allocated":981643264,"data_stored":979124911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":310008745},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126293,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1644036096,"data_stored":1644036096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":962859008,"data_stored":962854936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":708640768,"data_stored":708640768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":820973568,"data_stored":820969496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":434176000,"data_stored":434176000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":307499008,"data_stored":307494936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":647364608,"data_stored":647364608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:06:45.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:45.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:45.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:45 smithi076 ceph-mon[106482]: pgmap v1563: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.0 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 387 op/s; 26097/51357 objects degraded (50.815%); 1226/51357 objects misplaced (2.387%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:06:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:45 smithi076 ceph-mon[110963]: pgmap v1563: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.0 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 387 op/s; 26097/51357 objects degraded (50.815%); 1226/51357 objects misplaced (2.387%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:06:45.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:45.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:45.975 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:45.996 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:45.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:46.088 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:46.088 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:46.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:45 smithi191 ceph-mon[102206]: pgmap v1563: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.0 GiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 387 op/s; 26097/51357 objects degraded (50.815%); 1226/51357 objects misplaced (2.387%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:06:46.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:46.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:46.246 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:46.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:46.348 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:46.377 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:46.474 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:46.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:46.502 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:46.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:46.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:46.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:46.733 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:46.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:46.865 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:46.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:46.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:46.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:46.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:47.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:47.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:47.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:47.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:47.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:47.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:47.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:47.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:47.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:47.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:47.543 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:47.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:47.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:47.669 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:47.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:47.771 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:47.801 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:47 smithi191 ceph-mon[102206]: pgmap v1564: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.0 GiB data, 4.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 387 op/s; 26097/51357 objects degraded (50.815%); 1226/51357 objects misplaced (2.387%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:06:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:47.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[106482]: pgmap v1564: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.0 GiB data, 4.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 387 op/s; 26097/51357 objects degraded (50.815%); 1226/51357 objects misplaced (2.387%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[110963]: pgmap v1564: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 1.0 GiB data, 4.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 387 op/s; 26097/51357 objects degraded (50.815%); 1226/51357 objects misplaced (2.387%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:06:47.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:47.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:47.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:06:47.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:47.931 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:47.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:47.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:48.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:48.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:48.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:48.189 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:48.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:48.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:48.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:48.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:48.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:48.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:48.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.570 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:48.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:48.661 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:06:48.671 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:48.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:48.834 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:48.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:48.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:48.993 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:48.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:49.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:49.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:49.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:49.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:49.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:49.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:49.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:49.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:49.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:49.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:49.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:49.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:49.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:49.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:49.716 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:49.746 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:49.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:49.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:49.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:49.975 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:49.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:49.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:50.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:49 smithi191 ceph-mon[102206]: pgmap v1565: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 935 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 25631/44952 objects degraded (57.019%); 731/44952 objects misplaced (1.626%); 9.4 MiB/s, 156 objects/s recovering 2024-09-21T10:06:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 25631/44952 objects degraded (57.019%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.123 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:50.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:50.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:49 smithi076 ceph-mon[110963]: pgmap v1565: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 935 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 25631/44952 objects degraded (57.019%); 731/44952 objects misplaced (1.626%); 9.4 MiB/s, 156 objects/s recovering 2024-09-21T10:06:50.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 25631/44952 objects degraded (57.019%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:49 smithi076 ceph-mon[106482]: pgmap v1565: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 935 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 25631/44952 objects degraded (57.019%); 731/44952 objects misplaced (1.626%); 9.4 MiB/s, 156 objects/s recovering 2024-09-21T10:06:50.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 25631/44952 objects degraded (57.019%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:50.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:50.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:50.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:50.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:50.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:50.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:50.507 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:50.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:50.631 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:50.733 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:50.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:50.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:50.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:50.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:50.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:50.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:51.004 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:51.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:51.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:51.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:51.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:51.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:51.389 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:51.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:51.491 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:51.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:51.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:51.644 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:51.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:51.746 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:51.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:51.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:51.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:51.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:51.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:51.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:52.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:52.051 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:51 smithi191 ceph-mon[102206]: pgmap v1566: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 935 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 25631/44952 objects degraded (57.019%); 731/44952 objects misplaced (1.626%); 7.0 MiB/s, 117 objects/s recovering 2024-09-21T10:06:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:51 smithi076 ceph-mon[106482]: pgmap v1566: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 935 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 25631/44952 objects degraded (57.019%); 731/44952 objects misplaced (1.626%); 7.0 MiB/s, 117 objects/s recovering 2024-09-21T10:06:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:51 smithi076 ceph-mon[110963]: pgmap v1566: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 935 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 25631/44952 objects degraded (57.019%); 731/44952 objects misplaced (1.626%); 7.0 MiB/s, 117 objects/s recovering 2024-09-21T10:06:52.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:52.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:52.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:52.249 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:52.275 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:52.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:52.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.399 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.525 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.627 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:52.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:52.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:52.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:52.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:52.907 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:52.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:53.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:53.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:53.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:53.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:53.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:53.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:53.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:53.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:53.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:53.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:53.513 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:06:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:06:53] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:06:53.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:53.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:53.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:53.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:53.762 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:53.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:53.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:53.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:53 smithi076 ceph-mon[106482]: pgmap v1567: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 825 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 500 op/s; 25128/39666 objects degraded (63.349%); 647/39666 objects misplaced (1.631%); 8.8 MiB/s, 145 objects/s recovering 2024-09-21T10:06:53.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:53 smithi076 ceph-mon[110963]: pgmap v1567: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 825 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 500 op/s; 25128/39666 objects degraded (63.349%); 647/39666 objects misplaced (1.631%); 8.8 MiB/s, 145 objects/s recovering 2024-09-21T10:06:53.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:53.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:53.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:54.015 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:54.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:53 smithi191 ceph-mon[102206]: pgmap v1567: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 825 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 500 op/s; 25128/39666 objects degraded (63.349%); 647/39666 objects misplaced (1.631%); 8.8 MiB/s, 145 objects/s recovering 2024-09-21T10:06:54.116 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:06:54.116 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:06:54.141 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:06:54.141 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:06:54.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:54.164 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:06:54.164 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:06:54.175 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:54.197 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:06:54.197 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:06:54.222 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:06:54.222 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:06:54.247 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:06:54.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:06:54.272 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:06:54.272 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:06:54.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:54.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:06:54.301 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:06:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:54.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:54.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:54.440 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:54.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:54.542 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:54.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:54.672 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:54.695 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:54.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:54.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:54.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:54.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:54.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:54.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:55.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:55.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:55.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:55.108 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 25128/39666 objects degraded (63.349%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:55.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2098660272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:55.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 25128/39666 objects degraded (63.349%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:55.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2098660272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 25128/39666 objects degraded (63.349%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2098660272' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:06:55.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:55.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:55.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:55.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:55.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:55.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:55.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:55.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:55.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:55.724 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:55.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:55.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:55.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:55.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:55.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:55.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:55.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:55.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:56.076 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:56.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:55 smithi191 ceph-mon[102206]: pgmap v1568: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 809 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 370 op/s; 25128/38886 objects degraded (64.620%); 647/38886 objects misplaced (1.664%); 6.9 MiB/s, 163 objects/s recovering 2024-09-21T10:06:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:55 smithi076 ceph-mon[106482]: pgmap v1568: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 809 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 370 op/s; 25128/38886 objects degraded (64.620%); 647/38886 objects misplaced (1.664%); 6.9 MiB/s, 163 objects/s recovering 2024-09-21T10:06:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:55 smithi076 ceph-mon[110963]: pgmap v1568: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 809 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 370 op/s; 25128/38886 objects degraded (64.620%); 647/38886 objects misplaced (1.664%); 6.9 MiB/s, 163 objects/s recovering 2024-09-21T10:06:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:56.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:56.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:56.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:56.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:56.345 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:56.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:56.446 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:56.487 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:56.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:56.625 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:56.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:56.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:56.753 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:56.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:56.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:56.878 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:56.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:56.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:56.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:57.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:57.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:57.125 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:57.227 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:57.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:57.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:57.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:57.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:57.474 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:57.496 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:57.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:57.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:57.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:57.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:57.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:57.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:57.870 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:57.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:57.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:06:57.972 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:06:58.010 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:06:58.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:57 smithi191 ceph-mon[102206]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 809 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 346 op/s; 25128/38886 objects degraded (64.620%); 647/38886 objects misplaced (1.664%); 5.5 MiB/s, 140 objects/s recovering 2024-09-21T10:06:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:57 smithi191 ceph-mon[102206]: osdmap e1198: 8 total, 8 up, 5 in 2024-09-21T10:06:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:57 smithi076 ceph-mon[106482]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 809 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 346 op/s; 25128/38886 objects degraded (64.620%); 647/38886 objects misplaced (1.664%); 5.5 MiB/s, 140 objects/s recovering 2024-09-21T10:06:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:57 smithi076 ceph-mon[106482]: osdmap e1198: 8 total, 8 up, 5 in 2024-09-21T10:06:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:57 smithi076 ceph-mon[110963]: pgmap v1569: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 809 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 346 op/s; 25128/38886 objects degraded (64.620%); 647/38886 objects misplaced (1.664%); 5.5 MiB/s, 140 objects/s recovering 2024-09-21T10:06:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:57 smithi076 ceph-mon[110963]: osdmap e1198: 8 total, 8 up, 5 in 2024-09-21T10:06:58.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:58.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:58.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:58.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:58.362 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:06:58.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:06:58.486 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.522 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:58.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:58.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:58.646 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:58.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:58.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:06:58.789 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:58.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:06:58.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:06:58.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:58.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:06:59.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.055 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:59.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:59.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:58 smithi191 ceph-mon[102206]: osdmap e1199: 8 total, 8 up, 5 in 2024-09-21T10:06:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:58 smithi076 ceph-mon[106482]: osdmap e1199: 8 total, 8 up, 5 in 2024-09-21T10:06:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:58 smithi076 ceph-mon[110963]: osdmap e1199: 8 total, 8 up, 5 in 2024-09-21T10:06:59.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:59.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:59.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:59.348 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:59.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:06:59.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:06:59.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:06:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:59.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:06:59.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:06:59.741 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:06:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:06:59.821 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:06:59.843 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:06:59.898 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:06:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:06:59.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:06:59.951 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:59 smithi076 ceph-mon[106482]: pgmap v1572: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 647 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 24894/31128 objects degraded (79.973%); 234/31128 objects misplaced (0.752%); 6.8 MiB/s, 207 objects/s recovering 2024-09-21T10:06:59.951 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:06:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 24894/31128 objects degraded (79.973%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:06:59.951 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:59 smithi076 ceph-mon[110963]: pgmap v1572: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 647 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 24894/31128 objects degraded (79.973%); 234/31128 objects misplaced (0.752%); 6.8 MiB/s, 207 objects/s recovering 2024-09-21T10:06:59.951 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:06:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 24894/31128 objects degraded (79.973%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:00.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:00.026 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:00.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:59 smithi191 ceph-mon[102206]: pgmap v1572: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 647 MiB data, 3.7 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 24894/31128 objects degraded (79.973%); 234/31128 objects misplaced (0.752%); 6.8 MiB/s, 207 objects/s recovering 2024-09-21T10:07:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:06:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 24894/31128 objects degraded (79.973%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:00.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:00.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:00.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:00.304 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:00.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:00.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:00.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:00.547 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:00.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:00.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:00.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:00.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:00.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:00.878 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:00.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:00.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:01.006 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:01.076 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:07:01.076 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:07:01.076 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:07:01.076 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:07:01.076 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:07:01.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:01.130 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:01.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:01.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:01.372 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:01.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:01.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.550 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:01.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:01.651 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:01.685 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:01.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:01.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:01.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:01.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:01.933 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:01.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:02.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:02.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:01 smithi191 ceph-mon[102206]: pgmap v1573: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 647 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 24894/31128 objects degraded (79.973%); 234/31128 objects misplaced (0.752%); 4.2 MiB/s, 165 objects/s recovering 2024-09-21T10:07:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:01 smithi076 ceph-mon[106482]: pgmap v1573: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 647 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 24894/31128 objects degraded (79.973%); 234/31128 objects misplaced (0.752%); 4.2 MiB/s, 165 objects/s recovering 2024-09-21T10:07:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:01 smithi076 ceph-mon[110963]: pgmap v1573: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 647 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 355 op/s; 24894/31128 objects degraded (79.973%); 234/31128 objects misplaced (0.752%); 4.2 MiB/s, 165 objects/s recovering 2024-09-21T10:07:02.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:02.182 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:02.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:02.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:02.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:02.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:02.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:02.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:02.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:02.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:02.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:02.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:02.698 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:02.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:02.784 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:07:02.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:02.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:02.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:02.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:02.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:03.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.077 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2939027372' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:02 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2939027372' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:02 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2939027372' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:02 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:07:03.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:03.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:03.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:03.304 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:03.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:03.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:07:03] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.51.0" 2024-09-21T10:07:03.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:03.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:03.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:03.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:03.867 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:03.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:03.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:03.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:04.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:03 smithi191 ceph-mon[102206]: pgmap v1574: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 465 MiB data, 3.3 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 688 op/s; 24773/22350 objects degraded (110.841%); 6.0 MiB/s, 139 objects/s recovering 2024-09-21T10:07:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:03 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:03 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:03 smithi191 ceph-mon[102206]: osdmap e1200: 8 total, 8 up, 5 in 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[106482]: pgmap v1574: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 465 MiB data, 3.3 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 688 op/s; 24773/22350 objects degraded (110.841%); 6.0 MiB/s, 139 objects/s recovering 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[106482]: osdmap e1200: 8 total, 8 up, 5 in 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[110963]: pgmap v1574: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 465 MiB data, 3.3 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 688 op/s; 24773/22350 objects degraded (110.841%); 6.0 MiB/s, 139 objects/s recovering 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:07:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:03 smithi076 ceph-mon[110963]: osdmap e1200: 8 total, 8 up, 5 in 2024-09-21T10:07:04.133 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:04.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:04.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:04.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:04.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:04.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:04.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:04.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:04.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:04.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:04.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:04.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:04.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:04.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:04.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:04.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:04.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:05.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:05.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:05.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:05.098 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 24773/22350 objects degraded (110.841%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:05.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 24773/22350 objects degraded (110.841%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 24773/22350 objects degraded (110.841%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:05.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:05.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:05.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:05.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:05.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:05.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:05.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:05.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:05.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:05.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:05.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:05.814 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:05.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:05.942 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:05.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:05.971 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:05.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:06.072 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:06.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:05 smithi076 ceph-mon[106482]: pgmap v1576: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 435 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 755 op/s; 24773/20916 objects degraded (118.440%); 6.1 MiB/s, 140 objects/s recovering 2024-09-21T10:07:06.101 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:05 smithi076 ceph-mon[110963]: pgmap v1576: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 435 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 755 op/s; 24773/20916 objects degraded (118.440%); 6.1 MiB/s, 140 objects/s recovering 2024-09-21T10:07:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:05 smithi191 ceph-mon[102206]: pgmap v1576: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 435 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 755 op/s; 24773/20916 objects degraded (118.440%); 6.1 MiB/s, 140 objects/s recovering 2024-09-21T10:07:06.134 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:06.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:06.280 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:06.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:06.382 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:06.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:06.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:06.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:06.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:06.620 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:07:06.621 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:07:06.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:06.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:06.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:07:06.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:06.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:06.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:06.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:07.011 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:07.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:07.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:07.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:07.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:07.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:07.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:07.400 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:07.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:07.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:07.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:07.680 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:07.782 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:07.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:07.907 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:07.935 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:07.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:07.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:07.972 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1576,"stamp":"2024-09-21T10:07:04.661320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":455743485,"num_objects":6972,"num_object_clones":0,"num_object_copies":20916,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":24773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6972,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":137307,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44107,"num_bytes_recovered":2452960798,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77502,"ondisk_log_size":77502,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":3372352,"kb_used_data":1617172,"kb_used_omap":119,"kb_used_meta":1754696,"kb_avail":465353408,"statfs":{"total":479975178240,"available":476521889792,"internally_reserved":0,"allocated":1655984128,"data_stored":1643312260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1796808904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,6,6,3,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-392560640,"num_objects":-5990,"num_object_clones":0,"num_object_copies":-17970,"num_objects_missing_on_primary":-275,"num_objects_missing":-275,"num_objects_degraded":-355,"num_objects_misplaced":-647,"num_objects_unfound":0,"num_objects_dirty":-5990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5990,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1117,"num_bytes_recovered":50462720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.925951"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.747990+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:06:57.747990+0000","last_peered":"2024-09-21T10:06:57.747990+0000","last_clean":"2024-09-21T10:06:57.747990+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:06:57.747990+0000","last_undegraded":"2024-09-21T10:06:57.747990+0000","last_fullsized":"2024-09-21T10:06:57.747990+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":56,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739144+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:06:57.739144+0000","last_peered":"2024-09-21T10:06:57.739144+0000","last_clean":"2024-09-21T10:06:57.739144+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:06:57.739144+0000","last_undegraded":"2024-09-21T10:06:57.739144+0000","last_fullsized":"2024-09-21T10:06:57.739144+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1220,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745323+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:06:57.745323+0000","last_peered":"2024-09-21T10:06:57.745323+0000","last_clean":"2024-09-21T10:06:57.745323+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:06:57.745323+0000","last_undegraded":"2024-09-21T10:06:57.745323+0000","last_fullsized":"2024-09-21T10:06:57.745323+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1551,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745042+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:06:57.745042+0000","last_peered":"2024-09-21T10:06:57.745042+0000","last_clean":"2024-09-21T10:06:57.745042+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:06:57.745042+0000","last_undegraded":"2024-09-21T10:06:57.745042+0000","last_fullsized":"2024-09-21T10:06:57.745042+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1551,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.749088+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:06:57.749088+0000","last_peered":"2024-09-21T10:06:57.749088+0000","last_clean":"2024-09-21T10:06:57.749088+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:06:57.749088+0000","last_undegraded":"2024-09-21T10:06:57.749088+0000","last_fullsized":"2024-09-21T10:06:57.749088+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":59,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.738433+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:06:57.738433+0000","last_peered":"2024-09-21T10:06:57.738433+0000","last_clean":"2024-09-21T10:06:57.738433+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:06:57.738433+0000","last_undegraded":"2024-09-21T10:06:57.738433+0000","last_fullsized":"2024-09-21T10:06:57.738433+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":235,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.746537+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:06:57.746537+0000","last_peered":"2024-09-21T10:06:57.746537+0000","last_clean":"2024-09-21T10:06:57.746537+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:06:57.746537+0000","last_undegraded":"2024-09-21T10:06:57.746537+0000","last_fullsized":"2024-09-21T10:06:57.746537+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":79,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739025+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:06:57.739025+0000","last_peered":"2024-09-21T10:06:57.739025+0000","last_clean":"2024-09-21T10:06:57.739025+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:06:57.739025+0000","last_undegraded":"2024-09-21T10:06:57.739025+0000","last_fullsized":"2024-09-21T10:06:57.739025+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1199'8569","reported_seq":17437,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.377484+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:07:02.377484+0000","last_peered":"2024-09-21T10:07:02.377484+0000","last_clean":"2024-09-21T10:07:02.377484+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:07:02.377484+0000","last_undegraded":"2024-09-21T10:07:02.377484+0000","last_fullsized":"2024-09-21T10:07:02.377484+0000","mapping_epoch":1187,"log_start":"1157'3831","ondisk_log_start":"1157'3831","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4738,"log_dups_size":0,"ondisk_log_size":4738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31260672,"num_objects":477,"num_object_clones":0,"num_object_copies":1431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8569,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":90,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.742989+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:06:57.742989+0000","last_peered":"2024-09-21T10:06:57.742989+0000","last_clean":"2024-09-21T10:06:57.742989+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:06:57.742989+0000","last_undegraded":"2024-09-21T10:06:57.742989+0000","last_fullsized":"2024-09-21T10:06:57.742989+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":1200,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.787379+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:07:02.787379+0000","last_peered":"2024-09-21T10:07:02.787379+0000","last_clean":"2024-09-21T10:07:02.787379+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:07:02.787379+0000","last_undegraded":"2024-09-21T10:07:02.787379+0000","last_fullsized":"2024-09-21T10:07:02.787379+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1199'8618","reported_seq":16339,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.332135+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:07:02.332135+0000","last_peered":"2024-09-21T10:07:02.332135+0000","last_clean":"2024-09-21T10:07:02.332135+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:07:02.332135+0000","last_undegraded":"2024-09-21T10:07:02.332135+0000","last_fullsized":"2024-09-21T10:07:02.332135+0000","mapping_epoch":1180,"log_start":"1158'4003","ondisk_log_start":"1158'4003","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4615,"log_dups_size":0,"ondisk_log_size":4615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26869760,"num_objects":410,"num_object_clones":0,"num_object_copies":1230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8618,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":134,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743811+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:06:57.743811+0000","last_peered":"2024-09-21T10:06:57.743811+0000","last_clean":"2024-09-21T10:06:57.743811+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:06:57.743811+0000","last_undegraded":"2024-09-21T10:06:57.743811+0000","last_fullsized":"2024-09-21T10:06:57.743811+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1175,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743566+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:06:57.743566+0000","last_peered":"2024-09-21T10:06:57.743566+0000","last_clean":"2024-09-21T10:06:57.743566+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:06:57.743566+0000","last_undegraded":"2024-09-21T10:06:57.743566+0000","last_fullsized":"2024-09-21T10:06:57.743566+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1199'8740","reported_seq":8881,"reported_epoch":1199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.357983+0000","last_change":"2024-09-21T10:03:30.139654+0000","last_active":"2024-09-21T10:07:02.357983+0000","last_peered":"2024-09-21T10:07:02.357983+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:07:02.357983+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1158'3982","ondisk_log_start":"1158'3982","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4758,"log_dups_size":0,"ondisk_log_size":4758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30670848,"num_objects":468,"num_object_clones":0,"num_object_copies":1404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8740,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":468},{"shards":"1,4","objects":-3271}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":54,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739515+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:06:57.739515+0000","last_peered":"2024-09-21T10:06:57.739515+0000","last_clean":"2024-09-21T10:06:57.739515+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:06:57.739515+0000","last_undegraded":"2024-09-21T10:06:57.739515+0000","last_fullsized":"2024-09-21T10:06:57.739515+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":1200,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.787649+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:07:02.787649+0000","last_peered":"2024-09-21T10:07:02.787649+0000","last_clean":"2024-09-21T10:07:02.787649+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:07:02.787649+0000","last_undegraded":"2024-09-21T10:07:02.787649+0000","last_fullsized":"2024-09-21T10:07:02.787649+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1199'8517","reported_seq":14177,"reported_epoch":1199,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:07:02.268967+0000","last_change":"2024-09-21T10:06:10.765843+0000","last_active":"2024-09-21T10:07:02.268967+0000","last_peered":"2024-09-21T10:07:02.268967+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:07:02.268967+0000","last_undegraded":"2024-09-21T10:07:02.268967+0000","last_fullsized":"2024-09-21T10:07:02.268967+0000","mapping_epoch":1151,"log_start":"1158'3936","ondisk_log_start":"1158'3936","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4581,"log_dups_size":0,"ondisk_log_size":4581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30736384,"num_objects":469,"num_object_clones":0,"num_object_copies":1407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8517,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2755,"num_bytes_recovered":180551680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","2","4","7"],"object_location_counts":[{"shards":"2,3,4","objects":469}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1205,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745732+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:06:57.745732+0000","last_peered":"2024-09-21T10:06:57.745732+0000","last_clean":"2024-09-21T10:06:57.745732+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:06:57.745732+0000","last_undegraded":"2024-09-21T10:06:57.745732+0000","last_fullsized":"2024-09-21T10:06:57.745732+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1561,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741930+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:06:57.741930+0000","last_peered":"2024-09-21T10:06:57.741930+0000","last_clean":"2024-09-21T10:06:57.741930+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:06:57.741930+0000","last_undegraded":"2024-09-21T10:06:57.741930+0000","last_fullsized":"2024-09-21T10:06:57.741930+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1199'8561","reported_seq":14331,"reported_epoch":1199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.317421+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:07:02.317421+0000","last_peered":"2024-09-21T10:07:02.317421+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:07:02.317421+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1158'3948","ondisk_log_start":"1158'3948","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4613,"log_dups_size":0,"ondisk_log_size":4613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29687808,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8561,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"0,3,4","objects":453},{"shards":"3,4","objects":-3168}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":243,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744464+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:06:57.744464+0000","last_peered":"2024-09-21T10:06:57.744464+0000","last_clean":"2024-09-21T10:06:57.744464+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:06:57.744464+0000","last_undegraded":"2024-09-21T10:06:57.744464+0000","last_fullsized":"2024-09-21T10:06:57.744464+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1220,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745552+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:06:57.745552+0000","last_peered":"2024-09-21T10:06:57.745552+0000","last_clean":"2024-09-21T10:06:57.745552+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:06:57.745552+0000","last_undegraded":"2024-09-21T10:06:57.745552+0000","last_fullsized":"2024-09-21T10:06:57.745552+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1199'8461","reported_seq":17074,"reported_epoch":1199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.362245+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:07:02.362245+0000","last_peered":"2024-09-21T10:07:02.362245+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:07:02.362245+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1157'3853","ondisk_log_start":"1157'3853","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4608,"log_dups_size":0,"ondisk_log_size":4608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26935296,"num_objects":411,"num_object_clones":0,"num_object_copies":1233,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":2474,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8461,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,3","objects":411},{"shards":"2,3","objects":-339}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":51,"reported_epoch":1200,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.787509+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:07:02.787509+0000","last_peered":"2024-09-21T10:07:02.787509+0000","last_clean":"2024-09-21T10:07:02.787509+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:07:02.787509+0000","last_undegraded":"2024-09-21T10:07:02.787509+0000","last_fullsized":"2024-09-21T10:07:02.787509+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":863,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743508+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:06:57.743508+0000","last_peered":"2024-09-21T10:06:57.743508+0000","last_clean":"2024-09-21T10:06:57.743508+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:06:57.743508+0000","last_undegraded":"2024-09-21T10:06:57.743508+0000","last_fullsized":"2024-09-21T10:06:57.743508+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1200'8727","reported_seq":18885,"reported_epoch":1200,"state":"active+clean","last_fresh":"2024-09-21T10:07:03.128785+0000","last_change":"2024-09-21T10:06:23.957394+0000","last_active":"2024-09-21T10:07:03.128785+0000","last_peered":"2024-09-21T10:07:03.128785+0000","last_clean":"2024-09-21T10:07:03.128785+0000","last_became_active":"2024-09-21T10:06:23.937063+0000","last_became_peered":"2024-09-21T10:06:23.937063+0000","last_unstale":"2024-09-21T10:07:03.128785+0000","last_undegraded":"2024-09-21T10:07:03.128785+0000","last_fullsized":"2024-09-21T10:07:03.128785+0000","mapping_epoch":1194,"log_start":"1143'2700","ondisk_log_start":"1143'2700","created":1137,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:57.739037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28180504,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8727,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4579,"num_bytes_recovered":242810904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1206,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741630+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:06:57.741630+0000","last_peered":"2024-09-21T10:06:57.741630+0000","last_clean":"2024-09-21T10:06:57.741630+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:06:57.741630+0000","last_undegraded":"2024-09-21T10:06:57.741630+0000","last_fullsized":"2024-09-21T10:06:57.741630+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":50,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739580+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:06:57.739580+0000","last_peered":"2024-09-21T10:06:57.739580+0000","last_clean":"2024-09-21T10:06:57.739580+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:06:57.739580+0000","last_undegraded":"2024-09-21T10:06:57.739580+0000","last_fullsized":"2024-09-21T10:06:57.739580+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1199'8866","reported_seq":16975,"reported_epoch":1199,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.360812+0000","last_change":"2024-09-21T10:05:36.090756+0000","last_active":"2024-09-21T10:07:02.360812+0000","last_peered":"2024-09-21T10:07:02.360812+0000","last_clean":"2024-09-21T10:01:39.013739+0000","last_became_active":"2024-09-21T10:02:39.193544+0000","last_became_peered":"2024-09-21T10:02:39.193544+0000","last_unstale":"2024-09-21T10:07:02.360812+0000","last_undegraded":"2024-09-21T10:02:39.094756+0000","last_fullsized":"2024-09-21T10:02:38.973332+0000","mapping_epoch":1161,"log_start":"1158'4225","ondisk_log_start":"1158'4225","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4641,"log_dups_size":0,"ondisk_log_size":4641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28180480,"num_objects":430,"num_object_clones":0,"num_object_copies":1290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2275,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8866,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3981,"num_bytes_recovered":252968960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[3,2],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3","objects":-1845}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":235,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741415+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:06:57.741415+0000","last_peered":"2024-09-21T10:06:57.741415+0000","last_clean":"2024-09-21T10:06:57.741415+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:06:57.741415+0000","last_undegraded":"2024-09-21T10:06:57.741415+0000","last_fullsized":"2024-09-21T10:06:57.741415+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1565,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743146+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:06:57.743146+0000","last_peered":"2024-09-21T10:06:57.743146+0000","last_clean":"2024-09-21T10:06:57.743146+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:06:57.743146+0000","last_undegraded":"2024-09-21T10:06:57.743146+0000","last_fullsized":"2024-09-21T10:06:57.743146+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1199'8418","reported_seq":18668,"reported_epoch":1199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.353631+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:07:02.353631+0000","last_peered":"2024-09-21T10:07:02.353631+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:07:02.353631+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1158'3847","ondisk_log_start":"1158'3847","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4571,"log_dups_size":0,"ondisk_log_size":4571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27656192,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8418,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3642,"num_bytes_recovered":238682112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,2,3","objects":422},{"shards":"2,3","objects":-310}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":723,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.738877+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:06:57.738877+0000","last_peered":"2024-09-21T10:06:57.738877+0000","last_clean":"2024-09-21T10:06:57.738877+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:06:57.738877+0000","last_undegraded":"2024-09-21T10:06:57.738877+0000","last_fullsized":"2024-09-21T10:06:57.738877+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1219,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744220+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:06:57.744220+0000","last_peered":"2024-09-21T10:06:57.744220+0000","last_clean":"2024-09-21T10:06:57.744220+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:06:57.744220+0000","last_undegraded":"2024-09-21T10:06:57.744220+0000","last_fullsized":"2024-09-21T10:06:57.744220+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":249,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745180+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:06:57.745180+0000","last_peered":"2024-09-21T10:06:57.745180+0000","last_clean":"2024-09-21T10:06:57.745180+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:06:57.745180+0000","last_undegraded":"2024-09-21T10:06:57.745180+0000","last_fullsized":"2024-09-21T10:06:57.745180+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739446+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:06:57.739446+0000","last_peered":"2024-09-21T10:06:57.739446+0000","last_clean":"2024-09-21T10:06:57.739446+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:06:57.739446+0000","last_undegraded":"2024-09-21T10:06:57.739446+0000","last_fullsized":"2024-09-21T10:06:57.739446+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1199'8538","reported_seq":14183,"reported_epoch":1199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.389130+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:07:02.389130+0000","last_peered":"2024-09-21T10:07:02.389130+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:07:02.389130+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1157'3754","ondisk_log_start":"1157'3754","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4784,"log_dups_size":0,"ondisk_log_size":4784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26738688,"num_objects":408,"num_object_clones":0,"num_object_copies":1224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8538,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"0,1,2","objects":408},{"shards":"1,2","objects":-3197}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":849,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741311+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:06:57.741311+0000","last_peered":"2024-09-21T10:06:57.741311+0000","last_clean":"2024-09-21T10:06:57.741311+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:06:57.741311+0000","last_undegraded":"2024-09-21T10:06:57.741311+0000","last_fullsized":"2024-09-21T10:06:57.741311+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1438,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.748278+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:06:57.748278+0000","last_peered":"2024-09-21T10:06:57.748278+0000","last_clean":"2024-09-21T10:06:57.748278+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:06:57.748278+0000","last_undegraded":"2024-09-21T10:06:57.748278+0000","last_fullsized":"2024-09-21T10:06:57.748278+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":847,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743604+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:06:57.743604+0000","last_peered":"2024-09-21T10:06:57.743604+0000","last_clean":"2024-09-21T10:06:57.743604+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:06:57.743604+0000","last_undegraded":"2024-09-21T10:06:57.743604+0000","last_fullsized":"2024-09-21T10:06:57.743604+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1199'8174","reported_seq":16946,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.378227+0000","last_change":"2024-09-21T10:05:37.849809+0000","last_active":"2024-09-21T10:07:02.378227+0000","last_peered":"2024-09-21T10:07:02.378227+0000","last_clean":"2024-09-21T10:07:02.378227+0000","last_became_active":"2024-09-21T10:05:37.830539+0000","last_became_peered":"2024-09-21T10:05:37.830539+0000","last_unstale":"2024-09-21T10:07:02.378227+0000","last_undegraded":"2024-09-21T10:07:02.378227+0000","last_fullsized":"2024-09-21T10:07:02.378227+0000","mapping_epoch":1190,"log_start":"1153'3437","ondisk_log_start":"1153'3437","created":1137,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4737,"log_dups_size":0,"ondisk_log_size":4737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:35.172792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8174,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4311,"num_bytes_recovered":174260224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1479,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.748567+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:06:57.748567+0000","last_peered":"2024-09-21T10:06:57.748567+0000","last_clean":"2024-09-21T10:06:57.748567+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:06:57.748567+0000","last_undegraded":"2024-09-21T10:06:57.748567+0000","last_fullsized":"2024-09-21T10:06:57.748567+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1566,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741450+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:06:57.741450+0000","last_peered":"2024-09-21T10:06:57.741450+0000","last_clean":"2024-09-21T10:06:57.741450+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:06:57.741450+0000","last_undegraded":"2024-09-21T10:06:57.741450+0000","last_fullsized":"2024-09-21T10:06:57.741450+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1574,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743086+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:06:57.743086+0000","last_peered":"2024-09-21T10:06:57.743086+0000","last_clean":"2024-09-21T10:06:57.743086+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:06:57.743086+0000","last_undegraded":"2024-09-21T10:06:57.743086+0000","last_fullsized":"2024-09-21T10:06:57.743086+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1200'8618","reported_seq":18612,"reported_epoch":1200,"state":"active+clean","last_fresh":"2024-09-21T10:07:03.123822+0000","last_change":"2024-09-21T10:06:57.766690+0000","last_active":"2024-09-21T10:07:03.123822+0000","last_peered":"2024-09-21T10:07:03.123822+0000","last_clean":"2024-09-21T10:07:03.123822+0000","last_became_active":"2024-09-21T10:06:57.745326+0000","last_became_peered":"2024-09-21T10:06:57.745326+0000","last_unstale":"2024-09-21T10:07:03.123822+0000","last_undegraded":"2024-09-21T10:07:03.123822+0000","last_fullsized":"2024-09-21T10:07:03.123822+0000","mapping_epoch":1198,"log_start":"1143'2600","ondisk_log_start":"1143'2600","created":1137,"last_epoch_clean":1199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6018,"log_dups_size":0,"ondisk_log_size":6018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:11.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26214400,"num_objects":400,"num_object_clones":0,"num_object_copies":1200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8618,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4509,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":143,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741985+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:06:57.741985+0000","last_peered":"2024-09-21T10:06:57.741985+0000","last_clean":"2024-09-21T10:06:57.741985+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:06:57.741985+0000","last_undegraded":"2024-09-21T10:06:57.741985+0000","last_fullsized":"2024-09-21T10:06:57.741985+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1161,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.738989+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:06:57.738989+0000","last_peered":"2024-09-21T10:06:57.738989+0000","last_clean":"2024-09-21T10:06:57.738989+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:06:57.738989+0000","last_undegraded":"2024-09-21T10:06:57.738989+0000","last_fullsized":"2024-09-21T10:06:57.738989+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1577,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741920+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:06:57.741920+0000","last_peered":"2024-09-21T10:06:57.741920+0000","last_clean":"2024-09-21T10:06:57.741920+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:06:57.741920+0000","last_undegraded":"2024-09-21T10:06:57.741920+0000","last_fullsized":"2024-09-21T10:06:57.741920+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1199'8604","reported_seq":8737,"reported_epoch":1199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:02.357240+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:07:02.357240+0000","last_peered":"2024-09-21T10:07:02.357240+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:07:02.357240+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1158'3985","ondisk_log_start":"1158'3985","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28835840,"num_objects":440,"num_object_clones":0,"num_object_copies":1320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8604,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":285},{"shards":"3,4","objects":-3927}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1167,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745127+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:06:57.745127+0000","last_peered":"2024-09-21T10:06:57.745127+0000","last_clean":"2024-09-21T10:06:57.745127+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:06:57.745127+0000","last_undegraded":"2024-09-21T10:06:57.745127+0000","last_fullsized":"2024-09-21T10:06:57.745127+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1588,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.742011+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:06:57.742011+0000","last_peered":"2024-09-21T10:06:57.742011+0000","last_clean":"2024-09-21T10:06:57.742011+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:06:57.742011+0000","last_undegraded":"2024-09-21T10:06:57.742011+0000","last_fullsized":"2024-09-21T10:06:57.742011+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741599+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:06:57.741599+0000","last_peered":"2024-09-21T10:06:57.741599+0000","last_clean":"2024-09-21T10:06:57.741599+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:06:57.741599+0000","last_undegraded":"2024-09-21T10:06:57.741599+0000","last_fullsized":"2024-09-21T10:06:57.741599+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1199'8465","reported_seq":16072,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.335352+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:07:02.335352+0000","last_peered":"2024-09-21T10:07:02.335352+0000","last_clean":"2024-09-21T10:07:02.335352+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:07:02.335352+0000","last_undegraded":"2024-09-21T10:07:02.335352+0000","last_fullsized":"2024-09-21T10:07:02.335352+0000","mapping_epoch":1174,"log_start":"1154'3694","ondisk_log_start":"1154'3694","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4771,"log_dups_size":0,"ondisk_log_size":4771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26935296,"num_objects":411,"num_object_clones":0,"num_object_copies":1233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8465,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1565,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743344+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:06:57.743344+0000","last_peered":"2024-09-21T10:06:57.743344+0000","last_clean":"2024-09-21T10:06:57.743344+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:06:57.743344+0000","last_undegraded":"2024-09-21T10:06:57.743344+0000","last_fullsized":"2024-09-21T10:06:57.743344+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":745,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741893+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:06:57.741893+0000","last_peered":"2024-09-21T10:06:57.741893+0000","last_clean":"2024-09-21T10:06:57.741893+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:06:57.741893+0000","last_undegraded":"2024-09-21T10:06:57.741893+0000","last_fullsized":"2024-09-21T10:06:57.741893+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1557,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744624+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:06:57.744624+0000","last_peered":"2024-09-21T10:06:57.744624+0000","last_clean":"2024-09-21T10:06:57.744624+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:06:57.744624+0000","last_undegraded":"2024-09-21T10:06:57.744624+0000","last_fullsized":"2024-09-21T10:06:57.744624+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1199'8763","reported_seq":8901,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.283835+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:07:02.283835+0000","last_peered":"2024-09-21T10:07:02.283835+0000","last_clean":"2024-09-21T10:07:02.283835+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:07:02.283835+0000","last_undegraded":"2024-09-21T10:07:02.283835+0000","last_fullsized":"2024-09-21T10:07:02.283835+0000","mapping_epoch":1137,"log_start":"1158'4126","ondisk_log_start":"1158'4126","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4637,"log_dups_size":0,"ondisk_log_size":4637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26804224,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8763,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":265,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.746614+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:06:57.746614+0000","last_peered":"2024-09-21T10:06:57.746614+0000","last_clean":"2024-09-21T10:06:57.746614+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:06:57.746614+0000","last_undegraded":"2024-09-21T10:06:57.746614+0000","last_fullsized":"2024-09-21T10:06:57.746614+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1566,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743894+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:06:57.743894+0000","last_peered":"2024-09-21T10:06:57.743894+0000","last_clean":"2024-09-21T10:06:57.743894+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:06:57.743894+0000","last_undegraded":"2024-09-21T10:06:57.743894+0000","last_fullsized":"2024-09-21T10:06:57.743894+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":243,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741346+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:06:57.741346+0000","last_peered":"2024-09-21T10:06:57.741346+0000","last_clean":"2024-09-21T10:06:57.741346+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:06:57.741346+0000","last_undegraded":"2024-09-21T10:06:57.741346+0000","last_fullsized":"2024-09-21T10:06:57.741346+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1199'8389","reported_seq":8526,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.357802+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:07:02.357802+0000","last_peered":"2024-09-21T10:07:02.357802+0000","last_clean":"2024-09-21T10:07:02.357802+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:07:02.357802+0000","last_undegraded":"2024-09-21T10:07:02.357802+0000","last_fullsized":"2024-09-21T10:07:02.357802+0000","mapping_epoch":1137,"log_start":"1157'3813","ondisk_log_start":"1157'3813","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4576,"log_dups_size":0,"ondisk_log_size":4576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30081024,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8389,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":106,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743682+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:06:57.743682+0000","last_peered":"2024-09-21T10:06:57.743682+0000","last_clean":"2024-09-21T10:06:57.743682+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:06:57.743682+0000","last_undegraded":"2024-09-21T10:06:57.743682+0000","last_fullsized":"2024-09-21T10:06:57.743682+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":51,"reported_epoch":1200,"state":"active+clean","last_fresh":"2024-09-21T10:07:02.787784+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:07:02.787784+0000","last_peered":"2024-09-21T10:07:02.787784+0000","last_clean":"2024-09-21T10:07:02.787784+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:07:02.787784+0000","last_undegraded":"2024-09-21T10:07:02.787784+0000","last_fullsized":"2024-09-21T10:07:02.787784+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.738345+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:06:57.738345+0000","last_peered":"2024-09-21T10:06:57.738345+0000","last_clean":"2024-09-21T10:06:57.738345+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:06:57.738345+0000","last_undegraded":"2024-09-21T10:06:57.738345+0000","last_fullsized":"2024-09-21T10:06:57.738345+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":50,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739062+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:06:57.739062+0000","last_peered":"2024-09-21T10:06:57.739062+0000","last_clean":"2024-09-21T10:06:57.739062+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:06:57.739062+0000","last_undegraded":"2024-09-21T10:06:57.739062+0000","last_fullsized":"2024-09-21T10:06:57.739062+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":60,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.740483+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:06:57.740483+0000","last_peered":"2024-09-21T10:06:57.740483+0000","last_clean":"2024-09-21T10:06:57.740483+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:06:57.740483+0000","last_undegraded":"2024-09-21T10:06:57.740483+0000","last_fullsized":"2024-09-21T10:06:57.740483+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1565,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744171+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:06:57.744171+0000","last_peered":"2024-09-21T10:06:57.744171+0000","last_clean":"2024-09-21T10:06:57.744171+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:06:57.744171+0000","last_undegraded":"2024-09-21T10:06:57.744171+0000","last_fullsized":"2024-09-21T10:06:57.744171+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1178,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745374+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:06:57.745374+0000","last_peered":"2024-09-21T10:06:57.745374+0000","last_clean":"2024-09-21T10:06:57.745374+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:06:57.745374+0000","last_undegraded":"2024-09-21T10:06:57.745374+0000","last_fullsized":"2024-09-21T10:06:57.745374+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1189,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744066+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:06:57.744066+0000","last_peered":"2024-09-21T10:06:57.744066+0000","last_clean":"2024-09-21T10:06:57.744066+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:06:57.744066+0000","last_undegraded":"2024-09-21T10:06:57.744066+0000","last_fullsized":"2024-09-21T10:06:57.744066+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":263,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.748483+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:06:57.748483+0000","last_peered":"2024-09-21T10:06:57.748483+0000","last_clean":"2024-09-21T10:06:57.748483+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:06:57.748483+0000","last_undegraded":"2024-09-21T10:06:57.748483+0000","last_fullsized":"2024-09-21T10:06:57.748483+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1591,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.746418+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:06:57.746418+0000","last_peered":"2024-09-21T10:06:57.746418+0000","last_clean":"2024-09-21T10:06:57.746418+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:06:57.746418+0000","last_undegraded":"2024-09-21T10:06:57.746418+0000","last_fullsized":"2024-09-21T10:06:57.746418+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":242,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745582+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:06:57.745582+0000","last_peered":"2024-09-21T10:06:57.745582+0000","last_clean":"2024-09-21T10:06:57.745582+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:06:57.745582+0000","last_undegraded":"2024-09-21T10:06:57.745582+0000","last_fullsized":"2024-09-21T10:06:57.745582+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739678+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:06:57.739678+0000","last_peered":"2024-09-21T10:06:57.739678+0000","last_clean":"2024-09-21T10:06:57.739678+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:06:57.739678+0000","last_undegraded":"2024-09-21T10:06:57.739678+0000","last_fullsized":"2024-09-21T10:06:57.739678+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.741567+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:06:57.741567+0000","last_peered":"2024-09-21T10:06:57.741567+0000","last_clean":"2024-09-21T10:06:57.741567+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:06:57.741567+0000","last_undegraded":"2024-09-21T10:06:57.741567+0000","last_fullsized":"2024-09-21T10:06:57.741567+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1553,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.749235+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:06:57.749235+0000","last_peered":"2024-09-21T10:06:57.749235+0000","last_clean":"2024-09-21T10:06:57.749235+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:06:57.749235+0000","last_undegraded":"2024-09-21T10:06:57.749235+0000","last_fullsized":"2024-09-21T10:06:57.749235+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1564,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743632+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:06:57.743632+0000","last_peered":"2024-09-21T10:06:57.743632+0000","last_clean":"2024-09-21T10:06:57.743632+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:06:57.743632+0000","last_undegraded":"2024-09-21T10:06:57.743632+0000","last_fullsized":"2024-09-21T10:06:57.743632+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":228,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.745465+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:06:57.745465+0000","last_peered":"2024-09-21T10:06:57.745465+0000","last_clean":"2024-09-21T10:06:57.745465+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:06:57.745465+0000","last_undegraded":"2024-09-21T10:06:57.745465+0000","last_fullsized":"2024-09-21T10:06:57.745465+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":112,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744296+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:06:57.744296+0000","last_peered":"2024-09-21T10:06:57.744296+0000","last_clean":"2024-09-21T10:06:57.744296+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:06:57.744296+0000","last_undegraded":"2024-09-21T10:06:57.744296+0000","last_fullsized":"2024-09-21T10:06:57.744296+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1448,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743339+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:06:57.743339+0000","last_peered":"2024-09-21T10:06:57.743339+0000","last_clean":"2024-09-21T10:06:57.743339+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:06:57.743339+0000","last_undegraded":"2024-09-21T10:06:57.743339+0000","last_fullsized":"2024-09-21T10:06:57.743339+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1529,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.742024+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:06:57.742024+0000","last_peered":"2024-09-21T10:06:57.742024+0000","last_clean":"2024-09-21T10:06:57.742024+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:06:57.742024+0000","last_undegraded":"2024-09-21T10:06:57.742024+0000","last_fullsized":"2024-09-21T10:06:57.742024+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1174,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744389+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:06:57.744389+0000","last_peered":"2024-09-21T10:06:57.744389+0000","last_clean":"2024-09-21T10:06:57.744389+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:06:57.744389+0000","last_undegraded":"2024-09-21T10:06:57.744389+0000","last_fullsized":"2024-09-21T10:06:57.744389+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":127,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.743413+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:06:57.743413+0000","last_peered":"2024-09-21T10:06:57.743413+0000","last_clean":"2024-09-21T10:06:57.743413+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:06:57.743413+0000","last_undegraded":"2024-09-21T10:06:57.743413+0000","last_fullsized":"2024-09-21T10:06:57.743413+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1181,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.744964+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:06:57.744964+0000","last_peered":"2024-09-21T10:06:57.744964+0000","last_clean":"2024-09-21T10:06:57.744964+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:06:57.744964+0000","last_undegraded":"2024-09-21T10:06:57.744964+0000","last_fullsized":"2024-09-21T10:06:57.744964+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":54,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739499+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:06:57.739499+0000","last_peered":"2024-09-21T10:06:57.739499+0000","last_clean":"2024-09-21T10:06:57.739499+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:06:57.739499+0000","last_undegraded":"2024-09-21T10:06:57.739499+0000","last_fullsized":"2024-09-21T10:06:57.739499+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":235,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.748653+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:06:57.748653+0000","last_peered":"2024-09-21T10:06:57.748653+0000","last_clean":"2024-09-21T10:06:57.748653+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:06:57.748653+0000","last_undegraded":"2024-09-21T10:06:57.748653+0000","last_fullsized":"2024-09-21T10:06:57.748653+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":243,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.749136+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:06:57.749136+0000","last_peered":"2024-09-21T10:06:57.749136+0000","last_clean":"2024-09-21T10:06:57.749136+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:06:57.749136+0000","last_undegraded":"2024-09-21T10:06:57.749136+0000","last_fullsized":"2024-09-21T10:06:57.749136+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":50,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739162+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:06:57.739162+0000","last_peered":"2024-09-21T10:06:57.739162+0000","last_clean":"2024-09-21T10:06:57.739162+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:06:57.739162+0000","last_undegraded":"2024-09-21T10:06:57.739162+0000","last_fullsized":"2024-09-21T10:06:57.739162+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1569,"reported_epoch":1199,"state":"active+clean","last_fresh":"2024-09-21T10:06:57.739613+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:06:57.739613+0000","last_peered":"2024-09-21T10:06:57.739613+0000","last_clean":"2024-09-21T10:06:57.739613+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:06:57.739613+0000","last_undegraded":"2024-09-21T10:06:57.739613+0000","last_fullsized":"2024-09-21T10:06:57.739613+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":455147544,"num_objects":6946,"num_object_clones":0,"num_object_copies":20838,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":24773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6946,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137028,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44042,"num_bytes_recovered":2449997848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3946393600,"data_stored":3946381384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77294,"ondisk_log_size":77294,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234374,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496010,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":472500,"kb_used_data":152680,"kb_used_omap":1,"kb_used_meta":319806,"kb_avail":93272652,"statfs":{"total":95995035648,"available":95511195648,"internally_reserved":0,"allocated":156344320,"data_stored":153795364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":327481799},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757649,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019285,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":460892,"kb_used_data":195156,"kb_used_omap":1,"kb_used_meta":265726,"kb_avail":93284260,"statfs":{"total":95995035648,"available":95523082240,"internally_reserved":0,"allocated":199839744,"data_stored":197302269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":272103878},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888581,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846416,"kb_used_data":354888,"kb_used_omap":34,"kb_used_meta":491485,"kb_avail":92898736,"statfs":{"total":95995035648,"available":95128305664,"internally_reserved":0,"allocated":363405312,"data_stored":360874808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":503281651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,3,3,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967820,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":724500,"kb_used_data":371596,"kb_used_omap":41,"kb_used_meta":352726,"kb_avail":93020652,"statfs":{"total":95995035648,"available":95253147648,"internally_reserved":0,"allocated":380514304,"data_stored":377985395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":361191839},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047059,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":868044,"kb_used_data":542852,"kb_used_omap":41,"kb_used_meta":324950,"kb_avail":92877108,"statfs":{"total":95995035648,"available":95106158592,"internally_reserved":0,"allocated":555880448,"data_stored":553354424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":332749737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126298,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1404370944,"data_stored":1404370944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":546836480,"data_stored":546832408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":341573632,"data_stored":341573632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":341643264,"data_stored":341639192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":186515456,"data_stored":186515456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":145887232,"data_stored":145883160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":647364608,"data_stored":647364608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:07:07.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:07:08.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:08.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:08.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:07 smithi076 ceph-mon[106482]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:08.074 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:07 smithi076 ceph-mon[106482]: pgmap v1577: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 435 MiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 415 op/s; 24773/20916 objects degraded (118.440%); 2.6 MiB/s, 79 objects/s recovering 2024-09-21T10:07:08.074 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:07 smithi076 ceph-mon[110963]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:08.074 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:07 smithi076 ceph-mon[110963]: pgmap v1577: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 435 MiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 415 op/s; 24773/20916 objects degraded (118.440%); 2.6 MiB/s, 79 objects/s recovering 2024-09-21T10:07:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:07 smithi191 ceph-mon[102206]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:07 smithi191 ceph-mon[102206]: pgmap v1577: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 435 MiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 415 op/s; 24773/20916 objects degraded (118.440%); 2.6 MiB/s, 79 objects/s recovering 2024-09-21T10:07:08.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:08.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:08.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:08.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:08.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:08.448 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:08.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:08.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:08.589 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:08.594 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:08.605 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:07:08.605 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:08.652 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:07:08.653 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:08.677 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:08.683 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:07:08.683 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:08.721 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:07:08.721 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:08.753 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:07:08.753 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:08.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:08.786 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:07:08.786 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:08.828 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:07:08.828 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:08.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:08.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:08.860 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:07:08.860 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:08.915 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:08 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/343655142' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:08.915 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:08 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/343655142' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:08.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:08.966 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:08.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:09.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:09.109 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:09.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:08 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/343655142' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:09.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.239 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:09.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:09.340 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:09.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:09.478 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:09.513 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:09.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:07:09.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:09.660 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:09.762 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:09.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:09.870 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:09 smithi076 ceph-mon[106482]: pgmap v1578: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 159 MiB data, 2.6 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 24465/7677 objects degraded (318.679%); 1.5 MiB/s, 66 objects/s recovering 2024-09-21T10:07:09.870 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 24465/7677 objects degraded (318.679%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:09.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:09 smithi076 ceph-mon[110963]: pgmap v1578: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 159 MiB data, 2.6 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 24465/7677 objects degraded (318.679%); 1.5 MiB/s, 66 objects/s recovering 2024-09-21T10:07:09.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 24465/7677 objects degraded (318.679%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:09.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:09.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:09.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:09.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:10.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:10.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:10.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:09 smithi191 ceph-mon[102206]: pgmap v1578: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 159 MiB data, 2.6 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 24465/7677 objects degraded (318.679%); 1.5 MiB/s, 66 objects/s recovering 2024-09-21T10:07:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 24465/7677 objects degraded (318.679%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:10.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:10.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:10.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:10.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:10.327 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Removed 71986 objects 2024-09-21T10:07:10.327 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Clean up completed and total clean up time :264.594 2024-09-21T10:07:10.332 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:10.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:10.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:10.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:10.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:10.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:10.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:10.702 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:10.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:10.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:10.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:10.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:10.857 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:10.923 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:11.268 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:11.269 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:11.270 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:11.270 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:11.289 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:11.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:11.416 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:11.517 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:11.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:11.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:11.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:11.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:11.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:11.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:11.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:11.823 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:11.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:11.907 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:11.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:12.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.030 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:11 smithi191 ceph-mon[102206]: pgmap v1579: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 159 MiB data, 2.4 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 24465/7677 objects degraded (318.679%); 1.5 MiB/s, 66 objects/s recovering 2024-09-21T10:07:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:11 smithi076 ceph-mon[106482]: pgmap v1579: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 159 MiB data, 2.4 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 24465/7677 objects degraded (318.679%); 1.5 MiB/s, 66 objects/s recovering 2024-09-21T10:07:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:11 smithi076 ceph-mon[110963]: pgmap v1579: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 159 MiB data, 2.4 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 781 op/s; 24465/7677 objects degraded (318.679%); 1.5 MiB/s, 66 objects/s recovering 2024-09-21T10:07:12.131 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:12.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:12.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:12.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:12.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:12.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:12.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:12.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:12.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:12.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:12.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:12.766 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:12.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:12.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:12.890 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:12.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:12.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:12.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:13.013 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:13.114 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:13.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:13.237 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:13.258 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:13.360 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:13.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:13.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.512 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:07:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:07:13] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:07:13.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:13.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:13.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:13.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:13.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:13.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:13.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:13.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:13.998 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:13.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:14.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:13 smithi191 ceph-mon[102206]: pgmap v1580: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 16 MiB data, 2.1 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 717 op/s; 23678/807 objects degraded (2934.077%); 0 B/s, 109 objects/s recovering 2024-09-21T10:07:14.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:13 smithi076 ceph-mon[106482]: pgmap v1580: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 16 MiB data, 2.1 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 717 op/s; 23678/807 objects degraded (2934.077%); 0 B/s, 109 objects/s recovering 2024-09-21T10:07:14.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:13 smithi076 ceph-mon[110963]: pgmap v1580: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 16 MiB data, 2.1 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 717 op/s; 23678/807 objects degraded (2934.077%); 0 B/s, 109 objects/s recovering 2024-09-21T10:07:14.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:14.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:14.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:14.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:14.369 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:14.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:14.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:14.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:14.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:14.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:14.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:14.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:14.685 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-21T10:07:14.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-21T10:07:14.716 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:14.751 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:14.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:14.874 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:14.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:14.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:14.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:14.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:15.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 23678/807 objects degraded (2934.077%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:15.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:15.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 23678/807 objects degraded (2934.077%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:15.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 23678/807 objects degraded (2934.077%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:15.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:15.243 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:15.271 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:07:15.271 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23678/78 objects degraded (30356.410%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1337,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1200,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912243,"num_in_osds":5,"osd_in_since":1726913007,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":26,"data_bytes":595941,"bytes_used":2164555776,"bytes_avail":477810622464,"bytes_total":479975178240,"degraded_objects":23678,"degraded_total":78,"degraded_ratio":303.56410256410254,"recovering_objects_per_sec":92,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1095,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":624},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-21T10:05:04.625691+0000","services":{}},"progress_events":{}} 2024-09-21T10:07:15.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:15.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:15.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:15.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:15.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:15.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:15.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:15.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:15.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:15.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:15.856 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:15.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:15.978 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:15.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:16.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:16.101 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:15 smithi191 ceph-mon[102206]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 624 op/s; 23678/78 objects degraded (30356.410%); 0 B/s, 92 objects/s recovering 2024-09-21T10:07:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/758765930' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:07:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:15 smithi076 ceph-mon[106482]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 624 op/s; 23678/78 objects degraded (30356.410%); 0 B/s, 92 objects/s recovering 2024-09-21T10:07:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/758765930' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:07:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:15 smithi076 ceph-mon[110963]: pgmap v1581: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 624 op/s; 23678/78 objects degraded (30356.410%); 0 B/s, 92 objects/s recovering 2024-09-21T10:07:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/758765930' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:07:16.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.226 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:16.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:16.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:16.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:16.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:16.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:16.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:16.575 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:16.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:16.697 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:16.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:16.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:16.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:16.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:16.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:16.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:16.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:17.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:17.095 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:17.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:16 smithi191 ceph-mon[102206]: osdmap e1201: 8 total, 8 up, 5 in 2024-09-21T10:07:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:16 smithi076 ceph-mon[106482]: osdmap e1201: 8 total, 8 up, 5 in 2024-09-21T10:07:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:16 smithi076 ceph-mon[110963]: osdmap e1201: 8 total, 8 up, 5 in 2024-09-21T10:07:17.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:17.221 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:17.322 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:17.344 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:17.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:17.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:17.467 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:17.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:17.569 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:17.592 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:17.594 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:07:17.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:07:17.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:17.716 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:17.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:17.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:17.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:17.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:17.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:18.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:18.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:18.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:17 smithi076 ceph-mon[106482]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 694 op/s; 23678/78 objects degraded (30356.410%); 0 B/s, 109 objects/s recovering 2024-09-21T10:07:18.090 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:17 smithi076 ceph-mon[106482]: osdmap e1202: 8 total, 8 up, 5 in 2024-09-21T10:07:18.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:17 smithi076 ceph-mon[110963]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 694 op/s; 23678/78 objects degraded (30356.410%); 0 B/s, 109 objects/s recovering 2024-09-21T10:07:18.091 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:17 smithi076 ceph-mon[110963]: osdmap e1202: 8 total, 8 up, 5 in 2024-09-21T10:07:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:17 smithi191 ceph-mon[102206]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 694 op/s; 23678/78 objects degraded (30356.410%); 0 B/s, 109 objects/s recovering 2024-09-21T10:07:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:17 smithi191 ceph-mon[102206]: osdmap e1202: 8 total, 8 up, 5 in 2024-09-21T10:07:18.188 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:18.211 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:18.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:18.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:18.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:18.459 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:18.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:18.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:18.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:18.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:18.808 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:18.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:18.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:18.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:18.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:18.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:19.053 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:19.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:19.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:19.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:19.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.444 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:19.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:19.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:19.566 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:19.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:19.668 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:19.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:19.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:19.794 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:07:19.794 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:07:19.800 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:19.801 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:19.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:19.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:19.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:19.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:20.039 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:20.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:19 smithi191 ceph-mon[102206]: pgmap v1585: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 21633/78 objects degraded (27734.615%); 0 B/s, 61 objects/s recovering 2024-09-21T10:07:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 21633/78 objects degraded (27734.615%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:19 smithi076 ceph-mon[106482]: pgmap v1585: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 21633/78 objects degraded (27734.615%); 0 B/s, 61 objects/s recovering 2024-09-21T10:07:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 21633/78 objects degraded (27734.615%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:19 smithi076 ceph-mon[110963]: pgmap v1585: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 316 op/s; 21633/78 objects degraded (27734.615%); 0 B/s, 61 objects/s recovering 2024-09-21T10:07:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 21633/78 objects degraded (27734.615%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:20.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:20.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.186 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:20.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:20.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:20.308 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:20.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:20.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:20.432 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:20.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:20.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:20.556 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:20.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:20.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:20.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:20.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:20.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:20.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:20.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:20.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:21.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:21.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.048 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:21.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:21.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:21.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:21.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:21.273 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:21.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:21.325 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:21.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:21.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:21.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:21.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.644 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:21.666 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:21.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:21.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.789 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:21.834 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:07:21.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:21.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:21.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:22.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:22.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:22.078 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:21 smithi076 ceph-mon[106482]: pgmap v1586: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 30 op/s; 21633/78 objects degraded (27734.615%) 2024-09-21T10:07:22.079 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1413564538' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:22.079 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:22.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:21 smithi076 ceph-mon[110963]: pgmap v1586: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 30 op/s; 21633/78 objects degraded (27734.615%) 2024-09-21T10:07:22.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1413564538' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:22.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:21 smithi191 ceph-mon[102206]: pgmap v1586: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 30 op/s; 21633/78 objects degraded (27734.615%) 2024-09-21T10:07:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1413564538' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:07:22.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:22.157 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:22.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:22.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:22.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:22.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:22.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:22.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:22.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:22.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:22.650 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:22.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:22.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:22.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:22.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:22.898 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:22.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:22.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:23.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:07:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:23.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:22 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:22 smithi191 ceph-mon[102206]: osdmap e1203: 8 total, 8 up, 5 in 2024-09-21T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:22 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:23.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:22 smithi076 ceph-mon[106482]: osdmap e1203: 8 total, 8 up, 5 in 2024-09-21T10:07:23.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:22 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:07:23.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:07:23.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:22 smithi076 ceph-mon[110963]: osdmap e1203: 8 total, 8 up, 5 in 2024-09-21T10:07:23.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:23.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:23.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:23.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:23.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:07:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:07:23] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:07:23.392 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:23.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:23.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:23.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:23.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:23.741 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:23.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:23.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:23.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:23.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:23.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:23.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:24.008 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:24.109 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:23 smithi191 ceph-mon[102206]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 71 objects/s recovering 2024-09-21T10:07:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:23 smithi076 ceph-mon[106482]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 71 objects/s recovering 2024-09-21T10:07:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:23 smithi076 ceph-mon[110963]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 71 objects/s recovering 2024-09-21T10:07:24.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:24.233 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:24.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.255 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:24.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:24.356 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:24.377 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:24.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:24.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:24.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:24.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:24.625 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:24.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:24.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:24.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:24.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:24.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:24.873 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:24.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:24.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:24.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:25.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 21417/78 objects degraded (27457.692%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:25.120 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:25.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:25.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 21417/78 objects degraded (27457.692%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:25.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 21417/78 objects degraded (27457.692%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:25.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.244 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:25.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:25.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:25.366 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:25.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:25.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:25.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:25.491 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:07:25.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:25.612 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:25.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:25.735 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:25.837 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:25.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:25.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:25.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:25.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:26.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:26.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:25 smithi191 ceph-mon[102206]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 61 objects/s recovering 2024-09-21T10:07:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:25 smithi076 ceph-mon[106482]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 61 objects/s recovering 2024-09-21T10:07:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:25 smithi076 ceph-mon[110963]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 61 objects/s recovering 2024-09-21T10:07:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:26.208 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:26.229 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:26.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:26.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:26.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:26.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:26.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:26.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:26.598 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:26.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:26.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:26.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:26.846 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:26.948 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:26.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:26.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:27.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:27.094 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:27.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:27.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:27.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:27.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:27.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.345 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:27.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.446 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.473 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:27.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:27.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:27.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:27.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:27.697 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:27.719 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:27.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:27.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:27.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:27.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:27.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:27.965 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:27.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:28.066 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:28.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:27 smithi191 ceph-mon[102206]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 50 objects/s recovering 2024-09-21T10:07:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:27 smithi076 ceph-mon[106482]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 50 objects/s recovering 2024-09-21T10:07:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:27 smithi076 ceph-mon[110963]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 21417/78 objects degraded (27457.692%); 0 B/s, 50 objects/s recovering 2024-09-21T10:07:28.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:28.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:28.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:28.333 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:28.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:28.457 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:28.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:28.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:28.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:28.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:28.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:28.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:28.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:28.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:28.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:28.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:28.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:28.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:28.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:29.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:29.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:29.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:29.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:29.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:29.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:29.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:29.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:29.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:29.423 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:29.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:29.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:29.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:29.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:29.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:29.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:29.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:29.814 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:29.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:29.938 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:29.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:29.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:30.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:30.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:29 smithi191 ceph-mon[102206]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 20463/78 objects degraded (26234.615%); 0 B/s, 173 objects/s recovering 2024-09-21T10:07:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 20463/78 objects degraded (26234.615%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:29 smithi076 ceph-mon[106482]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 20463/78 objects degraded (26234.615%); 0 B/s, 173 objects/s recovering 2024-09-21T10:07:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 20463/78 objects degraded (26234.615%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:29 smithi076 ceph-mon[110963]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 20463/78 objects degraded (26234.615%); 0 B/s, 173 objects/s recovering 2024-09-21T10:07:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 20463/78 objects degraded (26234.615%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:30.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.186 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:30.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:30.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:30.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:30.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:30.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:30.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:30.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:30.659 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:30.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.683 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:30.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:30.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:30.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:30.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:30.928 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:30.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.030 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:31.051 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.176 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:31.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.277 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:31.300 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:31.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:31.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:31.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:31.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:31.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:31.772 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:31.793 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:31.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:31.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:31.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:32.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:32.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:32.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:31 smithi191 ceph-mon[102206]: pgmap v1592: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 20463/78 objects degraded (26234.615%); 0 B/s, 173 objects/s recovering 2024-09-21T10:07:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:31 smithi076 ceph-mon[106482]: pgmap v1592: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 20463/78 objects degraded (26234.615%); 0 B/s, 173 objects/s recovering 2024-09-21T10:07:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:31 smithi076 ceph-mon[110963]: pgmap v1592: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 20463/78 objects degraded (26234.615%); 0 B/s, 173 objects/s recovering 2024-09-21T10:07:32.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:32.165 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:32.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:32.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:32.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:32.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:32.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:32.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:32.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:32.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:32.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:32.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:32.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:32.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:32.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:32.902 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:32.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:32.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:33.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:33.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:33.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:33.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:33.151 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:33.253 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:33.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:33.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:33.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:33.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:33.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:33.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:33.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:07:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:07:33] "GET /metrics HTTP/1.1" 200 36478 "" "Prometheus/2.51.0" 2024-09-21T10:07:33.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:33.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:33.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:33.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:33.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:33.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:33.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:33.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:33.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:34.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:34.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:33 smithi191 ceph-mon[102206]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 276 objects/s recovering 2024-09-21T10:07:34.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:33 smithi076 ceph-mon[106482]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 276 objects/s recovering 2024-09-21T10:07:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:33 smithi076 ceph-mon[110963]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 276 objects/s recovering 2024-09-21T10:07:34.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:34.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:34.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:34.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.266 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:34.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:34.367 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:34.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:34.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:34.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:34.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:34.637 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:34.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:34.739 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:34.761 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:34.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:34.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:34.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:34.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:34.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:35.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:35.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:35.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 19208/78 objects degraded (24625.641%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:35.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 19208/78 objects degraded (24625.641%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:35.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 19208/78 objects degraded (24625.641%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:35.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.131 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:35.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:35.233 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:35.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:35.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:35.262 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:07:35.264 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:07:35.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:35.378 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:35.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:35.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:35.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:35.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:35.726 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:35.747 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:35.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:35.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:35.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:35.973 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:35.995 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:35.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:36.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:36.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:35 smithi191 ceph-mon[102206]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:35 smithi191 ceph-mon[102206]: pgmap v1594: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 184 objects/s recovering 2024-09-21T10:07:36.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:36.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:35 smithi076 ceph-mon[106482]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:35 smithi076 ceph-mon[106482]: pgmap v1594: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 184 objects/s recovering 2024-09-21T10:07:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:35 smithi076 ceph-mon[110963]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:36.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:35 smithi076 ceph-mon[110963]: pgmap v1594: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 184 objects/s recovering 2024-09-21T10:07:36.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:36.242 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:36.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:36.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:36.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:36.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:36.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:36.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:36.610 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:36.712 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:36.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:36.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:36.860 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:36.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:36.985 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:36.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:37.087 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:37.108 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:37.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:37.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:37.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:37.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:37.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:37.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:37.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:37.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:37.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:37.601 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:37.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:37.724 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:37.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:37.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:37.847 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:37.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:37.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:37.970 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:37.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:38.013 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1593,"stamp":"2024-09-21T10:07:32.669043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":19208,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":144253,"num_write_kb":4609554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47904,"num_bytes_recovered":2452960798,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77248,"ondisk_log_size":77248,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":2123236,"kb_used_data":319916,"kb_used_omap":119,"kb_used_meta":1803144,"kb_avail":466602524,"statfs":{"total":479975178240,"available":477800984576,"internally_reserved":0,"allocated":327593984,"data_stored":314860856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1846419656},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2425,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2994,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.838653"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839477+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:07:21.839477+0000","last_peered":"2024-09-21T10:07:21.839477+0000","last_clean":"2024-09-21T10:07:21.839477+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:07:21.839477+0000","last_undegraded":"2024-09-21T10:07:21.839477+0000","last_fullsized":"2024-09-21T10:07:21.839477+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":60,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.849524+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:07:21.849524+0000","last_peered":"2024-09-21T10:07:21.849524+0000","last_clean":"2024-09-21T10:07:21.849524+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:07:21.849524+0000","last_undegraded":"2024-09-21T10:07:21.849524+0000","last_fullsized":"2024-09-21T10:07:21.849524+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1228,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.838208+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:07:21.838208+0000","last_peered":"2024-09-21T10:07:21.838208+0000","last_clean":"2024-09-21T10:07:21.838208+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:07:21.838208+0000","last_undegraded":"2024-09-21T10:07:21.838208+0000","last_fullsized":"2024-09-21T10:07:21.838208+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1559,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837282+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:07:21.837282+0000","last_peered":"2024-09-21T10:07:21.837282+0000","last_clean":"2024-09-21T10:07:21.837282+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:07:21.837282+0000","last_undegraded":"2024-09-21T10:07:21.837282+0000","last_fullsized":"2024-09-21T10:07:21.837282+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1559,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840312+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:07:21.840312+0000","last_peered":"2024-09-21T10:07:21.840312+0000","last_clean":"2024-09-21T10:07:21.840312+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:07:21.840312+0000","last_undegraded":"2024-09-21T10:07:21.840312+0000","last_fullsized":"2024-09-21T10:07:21.840312+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":63,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.849593+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:07:21.849593+0000","last_peered":"2024-09-21T10:07:21.849593+0000","last_clean":"2024-09-21T10:07:21.849593+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:07:21.849593+0000","last_undegraded":"2024-09-21T10:07:21.849593+0000","last_fullsized":"2024-09-21T10:07:21.849593+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":243,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.838561+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:07:21.838561+0000","last_peered":"2024-09-21T10:07:21.838561+0000","last_clean":"2024-09-21T10:07:21.838561+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:07:21.838561+0000","last_undegraded":"2024-09-21T10:07:21.838561+0000","last_fullsized":"2024-09-21T10:07:21.838561+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":83,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.150815+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:07:18.150815+0000","last_peered":"2024-09-21T10:07:18.150815+0000","last_clean":"2024-09-21T10:07:18.150815+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:07:18.150815+0000","last_undegraded":"2024-09-21T10:07:18.150815+0000","last_fullsized":"2024-09-21T10:07:18.150815+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1200'9046","reported_seq":17922,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840594+0000","last_change":"2024-09-21T10:05:04.851742+0000","last_active":"2024-09-21T10:07:21.840594+0000","last_peered":"2024-09-21T10:07:21.840594+0000","last_clean":"2024-09-21T10:07:21.840594+0000","last_became_active":"2024-09-21T10:05:04.833027+0000","last_became_peered":"2024-09-21T10:05:04.833027+0000","last_unstale":"2024-09-21T10:07:21.840594+0000","last_undegraded":"2024-09-21T10:07:21.840594+0000","last_fullsized":"2024-09-21T10:07:21.840594+0000","mapping_epoch":1187,"log_start":"1160'4331","ondisk_log_start":"1160'4331","created":1137,"last_epoch_clean":1188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4715,"log_dups_size":0,"ondisk_log_size":4715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:21:19.012712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9046,"num_write_kb":289472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4356,"num_bytes_recovered":194969600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":98,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841288+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:07:21.841288+0000","last_peered":"2024-09-21T10:07:21.841288+0000","last_clean":"2024-09-21T10:07:21.841288+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:07:21.841288+0000","last_undegraded":"2024-09-21T10:07:21.841288+0000","last_fullsized":"2024-09-21T10:07:21.841288+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":54,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.824978+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:07:16.824978+0000","last_peered":"2024-09-21T10:07:16.824978+0000","last_clean":"2024-09-21T10:07:16.824978+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:07:16.824978+0000","last_undegraded":"2024-09-21T10:07:16.824978+0000","last_fullsized":"2024-09-21T10:07:16.824978+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.a","version":"1200'9028","reported_seq":16758,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837022+0000","last_change":"2024-09-21T10:04:26.723810+0000","last_active":"2024-09-21T10:07:21.837022+0000","last_peered":"2024-09-21T10:07:21.837022+0000","last_clean":"2024-09-21T10:07:21.837022+0000","last_became_active":"2024-09-21T10:04:26.708414+0000","last_became_peered":"2024-09-21T10:04:26.708414+0000","last_unstale":"2024-09-21T10:07:21.837022+0000","last_undegraded":"2024-09-21T10:07:21.837022+0000","last_fullsized":"2024-09-21T10:07:21.837022+0000","mapping_epoch":1180,"log_start":"1163'4403","ondisk_log_start":"1163'4403","created":1137,"last_epoch_clean":1181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4625,"log_dups_size":0,"ondisk_log_size":4625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:31.203529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9028,"num_write_kb":288896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3787,"num_bytes_recovered":186974208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":142,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840974+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:07:21.840974+0000","last_peered":"2024-09-21T10:07:21.840974+0000","last_clean":"2024-09-21T10:07:21.840974+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:07:21.840974+0000","last_undegraded":"2024-09-21T10:07:21.840974+0000","last_fullsized":"2024-09-21T10:07:21.840974+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1183,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.842175+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:07:21.842175+0000","last_peered":"2024-09-21T10:07:21.842175+0000","last_clean":"2024-09-21T10:07:21.842175+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:07:21.842175+0000","last_undegraded":"2024-09-21T10:07:21.842175+0000","last_fullsized":"2024-09-21T10:07:21.842175+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"1200'9208","reported_seq":15938,"reported_epoch":1203,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:32.450629+0000","last_change":"2024-09-21T10:07:15.378433+0000","last_active":"2024-09-21T10:07:32.450629+0000","last_peered":"2024-09-21T10:07:32.450629+0000","last_clean":"2024-09-21T10:03:27.679676+0000","last_became_active":"2024-09-21T10:03:30.089547+0000","last_became_peered":"2024-09-21T10:03:30.089547+0000","last_unstale":"2024-09-21T10:07:32.450629+0000","last_undegraded":"2024-09-21T10:03:29.968867+0000","last_fullsized":"2024-09-21T10:03:29.835533+0000","mapping_epoch":1170,"log_start":"1163'4482","ondisk_log_start":"1163'4482","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4726,"log_dups_size":0,"ondisk_log_size":4726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1314,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9208,"num_write_kb":294656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3290,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"1,4","objects":-1314}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":58,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825508+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:07:16.825508+0000","last_peered":"2024-09-21T10:07:16.825508+0000","last_clean":"2024-09-21T10:07:16.825508+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:07:16.825508+0000","last_undegraded":"2024-09-21T10:07:16.825508+0000","last_fullsized":"2024-09-21T10:07:16.825508+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":63,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825111+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:07:16.825111+0000","last_peered":"2024-09-21T10:07:16.825111+0000","last_clean":"2024-09-21T10:07:16.825111+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:07:16.825111+0000","last_undegraded":"2024-09-21T10:07:16.825111+0000","last_fullsized":"2024-09-21T10:07:16.825111+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1200'8986","reported_seq":14654,"reported_epoch":1203,"state":"active+recovery_wait","last_fresh":"2024-09-21T10:07:21.836403+0000","last_change":"2024-09-21T10:06:10.765843+0000","last_active":"2024-09-21T10:07:21.836403+0000","last_peered":"2024-09-21T10:07:21.836403+0000","last_clean":"2024-09-21T10:01:38.987142+0000","last_became_active":"2024-09-21T10:01:54.159657+0000","last_became_peered":"2024-09-21T10:01:54.159657+0000","last_unstale":"2024-09-21T10:07:21.836403+0000","last_undegraded":"2024-09-21T10:07:21.836403+0000","last_fullsized":"2024-09-21T10:07:21.836403+0000","mapping_epoch":1151,"log_start":"1163'4436","ondisk_log_start":"1163'4436","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8986,"num_write_kb":287552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2755,"num_bytes_recovered":180551680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":["3","2","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1213,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839228+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:07:21.839228+0000","last_peered":"2024-09-21T10:07:21.839228+0000","last_clean":"2024-09-21T10:07:21.839228+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:07:21.839228+0000","last_undegraded":"2024-09-21T10:07:21.839228+0000","last_fullsized":"2024-09-21T10:07:21.839228+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1569,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840311+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:07:21.840311+0000","last_peered":"2024-09-21T10:07:21.840311+0000","last_clean":"2024-09-21T10:07:21.840311+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:07:21.840311+0000","last_undegraded":"2024-09-21T10:07:21.840311+0000","last_fullsized":"2024-09-21T10:07:21.840311+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.f","version":"1200'9014","reported_seq":14792,"reported_epoch":1203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:21.836350+0000","last_change":"2024-09-21T10:03:36.324735+0000","last_active":"2024-09-21T10:07:21.836350+0000","last_peered":"2024-09-21T10:07:21.836350+0000","last_clean":"2024-09-21T10:03:27.763888+0000","last_became_active":"2024-09-21T10:03:36.310727+0000","last_became_peered":"2024-09-21T10:03:36.310727+0000","last_unstale":"2024-09-21T10:07:21.836350+0000","last_undegraded":"2024-09-21T10:03:29.952560+0000","last_fullsized":"2024-09-21T10:03:29.836480+0000","mapping_epoch":1170,"log_start":"1163'4448","ondisk_log_start":"1163'4448","created":1137,"last_epoch_clean":1154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4566,"log_dups_size":0,"ondisk_log_size":4566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4506,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9014,"num_write_kb":288448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2809,"num_bytes_recovered":184025088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","0","4","7"],"object_location_counts":[{"shards":"3,4","objects":-3621}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":251,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837883+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:07:21.837883+0000","last_peered":"2024-09-21T10:07:21.837883+0000","last_clean":"2024-09-21T10:07:21.837883+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:07:21.837883+0000","last_undegraded":"2024-09-21T10:07:21.837883+0000","last_fullsized":"2024-09-21T10:07:21.837883+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1228,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839059+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:07:21.839059+0000","last_peered":"2024-09-21T10:07:21.839059+0000","last_clean":"2024-09-21T10:07:21.839059+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:07:21.839059+0000","last_undegraded":"2024-09-21T10:07:21.839059+0000","last_fullsized":"2024-09-21T10:07:21.839059+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"1200'8872","reported_seq":17493,"reported_epoch":1203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:21.838255+0000","last_change":"2024-09-21T10:03:38.320428+0000","last_active":"2024-09-21T10:07:21.838255+0000","last_peered":"2024-09-21T10:07:21.838255+0000","last_clean":"2024-09-21T10:02:10.032942+0000","last_became_active":"2024-09-21T10:03:36.060192+0000","last_became_peered":"2024-09-21T10:03:36.060192+0000","last_unstale":"2024-09-21T10:07:21.838255+0000","last_undegraded":"2024-09-21T10:03:35.905760+0000","last_fullsized":"2024-09-21T10:03:29.835235+0000","mapping_epoch":1170,"log_start":"1160'4253","ondisk_log_start":"1160'4253","created":1137,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4619,"log_dups_size":0,"ondisk_log_size":4619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":2474,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8872,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":185204736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"2,3","objects":-750}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":54,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825017+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:07:16.825017+0000","last_peered":"2024-09-21T10:07:16.825017+0000","last_clean":"2024-09-21T10:07:16.825017+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:07:16.825017+0000","last_undegraded":"2024-09-21T10:07:16.825017+0000","last_fullsized":"2024-09-21T10:07:16.825017+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":871,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840920+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:07:21.840920+0000","last_peered":"2024-09-21T10:07:21.840920+0000","last_clean":"2024-09-21T10:07:21.840920+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:07:21.840920+0000","last_undegraded":"2024-09-21T10:07:21.840920+0000","last_fullsized":"2024-09-21T10:07:21.840920+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.d","version":"1200'9158","reported_seq":19319,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825347+0000","last_change":"2024-09-21T10:06:23.957394+0000","last_active":"2024-09-21T10:07:16.825347+0000","last_peered":"2024-09-21T10:07:16.825347+0000","last_clean":"2024-09-21T10:07:16.825347+0000","last_became_active":"2024-09-21T10:06:23.937063+0000","last_became_peered":"2024-09-21T10:06:23.937063+0000","last_unstale":"2024-09-21T10:07:16.825347+0000","last_undegraded":"2024-09-21T10:07:16.825347+0000","last_fullsized":"2024-09-21T10:07:16.825347+0000","mapping_epoch":1194,"log_start":"1148'3100","ondisk_log_start":"1148'3100","created":1137,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6058,"log_dups_size":0,"ondisk_log_size":6058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:57.739037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9158,"num_write_kb":292993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4579,"num_bytes_recovered":242810904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1214,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.836728+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:07:21.836728+0000","last_peered":"2024-09-21T10:07:21.836728+0000","last_clean":"2024-09-21T10:07:21.836728+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:07:21.836728+0000","last_undegraded":"2024-09-21T10:07:21.836728+0000","last_fullsized":"2024-09-21T10:07:21.836728+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825466+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:07:16.825466+0000","last_peered":"2024-09-21T10:07:16.825466+0000","last_clean":"2024-09-21T10:07:16.825466+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:07:16.825466+0000","last_undegraded":"2024-09-21T10:07:16.825466+0000","last_fullsized":"2024-09-21T10:07:16.825466+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"1200'9296","reported_seq":18435,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:18.150886+0000","last_change":"2024-09-21T10:07:16.854144+0000","last_active":"2024-09-21T10:07:18.150886+0000","last_peered":"2024-09-21T10:07:18.150886+0000","last_clean":"2024-09-21T10:07:18.150886+0000","last_became_active":"2024-09-21T10:07:16.827359+0000","last_became_peered":"2024-09-21T10:07:16.827359+0000","last_unstale":"2024-09-21T10:07:18.150886+0000","last_undegraded":"2024-09-21T10:07:18.150886+0000","last_fullsized":"2024-09-21T10:07:18.150886+0000","mapping_epoch":1201,"log_start":"1164'4725","ondisk_log_start":"1164'4725","created":1137,"last_epoch_clean":1202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4571,"log_dups_size":0,"ondisk_log_size":4571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:21.769651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9296,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4488,"num_bytes_recovered":252968960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":243,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837089+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:07:21.837089+0000","last_peered":"2024-09-21T10:07:21.837089+0000","last_clean":"2024-09-21T10:07:21.837089+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:07:21.837089+0000","last_undegraded":"2024-09-21T10:07:21.837089+0000","last_fullsized":"2024-09-21T10:07:21.837089+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1573,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841436+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:07:21.841436+0000","last_peered":"2024-09-21T10:07:21.841436+0000","last_clean":"2024-09-21T10:07:21.841436+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:07:21.841436+0000","last_undegraded":"2024-09-21T10:07:21.841436+0000","last_fullsized":"2024-09-21T10:07:21.841436+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"1200'8840","reported_seq":19098,"reported_epoch":1203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:21.839610+0000","last_change":"2024-09-21T10:03:38.330361+0000","last_active":"2024-09-21T10:07:21.839610+0000","last_peered":"2024-09-21T10:07:21.839610+0000","last_clean":"2024-09-21T10:01:39.014534+0000","last_became_active":"2024-09-21T10:03:36.039081+0000","last_became_peered":"2024-09-21T10:03:36.039081+0000","last_unstale":"2024-09-21T10:07:21.839610+0000","last_undegraded":"2024-09-21T10:03:30.052328+0000","last_fullsized":"2024-09-21T10:03:29.834353+0000","mapping_epoch":1170,"log_start":"1160'4247","ondisk_log_start":"1160'4247","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4593,"log_dups_size":0,"ondisk_log_size":4593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8840,"num_write_kb":282880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3642,"num_bytes_recovered":238682112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"2,3","objects":-732}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":726,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.824794+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:07:16.824794+0000","last_peered":"2024-09-21T10:07:16.824794+0000","last_clean":"2024-09-21T10:07:16.824794+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:07:16.824794+0000","last_undegraded":"2024-09-21T10:07:16.824794+0000","last_fullsized":"2024-09-21T10:07:16.824794+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1227,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837245+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:07:21.837245+0000","last_peered":"2024-09-21T10:07:21.837245+0000","last_clean":"2024-09-21T10:07:21.837245+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:07:21.837245+0000","last_undegraded":"2024-09-21T10:07:21.837245+0000","last_fullsized":"2024-09-21T10:07:21.837245+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837404+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:07:21.837404+0000","last_peered":"2024-09-21T10:07:21.837404+0000","last_clean":"2024-09-21T10:07:21.837404+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:07:21.837404+0000","last_undegraded":"2024-09-21T10:07:21.837404+0000","last_fullsized":"2024-09-21T10:07:21.837404+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":63,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825440+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:07:16.825440+0000","last_peered":"2024-09-21T10:07:16.825440+0000","last_clean":"2024-09-21T10:07:16.825440+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:07:16.825440+0000","last_undegraded":"2024-09-21T10:07:16.825440+0000","last_fullsized":"2024-09-21T10:07:16.825440+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1200'8946","reported_seq":14599,"reported_epoch":1203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:21.838056+0000","last_change":"2024-09-21T10:03:36.285639+0000","last_active":"2024-09-21T10:07:21.838056+0000","last_peered":"2024-09-21T10:07:21.838056+0000","last_clean":"2024-09-21T10:03:27.600116+0000","last_became_active":"2024-09-21T10:03:36.268983+0000","last_became_peered":"2024-09-21T10:03:36.268983+0000","last_unstale":"2024-09-21T10:07:21.838056+0000","last_undegraded":"2024-09-21T10:03:29.987723+0000","last_fullsized":"2024-09-21T10:03:29.835979+0000","mapping_epoch":1170,"log_start":"1160'4254","ondisk_log_start":"1160'4254","created":1137,"last_epoch_clean":1160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4692,"log_dups_size":0,"ondisk_log_size":4692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8946,"num_write_kb":286272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2749,"num_bytes_recovered":180158464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2],"avail_no_missing":["1","0","2","7"],"object_location_counts":[{"shards":"1,2","objects":-3605}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":857,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837434+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:07:21.837434+0000","last_peered":"2024-09-21T10:07:21.837434+0000","last_clean":"2024-09-21T10:07:21.837434+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:07:21.837434+0000","last_undegraded":"2024-09-21T10:07:21.837434+0000","last_fullsized":"2024-09-21T10:07:21.837434+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1446,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.838998+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:07:21.838998+0000","last_peered":"2024-09-21T10:07:21.838998+0000","last_clean":"2024-09-21T10:07:21.838998+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:07:21.838998+0000","last_undegraded":"2024-09-21T10:07:21.838998+0000","last_fullsized":"2024-09-21T10:07:21.838998+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":855,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840699+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:07:21.840699+0000","last_peered":"2024-09-21T10:07:21.840699+0000","last_clean":"2024-09-21T10:07:21.840699+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:07:21.840699+0000","last_undegraded":"2024-09-21T10:07:21.840699+0000","last_fullsized":"2024-09-21T10:07:21.840699+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"1200'8622","reported_seq":17402,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840982+0000","last_change":"2024-09-21T10:05:37.849809+0000","last_active":"2024-09-21T10:07:21.840982+0000","last_peered":"2024-09-21T10:07:21.840982+0000","last_clean":"2024-09-21T10:07:21.840982+0000","last_became_active":"2024-09-21T10:05:37.830539+0000","last_became_peered":"2024-09-21T10:05:37.830539+0000","last_unstale":"2024-09-21T10:07:21.840982+0000","last_undegraded":"2024-09-21T10:07:21.840982+0000","last_fullsized":"2024-09-21T10:07:21.840982+0000","mapping_epoch":1190,"log_start":"1158'3837","ondisk_log_start":"1158'3837","created":1137,"last_epoch_clean":1191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4785,"log_dups_size":0,"ondisk_log_size":4785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:35.172792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8622,"num_write_kb":275904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4311,"num_bytes_recovered":174260224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1487,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839406+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:07:21.839406+0000","last_peered":"2024-09-21T10:07:21.839406+0000","last_clean":"2024-09-21T10:07:21.839406+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:07:21.839406+0000","last_undegraded":"2024-09-21T10:07:21.839406+0000","last_fullsized":"2024-09-21T10:07:21.839406+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1574,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837696+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:07:21.837696+0000","last_peered":"2024-09-21T10:07:21.837696+0000","last_clean":"2024-09-21T10:07:21.837696+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:07:21.837696+0000","last_undegraded":"2024-09-21T10:07:21.837696+0000","last_fullsized":"2024-09-21T10:07:21.837696+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1582,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841352+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:07:21.841352+0000","last_peered":"2024-09-21T10:07:21.841352+0000","last_clean":"2024-09-21T10:07:21.841352+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:07:21.841352+0000","last_undegraded":"2024-09-21T10:07:21.841352+0000","last_fullsized":"2024-09-21T10:07:21.841352+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"1200'9018","reported_seq":19015,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825505+0000","last_change":"2024-09-21T10:06:57.766690+0000","last_active":"2024-09-21T10:07:16.825505+0000","last_peered":"2024-09-21T10:07:16.825505+0000","last_clean":"2024-09-21T10:07:16.825505+0000","last_became_active":"2024-09-21T10:06:57.745326+0000","last_became_peered":"2024-09-21T10:06:57.745326+0000","last_unstale":"2024-09-21T10:07:16.825505+0000","last_undegraded":"2024-09-21T10:07:16.825505+0000","last_fullsized":"2024-09-21T10:07:16.825505+0000","mapping_epoch":1198,"log_start":"1148'3000","ondisk_log_start":"1148'3000","created":1137,"last_epoch_clean":1199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":6018,"log_dups_size":0,"ondisk_log_size":6018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:11.695500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9018,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4509,"num_bytes_recovered":239665152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":151,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839963+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:07:21.839963+0000","last_peered":"2024-09-21T10:07:21.839963+0000","last_clean":"2024-09-21T10:07:21.839963+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:07:21.839963+0000","last_undegraded":"2024-09-21T10:07:21.839963+0000","last_fullsized":"2024-09-21T10:07:21.839963+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1164,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825057+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:07:16.825057+0000","last_peered":"2024-09-21T10:07:16.825057+0000","last_clean":"2024-09-21T10:07:16.825057+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:07:16.825057+0000","last_undegraded":"2024-09-21T10:07:16.825057+0000","last_fullsized":"2024-09-21T10:07:16.825057+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1585,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840075+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:07:21.840075+0000","last_peered":"2024-09-21T10:07:21.840075+0000","last_clean":"2024-09-21T10:07:21.840075+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:07:21.840075+0000","last_undegraded":"2024-09-21T10:07:21.840075+0000","last_fullsized":"2024-09-21T10:07:21.840075+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"1200'9044","reported_seq":9185,"reported_epoch":1203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-21T10:07:21.837045+0000","last_change":"2024-09-21T10:02:40.122227+0000","last_active":"2024-09-21T10:07:21.837045+0000","last_peered":"2024-09-21T10:07:21.837045+0000","last_clean":"2024-09-21T10:02:38.276244+0000","last_became_active":"2024-09-21T10:02:40.112750+0000","last_became_peered":"2024-09-21T10:02:40.112750+0000","last_unstale":"2024-09-21T10:07:21.837045+0000","last_undegraded":"2024-09-21T10:02:40.002835+0000","last_fullsized":"2024-09-21T10:02:39.969477+0000","mapping_epoch":1162,"log_start":"1163'4485","ondisk_log_start":"1163'4485","created":1137,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9044,"num_write_kb":289408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"3,4","objects":-4367}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1175,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837611+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:07:21.837611+0000","last_peered":"2024-09-21T10:07:21.837611+0000","last_clean":"2024-09-21T10:07:21.837611+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:07:21.837611+0000","last_undegraded":"2024-09-21T10:07:21.837611+0000","last_fullsized":"2024-09-21T10:07:21.837611+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1596,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840878+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:07:21.840878+0000","last_peered":"2024-09-21T10:07:21.840878+0000","last_clean":"2024-09-21T10:07:21.840878+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:07:21.840878+0000","last_undegraded":"2024-09-21T10:07:21.840878+0000","last_fullsized":"2024-09-21T10:07:21.840878+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.838602+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:07:21.838602+0000","last_peered":"2024-09-21T10:07:21.838602+0000","last_clean":"2024-09-21T10:07:21.838602+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:07:21.838602+0000","last_undegraded":"2024-09-21T10:07:21.838602+0000","last_fullsized":"2024-09-21T10:07:21.838602+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"1200'8876","reported_seq":16491,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.836493+0000","last_change":"2024-09-21T10:03:56.968095+0000","last_active":"2024-09-21T10:07:21.836493+0000","last_peered":"2024-09-21T10:07:21.836493+0000","last_clean":"2024-09-21T10:07:21.836493+0000","last_became_active":"2024-09-21T10:03:56.957809+0000","last_became_peered":"2024-09-21T10:03:56.957809+0000","last_unstale":"2024-09-21T10:07:21.836493+0000","last_undegraded":"2024-09-21T10:07:21.836493+0000","last_fullsized":"2024-09-21T10:07:21.836493+0000","mapping_epoch":1174,"log_start":"1158'4094","ondisk_log_start":"1158'4094","created":1137,"last_epoch_clean":1175,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:48:36.777067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8876,"num_write_kb":284032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3738,"num_bytes_recovered":189726720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1573,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841084+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:07:21.841084+0000","last_peered":"2024-09-21T10:07:21.841084+0000","last_clean":"2024-09-21T10:07:21.841084+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:07:21.841084+0000","last_undegraded":"2024-09-21T10:07:21.841084+0000","last_fullsized":"2024-09-21T10:07:21.841084+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":753,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840585+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:07:21.840585+0000","last_peered":"2024-09-21T10:07:21.840585+0000","last_clean":"2024-09-21T10:07:21.840585+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:07:21.840585+0000","last_undegraded":"2024-09-21T10:07:21.840585+0000","last_fullsized":"2024-09-21T10:07:21.840585+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1565,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840510+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:07:21.840510+0000","last_peered":"2024-09-21T10:07:21.840510+0000","last_clean":"2024-09-21T10:07:21.840510+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:07:21.840510+0000","last_undegraded":"2024-09-21T10:07:21.840510+0000","last_fullsized":"2024-09-21T10:07:21.840510+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.3","version":"1200'9172","reported_seq":9319,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837784+0000","last_change":"2024-09-21T10:01:05.963714+0000","last_active":"2024-09-21T10:07:21.837784+0000","last_peered":"2024-09-21T10:07:21.837784+0000","last_clean":"2024-09-21T10:07:21.837784+0000","last_became_active":"2024-09-21T10:01:05.963421+0000","last_became_peered":"2024-09-21T10:01:05.963421+0000","last_unstale":"2024-09-21T10:07:21.837784+0000","last_undegraded":"2024-09-21T10:07:21.837784+0000","last_fullsized":"2024-09-21T10:07:21.837784+0000","mapping_epoch":1137,"log_start":"1164'4626","ondisk_log_start":"1164'4626","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4546,"log_dups_size":0,"ondisk_log_size":4546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:39.509400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9172,"num_write_kb":293504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":273,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.838642+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:07:21.838642+0000","last_peered":"2024-09-21T10:07:21.838642+0000","last_clean":"2024-09-21T10:07:21.838642+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:07:21.838642+0000","last_undegraded":"2024-09-21T10:07:21.838642+0000","last_fullsized":"2024-09-21T10:07:21.838642+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1574,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841067+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:07:21.841067+0000","last_peered":"2024-09-21T10:07:21.841067+0000","last_clean":"2024-09-21T10:07:21.841067+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:07:21.841067+0000","last_undegraded":"2024-09-21T10:07:21.841067+0000","last_fullsized":"2024-09-21T10:07:21.841067+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":251,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.836990+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:07:21.836990+0000","last_peered":"2024-09-21T10:07:21.836990+0000","last_clean":"2024-09-21T10:07:21.836990+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:07:21.836990+0000","last_undegraded":"2024-09-21T10:07:21.836990+0000","last_fullsized":"2024-09-21T10:07:21.836990+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1200'8848","reported_seq":8995,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837340+0000","last_change":"2024-09-21T10:01:05.964616+0000","last_active":"2024-09-21T10:07:21.837340+0000","last_peered":"2024-09-21T10:07:21.837340+0000","last_clean":"2024-09-21T10:07:21.837340+0000","last_became_active":"2024-09-21T10:01:05.964343+0000","last_became_peered":"2024-09-21T10:01:05.964343+0000","last_unstale":"2024-09-21T10:07:21.837340+0000","last_undegraded":"2024-09-21T10:07:21.837340+0000","last_fullsized":"2024-09-21T10:07:21.837340+0000","mapping_epoch":1137,"log_start":"1160'4213","ondisk_log_start":"1160'4213","created":1137,"last_epoch_clean":1138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:01:04.944137+0000","last_clean_scrub_stamp":"2024-09-21T10:01:04.944137+0000","objects_scrubbed":0,"log_size":4635,"log_dups_size":0,"ondisk_log_size":4635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:36:01.725836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8848,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":114,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840787+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:07:21.840787+0000","last_peered":"2024-09-21T10:07:21.840787+0000","last_clean":"2024-09-21T10:07:21.840787+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:07:21.840787+0000","last_undegraded":"2024-09-21T10:07:21.840787+0000","last_fullsized":"2024-09-21T10:07:21.840787+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825584+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:07:16.825584+0000","last_peered":"2024-09-21T10:07:16.825584+0000","last_clean":"2024-09-21T10:07:16.825584+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:07:16.825584+0000","last_undegraded":"2024-09-21T10:07:16.825584+0000","last_fullsized":"2024-09-21T10:07:16.825584+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":63,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.824818+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:07:16.824818+0000","last_peered":"2024-09-21T10:07:16.824818+0000","last_clean":"2024-09-21T10:07:16.824818+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:07:16.824818+0000","last_undegraded":"2024-09-21T10:07:16.824818+0000","last_fullsized":"2024-09-21T10:07:16.824818+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":54,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.849388+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:07:21.849388+0000","last_peered":"2024-09-21T10:07:21.849388+0000","last_clean":"2024-09-21T10:07:21.849388+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:07:21.849388+0000","last_undegraded":"2024-09-21T10:07:21.849388+0000","last_fullsized":"2024-09-21T10:07:21.849388+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":64,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.849964+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:07:21.849964+0000","last_peered":"2024-09-21T10:07:21.849964+0000","last_clean":"2024-09-21T10:07:21.849964+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:07:21.849964+0000","last_undegraded":"2024-09-21T10:07:21.849964+0000","last_fullsized":"2024-09-21T10:07:21.849964+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1573,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839976+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:07:21.839976+0000","last_peered":"2024-09-21T10:07:21.839976+0000","last_clean":"2024-09-21T10:07:21.839976+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:07:21.839976+0000","last_undegraded":"2024-09-21T10:07:21.839976+0000","last_fullsized":"2024-09-21T10:07:21.839976+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1186,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837830+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:07:21.837830+0000","last_peered":"2024-09-21T10:07:21.837830+0000","last_clean":"2024-09-21T10:07:21.837830+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:07:21.837830+0000","last_undegraded":"2024-09-21T10:07:21.837830+0000","last_fullsized":"2024-09-21T10:07:21.837830+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1197,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841177+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:07:21.841177+0000","last_peered":"2024-09-21T10:07:21.841177+0000","last_clean":"2024-09-21T10:07:21.841177+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:07:21.841177+0000","last_undegraded":"2024-09-21T10:07:21.841177+0000","last_fullsized":"2024-09-21T10:07:21.841177+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":271,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839348+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:07:21.839348+0000","last_peered":"2024-09-21T10:07:21.839348+0000","last_clean":"2024-09-21T10:07:21.839348+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:07:21.839348+0000","last_undegraded":"2024-09-21T10:07:21.839348+0000","last_fullsized":"2024-09-21T10:07:21.839348+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1599,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.838430+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:07:21.838430+0000","last_peered":"2024-09-21T10:07:21.838430+0000","last_clean":"2024-09-21T10:07:21.838430+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:07:21.838430+0000","last_undegraded":"2024-09-21T10:07:21.838430+0000","last_fullsized":"2024-09-21T10:07:21.838430+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":250,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837258+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:07:21.837258+0000","last_peered":"2024-09-21T10:07:21.837258+0000","last_clean":"2024-09-21T10:07:21.837258+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:07:21.837258+0000","last_undegraded":"2024-09-21T10:07:21.837258+0000","last_fullsized":"2024-09-21T10:07:21.837258+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":63,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825561+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:07:16.825561+0000","last_peered":"2024-09-21T10:07:16.825561+0000","last_clean":"2024-09-21T10:07:16.825561+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:07:16.825561+0000","last_undegraded":"2024-09-21T10:07:16.825561+0000","last_fullsized":"2024-09-21T10:07:16.825561+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":243,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.836874+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:07:21.836874+0000","last_peered":"2024-09-21T10:07:21.836874+0000","last_clean":"2024-09-21T10:07:21.836874+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:07:21.836874+0000","last_undegraded":"2024-09-21T10:07:21.836874+0000","last_fullsized":"2024-09-21T10:07:21.836874+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1562,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840555+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:07:21.840555+0000","last_peered":"2024-09-21T10:07:21.840555+0000","last_clean":"2024-09-21T10:07:21.840555+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:07:21.840555+0000","last_undegraded":"2024-09-21T10:07:21.840555+0000","last_fullsized":"2024-09-21T10:07:21.840555+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1572,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.842223+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:07:21.842223+0000","last_peered":"2024-09-21T10:07:21.842223+0000","last_clean":"2024-09-21T10:07:21.842223+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:07:21.842223+0000","last_undegraded":"2024-09-21T10:07:21.842223+0000","last_fullsized":"2024-09-21T10:07:21.842223+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837941+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:07:21.837941+0000","last_peered":"2024-09-21T10:07:21.837941+0000","last_clean":"2024-09-21T10:07:21.837941+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:07:21.837941+0000","last_undegraded":"2024-09-21T10:07:21.837941+0000","last_fullsized":"2024-09-21T10:07:21.837941+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":120,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840130+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:07:21.840130+0000","last_peered":"2024-09-21T10:07:21.840130+0000","last_clean":"2024-09-21T10:07:21.840130+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:07:21.840130+0000","last_undegraded":"2024-09-21T10:07:21.840130+0000","last_fullsized":"2024-09-21T10:07:21.840130+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1456,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841036+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:07:21.841036+0000","last_peered":"2024-09-21T10:07:21.841036+0000","last_clean":"2024-09-21T10:07:21.841036+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:07:21.841036+0000","last_undegraded":"2024-09-21T10:07:21.841036+0000","last_fullsized":"2024-09-21T10:07:21.841036+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1537,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840399+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:07:21.840399+0000","last_peered":"2024-09-21T10:07:21.840399+0000","last_clean":"2024-09-21T10:07:21.840399+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:07:21.840399+0000","last_undegraded":"2024-09-21T10:07:21.840399+0000","last_fullsized":"2024-09-21T10:07:21.840399+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1182,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837832+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:07:21.837832+0000","last_peered":"2024-09-21T10:07:21.837832+0000","last_clean":"2024-09-21T10:07:21.837832+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:07:21.837832+0000","last_undegraded":"2024-09-21T10:07:21.837832+0000","last_fullsized":"2024-09-21T10:07:21.837832+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":136,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.841125+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:07:21.841125+0000","last_peered":"2024-09-21T10:07:21.841125+0000","last_clean":"2024-09-21T10:07:21.841125+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:07:21.841125+0000","last_undegraded":"2024-09-21T10:07:21.841125+0000","last_fullsized":"2024-09-21T10:07:21.841125+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1189,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.837192+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:07:21.837192+0000","last_peered":"2024-09-21T10:07:21.837192+0000","last_clean":"2024-09-21T10:07:21.837192+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:07:21.837192+0000","last_undegraded":"2024-09-21T10:07:21.837192+0000","last_fullsized":"2024-09-21T10:07:21.837192+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":57,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825419+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:07:16.825419+0000","last_peered":"2024-09-21T10:07:16.825419+0000","last_clean":"2024-09-21T10:07:16.825419+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:07:16.825419+0000","last_undegraded":"2024-09-21T10:07:16.825419+0000","last_fullsized":"2024-09-21T10:07:16.825419+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":243,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.839439+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:07:21.839439+0000","last_peered":"2024-09-21T10:07:21.839439+0000","last_clean":"2024-09-21T10:07:21.839439+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:07:21.839439+0000","last_undegraded":"2024-09-21T10:07:21.839439+0000","last_fullsized":"2024-09-21T10:07:21.839439+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":251,"reported_epoch":1203,"state":"active+clean","last_fresh":"2024-09-21T10:07:21.840394+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:07:21.840394+0000","last_peered":"2024-09-21T10:07:21.840394+0000","last_clean":"2024-09-21T10:07:21.840394+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:07:21.840394+0000","last_undegraded":"2024-09-21T10:07:21.840394+0000","last_fullsized":"2024-09-21T10:07:21.840394+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.824941+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:07:16.824941+0000","last_peered":"2024-09-21T10:07:16.824941+0000","last_clean":"2024-09-21T10:07:16.824941+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:07:16.824941+0000","last_undegraded":"2024-09-21T10:07:16.824941+0000","last_fullsized":"2024-09-21T10:07:16.824941+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1572,"reported_epoch":1202,"state":"active+clean","last_fresh":"2024-09-21T10:07:16.825493+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:07:16.825493+0000","last_peered":"2024-09-21T10:07:16.825493+0000","last_clean":"2024-09-21T10:07:16.825493+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:07:16.825493+0000","last_undegraded":"2024-09-21T10:07:16.825493+0000","last_fullsized":"2024-09-21T10:07:16.825493+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":862,"num_objects_missing":862,"num_objects_degraded":19208,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143974,"num_write_kb":4607105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47839,"num_bytes_recovered":2449997848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2421686272,"data_stored":2421686272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77040,"ondisk_log_size":77040,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234380,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496015,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":346192,"kb_used_data":8328,"kb_used_omap":1,"kb_used_meta":337854,"kb_avail":93398960,"statfs":{"total":95995035648,"available":95640535040,"internally_reserved":0,"allocated":8527872,"data_stored":5971376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":345962951},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757655,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019291,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281680,"kb_used_data":7748,"kb_used_omap":1,"kb_used_meta":273918,"kb_avail":93463472,"statfs":{"total":95995035648,"available":95706595328,"internally_reserved":0,"allocated":7933952,"data_stored":5381011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":280492486},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888587,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":499264,"kb_used_data":7732,"kb_used_omap":34,"kb_used_meta":491485,"kb_avail":93245888,"statfs":{"total":95995035648,"available":95483789312,"internally_reserved":0,"allocated":7917568,"data_stored":5375670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":503281651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967826,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":383304,"kb_used_data":8316,"kb_used_omap":41,"kb_used_meta":374934,"kb_avail":93361848,"statfs":{"total":95995035648,"available":95602532352,"internally_reserved":0,"allocated":8515584,"data_stored":5971209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":383932831},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047065,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":612796,"kb_used_data":287792,"kb_used_omap":41,"kb_used_meta":324950,"kb_avail":93132356,"statfs":{"total":95995035648,"available":95367532544,"internally_reserved":0,"allocated":294699008,"data_stored":292161590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":332749737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126303,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1185349632,"data_stored":1185349632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":286195712,"data_stored":286195712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332201984,"data_stored":332201984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":617938944,"data_stored":617938944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:07:38.072 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:38.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:37 smithi076 ceph-mon[106482]: pgmap v1595: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 184 objects/s recovering 2024-09-21T10:07:38.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:37 smithi076 ceph-mon[106482]: osdmap e1204: 8 total, 8 up, 5 in 2024-09-21T10:07:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:37 smithi076 ceph-mon[110963]: pgmap v1595: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 184 objects/s recovering 2024-09-21T10:07:38.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:37 smithi076 ceph-mon[110963]: osdmap e1204: 8 total, 8 up, 5 in 2024-09-21T10:07:38.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:38.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:38.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:38.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:38.339 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:38.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:37 smithi191 ceph-mon[102206]: pgmap v1595: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 19208/78 objects degraded (24625.641%); 0 B/s, 184 objects/s recovering 2024-09-21T10:07:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:37 smithi191 ceph-mon[102206]: osdmap e1204: 8 total, 8 up, 5 in 2024-09-21T10:07:38.441 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:38.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:38.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:38.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:38.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:38.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:38.705 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:07:38.705 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:38.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:38.723 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:07:38.723 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:38.740 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:07:38.740 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:38.760 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:07:38.760 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:38.777 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:07:38.777 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:38.793 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:07:38.793 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:38.809 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:07:38.810 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:38.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:38.826 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:07:38.826 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:38.835 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:38.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:38.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:38.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:38.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:39.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.082 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.140 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:38 smithi076 ceph-mon[106482]: osdmap e1205: 8 total, 8 up, 5 in 2024-09-21T10:07:39.140 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/911967511' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:39.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:38 smithi076 ceph-mon[110963]: osdmap e1205: 8 total, 8 up, 5 in 2024-09-21T10:07:39.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/911967511' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:39.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:39.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:39.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:39.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:39.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:38 smithi191 ceph-mon[102206]: osdmap e1205: 8 total, 8 up, 5 in 2024-09-21T10:07:39.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/911967511' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:39.398 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:39.399 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:39.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:39.454 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:39.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:39.578 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:39.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:39.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:39.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:39.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:39.827 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:39.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:39.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:39.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:39.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:39.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:40.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:40.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:39 smithi076 ceph-mon[106482]: pgmap v1598: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 17668/78 objects degraded (22651.282%); 0 B/s, 349 objects/s recovering 2024-09-21T10:07:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 17668/78 objects degraded (22651.282%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:39 smithi076 ceph-mon[110963]: pgmap v1598: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 17668/78 objects degraded (22651.282%); 0 B/s, 349 objects/s recovering 2024-09-21T10:07:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 17668/78 objects degraded (22651.282%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:40.175 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:40.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:40.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:40.263 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:39 smithi191 ceph-mon[102206]: pgmap v1598: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 17668/78 objects degraded (22651.282%); 0 B/s, 349 objects/s recovering 2024-09-21T10:07:40.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 17668/78 objects degraded (22651.282%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:40.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:40.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:40.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:40.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:40.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:40.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:40.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:40.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:40.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:40.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:40.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:40.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:40.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:40.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:40.937 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:40.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:40.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:41.017 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:07:41.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:41.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:41.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:41.182 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:41.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:41.284 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:41.306 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:41.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:41.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:41.429 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:41.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:41.553 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:41.655 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:41.676 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:41.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:41.778 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:41.800 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:41.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:41.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:41.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:41.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:42.013 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:41 smithi076 ceph-mon[106482]: pgmap v1599: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 17668/78 objects degraded (22651.282%); 0 B/s, 192 objects/s recovering 2024-09-21T10:07:42.013 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:41 smithi076 ceph-mon[110963]: pgmap v1599: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 17668/78 objects degraded (22651.282%); 0 B/s, 192 objects/s recovering 2024-09-21T10:07:42.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:42.048 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:42.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:42.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:42.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:42.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:42.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:42.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:42.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:41 smithi191 ceph-mon[102206]: pgmap v1599: 89 pgs: 1 peering, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 17668/78 objects degraded (22651.282%); 0 B/s, 192 objects/s recovering 2024-09-21T10:07:42.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:42.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:42.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:42.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:42.540 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:42.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:42.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:42.663 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:42.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:42.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:42.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:42.894 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_2' removed 2024-09-21T10:07:42.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:42.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:43.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:43.051 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:43.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:43.174 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:43.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1785837351' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:07:43.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:42 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:07:43.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1785837351' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:07:43.251 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:42 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:07:43.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:43.297 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:43.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1785837351' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:07:43.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:42 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:07:43.399 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:43.420 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:43.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:43.521 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:43.543 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:43.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:07:43] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:07:43.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:43.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:43.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:43.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:43.789 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:43.891 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:43.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:43.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:43.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:44.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:44.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-mon[106482]: pgmap v1600: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 16847/78 objects degraded (21598.718%); 0 B/s, 295 objects/s recovering 2024-09-21T10:07:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:07:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-mon[106482]: osdmap e1206: 8 total, 8 up, 5 in 2024-09-21T10:07:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-mon[110963]: pgmap v1600: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 16847/78 objects degraded (21598.718%); 0 B/s, 295 objects/s recovering 2024-09-21T10:07:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:07:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:43 smithi076 ceph-mon[110963]: osdmap e1206: 8 total, 8 up, 5 in 2024-09-21T10:07:44.137 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:44.158 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:44.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:44.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:44.281 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:43 smithi191 ceph-mon[102206]: pgmap v1600: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 16847/78 objects degraded (21598.718%); 0 B/s, 295 objects/s recovering 2024-09-21T10:07:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:43 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:07:44.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:43 smithi191 ceph-mon[102206]: osdmap e1206: 8 total, 8 up, 5 in 2024-09-21T10:07:44.383 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:44.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:44.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:44.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:44.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:44.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:07:44.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:44.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:44.758 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:44.781 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:44.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:44.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:44.908 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:44.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:45.009 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:45.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:45.133 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:45.142 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:07:45.142 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:07:45.143 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:07:45.143 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:07:45.151 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 16847/78 objects degraded (21598.718%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:45.151 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 16847/78 objects degraded (21598.718%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:45.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:45.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.279 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:45.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 16847/78 objects degraded (21598.718%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-21T10:07:45.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:45.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:45.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:45.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:45.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:45.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:45.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:45.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:45.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:45.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:45.900 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:07:45.910 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:45.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:46.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:46.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:46.125 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:07:46.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:46.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:46.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:46.313 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:46.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:45 smithi191 ceph-mon[102206]: pgmap v1602: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:45 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16847/78 objects degraded (21598.718%), 5 pgs degraded, 5 pgs undersized) 2024-09-21T10:07:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:45 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:07:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3295380694' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[106482]: pgmap v1602: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16847/78 objects degraded (21598.718%), 5 pgs degraded, 5 pgs undersized) 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3295380694' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[110963]: pgmap v1602: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16847/78 objects degraded (21598.718%), 5 pgs degraded, 5 pgs undersized) 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:07:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3295380694' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:07:46.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:46.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:46.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:46.564 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:46.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:46.688 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:46.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:46.811 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:46.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:46.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:46.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:46.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:47.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:47.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:47.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:47.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.189 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:47.312 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:47.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:47.354 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:46 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:47.354 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:46 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3295380694' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:47.354 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:46 smithi076 ceph-mon[106482]: osdmap e1207: 8 total, 8 up, 5 in 2024-09-21T10:07:47.354 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:46 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:47.354 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:46 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3295380694' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:47.354 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:46 smithi076 ceph-mon[110963]: osdmap e1207: 8 total, 8 up, 5 in 2024-09-21T10:07:47.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:46 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:07:47.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:46 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3295380694' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:07:47.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:46 smithi191 ceph-mon[102206]: osdmap e1207: 8 total, 8 up, 5 in 2024-09-21T10:07:47.363 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:47.363 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:47.385 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:07:47.385 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:07:47.385 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-21T10:07:47.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_3 16 2024-09-21T10:07:47.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:47.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:47.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:47.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:47.696 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:47.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.820 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:47.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:47.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:47.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:47.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:48.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.066 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:48.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:48.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:48.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:48.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:48.227 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:47 smithi076 ceph-mon[106482]: pgmap v1604: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:48.227 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:48.227 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:47 smithi076 ceph-mon[110963]: pgmap v1604: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:48.227 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:48.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:48.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:48.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:47 smithi191 ceph-mon[102206]: pgmap v1604: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:48.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:48.437 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:48.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:48.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:48.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:48.685 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:48.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:48.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:48.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:48.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:48.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:48.918 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:07:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:48.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:48.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:49.038 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:49.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:49.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:49.187 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:49.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:49.314 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:49.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:49.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:49.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:49.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:49.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:49.687 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:49.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:49.815 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:49.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:49.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:49.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:49.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:50.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:50.066 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:50.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:49 smithi076 ceph-mon[106482]: pgmap v1605: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:50.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:49 smithi076 ceph-mon[110963]: pgmap v1605: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:50.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:50.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:50.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:50.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:50.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:50.346 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:49 smithi191 ceph-mon[102206]: pgmap v1605: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:50.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:50.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:50.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:50.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:50.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:50.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:50.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:50.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:50.818 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:50.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:50.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:50.941 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:50.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:51.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:51.064 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:51.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:51.166 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:51.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:51.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:51.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:51.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:51.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:51.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:51.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:51.544 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:51.570 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:51.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:51.693 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:51.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:51.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:51.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:51.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:51.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:51.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:52.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:52.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:52.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:52.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:52.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:52.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:52.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:51 smithi191 ceph-mon[102206]: pgmap v1606: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:51 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:52.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:51 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:51 smithi076 ceph-mon[106482]: pgmap v1606: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:52.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:51 smithi076 ceph-mon[110963]: pgmap v1606: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:51 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:51 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:52.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:52.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:52.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:52.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:52.567 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:52.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:52.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:52.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:52.818 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:52.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:52.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:52.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:52.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:53.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:53.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:53.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:53.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:53.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:53.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:53.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:53.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:53.416 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:53.440 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:53.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:53.539 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:53.542 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:53.565 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:07:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:07:53] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:07:53.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:53.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:53.791 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:53.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:53.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:53.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:53.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:53.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:54.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:54.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:54.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:54.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:54.186 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:53 smithi076 ceph-mon[106482]: pgmap v1607: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:54.187 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:53 smithi076 ceph-mon[110963]: pgmap v1607: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:54.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:54.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:54.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:54.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:54.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:53 smithi191 ceph-mon[102206]: pgmap v1607: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:54.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:54.448 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:54.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:54.512 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:07:54.513 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:07:54.514 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:07:54.514 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:07:54.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:07:54.514 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:07:54.514 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:07:54.514 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:07:54.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:54.565 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:07:54.566 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:07:54.571 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:54.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:54.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:54.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:54.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:54.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:54.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:54.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:55.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:55.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:55.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:55.175 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:55.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:55.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.324 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:55.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2692827117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2692827117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2692827117' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:07:55.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:55.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:55.490 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1607,"stamp":"2024-09-21T10:07:52.673312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":2962950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":1839916,"kb_used_data":40688,"kb_used_omap":119,"kb_used_meta":1799048,"kb_avail":466885844,"statfs":{"total":479975178240,"available":478091104256,"internally_reserved":0,"allocated":41664512,"data_stored":28830139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":1842225352},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.783781"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348994+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:07:46.348994+0000","last_peered":"2024-09-21T10:07:46.348994+0000","last_clean":"2024-09-21T10:07:46.348994+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:07:46.348994+0000","last_undegraded":"2024-09-21T10:07:46.348994+0000","last_fullsized":"2024-09-21T10:07:46.348994+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":64,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.891593+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:07:42.891593+0000","last_peered":"2024-09-21T10:07:42.891593+0000","last_clean":"2024-09-21T10:07:42.891593+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:07:42.891593+0000","last_undegraded":"2024-09-21T10:07:42.891593+0000","last_fullsized":"2024-09-21T10:07:42.891593+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1236,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349283+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:07:46.349283+0000","last_peered":"2024-09-21T10:07:46.349283+0000","last_clean":"2024-09-21T10:07:46.349283+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:07:46.349283+0000","last_undegraded":"2024-09-21T10:07:46.349283+0000","last_fullsized":"2024-09-21T10:07:46.349283+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1567,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350175+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:07:46.350175+0000","last_peered":"2024-09-21T10:07:46.350175+0000","last_clean":"2024-09-21T10:07:46.350175+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:07:46.350175+0000","last_undegraded":"2024-09-21T10:07:46.350175+0000","last_fullsized":"2024-09-21T10:07:46.350175+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1567,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348555+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:07:46.348555+0000","last_peered":"2024-09-21T10:07:46.348555+0000","last_clean":"2024-09-21T10:07:46.348555+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:07:46.348555+0000","last_undegraded":"2024-09-21T10:07:46.348555+0000","last_fullsized":"2024-09-21T10:07:46.348555+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":67,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.891675+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:07:42.891675+0000","last_peered":"2024-09-21T10:07:42.891675+0000","last_clean":"2024-09-21T10:07:42.891675+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:07:42.891675+0000","last_undegraded":"2024-09-21T10:07:42.891675+0000","last_fullsized":"2024-09-21T10:07:42.891675+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349208+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:07:46.349208+0000","last_peered":"2024-09-21T10:07:46.349208+0000","last_clean":"2024-09-21T10:07:46.349208+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:07:46.349208+0000","last_undegraded":"2024-09-21T10:07:46.349208+0000","last_fullsized":"2024-09-21T10:07:46.349208+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":87,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.890690+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:07:42.890690+0000","last_peered":"2024-09-21T10:07:42.890690+0000","last_clean":"2024-09-21T10:07:42.890690+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:07:42.890690+0000","last_undegraded":"2024-09-21T10:07:42.890690+0000","last_fullsized":"2024-09-21T10:07:42.890690+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":106,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.351105+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:07:46.351105+0000","last_peered":"2024-09-21T10:07:46.351105+0000","last_clean":"2024-09-21T10:07:46.351105+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:07:46.351105+0000","last_undegraded":"2024-09-21T10:07:46.351105+0000","last_fullsized":"2024-09-21T10:07:46.351105+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":58,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.890868+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:07:42.890868+0000","last_peered":"2024-09-21T10:07:42.890868+0000","last_clean":"2024-09-21T10:07:42.890868+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:07:42.890868+0000","last_undegraded":"2024-09-21T10:07:42.890868+0000","last_fullsized":"2024-09-21T10:07:42.890868+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":150,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.351484+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:07:46.351484+0000","last_peered":"2024-09-21T10:07:46.351484+0000","last_clean":"2024-09-21T10:07:46.351484+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:07:46.351484+0000","last_undegraded":"2024-09-21T10:07:46.351484+0000","last_fullsized":"2024-09-21T10:07:46.351484+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1191,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350705+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:07:46.350705+0000","last_peered":"2024-09-21T10:07:46.350705+0000","last_clean":"2024-09-21T10:07:46.350705+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:07:46.350705+0000","last_undegraded":"2024-09-21T10:07:46.350705+0000","last_fullsized":"2024-09-21T10:07:46.350705+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":62,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.893252+0000","last_change":"2024-09-21T10:02:39.012908+0000","last_active":"2024-09-21T10:07:42.893252+0000","last_peered":"2024-09-21T10:07:42.893252+0000","last_clean":"2024-09-21T10:07:42.893252+0000","last_became_active":"2024-09-21T10:02:38.992739+0000","last_became_peered":"2024-09-21T10:02:38.992739+0000","last_unstale":"2024-09-21T10:07:42.893252+0000","last_undegraded":"2024-09-21T10:07:42.893252+0000","last_fullsized":"2024-09-21T10:07:42.893252+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:28.811028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":67,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.893446+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:07:42.893446+0000","last_peered":"2024-09-21T10:07:42.893446+0000","last_clean":"2024-09-21T10:07:42.893446+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:07:42.893446+0000","last_undegraded":"2024-09-21T10:07:42.893446+0000","last_fullsized":"2024-09-21T10:07:42.893446+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1221,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348829+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:07:46.348829+0000","last_peered":"2024-09-21T10:07:46.348829+0000","last_clean":"2024-09-21T10:07:46.348829+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:07:46.348829+0000","last_undegraded":"2024-09-21T10:07:46.348829+0000","last_fullsized":"2024-09-21T10:07:46.348829+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1577,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349875+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:07:46.349875+0000","last_peered":"2024-09-21T10:07:46.349875+0000","last_clean":"2024-09-21T10:07:46.349875+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:07:46.349875+0000","last_undegraded":"2024-09-21T10:07:46.349875+0000","last_fullsized":"2024-09-21T10:07:46.349875+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":259,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348859+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:07:46.348859+0000","last_peered":"2024-09-21T10:07:46.348859+0000","last_clean":"2024-09-21T10:07:46.348859+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:07:46.348859+0000","last_undegraded":"2024-09-21T10:07:46.348859+0000","last_fullsized":"2024-09-21T10:07:46.348859+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1236,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348737+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:07:46.348737+0000","last_peered":"2024-09-21T10:07:46.348737+0000","last_clean":"2024-09-21T10:07:46.348737+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:07:46.348737+0000","last_undegraded":"2024-09-21T10:07:46.348737+0000","last_fullsized":"2024-09-21T10:07:46.348737+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":58,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.892963+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:07:42.892963+0000","last_peered":"2024-09-21T10:07:42.892963+0000","last_clean":"2024-09-21T10:07:42.892963+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:07:42.892963+0000","last_undegraded":"2024-09-21T10:07:42.892963+0000","last_fullsized":"2024-09-21T10:07:42.892963+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":879,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350759+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:07:46.350759+0000","last_peered":"2024-09-21T10:07:46.350759+0000","last_clean":"2024-09-21T10:07:46.350759+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:07:46.350759+0000","last_undegraded":"2024-09-21T10:07:46.350759+0000","last_fullsized":"2024-09-21T10:07:46.350759+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1222,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349633+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:07:46.349633+0000","last_peered":"2024-09-21T10:07:46.349633+0000","last_clean":"2024-09-21T10:07:46.349633+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:07:46.349633+0000","last_undegraded":"2024-09-21T10:07:46.349633+0000","last_fullsized":"2024-09-21T10:07:46.349633+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.893262+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:07:42.893262+0000","last_peered":"2024-09-21T10:07:42.893262+0000","last_clean":"2024-09-21T10:07:42.893262+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:07:42.893262+0000","last_undegraded":"2024-09-21T10:07:42.893262+0000","last_fullsized":"2024-09-21T10:07:42.893262+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348452+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:07:46.348452+0000","last_peered":"2024-09-21T10:07:46.348452+0000","last_clean":"2024-09-21T10:07:46.348452+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:07:46.348452+0000","last_undegraded":"2024-09-21T10:07:46.348452+0000","last_fullsized":"2024-09-21T10:07:46.348452+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1582,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.351322+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:07:46.351322+0000","last_peered":"2024-09-21T10:07:46.351322+0000","last_clean":"2024-09-21T10:07:46.351322+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:07:46.351322+0000","last_undegraded":"2024-09-21T10:07:46.351322+0000","last_fullsized":"2024-09-21T10:07:46.351322+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":731,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.890644+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:07:42.890644+0000","last_peered":"2024-09-21T10:07:42.890644+0000","last_clean":"2024-09-21T10:07:42.890644+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:07:42.890644+0000","last_undegraded":"2024-09-21T10:07:42.890644+0000","last_fullsized":"2024-09-21T10:07:42.890644+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1235,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348664+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:07:46.348664+0000","last_peered":"2024-09-21T10:07:46.348664+0000","last_clean":"2024-09-21T10:07:46.348664+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:07:46.348664+0000","last_undegraded":"2024-09-21T10:07:46.348664+0000","last_fullsized":"2024-09-21T10:07:46.348664+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":266,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350324+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:07:46.350324+0000","last_peered":"2024-09-21T10:07:46.350324+0000","last_clean":"2024-09-21T10:07:46.350324+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:07:46.350324+0000","last_undegraded":"2024-09-21T10:07:46.350324+0000","last_fullsized":"2024-09-21T10:07:46.350324+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":67,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.893159+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:07:42.893159+0000","last_peered":"2024-09-21T10:07:42.893159+0000","last_clean":"2024-09-21T10:07:42.893159+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:07:42.893159+0000","last_undegraded":"2024-09-21T10:07:42.893159+0000","last_fullsized":"2024-09-21T10:07:42.893159+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":866,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349226+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:07:46.349226+0000","last_peered":"2024-09-21T10:07:46.349226+0000","last_clean":"2024-09-21T10:07:46.349226+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:07:46.349226+0000","last_undegraded":"2024-09-21T10:07:46.349226+0000","last_fullsized":"2024-09-21T10:07:46.349226+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1455,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348454+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:07:46.348454+0000","last_peered":"2024-09-21T10:07:46.348454+0000","last_clean":"2024-09-21T10:07:46.348454+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:07:46.348454+0000","last_undegraded":"2024-09-21T10:07:46.348454+0000","last_fullsized":"2024-09-21T10:07:46.348454+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":864,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350831+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:07:46.350831+0000","last_peered":"2024-09-21T10:07:46.350831+0000","last_clean":"2024-09-21T10:07:46.350831+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:07:46.350831+0000","last_undegraded":"2024-09-21T10:07:46.350831+0000","last_fullsized":"2024-09-21T10:07:46.350831+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1496,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348608+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:07:46.348608+0000","last_peered":"2024-09-21T10:07:46.348608+0000","last_clean":"2024-09-21T10:07:46.348608+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:07:46.348608+0000","last_undegraded":"2024-09-21T10:07:46.348608+0000","last_fullsized":"2024-09-21T10:07:46.348608+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1583,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349555+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:07:46.349555+0000","last_peered":"2024-09-21T10:07:46.349555+0000","last_clean":"2024-09-21T10:07:46.349555+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:07:46.349555+0000","last_undegraded":"2024-09-21T10:07:46.349555+0000","last_fullsized":"2024-09-21T10:07:46.349555+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1590,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.351197+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:07:46.351197+0000","last_peered":"2024-09-21T10:07:46.351197+0000","last_clean":"2024-09-21T10:07:46.351197+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:07:46.351197+0000","last_undegraded":"2024-09-21T10:07:46.351197+0000","last_fullsized":"2024-09-21T10:07:46.351197+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":159,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349903+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:07:46.349903+0000","last_peered":"2024-09-21T10:07:46.349903+0000","last_clean":"2024-09-21T10:07:46.349903+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:07:46.349903+0000","last_undegraded":"2024-09-21T10:07:46.349903+0000","last_fullsized":"2024-09-21T10:07:46.349903+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1169,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348885+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:07:46.348885+0000","last_peered":"2024-09-21T10:07:46.348885+0000","last_clean":"2024-09-21T10:07:46.348885+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:07:46.348885+0000","last_undegraded":"2024-09-21T10:07:46.348885+0000","last_fullsized":"2024-09-21T10:07:46.348885+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1593,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350012+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:07:46.350012+0000","last_peered":"2024-09-21T10:07:46.350012+0000","last_clean":"2024-09-21T10:07:46.350012+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:07:46.350012+0000","last_undegraded":"2024-09-21T10:07:46.350012+0000","last_fullsized":"2024-09-21T10:07:46.350012+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1183,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349191+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:07:46.349191+0000","last_peered":"2024-09-21T10:07:46.349191+0000","last_clean":"2024-09-21T10:07:46.349191+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:07:46.349191+0000","last_undegraded":"2024-09-21T10:07:46.349191+0000","last_fullsized":"2024-09-21T10:07:46.349191+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1604,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350091+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:07:46.350091+0000","last_peered":"2024-09-21T10:07:46.350091+0000","last_clean":"2024-09-21T10:07:46.350091+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:07:46.350091+0000","last_undegraded":"2024-09-21T10:07:46.350091+0000","last_fullsized":"2024-09-21T10:07:46.350091+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349685+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:07:46.349685+0000","last_peered":"2024-09-21T10:07:46.349685+0000","last_clean":"2024-09-21T10:07:46.349685+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:07:46.349685+0000","last_undegraded":"2024-09-21T10:07:46.349685+0000","last_fullsized":"2024-09-21T10:07:46.349685+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1581,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350300+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:07:46.350300+0000","last_peered":"2024-09-21T10:07:46.350300+0000","last_clean":"2024-09-21T10:07:46.350300+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:07:46.350300+0000","last_undegraded":"2024-09-21T10:07:46.350300+0000","last_fullsized":"2024-09-21T10:07:46.350300+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":762,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349818+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:07:46.349818+0000","last_peered":"2024-09-21T10:07:46.349818+0000","last_clean":"2024-09-21T10:07:46.349818+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:07:46.349818+0000","last_undegraded":"2024-09-21T10:07:46.349818+0000","last_fullsized":"2024-09-21T10:07:46.349818+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1574,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350344+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:07:46.350344+0000","last_peered":"2024-09-21T10:07:46.350344+0000","last_clean":"2024-09-21T10:07:46.350344+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:07:46.350344+0000","last_undegraded":"2024-09-21T10:07:46.350344+0000","last_fullsized":"2024-09-21T10:07:46.350344+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":282,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349235+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:07:46.349235+0000","last_peered":"2024-09-21T10:07:46.349235+0000","last_clean":"2024-09-21T10:07:46.349235+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:07:46.349235+0000","last_undegraded":"2024-09-21T10:07:46.349235+0000","last_fullsized":"2024-09-21T10:07:46.349235+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1582,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.351552+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:07:46.351552+0000","last_peered":"2024-09-21T10:07:46.351552+0000","last_clean":"2024-09-21T10:07:46.351552+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:07:46.351552+0000","last_undegraded":"2024-09-21T10:07:46.351552+0000","last_fullsized":"2024-09-21T10:07:46.351552+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348369+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:07:46.348369+0000","last_peered":"2024-09-21T10:07:46.348369+0000","last_clean":"2024-09-21T10:07:46.348369+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:07:46.348369+0000","last_undegraded":"2024-09-21T10:07:46.348369+0000","last_fullsized":"2024-09-21T10:07:46.348369+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":123,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350851+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:07:46.350851+0000","last_peered":"2024-09-21T10:07:46.350851+0000","last_clean":"2024-09-21T10:07:46.350851+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:07:46.350851+0000","last_undegraded":"2024-09-21T10:07:46.350851+0000","last_fullsized":"2024-09-21T10:07:46.350851+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":58,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.891997+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:07:42.891997+0000","last_peered":"2024-09-21T10:07:42.891997+0000","last_clean":"2024-09-21T10:07:42.891997+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:07:42.891997+0000","last_undegraded":"2024-09-21T10:07:42.891997+0000","last_fullsized":"2024-09-21T10:07:42.891997+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":67,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.891612+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:07:42.891612+0000","last_peered":"2024-09-21T10:07:42.891612+0000","last_clean":"2024-09-21T10:07:42.891612+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:07:42.891612+0000","last_undegraded":"2024-09-21T10:07:42.891612+0000","last_fullsized":"2024-09-21T10:07:42.891612+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.891531+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:07:42.891531+0000","last_peered":"2024-09-21T10:07:42.891531+0000","last_clean":"2024-09-21T10:07:42.891531+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:07:42.891531+0000","last_undegraded":"2024-09-21T10:07:42.891531+0000","last_fullsized":"2024-09-21T10:07:42.891531+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":68,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.893256+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:07:42.893256+0000","last_peered":"2024-09-21T10:07:42.893256+0000","last_clean":"2024-09-21T10:07:42.893256+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:07:42.893256+0000","last_undegraded":"2024-09-21T10:07:42.893256+0000","last_fullsized":"2024-09-21T10:07:42.893256+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1581,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350019+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:07:46.350019+0000","last_peered":"2024-09-21T10:07:46.350019+0000","last_clean":"2024-09-21T10:07:46.350019+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:07:46.350019+0000","last_undegraded":"2024-09-21T10:07:46.350019+0000","last_fullsized":"2024-09-21T10:07:46.350019+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1194,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349355+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:07:46.349355+0000","last_peered":"2024-09-21T10:07:46.349355+0000","last_clean":"2024-09-21T10:07:46.349355+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:07:46.349355+0000","last_undegraded":"2024-09-21T10:07:46.349355+0000","last_fullsized":"2024-09-21T10:07:46.349355+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1205,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.351583+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:07:46.351583+0000","last_peered":"2024-09-21T10:07:46.351583+0000","last_clean":"2024-09-21T10:07:46.351583+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:07:46.351583+0000","last_undegraded":"2024-09-21T10:07:46.351583+0000","last_fullsized":"2024-09-21T10:07:46.351583+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":279,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348588+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:07:46.348588+0000","last_peered":"2024-09-21T10:07:46.348588+0000","last_clean":"2024-09-21T10:07:46.348588+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:07:46.348588+0000","last_undegraded":"2024-09-21T10:07:46.348588+0000","last_fullsized":"2024-09-21T10:07:46.348588+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1607,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349141+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:07:46.349141+0000","last_peered":"2024-09-21T10:07:46.349141+0000","last_clean":"2024-09-21T10:07:46.349141+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:07:46.349141+0000","last_undegraded":"2024-09-21T10:07:46.349141+0000","last_fullsized":"2024-09-21T10:07:46.349141+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":258,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348584+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:07:46.348584+0000","last_peered":"2024-09-21T10:07:46.348584+0000","last_clean":"2024-09-21T10:07:46.348584+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:07:46.348584+0000","last_undegraded":"2024-09-21T10:07:46.348584+0000","last_fullsized":"2024-09-21T10:07:46.348584+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":67,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.891916+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:07:42.891916+0000","last_peered":"2024-09-21T10:07:42.891916+0000","last_clean":"2024-09-21T10:07:42.891916+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:07:42.891916+0000","last_undegraded":"2024-09-21T10:07:42.891916+0000","last_fullsized":"2024-09-21T10:07:42.891916+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":251,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348260+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:07:46.348260+0000","last_peered":"2024-09-21T10:07:46.348260+0000","last_clean":"2024-09-21T10:07:46.348260+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:07:46.348260+0000","last_undegraded":"2024-09-21T10:07:46.348260+0000","last_fullsized":"2024-09-21T10:07:46.348260+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1570,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349021+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:07:46.349021+0000","last_peered":"2024-09-21T10:07:46.349021+0000","last_clean":"2024-09-21T10:07:46.349021+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:07:46.349021+0000","last_undegraded":"2024-09-21T10:07:46.349021+0000","last_fullsized":"2024-09-21T10:07:46.349021+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1580,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350823+0000","last_change":"2024-09-21T09:56:45.288509+0000","last_active":"2024-09-21T10:07:46.350823+0000","last_peered":"2024-09-21T10:07:46.350823+0000","last_clean":"2024-09-21T10:07:46.350823+0000","last_became_active":"2024-09-21T09:56:45.287583+0000","last_became_peered":"2024-09-21T09:56:45.287583+0000","last_unstale":"2024-09-21T10:07:46.350823+0000","last_undegraded":"2024-09-21T10:07:46.350823+0000","last_fullsized":"2024-09-21T10:07:46.350823+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:07:37.440682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":244,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.349441+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:07:46.349441+0000","last_peered":"2024-09-21T10:07:46.349441+0000","last_clean":"2024-09-21T10:07:46.349441+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:07:46.349441+0000","last_undegraded":"2024-09-21T10:07:46.349441+0000","last_fullsized":"2024-09-21T10:07:46.349441+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":128,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350169+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:07:46.350169+0000","last_peered":"2024-09-21T10:07:46.350169+0000","last_clean":"2024-09-21T10:07:46.350169+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:07:46.350169+0000","last_undegraded":"2024-09-21T10:07:46.350169+0000","last_fullsized":"2024-09-21T10:07:46.350169+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1464,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350214+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:07:46.350214+0000","last_peered":"2024-09-21T10:07:46.350214+0000","last_clean":"2024-09-21T10:07:46.350214+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:07:46.350214+0000","last_undegraded":"2024-09-21T10:07:46.350214+0000","last_fullsized":"2024-09-21T10:07:46.350214+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1545,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350298+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:07:46.350298+0000","last_peered":"2024-09-21T10:07:46.350298+0000","last_clean":"2024-09-21T10:07:46.350298+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:07:46.350298+0000","last_undegraded":"2024-09-21T10:07:46.350298+0000","last_fullsized":"2024-09-21T10:07:46.350298+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1190,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348765+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:07:46.348765+0000","last_peered":"2024-09-21T10:07:46.348765+0000","last_clean":"2024-09-21T10:07:46.348765+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:07:46.348765+0000","last_undegraded":"2024-09-21T10:07:46.348765+0000","last_fullsized":"2024-09-21T10:07:46.348765+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":144,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350495+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:07:46.350495+0000","last_peered":"2024-09-21T10:07:46.350495+0000","last_clean":"2024-09-21T10:07:46.350495+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:07:46.350495+0000","last_undegraded":"2024-09-21T10:07:46.350495+0000","last_fullsized":"2024-09-21T10:07:46.350495+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1197,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.350084+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:07:46.350084+0000","last_peered":"2024-09-21T10:07:46.350084+0000","last_clean":"2024-09-21T10:07:46.350084+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:07:46.350084+0000","last_undegraded":"2024-09-21T10:07:46.350084+0000","last_fullsized":"2024-09-21T10:07:46.350084+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":62,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:43.153777+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:07:43.153777+0000","last_peered":"2024-09-21T10:07:43.153777+0000","last_clean":"2024-09-21T10:07:43.153777+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:07:43.153777+0000","last_undegraded":"2024-09-21T10:07:43.153777+0000","last_fullsized":"2024-09-21T10:07:43.153777+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":251,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348694+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:07:46.348694+0000","last_peered":"2024-09-21T10:07:46.348694+0000","last_clean":"2024-09-21T10:07:46.348694+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:07:46.348694+0000","last_undegraded":"2024-09-21T10:07:46.348694+0000","last_fullsized":"2024-09-21T10:07:46.348694+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":259,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-09-21T10:07:46.348849+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:07:46.348849+0000","last_peered":"2024-09-21T10:07:46.348849+0000","last_clean":"2024-09-21T10:07:46.348849+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:07:46.348849+0000","last_undegraded":"2024-09-21T10:07:46.348849+0000","last_fullsized":"2024-09-21T10:07:46.348849+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:42.890788+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:07:42.890788+0000","last_peered":"2024-09-21T10:07:42.890788+0000","last_clean":"2024-09-21T10:07:42.890788+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:07:42.890788+0000","last_undegraded":"2024-09-21T10:07:42.890788+0000","last_fullsized":"2024-09-21T10:07:42.890788+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1577,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-09-21T10:07:43.153816+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:07:43.153816+0000","last_peered":"2024-09-21T10:07:43.153816+0000","last_clean":"2024-09-21T10:07:43.153816+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:07:43.153816+0000","last_undegraded":"2024-09-21T10:07:43.153816+0000","last_fullsized":"2024-09-21T10:07:43.153816+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":11072,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234384,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496019,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342152,"kb_used_data":8380,"kb_used_omap":1,"kb_used_meta":333758,"kb_avail":93403000,"statfs":{"total":95995035648,"available":95644672000,"internally_reserved":0,"allocated":8581120,"data_stored":6004375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":341768647},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757659,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019295,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281732,"kb_used_data":7800,"kb_used_omap":1,"kb_used_meta":273918,"kb_avail":93463420,"statfs":{"total":95995035648,"available":95706542080,"internally_reserved":0,"allocated":7987200,"data_stored":5414010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":280492486},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888591,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":499316,"kb_used_data":7784,"kb_used_omap":34,"kb_used_meta":491485,"kb_avail":93245836,"statfs":{"total":95995035648,"available":95483736064,"internally_reserved":0,"allocated":7970816,"data_stored":5408669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":503281651},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967830,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":383356,"kb_used_data":8368,"kb_used_omap":41,"kb_used_meta":374934,"kb_avail":93361796,"statfs":{"total":95995035648,"available":95602479104,"internally_reserved":0,"allocated":8568832,"data_stored":6004208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":383932831},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047069,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333360,"kb_used_data":8356,"kb_used_omap":41,"kb_used_meta":324950,"kb_avail":93411792,"statfs":{"total":95995035648,"available":95653675008,"internally_reserved":0,"allocated":8556544,"data_stored":5998877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":332749737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126307,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:07:55.494 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:07:55.494 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:07:55.494 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-21T10:07:55.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-21T10:07:55.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:55.575 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:55.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.698 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:55.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:55.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:55.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:55.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:55.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:55.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:56.050 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:56.050 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:56.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:56.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:56.182 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:56.204 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:56.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:56.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[110963]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:56.279 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[110963]: pgmap v1608: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[106482]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[106482]: pgmap v1608: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:56.306 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:56.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:55 smithi191 ceph-mon[102206]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:07:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:55 smithi191 ceph-mon[102206]: pgmap v1608: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:07:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:56.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:56.461 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:56.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:56.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:56.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:07:56.687 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:56.708 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:56.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:56.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:56.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:56.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:56.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:56.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:57.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:57.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:57.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:57.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:57.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:57.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:57.371 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:57.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:57.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:57.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:57.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:57.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:57.726 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:57.748 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:57.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:57.876 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:57.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:57.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:57.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:07:57.977 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:58.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:57 smithi076 ceph-mon[106482]: pgmap v1609: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:57 smithi076 ceph-mon[110963]: pgmap v1609: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:58.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:07:58.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:58.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:07:58.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:58.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:57 smithi191 ceph-mon[102206]: pgmap v1609: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:07:58.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:58.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:58.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:58.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:58.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:58.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.658 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:58.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:58.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:07:58.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:58.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:07:58.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:58.907 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:58.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:58.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:58.935 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:07:59.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:59.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:59.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:59.136 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:07:59.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:07:59.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:59.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:59.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:07:59.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:07:59.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:07:59.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:07:59.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:07:59.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:07:59.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:07:59.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:07:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:07:59.887 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:07:59.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:07:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:07:59.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:00.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:00.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:00.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:00.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:00.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:00.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:00.292 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:00.341 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:07:59 smithi076 ceph-mon[106482]: pgmap v1610: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:00.341 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:07:59 smithi076 ceph-mon[110963]: pgmap v1610: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:07:59 smithi191 ceph-mon[102206]: pgmap v1610: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:00.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:00.420 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:00.522 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:00.544 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:00.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:00.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:00.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:00.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:00.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:00.920 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:00.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:01.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:01.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:01.150 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:01.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:01.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:01.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:01.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:01.427 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:01.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.655 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:01.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:01.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:01.806 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:01.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:01.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:01.929 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:01.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:01.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:01.935 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_3' created 2024-09-21T10:08:02.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:02.057 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:02.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:02.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:02.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:01 smithi076 ceph-mon[106482]: pgmap v1611: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:02.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/382300923' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:08:02.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:01 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:08:02.267 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:01 smithi076 ceph-mon[110963]: pgmap v1611: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:02.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/382300923' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:08:02.268 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:01 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:08:02.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:02.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:01 smithi191 ceph-mon[102206]: pgmap v1611: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/382300923' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:08:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:01 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-21T10:08:02.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:02.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.432 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:02.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:02.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:02.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:02.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:02.682 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:02.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:02.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:02.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:02.912 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:02.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:02.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:02.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:02.948 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:08:03.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:03.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:03.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:03.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:03.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:08:03.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[106482]: osdmap e1208: 8 total, 8 up, 5 in 2024-09-21T10:08:03.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4120420811' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:03.221 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:03.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:08:03.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[110963]: osdmap e1208: 8 total, 8 up, 5 in 2024-09-21T10:08:03.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4120420811' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:03.222 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:02 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:03.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-21T10:08:03.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:03.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:03.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:02 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-21T10:08:03.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:02 smithi191 ceph-mon[102206]: osdmap e1208: 8 total, 8 up, 5 in 2024-09-21T10:08:03.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4120420811' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:03.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:02 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:03.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:03.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:03.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:03.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:03.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:08:03] "GET /metrics HTTP/1.1" 200 34122 "" "Prometheus/2.51.0" 2024-09-21T10:08:03.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:03.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:03.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:03.825 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:03.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:03.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:03.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:03.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:03.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:04.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:04.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:04.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:04.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:04.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:04.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[106482]: pgmap v1613: 89 pgs: 11 creating+peering, 5 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[106482]: osdmap e1209: 8 total, 8 up, 5 in 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[110963]: pgmap v1613: 89 pgs: 11 creating+peering, 5 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:04.329 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:04.330 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:03 smithi076 ceph-mon[110963]: osdmap e1209: 8 total, 8 up, 5 in 2024-09-21T10:08:04.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:03 smithi191 ceph-mon[102206]: pgmap v1613: 89 pgs: 11 creating+peering, 5 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:03 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:03 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:03 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:03 smithi191 ceph-mon[102206]: osdmap e1209: 8 total, 8 up, 5 in 2024-09-21T10:08:04.430 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:04.430 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:04.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:04.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:04.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:04.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:04.603 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:04.705 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:04.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:04.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:04.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:04.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:04.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:04.966 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:04 smithi076 ceph-mon[106482]: osdmap e1210: 8 total, 8 up, 5 in 2024-09-21T10:08:04.966 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:04 smithi076 ceph-mon[110963]: osdmap e1210: 8 total, 8 up, 5 in 2024-09-21T10:08:04.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:04.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:04.989 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:04.989 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:bench [] [] [] [] 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator dump block 2024-09-21T10:08:04.990 INFO:teuthology.orchestra.run.smithi076.stderr:Error EINVAL: invalid command 2024-09-21T10:08:05.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:05.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:05.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:05.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:05.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:05.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:05.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:04 smithi191 ceph-mon[102206]: osdmap e1210: 8 total, 8 up, 5 in 2024-09-21T10:08:05.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:05.465 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:05.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:05.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:05.613 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:05.620 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:08:05.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:05.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:05.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:05.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:05.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:05.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:05.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:05.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:06.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:06.115 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:06.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:06.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:06.243 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:06.335 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:05 smithi076 ceph-mon[106482]: pgmap v1616: 89 pgs: 3 creating+activating, 11 creating+peering, 75 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:06.335 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:05 smithi076 ceph-mon[110963]: pgmap v1616: 89 pgs: 3 creating+activating, 11 creating+peering, 75 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:06.345 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:06.357 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:08:06.357 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-21T10:08:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:05 smithi191 ceph-mon[102206]: pgmap v1616: 89 pgs: 3 creating+activating, 11 creating+peering, 75 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.367 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:06.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:06.468 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:06.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.500 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:06.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:06.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:06.624 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:06.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:06.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:06.745 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:06.746 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:06.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:06.810 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:06.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:06.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:06.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:06.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:07.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:07.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:07.130 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:07.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:07.255 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:07.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:07.357 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:07.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:07.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:07.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:07.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:07.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:07.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:07.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:07.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:07.757 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:07.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:07.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:07.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:07.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.005 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:08.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:08.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:08.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:08.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:08.294 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:07 smithi076 ceph-mon[106482]: pgmap v1617: 89 pgs: 3 creating+activating, 11 creating+peering, 75 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:08.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:07 smithi076 ceph-mon[110963]: pgmap v1617: 89 pgs: 3 creating+activating, 11 creating+peering, 75 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:08.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:07 smithi191 ceph-mon[102206]: pgmap v1617: 89 pgs: 3 creating+activating, 11 creating+peering, 75 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:08.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:08.480 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:08.504 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:08.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:08.627 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:08.728 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:08.750 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:08.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:08:08.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:08.873 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:08.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:08.996 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:08.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:09.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:09.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:09.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:09.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:09.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:09.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:09.350 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:09.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:09.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:09.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.502 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:09.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:09.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:09.627 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:09.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:09.753 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:09.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:09.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:09.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:09.980 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:10.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:10.033 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:09 smithi076 ceph-mon[106482]: pgmap v1618: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:10.033 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:09 smithi076 ceph-mon[110963]: pgmap v1618: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:10.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:10.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:10.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:10.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:10.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:09 smithi191 ceph-mon[102206]: pgmap v1618: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:10.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:10.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:10.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:10.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:10.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:10.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:10.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:10.751 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:10.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:10.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:10.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:10.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:10.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:11.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:11.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:11.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:11.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:11.345 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:11.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:11.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:11.485 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:11.485 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:11.491 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:11.593 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:11 smithi076 ceph-mon[106482]: pgmap v1619: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:11.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:11.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:11.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:11 smithi076 ceph-mon[110963]: pgmap v1619: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:11.709 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:11.717 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:11.739 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:11.842 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:11 smithi191 ceph-mon[102206]: pgmap v1619: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:11.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.865 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:11.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:11.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:11.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:11.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:12.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:12.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:12.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:08:12.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:12.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:12.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:12.352 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:12.361 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:12.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:12.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:12.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:12.606 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:12.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:12.730 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:12.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:12.832 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:12.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:12.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:12.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:12.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:13.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:13.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:13.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:13.346 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:13.448 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:13.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:13.529 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:08:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:08:13] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.51.0" 2024-09-21T10:08:13.572 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:13.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:13.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:13.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:13.737 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-21T10:08:13.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:13.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:13.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:13 smithi076 ceph-mon[106482]: pgmap v1620: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:13.854 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/38521976' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:13.854 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:13 smithi076 ceph-mon[110963]: pgmap v1620: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:13.854 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/38521976' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:13.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:13.965 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:13.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:13.987 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:13 smithi191 ceph-mon[102206]: pgmap v1620: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:13.987 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/38521976' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:14.068 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:14.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:14.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:14.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:14.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:14.221 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:14.221 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:14.315 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:14.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:14.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:14.461 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:14.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:14.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:14.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:14.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:14.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:14.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:14.720 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:08:14.721 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:14.737 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:08:14.738 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:14.755 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:08:14.756 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:14.772 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:08:14.772 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:14.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/38521976' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:14.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:14 smithi076 ceph-mon[106482]: osdmap e1211: 8 total, 8 up, 5 in 2024-09-21T10:08:14.773 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/864140556' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:14.773 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/38521976' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:14.773 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:14 smithi076 ceph-mon[110963]: osdmap e1211: 8 total, 8 up, 5 in 2024-09-21T10:08:14.773 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/864140556' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:14.788 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:08:14.788 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:14.804 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:08:14.805 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:14.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:14.821 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:08:14.821 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:14.830 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:14.834 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/38521976' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:14.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:14 smithi191 ceph-mon[102206]: osdmap e1211: 8 total, 8 up, 5 in 2024-09-21T10:08:14.835 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/864140556' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:14.838 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:08:14.838 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:14.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:08:14.932 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:14.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:14.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:14.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:15.052 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:15.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:15.077 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:15.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:15.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:15.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:15.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:15.425 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:15.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:15.532 DEBUG:teuthology.orchestra.run.smithi076:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-21T10:08:15.535 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:08:15.535 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:08:15.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:15.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:15.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:15.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:15.732 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:hints = 1 2024-09-21T10:08:15.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:08:15.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object prefix: benchmark_data_smithi076_241159 2024-09-21T10:08:15.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:08:15.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:08:15.774 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:15 smithi076 ceph-mon[106482]: pgmap v1622: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:15.774 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:15 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:15.774 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:15 smithi076 ceph-mon[110963]: pgmap v1622: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:15.774 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:15 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:15.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:15.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:15.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:15.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:15.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:16.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:16.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:15 smithi191 ceph-mon[102206]: pgmap v1622: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:15 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:16.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:16.210 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:bench [] [] [] [] 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator dump block 2024-09-21T10:08:16.215 INFO:teuthology.orchestra.run.smithi076.stderr:Error EINVAL: invalid command 2024-09-21T10:08:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:16.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:16.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:16.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:16.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:16.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:16.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.667 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:16.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:16.715 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:16.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 1 16 3489 3473 217.028 217.062 0.00292935 0.00458547 2024-09-21T10:08:16.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:16.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:16.886 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:08:16.886 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:08:16.886 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:08:16.886 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:08:16.886 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:08:16.902 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:16.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:16.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:17.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:17.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:17.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:17.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:17.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:17.295 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:17.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:17.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:17.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:17.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:17.599 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:17.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:17.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:17.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 2 16 6900 6884 215.098 213.188 0.00554748 0.00463893 2024-09-21T10:08:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:17.734 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:08:17.735 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-21T10:08:17.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:08:17.828 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:17.865 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:17.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:17.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:17.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:18.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:17 smithi076 ceph-mon[106482]: pgmap v1623: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:18.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:17 smithi076 ceph-mon[110963]: pgmap v1623: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:18.086 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:18.101 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:17 smithi191 ceph-mon[102206]: pgmap v1623: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 445 GiB / 447 GiB avail 2024-09-21T10:08:18.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:18.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:18.285 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:18.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:18.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:18.416 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:18.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.560 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:18.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:18.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:18.688 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:18.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:18.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 3 16 9862 9846 205.102 185.125 0.00535595 0.00486865 2024-09-21T10:08:18.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:18.813 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:18.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:18.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:18.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:18.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:19.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:19.078 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:19.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:19.179 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:19.215 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:19.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:19.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:19.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.366 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:19.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:19.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:19.510 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:19.612 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:19.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:19.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:19.733 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 4 16 12029 12013 187.681 135.438 0.00129573 0.00520363 2024-09-21T10:08:19.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:19.761 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:19.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:19.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:19 smithi191 ceph-mon[102206]: pgmap v1624: 89 pgs: 89 active+clean; 411 MiB data, 2.4 GiB used, 445 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s 2024-09-21T10:08:19.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:19.891 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:19.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:19.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:20.014 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:20.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:20.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:19 smithi076 ceph-mon[106482]: pgmap v1624: 89 pgs: 89 active+clean; 411 MiB data, 2.4 GiB used, 445 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s 2024-09-21T10:08:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:19 smithi076 ceph-mon[110963]: pgmap v1624: 89 pgs: 89 active+clean; 411 MiB data, 2.4 GiB used, 445 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s 2024-09-21T10:08:20.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:20.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:20.246 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:20.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:20.383 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:20.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:20.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:20.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:20.606 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:130966 6.e 6:7b277cd3:::benchmark_data_smithi076_154645_object17060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1112)", 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:58:26.597765+0000", 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 594.00374348399998, 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11216854499999999, 2024-09-21T10:08:20.607 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 130966 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597765+0000", 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597765+0000", 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597764+0000", 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597770+0000", 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0669999999999997e-06 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597787+0000", 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.73e-05 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.597795+0000", 2024-09-21T10:08:20.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1140000000000006e-06 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.598324+0000", 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00052861100000000001 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.598595+0000", 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00027137100000000001 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:08:20.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.598919+0000", 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00032355600000000002 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.603063+0000", 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041440340000000004 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.609715+0000", 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066514720000000003 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.709877+0000", 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.10016209700000001 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.709909+0000", 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2515999999999998e-05 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:58:26.709934+0000", 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4882999999999999e-05 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:26509 7.1 7:8f44b69a:::benchmark_data_smithi076_190198_object26508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:27.269314+0000", 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 413.33219508600001, 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.125801302, 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.614 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 26509 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269314+0000", 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269314+0000", 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269313+0000", 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269417+0000", 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010454300000000001 2024-09-21T10:08:20.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269430+0000", 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2095e-05 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269436+0000", 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5729999999999996e-06 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269471+0000", 2024-09-21T10:08:20.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5126999999999998e-05 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269576+0000", 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010443200000000001 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269689+0000", 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113649 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.618 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.271022+0000", 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013326729999999999 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.272351+0000", 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001329149 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.395020+0000", 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.122668663 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.395075+0000", 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5235999999999998e-05 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.395115+0000", 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0085e-05 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.620 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:26510 7.1 7:82ab11d7:::benchmark_data_smithi076_190198_object26509:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:27.269511+0000", 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 413.33199749200003, 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12572269599999999, 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 26510 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269511+0000", 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269511+0000", 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269511+0000", 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269574+0000", 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3564000000000003e-05 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269588+0000", 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3458999999999999e-05 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.269594+0000", 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.421e-06 2024-09-21T10:08:20.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.270285+0000", 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00069054299999999995 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.270354+0000", 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8960000000000004e-05 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.624 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.270467+0000", 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011352400000000001 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.271116+0000", 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00064929600000000005 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.272375+0000", 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0012587259999999999 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.395158+0000", 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.122782907 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.395200+0000", 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2280999999999997e-05 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:27.395234+0000", 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3846000000000001e-05 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:27787 7.1 7:8671834f:::benchmark_data_smithi076_190198_object27786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:28.036377+0000", 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 412.56513241300001, 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.150525565, 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.627 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 27787 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036377+0000", 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036377+0000", 2024-09-21T10:08:20.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036376+0000", 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036437+0000", 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1366999999999997e-05 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036451+0000", 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3648999999999999e-05 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036457+0000", 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4899999999999997e-06 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036490+0000", 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2879999999999997e-05 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036576+0000", 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6396000000000004e-05 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.036663+0000", 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6299999999999997e-05 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037532+0000", 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086928699999999995 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.040172+0000", 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002639766 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.186811+0000", 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14663880500000001 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.186862+0000", 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1418000000000001e-05 2024-09-21T10:08:20.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.186902+0000", 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0088e-05 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:27788 7.1 7:867aafd5:::benchmark_data_smithi076_190198_object27787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:28.037699+0000", 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 412.56381006700002, 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.150695676, 2024-09-21T10:08:20.633 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 27788 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037699+0000", 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037699+0000", 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037698+0000", 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037788+0000", 2024-09-21T10:08:20.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0062999999999995e-05 2024-09-21T10:08:20.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037799+0000", 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.099e-05 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037806+0000", 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0740000000000004e-06 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037836+0000", 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0062000000000001e-05 2024-09-21T10:08:20.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037913+0000", 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6433999999999994e-05 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.037993+0000", 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9913000000000006e-05 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.038749+0000", 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00075632800000000001 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.040452+0000", 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017026159999999999 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.188343+0000", 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14789100599999999 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.188372+0000", 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9646e-05 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.188395+0000", 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2271000000000001e-05 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:27798 7.1 7:8fe4daa8:::benchmark_data_smithi076_190198_object27797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.640 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:28.041366+0000", 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 412.560143283, 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14716831699999999, 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 27798 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041366+0000", 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041366+0000", 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041365+0000", 2024-09-21T10:08:20.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041421+0000", 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6546999999999999e-05 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041432+0000", 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0661e-05 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041438+0000", 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.626e-06 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041465+0000", 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7656000000000001e-05 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041548+0000", 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2552000000000005e-05 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.041632+0000", 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4067999999999995e-05 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.042394+0000", 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000762208 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.042941+0000", 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054662700000000005 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.188449+0000", 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.145508528 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.188498+0000", 2024-09-21T10:08:20.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8390999999999998e-05 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.188534+0000", 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6343000000000001e-05 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:28313 7.1 7:8fae72c7:::benchmark_data_smithi076_190198_object28312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:28.436317+0000", 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 412.16519236300002, 2024-09-21T10:08:20.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11652289, 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 28313 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436317+0000", 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436317+0000", 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436315+0000", 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436454+0000", 2024-09-21T10:08:20.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013873600000000001 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436466+0000", 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2255e-05 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436471+0000", 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0769999999999998e-06 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.650 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436524+0000", 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2706e-05 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436643+0000", 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011852499999999999 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.436725+0000", 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2436999999999997e-05 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.651 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.437827+0000", 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001102474 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.438972+0000", 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011442900000000001 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.552780+0000", 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11380823700000001 2024-09-21T10:08:20.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.552815+0000", 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4824999999999999e-05 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:28.552839+0000", 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4708000000000001e-05 2024-09-21T10:08:20.653 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:30066 7.1 7:8478c790:::benchmark_data_smithi076_190198_object30065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:29.614443+0000", 2024-09-21T10:08:20.654 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 410.98706624300002, 2024-09-21T10:08:20.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14725986799999999, 2024-09-21T10:08:20.655 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.655 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.655 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.655 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.655 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 30066 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614443+0000", 2024-09-21T10:08:20.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614443+0000", 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614442+0000", 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614498+0000", 2024-09-21T10:08:20.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6461e-05 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614506+0000", 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4139999999999994e-06 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614511+0000", 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2069999999999997e-06 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614536+0000", 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4938000000000001e-05 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614622+0000", 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6868999999999998e-05 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.614709+0000", 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6123000000000002e-05 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.615766+0000", 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010577690000000001 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.617173+0000", 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001406481 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.761638+0000", 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.144465492 2024-09-21T10:08:20.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.761675+0000", 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6461000000000002e-05 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.761703+0000", 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7838999999999999e-05 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:30074 7.1 7:8f30ed4f:::benchmark_data_smithi076_190198_object30073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:29.619255+0000", 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 410.98225356199998, 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14673358, 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 30074 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.663 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619255+0000", 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619255+0000", 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619254+0000", 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619313+0000", 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8924e-05 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619324+0000", 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1011e-05 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619329+0000", 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2499999999999997e-06 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619360+0000", 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0497999999999998e-05 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619448+0000", 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8517000000000006e-05 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.619544+0000", 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5569999999999995e-05 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.620475+0000", 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00093117700000000005 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.620862+0000", 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000386726 2024-09-21T10:08:20.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.765896+0000", 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.145034418 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.765950+0000", 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3758999999999998e-05 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.765989+0000", 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9206999999999999e-05 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:30388 7.1 7:8f16579d:::benchmark_data_smithi076_190198_object30387:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:29.920089+0000", 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 410.68142025200001, 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.12066988100000001, 2024-09-21T10:08:20.669 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.670 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.670 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.670 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.670 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.670 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 30388 2024-09-21T10:08:20.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920089+0000", 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920089+0000", 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920087+0000", 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920133+0000", 2024-09-21T10:08:20.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5912999999999999e-05 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920146+0000", 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3213999999999999e-05 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920153+0000", 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.99e-06 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920189+0000", 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5988000000000002e-05 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920268+0000", 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8781000000000003e-05 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.920395+0000", 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000126472 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921472+0000", 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001077256 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.922515+0000", 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010426739999999999 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:30.040648+0000", 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.118133058 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:30.040704+0000", 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6700999999999998e-05 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:30.040759+0000", 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4230000000000001e-05 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.676 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:30390 7.1 7:874720ee:::benchmark_data_smithi076_190198_object30389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:29.921613+0000", 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 410.67989551900001, 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.11924167099999999, 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 30390 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.677 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921613+0000", 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921613+0000", 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921613+0000", 2024-09-21T10:08:20.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921673+0000", 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0825000000000001e-05 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921683+0000", 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6309999999999998e-06 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921688+0000", 2024-09-21T10:08:20.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8770000000000001e-06 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921719+0000", 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0828999999999999e-05 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921832+0000", 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113212 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3", 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.921933+0000", 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100672 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.922606+0000", 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00067318599999999999 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:29.923469+0000", 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00086321200000000003 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:30.040802+0000", 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.117332663 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:30.040833+0000", 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1593000000000001e-05 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:30.040855+0000", 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1835e-05 2024-09-21T10:08:20.682 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52756 7.8 7.fd8f9308 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.165228+0000", 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.43628113099999, 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95696024000000002, 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.683 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52756 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165228+0000", 2024-09-21T10:08:20.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165228+0000", 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165226+0000", 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165254+0000", 2024-09-21T10:08:20.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8163000000000001e-05 2024-09-21T10:08:20.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165267+0000", 2024-09-21T10:08:20.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3325e-05 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165273+0000", 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.31e-06 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165336+0000", 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3285000000000001e-05 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.165352+0000", 2024-09-21T10:08:20.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5988e-05 2024-09-21T10:08:20.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122175+0000", 2024-09-21T10:08:20.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95682266900000001 2024-09-21T10:08:20.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.691 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122188+0000", 2024-09-21T10:08:20.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3405e-05 2024-09-21T10:08:20.691 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52768 7.8 7.22128a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.182421+0000", 2024-09-21T10:08:20.692 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.41908790899998, 2024-09-21T10:08:20.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94019540800000001, 2024-09-21T10:08:20.693 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.693 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.693 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.693 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.693 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52768 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182421+0000", 2024-09-21T10:08:20.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182421+0000", 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182420+0000", 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182481+0000", 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1891999999999996e-05 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182491+0000", 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9380000000000004e-06 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182496+0000", 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5499999999999996e-06 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182532+0000", 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6227999999999999e-05 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.182540+0000", 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8110000000000003e-06 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122606+0000", 2024-09-21T10:08:20.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94006586999999997 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122616+0000", 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0570999999999999e-05 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52784 7.8 7.cbb60358 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.191808+0000", 2024-09-21T10:08:20.698 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.409700509, 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93083997699999999, 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52784 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191808+0000", 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191808+0000", 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191806+0000", 2024-09-21T10:08:20.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191833+0000", 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6071000000000002e-05 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191844+0000", 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1922e-05 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191850+0000", 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3399999999999997e-06 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191889+0000", 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9561999999999998e-05 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.191897+0000", 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7440000000000007e-06 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122640+0000", 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93074263999999995 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122648+0000", 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.653e-06 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52790 7.8 7.59c0af38 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.231167+0000", 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.37034201699998, 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89151368799999997, 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52790 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231167+0000", 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231167+0000", 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231165+0000", 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.706 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231200+0000", 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4286999999999999e-05 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231215+0000", 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5191000000000001e-05 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231223+0000", 2024-09-21T10:08:20.707 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0909999999999993e-06 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231267+0000", 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3859999999999997e-05 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.231277+0000", 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0040999999999999e-05 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.708 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122672+0000", 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89139470700000001 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122681+0000", 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.115e-06 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52807 7.8 7.8993fae8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.709 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.244707+0000", 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.35680158500003, 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87800583399999999, 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52807 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244707+0000", 2024-09-21T10:08:20.710 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244707+0000", 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244703+0000", 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999957 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.711 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244731+0000", 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8002999999999999e-05 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244743+0000", 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1949e-05 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244750+0000", 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4429999999999997e-06 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.712 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244795+0000", 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4994000000000003e-05 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.244804+0000", 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6119999999999999e-06 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122705+0000", 2024-09-21T10:08:20.713 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87790051899999999 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122713+0000", 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6889999999999993e-06 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52808 7.8 7.d9e4d608 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.245450+0000", 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.356059071, 2024-09-21T10:08:20.714 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87729441100000005, 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52808 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245450+0000", 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.715 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245450+0000", 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245449+0000", 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245514+0000", 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4317000000000001e-05 2024-09-21T10:08:20.716 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245523+0000", 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5999999999999996e-06 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245529+0000", 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0560000000000001e-06 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245589+0000", 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0146999999999999e-05 2024-09-21T10:08:20.717 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.245601+0000", 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1311000000000001e-05 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122736+0000", 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.87713518800000001 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.718 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122744+0000", 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3620000000000006e-06 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52818 7.8 7.796eb9e8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.254333+0000", 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.347175623, 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86844151999999997, 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.719 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52818 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254333+0000", 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254333+0000", 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.720 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254332+0000", 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254361+0000", 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8736999999999999e-05 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254372+0000", 2024-09-21T10:08:20.721 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1661000000000001e-05 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254377+0000", 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0119999999999996e-06 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254430+0000", 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2139999999999999e-05 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.722 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.254440+0000", 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0729e-05 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122766+0000", 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.86832601600000003 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122775+0000", 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5639999999999997e-06 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.723 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52827 7.8 7.6fbb0f08 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.272465+0000", 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.32904392099999, 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85034090699999998, 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.724 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52827 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272465+0000", 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272465+0000", 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.725 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272463+0000", 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272535+0000", 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1868999999999999e-05 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272549+0000", 2024-09-21T10:08:20.726 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4124999999999999e-05 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272557+0000", 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3289999999999996e-06 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272596+0000", 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8855e-05 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.727 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.272603+0000", 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5979999999999999e-06 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122797+0000", 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85019413200000005 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122806+0000", 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5569999999999996e-06 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.728 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:52845 7.8 7.7c85e188 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:52.316322+0000", 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 388.28518694299999, 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80651602600000005, 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 52845 2024-09-21T10:08:20.729 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316322+0000", 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316322+0000", 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:20.730 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316320+0000", 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316345+0000", 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4567e-05 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316357+0000", 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2034e-05 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.731 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316364+0000", 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8689999999999998e-06 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316426+0000", 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2068000000000003e-05 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:52.316434+0000", 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0690000000000002e-06 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.732 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122829+0000", 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.80639559900000002 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:53.122838+0000", 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6340000000000007e-06 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:20.733 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:08:20.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 5 16 14618 14602 182.504 161.812 0.00881364 0.0054726 2024-09-21T10:08:20.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:20.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:20.906 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:20.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:20.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.008 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:21.044 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:21.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:21.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:21.273 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:21.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:21.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:21.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.443 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:21.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:21.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:08:21.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:21.599 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:21.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:21.701 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:21.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 6 16 17155 17139 178.497 158.562 0.00791358 0.00559498 2024-09-21T10:08:21.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:21.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:21.823 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:21.836 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:21.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:21.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:21.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.005 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:22.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:22.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:22.137 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:22.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:22.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:22.268 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:22.369 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:22.402 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:22 smithi191 ceph-mon[102206]: pgmap v1625: 89 pgs: 89 active+clean; 411 MiB data, 2.9 GiB used, 444 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s 2024-09-21T10:08:22.402 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.403 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.403 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:08:22.403 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:08:22.403 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:22.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:22.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[106482]: pgmap v1625: 89 pgs: 89 active+clean; 411 MiB data, 2.9 GiB used, 444 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[110963]: pgmap v1625: 89 pgs: 89 active+clean; 411 MiB data, 2.9 GiB used, 444 GiB / 447 GiB avail; 41 MiB/s wr, 656 op/s 2024-09-21T10:08:22.510 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.511 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.511 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:08:22.511 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:08:22.511 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:22.549 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:22.651 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.705 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:22.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:22.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 7 16 19294 19278 172.093 133.688 0.00380502 0.00580279 2024-09-21T10:08:22.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:22.832 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:22.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:22.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:22.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:22.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:23.062 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:23.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:23.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:23.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:23.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:23.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:23.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:08:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:08:23] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.51.0" 2024-09-21T10:08:23.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:23.506 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:23.608 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.644 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:23.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:23.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 8 16 21677 21661 169.197 148.938 0.00588922 0.00590483 2024-09-21T10:08:23.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:23.753 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:08:23.753 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:08:23.753 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:08:23.753 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:08:23.776 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:23.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:23.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:23.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:24.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:24.029 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:24.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:24.131 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.174 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:24.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:24.273 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:08:24.274 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:24 smithi076 ceph-mon[106482]: pgmap v1626: 89 pgs: 89 active+clean; 957 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-21T10:08:24.274 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3021017889' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:24.274 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:24 smithi076 ceph-mon[110963]: pgmap v1626: 89 pgs: 89 active+clean; 957 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-21T10:08:24.274 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3021017889' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:24.275 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:24.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:24 smithi191 ceph-mon[102206]: pgmap v1626: 89 pgs: 89 active+clean; 957 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-21T10:08:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3021017889' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:08:24.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:24.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:24.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:24.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:24.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:24.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:24.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:24.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 9 15 23925 23910 166.014 140.562 0.00445023 0.00601794 2024-09-21T10:08:24.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:24.847 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:24.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:24.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:24.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:25.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:25.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:25.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:25.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:25.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:25.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:25.378 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:25.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:25.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:25.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:25.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:25 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3021017889' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:25 smithi191 ceph-mon[102206]: osdmap e1212: 8 total, 8 up, 5 in 2024-09-21T10:08:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:25 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3021017889' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:25 smithi076 ceph-mon[106482]: osdmap e1212: 8 total, 8 up, 5 in 2024-09-21T10:08:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:25 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3021017889' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:08:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:25 smithi076 ceph-mon[110963]: osdmap e1212: 8 total, 8 up, 5 in 2024-09-21T10:08:25.638 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:25.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:25.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 10 16 25826 25810 161.287 118.75 0.00669962 0.00619314 2024-09-21T10:08:25.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:25.766 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:25.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:25.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:25.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:25.992 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:26.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:26.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:26.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:26.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:26.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:26.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:26 smithi076 ceph-mon[106482]: pgmap v1628: 89 pgs: 89 active+clean; 1.2 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:26 smithi076 ceph-mon[110963]: pgmap v1628: 89 pgs: 89 active+clean; 1.2 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:26.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:26.409 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:26.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:26.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:26.544 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:26.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:26 smithi191 ceph-mon[102206]: pgmap v1628: 89 pgs: 89 active+clean; 1.2 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:08:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:26.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:26.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:26.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 11 16 27546 27530 156.397 107.5 0.011767 0.00638762 2024-09-21T10:08:26.766 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:26.766 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:08:26.766 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:08:26.766 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:bench [] [] [] [] 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator dump block 2024-09-21T10:08:26.767 INFO:teuthology.orchestra.run.smithi076.stderr:Error EINVAL: invalid command 2024-09-21T10:08:26.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:26.811 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:26.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:08:26.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:26.951 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:26.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:26.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:27.053 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:27.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:27.135 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:27.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:27.233 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:27.334 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:27.386 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:27.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:27.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:27.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:27.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:27.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:27.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 12 16 29363 29347 152.826 113.562 0.00426029 0.00653692 2024-09-21T10:08:27.743 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:27.773 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:08:27.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:27.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:27.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:27.971 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:27.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:28.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.112 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:28.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:28.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.253 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:28.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:28.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:28.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:28 smithi191 ceph-mon[102206]: pgmap v1629: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:28.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:28.442 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:28 smithi076 ceph-mon[106482]: pgmap v1629: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:28.442 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:28 smithi076 ceph-mon[110963]: pgmap v1629: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:28.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:28.502 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:28.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:28.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:28.629 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:28.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:28.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:28.734 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 13 16 30734 30718 147.662 85.6875 0.00512109 0.00676606 2024-09-21T10:08:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.766 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:28.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:28.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:28.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:28.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:29.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:29.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:29.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:29.063 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:29.063 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:29.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:29.176 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:29.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:08:29.301 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:29.302 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:29.308 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:29.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:29.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:29.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:29.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:29.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:29 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/349894308' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:29 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/349894308' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:29.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:29 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/349894308' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:29.661 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:29.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:29.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:08:29.735 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 14 16 32511 32495 145.046 111.062 0.00184679 0.00686435 2024-09-21T10:08:29.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:29.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:29.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:29.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:29.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:29.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:29.987 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:30.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:30.093 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:30.195 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.232 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:30.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:30.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:30.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:30.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:30.440 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:30 smithi191 ceph-mon[102206]: pgmap v1630: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-21T10:08:30.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:30.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:30.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:30.617 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:30 smithi076 ceph-mon[106482]: pgmap v1630: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-21T10:08:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:30 smithi076 ceph-mon[110963]: pgmap v1630: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-21T10:08:30.719 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:30.735 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 15 16 34234 34218 142.552 107.688 0.00579254 0.00700995 2024-09-21T10:08:30.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:30.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:30.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:30.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:30.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:30.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:31.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:31.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:31.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:31.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:31.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:31.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:31.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:31.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:31.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:31.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:31.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:31.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:31.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.693 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:31.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:31.735 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 16 16 35967 35951 140.412 108.312 0.044775 0.00711005 2024-09-21T10:08:31.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:31.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:31.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:31.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:31.928 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:31.928 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:31.964 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:31.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:31.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:32.066 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:32.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:32.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:32.215 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:32.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:32.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:32.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:32.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:32.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:32.581 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:32 smithi191 ceph-mon[102206]: pgmap v1631: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-21T10:08:32.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:32.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:08:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:32 smithi076 ceph-mon[106482]: pgmap v1631: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-21T10:08:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:32 smithi076 ceph-mon[110963]: pgmap v1631: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-21T10:08:32.711 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:32.735 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 17 16 37510 37494 137.825 96.4375 0.00233673 0.00724843 2024-09-21T10:08:32.742 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:32.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:32.843 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:32.869 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:32.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:32.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:32.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:32.973 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:33.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:33.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:33.137 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:33.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:33.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:33.279 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:33.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:33.417 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:33.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:33.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:33.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:33.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:08:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:08:33] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-21T10:08:33.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:33.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:33.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:33.735 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 18 16 38831 38815 134.754 82.5625 0.0022229 0.00740396 2024-09-21T10:08:33.772 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:33.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:33.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:33.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:33.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:33.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:34.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:34.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:34.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:34.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:34.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:34.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:34.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:34.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:34.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:34.459 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:34 smithi191 ceph-mon[102206]: pgmap v1632: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:08:34.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:34.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:34.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:34 smithi076 ceph-mon[106482]: pgmap v1632: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:08:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:34 smithi076 ceph-mon[110963]: pgmap v1632: 89 pgs: 89 active+clean; 2.1 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:08:34.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:34.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:34.735 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 19 16 40406 40390 132.842 98.4375 0.00610229 0.00752334 2024-09-21T10:08:34.744 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:34.744 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:34.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:34.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:34.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:34.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:34.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:34.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:35.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:35.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:35.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:35.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:35.244 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:35.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:35.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:35.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:35.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:08:35.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:35.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:35.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:35.647 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:35.720 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:35.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:08:35.734845+0000 min lat: 0.00119425 max lat: 0.156011 avg lat: 0.00761228 2024-09-21T10:08:35.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:08:35.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 20 16 42026 42010 131.262 101.25 0.00572852 0.00761228 2024-09-21T10:08:35.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:35.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:35.873 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:35.908 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:35.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:36.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:36.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:36.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:36.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:36.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:36.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:36.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:36.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:36.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:36.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:36 smithi191 ceph-mon[102206]: pgmap v1633: 89 pgs: 89 active+clean; 2.3 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:08:36.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:36 smithi076 ceph-mon[106482]: pgmap v1633: 89 pgs: 89 active+clean; 2.3 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:08:36.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:36 smithi076 ceph-mon[110963]: pgmap v1633: 89 pgs: 89 active+clean; 2.3 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-21T10:08:36.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:36.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:36.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:36.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 21 16 43729 43713 130.08 106.438 0.00532797 0.00768207 2024-09-21T10:08:36.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:36.845 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:36.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:36.976 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:36.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:37.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:37.114 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:37.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:37.253 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:37.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:37.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:37.491 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:37.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:37.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:37.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:37.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:37.666 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:08:37.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-21T10:08:37.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 22 16 45307 45291 128.649 98.625 0.00862514 0.00776708 2024-09-21T10:08:37.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:37.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:37.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:37.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:37.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:38.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:38.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:38.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:38.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:38.234 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:38.234 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:38.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:38.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:38.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:38.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:38.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:38.447 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:38.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:38.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:38.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:38.589 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:38 smithi076 ceph-mon[110963]: pgmap v1634: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-21T10:08:38.589 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:38 smithi076 ceph-mon[106482]: pgmap v1634: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-21T10:08:38.608 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:38 smithi191 ceph-mon[102206]: pgmap v1634: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-21T10:08:38.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:38.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:38.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 23 16 46910 46894 127.411 100.188 0.00814901 0.00784325 2024-09-21T10:08:38.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:38.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:38.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:08:38.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:38.993 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:38.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:39.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.136 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:39.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:39.221 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:39.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:39.243 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:39.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:39.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:39.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:39.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:39.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:39.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:39.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:39.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:39.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 24 16 48518 48502 126.289 100.5 0.00784445 0.00791286 2024-09-21T10:08:39.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:39.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.820 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:39.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:39.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:39.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:39.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:40.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:40.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:40.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:40.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:40.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:40.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:40.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:08:40.464 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.511 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:40.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:40.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:40 smithi191 ceph-mon[102206]: pgmap v1635: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-21T10:08:40.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:40.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:40 smithi076 ceph-mon[106482]: pgmap v1635: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-21T10:08:40.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:40 smithi076 ceph-mon[110963]: pgmap v1635: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-21T10:08:40.633 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:40.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:40.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:40.736 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 25 16 49996 49980 124.933 92.375 0.0154969 0.00799885 2024-09-21T10:08:40.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:40.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:40.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:40.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:40.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:40.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:40.991 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:41.017 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:41.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:41.118 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:41.157 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:41.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:41.191 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:41.191 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:41.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:41.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:41.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:41.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:41.421 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:41.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:41.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:41.534 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:41.553 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:41.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:41.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:41.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:41.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:41.738 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 26 16 51397 51381 123.486 87.5625 0.0131801 0.00809183 2024-09-21T10:08:41.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:41.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:41.888 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:08:41.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:41.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:41.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:41.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:42.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:42.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:42.166 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:42.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:42.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:42.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:42.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:42.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:42.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:42.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:42.612 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:42 smithi076 ceph-mon[106482]: pgmap v1636: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-21T10:08:42.612 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]} : dispatch 2024-09-21T10:08:42.612 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-09-21T10:08:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:42 smithi191 ceph-mon[102206]: pgmap v1636: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-21T10:08:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]} : dispatch 2024-09-21T10:08:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-09-21T10:08:42.613 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:42 smithi076 ceph-mon[110963]: pgmap v1636: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-21T10:08:42.613 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]} : dispatch 2024-09-21T10:08:42.613 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-09-21T10:08:42.631 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:42.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:42.732 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:42.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 27 16 52715 52699 121.957 82.375 0.0210775 0.00819271 2024-09-21T10:08:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:42.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:42.808 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:42.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:42.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:42.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:42.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:43.021 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:43.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:43.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:43.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:43.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:43.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:43.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:43.367 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:08:43.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:43.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:43.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:43.591 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]': finished 2024-09-21T10:08:43.591 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]}]': finished 2024-09-21T10:08:43.591 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:43 smithi191 ceph-mon[102206]: osdmap e1213: 8 total, 8 up, 5 in 2024-09-21T10:08:43.591 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1866416926' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:43.592 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:08:43] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-21T10:08:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]': finished 2024-09-21T10:08:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]}]': finished 2024-09-21T10:08:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[106482]: osdmap e1213: 8 total, 8 up, 5 in 2024-09-21T10:08:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1866416926' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:43.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]': finished 2024-09-21T10:08:43.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]}]': finished 2024-09-21T10:08:43.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[110963]: osdmap e1213: 8 total, 8 up, 5 in 2024-09-21T10:08:43.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1866416926' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:08:43.694 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:43.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:43.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 28 16 54125 54109 120.748 88.125 0.00767232 0.00827658 2024-09-21T10:08:43.831 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:43.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:43.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:43.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:43.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:43.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:44.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:44.131 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:44.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:44.275 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:44.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:44.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[106482]: pgmap v1638: 89 pgs: 1 peering, 88 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:08:44.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[110963]: pgmap v1638: 89 pgs: 1 peering, 88 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:08:44.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:08:44.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:44.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:08:44.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1866416926' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:44.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[110963]: osdmap e1214: 8 total, 8 up, 5 in 2024-09-21T10:08:44.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:44.413 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:44.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:44.521 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:44 smithi191 ceph-mon[102206]: pgmap v1638: 89 pgs: 1 peering, 88 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:44 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:44 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:44 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1866416926' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:44.522 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:44 smithi191 ceph-mon[102206]: osdmap e1214: 8 total, 8 up, 5 in 2024-09-21T10:08:44.540 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:44.540 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:44.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:44.620 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:08:44.620 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:08:44.620 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:08:44.620 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1866416926' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:08:44.621 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:44 smithi076 ceph-mon[106482]: osdmap e1214: 8 total, 8 up, 5 in 2024-09-21T10:08:44.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:44.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:44.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 29 16 55648 55632 119.867 95.1875 0.00817539 0.00833748 2024-09-21T10:08:44.773 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:44.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:44.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:44.954 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:44.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:45.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:45.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:45.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.226 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:45.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:45.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:45.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:08:45.367 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:45.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:45.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.598 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:45.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:45.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:45.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 30 16 57100 57084 118.895 90.75 0.0162418 0.00840473 2024-09-21T10:08:45.753 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.807 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:45.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:45.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:45.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:45.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:45.992 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:45.992 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:08:45.992 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:08:45.992 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:08:45.992 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:bench [] [] [] [] 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator dump block 2024-09-21T10:08:45.993 INFO:teuthology.orchestra.run.smithi076.stderr:Error EINVAL: invalid command 2024-09-21T10:08:46.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:46.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:46.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:08:46.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:46.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:46.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:46.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:46.466 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:46.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:46.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:46.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:46 smithi076 ceph-mon[106482]: pgmap v1640: 89 pgs: 2 peering, 87 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:46.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:46 smithi076 ceph-mon[110963]: pgmap v1640: 89 pgs: 2 peering, 87 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:46.624 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:46.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:46.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 31 16 58414 58398 117.709 82.125 0.0128965 0.00848944 2024-09-21T10:08:46.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:46.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:46.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:46.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:46 smithi191 ceph-mon[102206]: pgmap v1640: 89 pgs: 2 peering, 87 active+clean; 3.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:08:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:46.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:46.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:46.985 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:47.011 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:47.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.149 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:47.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:47.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:47.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:47.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:47.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:47.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:47.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:47.529 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:47.529 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:47.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:47.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:47.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:47.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:47.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 32 15 59919 59904 116.972 94.125 0.0130958 0.00854376 2024-09-21T10:08:47.787 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:47.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:47.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:47.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:47.949 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:47.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:48.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:48.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:48.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:48.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:48.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:48.436 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:48.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:48.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:48.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:48.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:48 smithi076 ceph-mon[106482]: pgmap v1641: 89 pgs: 2 peering, 87 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 58 MiB/s wr, 933 op/s 2024-09-21T10:08:48.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:48 smithi076 ceph-mon[110963]: pgmap v1641: 89 pgs: 2 peering, 87 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 58 MiB/s wr, 933 op/s 2024-09-21T10:08:48.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:48.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.740 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 33 16 61179 61163 115.811 78.6875 0.0106209 0.00862894 2024-09-21T10:08:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:48.813 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:08:48.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-21T10:08:48.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:48.865 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:48 smithi191 ceph-mon[102206]: pgmap v1641: 89 pgs: 2 peering, 87 active+clean; 3.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 58 MiB/s wr, 933 op/s 2024-09-21T10:08:48.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:48.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:48.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:49.000 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:08:49.084 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:08:49.084 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:49.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:49.118 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:49.124 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:08:49.124 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:49.164 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:08:49.164 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:49.203 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:08:49.203 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:49.206 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:49.206 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:49.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:49.236 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:08:49.236 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:49.249 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:49.268 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:08:49.268 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:49.283 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:08:49.307 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:08:49.307 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:49.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:08:49.341 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:49.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:49.400 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:49.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.529 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:49.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:49.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:49 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1397273528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:49.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:49 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1397273528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:49.629 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:49.668 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:49.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:49.741 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 34 16 62348 62332 114.554 73.0625 0.0102747 0.0087235 2024-09-21T10:08:49.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:49.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:49.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:49.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:49 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1397273528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:49.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:49.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:49.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:50.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:50.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:50.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:50.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:50.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:50.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:50.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:50.451 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:50.491 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:50.593 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:50.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:50 smithi191 ceph-mon[102206]: pgmap v1642: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 114 MiB/s wr, 1.83k op/s; 8 B/s, 0 objects/s recovering 2024-09-21T10:08:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:50 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:08:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:50 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:08:50.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:50 smithi076 ceph-mon[110963]: pgmap v1642: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 114 MiB/s wr, 1.83k op/s; 8 B/s, 0 objects/s recovering 2024-09-21T10:08:50.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:50 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:08:50.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:50 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:08:50.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:50 smithi076 ceph-mon[106482]: pgmap v1642: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 114 MiB/s wr, 1.83k op/s; 8 B/s, 0 objects/s recovering 2024-09-21T10:08:50.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:50 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:08:50.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:50 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:08:50.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:50.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:50.741 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 35 16 63674 63658 113.649 82.875 0.00780895 0.00879385 2024-09-21T10:08:50.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:50.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:50.887 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:50.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:50.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:50.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:51.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:51.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:51.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:51.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:51.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:51.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:51.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:51.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:51.440 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:51.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:51.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:51.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:51.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:51.716 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:51.741 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 36 16 65045 65029 112.872 85.6875 0.0107774 0.00885515 2024-09-21T10:08:51.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:51.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:51.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:51.984 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:51.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:52.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:52.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:52.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:52.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:52.359 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:52.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:52.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:52.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:52.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:52.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:52 smithi191 ceph-mon[102206]: pgmap v1643: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 110 MiB/s wr, 1.75k op/s; 8 B/s, 0 objects/s recovering 2024-09-21T10:08:52.621 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:52 smithi076 ceph-mon[110963]: pgmap v1643: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 110 MiB/s wr, 1.75k op/s; 8 B/s, 0 objects/s recovering 2024-09-21T10:08:52.621 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:52 smithi076 ceph-mon[106482]: pgmap v1643: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 110 MiB/s wr, 1.75k op/s; 8 B/s, 0 objects/s recovering 2024-09-21T10:08:52.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.682 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:52.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:52.741 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 37 16 66383 66367 112.082 83.625 0.00633824 0.00891754 2024-09-21T10:08:52.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:52.831 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:52.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:52.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:52.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:08:52.974 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:52.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:53.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177812 6.5 6:a81a04cb:::benchmark_data_smithi076_154645_object63906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1119)", 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:27.942668+0000", 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.14663147600004, 2024-09-21T10:08:53.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.135777995, 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177812 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942668+0000", 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942668+0000", 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942667+0000", 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942672+0000", 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3140000000000003e-06 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942682+0000", 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9539999999999999e-06 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942687+0000", 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9949999999999996e-06 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942728+0000", 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1044999999999998e-05 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942864+0000", 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013567900000000001 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.942998+0000", 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013448999999999999 2024-09-21T10:08:53.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.947349+0000", 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0043506860000000003 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.947363+0000", 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4099e-05 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.960362+0000", 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.01299904 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.960372+0000", 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7229999999999999e-06 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.078349+0000", 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1179771919999999 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.078446+0000", 2024-09-21T10:08:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6928000000000004e-05 2024-09-21T10:08:53.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177818 6.5 6:a989fedd:::benchmark_data_smithi076_154645_object63912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1119)", 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:27.951341+0000", 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.13795853900001, 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.140315231, 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.102 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177818 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951341+0000", 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951341+0000", 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951339+0000", 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951344+0000", 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.006e-06 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951356+0000", 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2252999999999999e-05 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951364+0000", 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4399999999999999e-06 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951438+0000", 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4486999999999993e-05 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951575+0000", 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013677900000000001 2024-09-21T10:08:53.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.951697+0000", 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121828 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.961263+0000", 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0095656669999999999 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.961271+0000", 2024-09-21T10:08:53.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4989999999999994e-06 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.078692+0000", 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.117420418 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.078800+0000", 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108318 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091603+0000", 2024-09-21T10:08:53.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012802902999999999 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091656+0000", 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3072999999999998e-05 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177827 6.5 6:a27c631e:::benchmark_data_smithi076_154645_object63921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:27.969645+0000", 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.11965437699996, 2024-09-21T10:08:53.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1221964600000001, 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177827 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969645+0000", 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969645+0000", 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969644+0000", 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969649+0000", 2024-09-21T10:08:53.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7650000000000001e-06 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969659+0000", 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.044e-05 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969666+0000", 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4139999999999997e-06 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969707+0000", 2024-09-21T10:08:53.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0667000000000002e-05 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.969715+0000", 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5549999999999997e-06 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.078867+0000", 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1091523910000001 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.078949+0000", 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1487000000000001e-05 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.079078+0000", 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000129417 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.088591+0000", 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0095127179999999999 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091692+0000", 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031011540000000001 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091730+0000", 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7799000000000002e-05 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091757+0000", 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7614999999999999e-05 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091841+0000", 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4007999999999999e-05 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177833 6.5 6:aad501f9:::benchmark_data_smithi076_154645_object63927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:27.976481+0000", 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.11281835600005, 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1184572230000001, 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177833 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.116 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976481+0000", 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976481+0000", 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976480+0000", 2024-09-21T10:08:53.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976485+0000", 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6399999999999996e-06 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976496+0000", 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1413999999999999e-05 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976502+0000", 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2920000000000001e-06 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976543+0000", 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0828999999999998e-05 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.976550+0000", 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8739999999999997e-06 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.082345+0000", 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.105794924 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.082429+0000", 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3770999999999995e-05 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.082516+0000", 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7756000000000005e-05 2024-09-21T10:08:53.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091575+0000", 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.009058066 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091874+0000", 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000299203 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.094845+0000", 2024-09-21T10:08:53.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002971535 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.094872+0000", 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7044999999999999e-05 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.094938+0000", 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5735000000000007e-05 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.123 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177850 6.5 6:a2e983dd:::benchmark_data_smithi076_154645_object63944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:27.999519+0000", 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.08977978999997, 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.115437115, 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177850 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.124 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.999519+0000", 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.999519+0000", 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.999517+0000", 2024-09-21T10:08:53.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.999524+0000", 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.224e-06 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.999585+0000", 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1347999999999996e-05 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:27.999593+0000", 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5959999999999998e-06 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.004935+0000", 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0053420489999999998 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.004943+0000", 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0520000000000002e-06 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.085612+0000", 2024-09-21T10:08:53.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.080669726 2024-09-21T10:08:53.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.085712+0000", 2024-09-21T10:08:53.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9766000000000007e-05 2024-09-21T10:08:53.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.085784+0000", 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1568000000000003e-05 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091698+0000", 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059143870000000001 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091902+0000", 2024-09-21T10:08:53.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020347099999999999 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.114843+0000", 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.022941275000000001 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.114873+0000", 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0133000000000001e-05 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.114956+0000", 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3446e-05 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177855 6.5 6:aca56aac:::benchmark_data_smithi076_154645_object63949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.007069+0000", 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.08223062699994, 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.108052802, 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.132 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177855 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007069+0000", 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007069+0000", 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007067+0000", 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007075+0000", 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4730000000000003e-06 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007087+0000", 2024-09-21T10:08:53.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2564e-05 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007093+0000", 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7790000000000001e-06 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007123+0000", 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0395999999999998e-05 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.007131+0000", 2024-09-21T10:08:53.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2060000000000001e-06 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.088624+0000", 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.08149308 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.088690+0000", 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6503999999999999e-05 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.088774+0000", 2024-09-21T10:08:53.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4352999999999997e-05 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091878+0000", 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031037959999999998 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.094814+0000", 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029353000000000001 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115035+0000", 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020221323999999999 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115062+0000", 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6962999999999999e-05 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115121+0000", 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9453000000000002e-05 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.138 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177866 6.5 6:a3549aa5:::benchmark_data_smithi076_154645_object63960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.024178+0000", 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.06512153200003, 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0910908850000001, 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.139 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177866 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024178+0000", 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024178+0000", 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024177+0000", 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024180+0000", 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5509999999999998e-06 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024190+0000", 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0297e-05 2024-09-21T10:08:53.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024203+0000", 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3083e-05 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024316+0000", 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113065 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.024325+0000", 2024-09-21T10:08:53.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9430000000000006e-06 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.091928+0000", 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0676025220000001 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.092021+0000", 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2948999999999994e-05 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.092148+0000", 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012707800000000001 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.101770+0000", 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096226049999999994 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.101848+0000", 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7417999999999996e-05 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115183+0000", 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013335236 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115207+0000", 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4110000000000001e-05 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115269+0000", 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1440999999999997e-05 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.145 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177889 6.5 6:a9c06742:::benchmark_data_smithi076_154645_object63983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.056122+0000", 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.03317705799998, 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.059261676, 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177889 2024-09-21T10:08:53.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056122+0000", 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056122+0000", 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056121+0000", 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056127+0000", 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9179999999999999e-06 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056139+0000", 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2568999999999999e-05 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056145+0000", 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.361e-06 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056187+0000", 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2292000000000001e-05 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.056195+0000", 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9400000000000002e-06 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.094966+0000", 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.038771345 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.095039+0000", 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2367999999999996e-05 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.095108+0000", 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8881999999999995e-05 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.101853+0000", 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0067450829999999998 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.101896+0000", 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.32e-05 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115297+0000", 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013401547999999999 2024-09-21T10:08:53.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115324+0000", 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6928e-05 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115384+0000", 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9454999999999999e-05 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177907 6.5 6:ae6a9765:::benchmark_data_smithi076_154645_object64001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.082117+0000", 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.00718209800004, 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0461353840000001, 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177907 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082117+0000", 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082117+0000", 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082116+0000", 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082121+0000", 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5739999999999997e-06 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082131+0000", 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0404000000000001e-05 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082136+0000", 2024-09-21T10:08:53.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.203e-06 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082189+0000", 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2256000000000002e-05 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.082200+0000", 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.13e-05 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.098363+0000", 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0161629240000001 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.098477+0000", 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011425300000000001 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.098594+0000", 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117428 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.101916+0000", 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033218950000000001 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.111758+0000", 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0098415299999999994 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128105+0000", 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.016347256000000001 2024-09-21T10:08:53.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128152+0000", 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6981999999999998e-05 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128252+0000", 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100289 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.159 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177913 6.5 6:ae5bd656:::benchmark_data_smithi076_154645_object64007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.087579+0000", 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 565.00171972999999, 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.040923169, 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177913 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087579+0000", 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087579+0000", 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087578+0000", 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087584+0000", 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4929999999999998e-06 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087594+0000", 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.023e-05 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087601+0000", 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8009999999999998e-06 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087650+0000", 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9480000000000001e-05 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.087660+0000", 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2240000000000001e-06 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.101946+0000", 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0142863040000001 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.102070+0000", 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012428800000000001 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.102195+0000", 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012477399999999999 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.114965+0000", 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012770472 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115005+0000", 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0021000000000002e-05 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128359+0000", 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013353353 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128402+0000", 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2914000000000003e-05 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128503+0000", 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010088499999999999 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177922 6.5 6:a3f556d9:::benchmark_data_smithi076_154645_object64016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.100057+0000", 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 564.98924261399998, 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.028730251, 2024-09-21T10:08:53.166 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177922 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100057+0000", 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100057+0000", 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100055+0000", 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100061+0000", 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6189999999999999e-06 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100072+0000", 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0971999999999999e-05 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100077+0000", 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2190000000000003e-06 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100124+0000", 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6752000000000002e-05 2024-09-21T10:08:53.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.100135+0000", 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0917000000000001e-05 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.104937+0000", 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0048026830000001 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.105047+0000", 2024-09-21T10:08:53.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109875 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.105171+0000", 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000123218 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.114970+0000", 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0097990839999999996 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115156+0000", 2024-09-21T10:08:53.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000186856 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128645+0000", 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013488814999999999 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128692+0000", 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7126999999999998e-05 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128787+0000", 2024-09-21T10:08:53.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4359999999999998e-05 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177924 6.5 6:aa1ff023:::benchmark_data_smithi076_154645_object64018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.103417+0000", 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 564.98588217500003, 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.038315315, 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177924 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103417+0000", 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103417+0000", 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103416+0000", 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103422+0000", 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1009999999999997e-06 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103431+0000", 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7469999999999994e-06 2024-09-21T10:08:53.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103436+0000", 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8300000000000003e-06 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103478+0000", 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2076000000000002e-05 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.103487+0000", 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6719999999999999e-06 2024-09-21T10:08:53.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.108675+0000", 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0051877899999999 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.108785+0000", 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109584 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.108858+0000", 2024-09-21T10:08:53.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2892999999999995e-05 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115129+0000", 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062712030000000004 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.127990+0000", 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012861142000000001 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.141567+0000", 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013576794999999999 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.141618+0000", 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0856999999999998e-05 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.141732+0000", 2024-09-21T10:08:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011463099999999999 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177931 6.5 6:a17f0427:::benchmark_data_smithi076_154645_object64025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.181 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.112404+0000", 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 564.97689471299998, 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.031505355, 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177931 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112404+0000", 2024-09-21T10:08:53.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112404+0000", 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112403+0000", 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112417+0000", 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.375e-05 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112428+0000", 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1094000000000001e-05 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112434+0000", 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1179999999999997e-06 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112473+0000", 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8399999999999998e-05 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.112480+0000", 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0600000000000002e-06 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.111812+0000", 2024-09-21T10:08:53.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99933234199999998 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.111890+0000", 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8379000000000003e-05 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.111963+0000", 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2358000000000001e-05 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115133+0000", 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031700909999999999 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128042+0000", 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012909541 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.143733+0000", 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015690707000000002 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.143797+0000", 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3678999999999998e-05 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.143910+0000", 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011302 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34674.0:177960 6.5 6:a703be25:::benchmark_data_smithi076_154645_object64054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1120)", 2024-09-21T10:08:53.188 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T09:59:28.180564+0000", 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 564.90873517700004, 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96355410600000002, 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34674", 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/2948755423", 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177960 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180564+0000", 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180564+0000", 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180563+0000", 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180568+0000", 2024-09-21T10:08:53.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2039999999999996e-06 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180580+0000", 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1455e-05 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180587+0000", 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1069999999999999e-06 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180633+0000", 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6845999999999998e-05 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:28.180642+0000", 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0630000000000006e-06 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115415+0000", 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.934773199 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115509+0000", 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4300000000000002e-05 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,6", 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.115588+0000", 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8588e-05 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128265+0000", 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012677300000000001 2024-09-21T10:08:53.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.128318+0000", 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2802e-05 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.143956+0000", 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.015638761000000001 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.144022+0000", 2024-09-21T10:08:53.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5486000000000003e-05 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T09:59:29.144118+0000", 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6123000000000001e-05 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18335 7.8 7:15da7ed0:::benchmark_data_smithi076_190198_object18334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.661512+0000", 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 451.42778714600001, 2024-09-21T10:08:53.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.999778414, 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18335 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661512+0000", 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661512+0000", 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661511+0000", 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661598+0000", 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6918999999999999e-05 2024-09-21T10:08:53.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661609+0000", 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1175999999999999e-05 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661617+0000", 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5680000000000002e-06 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661662+0000", 2024-09-21T10:08:53.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5257999999999999e-05 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.661671+0000", 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3190000000000001e-06 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.660942+0000", 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99927068500000005 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.661290+0000", 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00034835399999999999 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18359 7.8 7:1976fab3:::benchmark_data_smithi076_190198_object18358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.677424+0000", 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 451.41187515399997, 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98853828899999996, 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.200 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18359 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677424+0000", 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677424+0000", 2024-09-21T10:08:53.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677423+0000", 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677567+0000", 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014406099999999999 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677587+0000", 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9843e-05 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677594+0000", 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3130000000000001e-06 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677658+0000", 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3638000000000003e-05 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.677669+0000", 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.098e-05 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.661561+0000", 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.983892039 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.661699+0000", 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000138713 2024-09-21T10:08:53.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.661807+0000", 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107835 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664184+0000", 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023771410000000001 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664193+0000", 2024-09-21T10:08:53.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8179999999999993e-06 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.665898+0000", 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017052899999999999 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.665927+0000", 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8912999999999999e-05 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.665962+0000", 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4929999999999999e-05 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18370 7.8 7:1b3fe5ca:::benchmark_data_smithi076_190198_object18369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.683757+0000", 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 451.405542637, 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98239566700000003, 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.207 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18370 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683757+0000", 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683757+0000", 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683756+0000", 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683828+0000", 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2428999999999994e-05 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683836+0000", 2024-09-21T10:08:53.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7489999999999998e-06 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683840+0000", 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3649999999999997e-06 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683882+0000", 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1575000000000001e-05 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.683889+0000", 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.34e-06 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.662496+0000", 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97860670299999997 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.662618+0000", 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000122197 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.662807+0000", 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000188584 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666010+0000", 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032032340000000001 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666021+0000", 2024-09-21T10:08:53.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.08e-05 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666069+0000", 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7797999999999999e-05 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666114+0000", 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5079e-05 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666152+0000", 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8432000000000001e-05 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18389 7.8 7:178b1344:::benchmark_data_smithi076_190198_object18388:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.696556+0000", 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 451.39274331600001, 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.969735244, 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.214 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18389 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696556+0000", 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696556+0000", 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696555+0000", 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696617+0000", 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2438000000000001e-05 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696629+0000", 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1755e-05 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696639+0000", 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0455000000000001e-05 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696691+0000", 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2024000000000002e-05 2024-09-21T10:08:53.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.696702+0000", 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0944000000000001e-05 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663434+0000", 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96673143500000003 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663543+0000", 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109509 2024-09-21T10:08:53.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.663641+0000", 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7471000000000002e-05 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666201+0000", 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025601439999999999 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666211+0000", 2024-09-21T10:08:53.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0156000000000001e-05 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666242+0000", 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1263000000000002e-05 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666270+0000", 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7432000000000001e-05 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666291+0000", 2024-09-21T10:08:53.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1350999999999999e-05 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18419 7.8 7:1ffcbf82:::benchmark_data_smithi076_190198_object18418:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.719593+0000", 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 451.36970644399997, 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94689748799999995, 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:53.221 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18419 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719593+0000", 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719593+0000", 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719592+0000", 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719663+0000", 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1188999999999999e-05 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719675+0000", 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2071e-05 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719682+0000", 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4560000000000003e-06 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719728+0000", 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5652999999999998e-05 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.719735+0000", 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.477e-06 2024-09-21T10:08:53.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664247+0000", 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94451226300000002 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664357+0000", 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109775 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.664492+0000", 2024-09-21T10:08:53.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013574799999999999 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666217+0000", 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0017245509999999999 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666345+0000", 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012815099999999999 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666420+0000", 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5284000000000006e-05 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666469+0000", 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8869999999999998e-05 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666490+0000", 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0992000000000002e-05 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.227 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.34992.0:18435 7.8 7:198f8dd0:::benchmark_data_smithi076_190198_object18434:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:01:21.728693+0000", 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 451.36060574200002, 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93880749299999999, 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.34992", 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 18435 2024-09-21T10:08:53.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728693+0000", 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728693+0000", 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.229 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728693+0000", 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728751+0000", 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8326999999999999e-05 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728759+0000", 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2269999999999993e-06 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728764+0000", 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.515e-06 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728796+0000", 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1844000000000002e-05 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:21.728802+0000", 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2790000000000004e-06 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.665110+0000", 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93630767299999995 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.665242+0000", 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013233 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.665376+0000", 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000133774 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.666356+0000", 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00098026800000000007 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667373+0000", 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010169020000000001 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667420+0000", 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6844000000000001e-05 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667476+0000", 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6716999999999999e-05 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:01:22.667501+0000", 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4501000000000001e-05 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:08:53.234 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:08:53.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:53.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:53.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:53.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:53.374 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:08:53.374 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:08:53.375 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:08:53.375 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:08:53.375 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:08:53.375 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:08:53.375 INFO:teuthology.orchestra.run.smithi076.stderr:bench [] [] [] [] 2024-09-21T10:08:53.375 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:08:53.376 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:08:53.376 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:08:53.376 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator dump block 2024-09-21T10:08:53.376 INFO:teuthology.orchestra.run.smithi076.stderr:Error EINVAL: invalid command 2024-09-21T10:08:53.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:53.490 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:53.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:53.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:53.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:08:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:08:53] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-21T10:08:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:53.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:53.741 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 38 15 67526 67511 111.014 71.5 0.0196063 0.00900207 2024-09-21T10:08:53.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:53.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:53.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:53.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:53.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:53.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:54.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:54.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:54.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:54.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:54.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:54.182 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:08:54.183 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-21T10:08:54.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:08:54.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:54.253 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:54.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:54.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:54.422 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:08:54.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.428 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:54.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:54.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:54.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:54 smithi191 ceph-mon[102206]: pgmap v1644: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 87 MiB/s wr, 1.38k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/141985978' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:54 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:54.674 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:54.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:54.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:54.744 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 39 16 68767 68751 110.149 77.5 0.0128 0.00907351 2024-09-21T10:08:54.775 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:54 smithi076 ceph-mon[106482]: pgmap v1644: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 87 MiB/s wr, 1.38k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:54.775 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/141985978' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:54.775 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:54 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:54.776 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:54 smithi076 ceph-mon[110963]: pgmap v1644: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 87 MiB/s wr, 1.38k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:54.776 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/141985978' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:54.776 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:54 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:08:54.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:54.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:54.953 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:54.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:54.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:55.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:55.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:55.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:55.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:55.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.270 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:55.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:55.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:55.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:55.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:55.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:55.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:55.639 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:55.680 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:55.743 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:08:55.742409+0000 min lat: 0.00119425 max lat: 0.156011 avg lat: 0.00913573 2024-09-21T10:08:55.743 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:08:55.743 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 40 16 70046 70030 109.393 79.9375 0.0112693 0.00913573 2024-09-21T10:08:55.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:55.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:55.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:55.851 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:55.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:55 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:55 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:55.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:55 smithi191 ceph-mon[102206]: osdmap e1215: 8 total, 8 up, 5 in 2024-09-21T10:08:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:55 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:55 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:55 smithi076 ceph-mon[110963]: osdmap e1215: 8 total, 8 up, 5 in 2024-09-21T10:08:55.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:55 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:08:55.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:55 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:08:55.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:55 smithi076 ceph-mon[106482]: osdmap e1215: 8 total, 8 up, 5 in 2024-09-21T10:08:55.909 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:55.935 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:55.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:56.037 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:56.078 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:56.180 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:56.213 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:56.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:56.341 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:56.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:56.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:56.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:56.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:56.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:56.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:56.707 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:56 smithi076 ceph-mon[106482]: pgmap v1646: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 85 MiB/s wr, 1.36k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:56.707 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:56.707 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:56 smithi076 ceph-mon[110963]: pgmap v1646: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 85 MiB/s wr, 1.36k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:56.707 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:56.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:56.743 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 41 16 71302 71286 108.639 78.5 0.012902 0.00919904 2024-09-21T10:08:56.755 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:56.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:56.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:56.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:56 smithi191 ceph-mon[102206]: pgmap v1646: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 85 MiB/s wr, 1.36k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:56.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:56.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:56.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:56.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:57.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:57.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:57.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:57.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:57.161 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:08:57.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.271 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:08:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:57.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:08:57.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.669 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:57.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.743 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 42 16 72410 72394 107.702 69.25 0.011266 0.00927964 2024-09-21T10:08:57.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:57.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:57.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:57.923 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:57.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:58.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:58.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:58.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:58.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:58.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:58.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:58.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:58.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:58.547 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:08:58.568 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:58 smithi076 ceph-mon[106482]: pgmap v1647: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 85 MiB/s wr, 1.36k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:58.568 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3272918517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:58.568 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:58 smithi076 ceph-mon[110963]: pgmap v1647: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 85 MiB/s wr, 1.36k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:58.568 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3272918517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:58.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:58.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:08:58.623 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:08:58.624 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:08:58.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:58.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:58.744 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 43 16 73412 73396 106.653 62.625 0.00805086 0.00937071 2024-09-21T10:08:58.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:58.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:58.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:58 smithi191 ceph-mon[102206]: pgmap v1647: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 85 MiB/s wr, 1.36k op/s; 6 B/s, 0 objects/s recovering 2024-09-21T10:08:58.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3272918517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:08:58.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:08:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:58.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:58.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:08:59.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:59.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:59.184 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 5, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:08:59.184 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:08:59.184 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-21T10:08:59.184 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:08:59.184 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:08:59.185 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-21T10:08:59.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:59.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:59.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:08:59.355 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:59.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:08:59.457 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:08:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.503 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:08:59.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:08:59.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:08:59.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:59.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:08:59.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:08:59.744 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 44 16 74500 74484 105.775 68 0.0108163 0.00944844 2024-09-21T10:08:59.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:08:59.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:08:59 smithi076 ceph-mon[106482]: pgmap v1648: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:08:59.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:08:59 smithi076 ceph-mon[110963]: pgmap v1648: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:08:59.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:08:59.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:08:59 smithi191 ceph-mon[102206]: pgmap v1648: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:08:59.865 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:08:59.865 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:08:59.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:08:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:59.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:08:59.985 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:00.013 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:00.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:00.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:00.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:00.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:00.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:00.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:00.415 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:00.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:00.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:00.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:00.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:00.654 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:00.701 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:09:00.719 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:00.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 45 16 75687 75671 105.071 74.1875 0.0161475 0.00951135 2024-09-21T10:09:00.745 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:00.754 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:00.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:00.866 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:00.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:00.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.018 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:01.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:01.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:01.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:01.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:01.246 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:01.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.292 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:01.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:01.393 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:01.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:01.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:01.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:01.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:01.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:09:01.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:01.694 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:01.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:01.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 46 16 76810 76794 104.312 70.1875 0.0108516 0.0095807 2024-09-21T10:09:01.795 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:01.847 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:01.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:01.987 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:01.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:02.012 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:01 smithi076 ceph-mon[106482]: pgmap v1649: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:09:02.012 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:01 smithi076 ceph-mon[110963]: pgmap v1649: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:09:02.089 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:01 smithi191 ceph-mon[102206]: pgmap v1649: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-21T10:09:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.128 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:02.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:02.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:02.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.279 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:02.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:02.380 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.423 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:02.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:02.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:02.549 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:02.651 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:02.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:02.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 47 16 77995 77979 103.669 74.0625 0.0058588 0.0096411 2024-09-21T10:09:02.788 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:02.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.825 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:02.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:02.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:02.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:02.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:03.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:03.076 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:03.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:03.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:03.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:03.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:03.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:03.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:03.462 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:03.506 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:03.608 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:09:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:09:03] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.51.0" 2024-09-21T10:09:03.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:03.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 48 16 79100 79084 102.948 69.0625 0.0224847 0.0097081 2024-09-21T10:09:03.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:03.791 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:03.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:03.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:03.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:04.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:04.043 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:03 smithi191 ceph-mon[102206]: pgmap v1650: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:09:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:03 smithi076 ceph-mon[106482]: pgmap v1650: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:09:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:03 smithi076 ceph-mon[110963]: pgmap v1650: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-21T10:09:04.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:04.186 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:04.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:04.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:04.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:04.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:04.441 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:04.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:04.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:04.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:04.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:04.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:04.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:04.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 49 16 80072 80056 102.087 60.75 0.013761 0.00979027 2024-09-21T10:09:04.792 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:04.830 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:04.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:04.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:04.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:04.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:05.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.187 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:05.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:05.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:05.350 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:05.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:05.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:05.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:05.617 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:05.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:05.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 50 16 81275 81259 101.549 75.1875 0.00290833 0.00984001 2024-09-21T10:09:05.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:05.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:05.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:05.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:05.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:06.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:05 smithi191 ceph-mon[102206]: pgmap v1651: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:09:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:05 smithi076 ceph-mon[106482]: pgmap v1651: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:09:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:05 smithi076 ceph-mon[110963]: pgmap v1651: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-21T10:09:06.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:06.227 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:06.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:06.364 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:06.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:06.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:06.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.526 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:06.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:06.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:06.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:06.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:06.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 51 16 82332 82316 100.853 66.0625 0.0107887 0.00991036 2024-09-21T10:09:06.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:06.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:06.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:06.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:06.931 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:06.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:07.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.060 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:07.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:07.188 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:07.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:07.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:07.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.442 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:07.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:07.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:07.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:07.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 52 16 83452 83436 100.26 70 0.0094545 0.00996802 2024-09-21T10:09:07.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:07.845 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:07.903 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:07 smithi076 ceph-mon[106482]: pgmap v1652: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 60 MiB/s wr, 967 op/s 2024-09-21T10:09:07.903 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:07 smithi076 ceph-mon[110963]: pgmap v1652: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 60 MiB/s wr, 967 op/s 2024-09-21T10:09:07.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:07.973 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:07.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:08.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:08.103 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:08.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:07 smithi191 ceph-mon[102206]: pgmap v1652: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 60 MiB/s wr, 967 op/s 2024-09-21T10:09:08.205 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:08.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:08.348 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:08.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:08.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:08.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:08.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:08.619 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:08.666 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:08.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 53 16 84577 84561 99.6944 70.3125 0.0192008 0.0100245 2024-09-21T10:09:08.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:08.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:08.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:08.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:08.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:08.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:08.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:09.061 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:09.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:09.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:09.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:09.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:09.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:09.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:09.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:09.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:09.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:09.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:09.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:09.618 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:09.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:09.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 54 16 85672 85656 99.1156 68.4375 0.0184262 0.0100837 2024-09-21T10:09:09.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:09.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:09.877 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:09.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:09.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:10.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:10.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:09 smithi191 ceph-mon[102206]: pgmap v1653: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:09:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:09 smithi076 ceph-mon[106482]: pgmap v1653: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:09:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:09 smithi076 ceph-mon[110963]: pgmap v1653: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:09:10.130 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:10.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:10.266 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:10.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:10.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:10.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:10.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:10.541 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:10.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:10.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:10.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:10.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:10.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 55 16 86767 86751 98.5578 68.4375 0.0128141 0.01014 2024-09-21T10:09:10.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:10.789 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:09:10.811 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:10.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:10.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:10.940 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:10.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:11.041 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:11.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:11.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:11.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1385442737' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1385442737' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1385442737' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:11.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:11.203 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:11.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:11.333 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:11.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:11.472 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:11.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:11.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:11.697 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:11.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:11.745 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 56 16 87549 87533 97.6706 48.875 0.00924042 0.0102338 2024-09-21T10:09:11.828 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:09:11.828 INFO:teuthology.orchestra.run.smithi076.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:09:11.828 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:09:11.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:11.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:11.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:11.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:11.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:12.024 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:11 smithi191 ceph-mon[102206]: pgmap v1654: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-21T10:09:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:11 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:11 smithi191 ceph-mon[102206]: osdmap e1216: 8 total, 8 up, 5 in 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[106482]: pgmap v1654: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[106482]: osdmap e1216: 8 total, 8 up, 5 in 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[110963]: pgmap v1654: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:12.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:11 smithi076 ceph-mon[110963]: osdmap e1216: 8 total, 8 up, 5 in 2024-09-21T10:09:12.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:12.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:12.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:12.291 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:12.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:12.393 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:12.423 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:12.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:12.555 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:12.657 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:12.698 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:12.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:12.746 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 57 16 88604 88588 97.1139 65.9375 0.0111743 0.0102918 2024-09-21T10:09:12.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:12.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:12.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:12.967 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:12.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:12.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:13.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:13.104 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:13.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:13.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:13.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:13.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:13.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:13.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:13.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:13.598 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:13.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:09:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:09:13] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-09-21T10:09:13.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:13.701 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:13.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:13.749 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 58 16 89741 89725 96.662 71.0625 0.0170512 0.0103398 2024-09-21T10:09:13.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:13.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:13.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:13.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:13.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:14.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:14.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:13 smithi191 ceph-mon[102206]: pgmap v1656: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-21T10:09:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:13 smithi076 ceph-mon[106482]: pgmap v1656: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-21T10:09:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:13 smithi076 ceph-mon[110963]: pgmap v1656: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-21T10:09:14.175 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:14.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:14.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:14.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:14.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:14.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:14.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:14.632 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:14.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:14.747 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 59 16 90884 90868 96.2344 71.4375 0.00843975 0.0103863 2024-09-21T10:09:14.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:14.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:14.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:14.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:15.025 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:15.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:15.157 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:15.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.199 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:15.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:15.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:15.342 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:15.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:15.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:15.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:15.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:15.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:15.747 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:09:15.746737+0000 min lat: 0.00119425 max lat: 0.193843 avg lat: 0.0104335 2024-09-21T10:09:15.748 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:09:15.748 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 60 15 92002 91987 95.7961 69.9375 0.00675328 0.0104335 2024-09-21T10:09:15.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:15.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:15.909 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:15.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:15.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:16.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:16.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:15 smithi191 ceph-mon[102206]: pgmap v1657: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:15 smithi076 ceph-mon[106482]: pgmap v1657: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:15 smithi076 ceph-mon[110963]: pgmap v1657: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:16.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:16.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:16.268 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:16.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.294 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:16.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:16.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:16.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:16.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:16.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:16.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:16.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:16.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:16.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:16.748 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 61 16 93159 93143 95.4101 72.25 0.0074118 0.0104759 2024-09-21T10:09:16.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:16.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:16.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:16.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:16.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:16.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:17.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:17.096 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:17.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:17.198 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:17.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:17.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:17.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:17.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:17.474 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:09:17.474 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:09:17.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:17.496 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:09:17.516 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:17.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:17.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:17.646 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:17.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:17.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:17.750 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 62 16 94131 94115 94.848 60.75 0.0254069 0.0105375 2024-09-21T10:09:17.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:17.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:17.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:17.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:17.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:17.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:18.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:18.049 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:17 smithi191 ceph-mon[102206]: pgmap v1658: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:17 smithi076 ceph-mon[106482]: pgmap v1658: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:17 smithi076 ceph-mon[110963]: pgmap v1658: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:18.152 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:18.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:18.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:18.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:18.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:18.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:18.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:18.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:18.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:09:18.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:18.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:18.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:18.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.742 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:18.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:18.750 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 63 16 95215 95199 94.4178 67.75 0.0278682 0.0105857 2024-09-21T10:09:18.762 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:09:18.763 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:18.802 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:09:18.802 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:18.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:18.845 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:09:18.845 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:18.855 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:18.864 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:18.881 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:09:18.882 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:18.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:18.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:18.915 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:09:18.915 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:18.947 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:09:18.948 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:18.948 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3250777861' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:18.948 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3250777861' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:18.979 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:09:18.979 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:18.991 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:19.011 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:09:19.011 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:19.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:19.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3250777861' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:19.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:19.173 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:19.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:19.316 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:19.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:19.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:19.444 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:19.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:19.473 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 5, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:09:19.473 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:09:19.473 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2024-09-21T10:09:19.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:09:19.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:19.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:19.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:19.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:19.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:19.750 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 64 16 96175 96159 93.88 60 0.00984964 0.0106461 2024-09-21T10:09:19.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:19.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:19.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:19.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:19.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:20.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:20.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:19 smithi076 ceph-mon[106482]: pgmap v1659: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:20.075 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:19 smithi076 ceph-mon[110963]: pgmap v1659: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:20.112 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:19 smithi191 ceph-mon[102206]: pgmap v1659: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:20.214 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:20.240 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:20.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:20.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:20.368 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:20.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:20.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:20.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:20.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:20.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:20.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:20.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:20.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:20.751 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 65 16 97286 97270 93.5039 69.4375 0.0150669 0.0106895 2024-09-21T10:09:20.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:20.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:20.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:20.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:20.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:20.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:21.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:21.066 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:21.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:21.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:21.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:21.339 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:21.440 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:21.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:21.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:21.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:21.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:21.705 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:21.738 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:21.750 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 66 16 98411 98395 93.1525 70.3125 0.0247893 0.0107301 2024-09-21T10:09:21.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:21.866 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:21.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:21.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:21.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:21.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:22.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:22.102 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:21 smithi076 ceph-mon[106482]: pgmap v1660: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:22.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:21 smithi076 ceph-mon[110963]: pgmap v1660: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:21 smithi191 ceph-mon[102206]: pgmap v1660: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:09:22.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:22.170 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44752 7.5 7:aa71fbe7:::benchmark_data_smithi076_190198_object44751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.236980+0000", 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.92886097399997, 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84928779700000001, 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.171 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44752 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.236980+0000", 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.236980+0000", 2024-09-21T10:09:22.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.236981+0000", 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1099999999999996e-07 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237022+0000", 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0970000000000002e-05 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237029+0000", 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.328e-06 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237034+0000", 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4299999999999997e-06 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.070323+0000", 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83328886099999999 2024-09-21T10:09:22.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.070479+0000", 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015550199999999999 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.070648+0000", 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016900199999999999 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.074462+0000", 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038139340000000002 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078194+0000", 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00373216 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084829+0000", 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066349779999999997 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084884+0000", 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4749e-05 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086268+0000", 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001384372 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44755 7.5 7:a24569dd:::benchmark_data_smithi076_190198_object44754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.237890+0000", 2024-09-21T10:09:22.177 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.92795090099997, 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86201483000000001, 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44755 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237890+0000", 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237890+0000", 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237891+0000", 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.24e-06 2024-09-21T10:09:22.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237939+0000", 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8021e-05 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237949+0000", 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6819999999999998e-06 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.237954+0000", 2024-09-21T10:09:22.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8670000000000002e-06 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.072413+0000", 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83445930700000004 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.072570+0000", 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015627800000000001 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.072781+0000", 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000211463 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078271+0000", 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054898880000000001 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.082585+0000", 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043141070000000002 2024-09-21T10:09:22.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.098267+0000", 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015681725000000001 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.098316+0000", 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9376000000000003e-05 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.099905+0000", 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015888759999999999 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44759 7.5 7:a88bd5e4:::benchmark_data_smithi076_190198_object44758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.243453+0000", 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.92238783400001, 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86711577200000001, 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.184 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.185 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.185 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.185 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.185 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44759 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243453+0000", 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243453+0000", 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243455+0000", 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5430000000000001e-06 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243508+0000", 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.312e-05 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243525+0000", 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6671e-05 2024-09-21T10:09:22.187 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243533+0000", 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078310+0000", 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.834777769 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078495+0000", 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018510799999999999 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.078691+0000", 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000195801 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084303+0000", 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056118959999999999 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086330+0000", 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002027067 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.108649+0000", 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.022319039999999998 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.108709+0000", 2024-09-21T10:09:22.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9670000000000003e-05 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.110569+0000", 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001860149 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44760 7.5 7:a5bffe95:::benchmark_data_smithi076_190198_object44759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.243549+0000", 2024-09-21T10:09:22.191 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.922291734, 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86892832200000003, 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44760 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243549+0000", 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243549+0000", 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243550+0000", 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3799999999999997e-07 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243592+0000", 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1882000000000003e-05 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243599+0000", 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4549999999999998e-06 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.243604+0000", 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1950000000000002e-06 2024-09-21T10:09:22.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.080531+0000", 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83692627200000003 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.080721+0000", 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000190693 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.080875+0000", 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015366499999999999 2024-09-21T10:09:22.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.084309+0000", 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034344050000000002 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.090160+0000", 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0058501669999999999 2024-09-21T10:09:22.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.110627+0000", 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020467793000000001 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.110975+0000", 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00034761299999999998 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.112478+0000", 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001502644 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:44762 7.d e1148/1137 7:b780d3c9:::benchmark_data_smithi076_190198_object44761:head v 1148'2804, mlcod=1146'2802)", 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.247040+0000", 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.91880102499999, 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77214165199999996, 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.198 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247040+0000", 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247040+0000", 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.199 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247041+0000", 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3700000000000001e-07 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247063+0000", 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2895e-05 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247071+0000", 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.055e-06 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.247076+0000", 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7679999999999999e-06 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.996010+0000", 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74893399599999999 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.996035+0000", 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5103e-05 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019154+0000", 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023118825999999999 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019182+0000", 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7572e-05 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.202 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44766 7.5 7:a0cdd0d8:::benchmark_data_smithi076_190198_object44765:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.249062+0000", 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.91677931599997, 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86492537199999997, 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44766 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249062+0000", 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249062+0000", 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249063+0000", 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5149999999999999e-06 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249093+0000", 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9734e-05 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249106+0000", 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3171e-05 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.249114+0000", 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7270000000000007e-06 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086370+0000", 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83725620000000001 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086493+0000", 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012246499999999999 2024-09-21T10:09:22.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.086659+0000", 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000166398 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.093835+0000", 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0071764580000000001 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.106600+0000", 2024-09-21T10:09:22.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012764559999999999 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.112529+0000", 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0059289649999999996 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.112584+0000", 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4654000000000002e-05 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.113987+0000", 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014035250000000001 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:44768 7.5 7:a3682ca2:::benchmark_data_smithi076_190198_object44767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1148)", 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.250180+0000", 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.91566101699999, 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87050010200000005, 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.209 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 44768 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250180+0000", 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250180+0000", 2024-09-21T10:09:22.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250182+0000", 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4219999999999999e-06 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250205+0000", 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3748999999999999e-05 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250218+0000", 2024-09-21T10:09:22.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2619000000000001e-05 2024-09-21T10:09:22.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.250225+0000", 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0079999999999996e-06 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.088311+0000", 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83808586100000004 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.088443+0000", 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013174000000000001 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.088606+0000", 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016336500000000001 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.100609+0000", 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012003565000000001 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.106606+0000", 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0059969589999999996 2024-09-21T10:09:22.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.118424+0000", 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011817246 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.118512+0000", 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8101000000000002e-05 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.120680+0000", 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021684669999999999 2024-09-21T10:09:22.217 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:44770 7.d e1148/1137 7:bab46a7f:::benchmark_data_smithi076_190198_object44769:head v 1148'2805, mlcod=1146'2802)", 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.254313+0000", 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.91152826799998, 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76489147199999996, 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254313+0000", 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254313+0000", 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254314+0000", 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1379999999999999e-06 2024-09-21T10:09:22.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254343+0000", 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.851e-05 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254356+0000", 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3869000000000001e-05 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.254364+0000", 2024-09-21T10:09:22.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4900000000000003e-06 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.997360+0000", 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.742996141 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.997380+0000", 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0123999999999999e-05 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019187+0000", 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021806447999999999 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019204+0000", 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7751999999999999e-05 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.222 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:44779 7.d e1148/1137 7:b6c25ae2:::benchmark_data_smithi076_190198_object44778:head v 1148'2806, mlcod=1146'2802)", 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:43.274461+0000", 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 458.89137993200001, 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74475925399999998, 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274461+0000", 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.223 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274461+0000", 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274462+0000", 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.92e-07 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274486+0000", 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4417999999999999e-05 2024-09-21T10:09:22.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274496+0000", 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0036e-05 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.274502+0000", 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1319999999999999e-06 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.998093+0000", 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.72359012099999998 2024-09-21T10:09:22.225 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:43.998111+0000", 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8046999999999998e-05 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019206+0000", 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021095137999999999 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:44.019220+0000", 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.467e-05 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67019 7.5 7:addb2ccf:::benchmark_data_smithi076_190198_object67018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.328853+0000", 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.83698781200002, 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.98297844300000004, 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.227 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67019 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328853+0000", 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328853+0000", 2024-09-21T10:09:22.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328854+0000", 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4499999999999995e-07 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328872+0000", 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7958e-05 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328881+0000", 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8470000000000002e-06 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328887+0000", 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3230000000000003e-06 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328956+0000", 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8629999999999999e-05 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.328963+0000", 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4710000000000001e-06 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.303405+0000", 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.97444187100000002 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.303556+0000", 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015055900000000001 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.303683+0000", 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000126947 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311598+0000", 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079149809999999998 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311659+0000", 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1092000000000004e-05 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311733+0000", 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3565999999999993e-05 2024-09-21T10:09:22.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311780+0000", 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7896999999999999e-05 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311832+0000", 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1356000000000001e-05 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.235 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67073 7.5 7:a0e58874:::benchmark_data_smithi076_190198_object67072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.468813+0000", 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.69702798200001, 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.84322499799999995, 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67073 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468813+0000", 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468813+0000", 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468815+0000", 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6449999999999999e-06 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468837+0000", 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1844999999999998e-05 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468850+0000", 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3105000000000001e-05 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468857+0000", 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3189999999999997e-06 2024-09-21T10:09:22.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468937+0000", 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0020999999999998e-05 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.468951+0000", 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3529999999999999e-05 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.306149+0000", 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.83719846499999995 2024-09-21T10:09:22.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.306280+0000", 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013107900000000001 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.306452+0000", 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000171428 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311714+0000", 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005262353 2024-09-21T10:09:22.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311868+0000", 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015407499999999999 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.311959+0000", 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0593999999999994e-05 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312002+0000", 2024-09-21T10:09:22.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.375e-05 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312038+0000", 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5788999999999999e-05 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67097 7.5 7:abaa65eb:::benchmark_data_smithi076_190198_object67096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.242 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.528415+0000", 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.63742621199998, 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.78379750800000003, 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67097 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528415+0000", 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528415+0000", 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528416+0000", 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3e-06 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528438+0000", 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1664000000000001e-05 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528450+0000", 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2489e-05 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528458+0000", 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2099999999999996e-06 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528544+0000", 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6278000000000003e-05 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.528555+0000", 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1453e-05 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.309199+0000", 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.78064412000000005 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.309329+0000", 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012979500000000001 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.309462+0000", 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013248399999999999 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312093+0000", 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002630945 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312132+0000", 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9480999999999997e-05 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312143+0000", 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0529e-05 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312180+0000", 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7468000000000001e-05 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312212+0000", 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2292000000000002e-05 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67099 7.5 7:ac6bfae4:::benchmark_data_smithi076_190198_object67098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.535830+0000", 2024-09-21T10:09:22.249 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.63001112299997, 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.791905674, 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67099 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535830+0000", 2024-09-21T10:09:22.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535830+0000", 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535831+0000", 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2309999999999999e-06 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535853+0000", 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2084999999999999e-05 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535863+0000", 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9250000000000007e-06 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535870+0000", 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1890000000000001e-06 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535932+0000", 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2063999999999994e-05 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.535941+0000", 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3280000000000006e-06 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312251+0000", 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77631040100000004 2024-09-21T10:09:22.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312383+0000", 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013156600000000001 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.312508+0000", 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000125036 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327510+0000", 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015002547999999999 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327562+0000", 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1341999999999997e-05 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327639+0000", 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7724000000000003e-05 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327690+0000", 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0467000000000003e-05 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327736+0000", 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5768e-05 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67106 7.5 7:a679e0fc:::benchmark_data_smithi076_190198_object67105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.256 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.603699+0000", 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.56214261999997, 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.73135408599999996, 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67106 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.257 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603699+0000", 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603699+0000", 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603700+0000", 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4190000000000001e-06 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603721+0000", 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1535999999999998e-05 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603734+0000", 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2483999999999999e-05 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603740+0000", 2024-09-21T10:09:22.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0979999999999999e-06 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603795+0000", 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5031000000000002e-05 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.603803+0000", 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8450000000000003e-06 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.314640+0000", 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.71083746299999995 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.314936+0000", 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00029561700000000003 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.316359+0000", 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001423047 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327515+0000", 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011156142000000001 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327603+0000", 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8134999999999996e-05 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.334958+0000", 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073542360000000001 2024-09-21T10:09:22.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335009+0000", 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1499999999999998e-05 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335053+0000", 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3532999999999998e-05 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.263 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67112 7.5 7:a6c7e4ba:::benchmark_data_smithi076_190198_object67111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.665613+0000", 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.50022829099998, 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.66975227199999998, 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67112 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.264 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665613+0000", 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665613+0000", 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665614+0000", 2024-09-21T10:09:22.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.124e-06 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665633+0000", 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8783e-05 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665642+0000", 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1039999999999996e-06 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665646+0000", 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5820000000000003e-06 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665707+0000", 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0228e-05 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.665719+0000", 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2174000000000001e-05 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.318684+0000", 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.65296505900000001 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.318806+0000", 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000121636 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.318945+0000", 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000139517 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327520+0000", 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0085746559999999999 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335115+0000", 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075948889999999996 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335284+0000", 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016905800000000001 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335331+0000", 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7706e-05 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335365+0000", 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3756e-05 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67138 7.5 7:a50cf1cf:::benchmark_data_smithi076_190198_object67137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.270 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.743306+0000", 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.42253476600001, 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.59216225700000003, 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67138 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743306+0000", 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743306+0000", 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743308+0000", 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4670000000000001e-06 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743329+0000", 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1608999999999999e-05 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743340+0000", 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.079e-05 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743347+0000", 2024-09-21T10:09:22.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4359999999999999e-06 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743406+0000", 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9503000000000003e-05 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.743415+0000", 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3380000000000005e-06 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.321215+0000", 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.57780047899999998 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.321355+0000", 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000139791 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.321499+0000", 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000144516 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327572+0000", 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0060731170000000003 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335153+0000", 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075807390000000004 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335398+0000", 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000244966 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335438+0000", 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0074999999999998e-05 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335469+0000", 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0431000000000001e-05 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67159 7.5 7:afaaa7d0:::benchmark_data_smithi076_190198_object67158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.793795+0000", 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.37204613900002, 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.54180674299999998, 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.278 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67159 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793795+0000", 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793795+0000", 2024-09-21T10:09:22.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793796+0000", 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1650000000000001e-06 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793816+0000", 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0100999999999999e-05 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793829+0000", 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2833000000000001e-05 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793836+0000", 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1019999999999999e-06 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793896+0000", 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9976999999999999e-05 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.793907+0000", 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0929e-05 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.324145+0000", 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.53023783800000002 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.324253+0000", 2024-09-21T10:09:22.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000107738 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.324405+0000", 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000152099 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.334917+0000", 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010512199999999999 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335182+0000", 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026484499999999999 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335533+0000", 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00035097999999999998 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335572+0000", 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9326999999999998e-05 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335602+0000", 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9609000000000001e-05 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67196 7.5 7:a910f998:::benchmark_data_smithi076_190198_object67195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.932286+0000", 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.23355470799999, 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.40341120400000002, 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.285 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67196 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932286+0000", 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932286+0000", 2024-09-21T10:09:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932288+0000", 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.747e-06 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932315+0000", 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6750999999999998e-05 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932330+0000", 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4773999999999999e-05 2024-09-21T10:09:22.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932338+0000", 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1289999999999996e-06 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932400+0000", 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2397000000000006e-05 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.932408+0000", 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2600000000000005e-06 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327766+0000", 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.39535715599999999 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327869+0000", 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0001037 2024-09-21T10:09:22.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.327998+0000", 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012844000000000001 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335063+0000", 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0070656149999999999 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335251+0000", 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018784199999999999 2024-09-21T10:09:22.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335631+0000", 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00037950400000000002 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335669+0000", 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8624999999999997e-05 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335698+0000", 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8263999999999999e-05 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67209 7.5 7:a1f6f0be:::benchmark_data_smithi076_190198_object67208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.964507+0000", 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.201333847, 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.37128402999999999, 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.293 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67209 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964507+0000", 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964507+0000", 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964509+0000", 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4500000000000001e-06 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964531+0000", 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2345999999999999e-05 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964543+0000", 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2074e-05 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964551+0000", 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4340000000000003e-06 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964614+0000", 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3429000000000005e-05 2024-09-21T10:09:22.296 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.964623+0000", 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0990000000000005e-06 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.330022+0000", 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.36539935200000001 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.330167+0000", 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000144437 2024-09-21T10:09:22.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.330315+0000", 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014777900000000001 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335499+0000", 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051839149999999999 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335508+0000", 2024-09-21T10:09:22.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6139999999999998e-06 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335727+0000", 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021863899999999999 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335764+0000", 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7558999999999997e-05 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335791+0000", 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6903e-05 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:67217 7.5 7:aa3c2a52:::benchmark_data_smithi076_190198_object67216:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1159)", 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:02:32.991131+0000", 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 409.17471025700002, 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.356302434, 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:22.300 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 67217 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991131+0000", 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991131+0000", 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991133+0000", 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.796e-06 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991181+0000", 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8643000000000002e-05 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991193+0000", 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.13e-05 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991200+0000", 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1149999999999996e-06 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991281+0000", 2024-09-21T10:09:22.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1487000000000001e-05 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:32.991292+0000", 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0635999999999999e-05 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335813+0000", 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.34452144299999998 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.335943+0000", 2024-09-21T10:09:22.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012933199999999999 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.336072+0000", 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012897399999999999 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347209+0000", 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011137651 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:22.305 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347307+0000", 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7627000000000006e-05 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347344+0000", 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7228000000000003e-05 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347390+0000", 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5884000000000003e-05 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:09:22.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:02:33.347433+0000", 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3318000000000001e-05 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:22.307 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:09:22.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:22.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:22.423 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:22.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:22.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:22.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:22.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:22.751 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 67 16 99406 99390 92.6898 62.1875 0.012837 0.0107836 2024-09-21T10:09:22.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:22.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:22.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:22.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:22.882 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:22.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:22.983 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:23.007 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:23.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:23.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:22 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:09:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:22 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:09:23.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:22 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:09:23.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:23.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.284 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:23.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:23.385 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:23.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:23.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:23.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:23.569 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:09:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:09:23] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-09-21T10:09:23.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:23.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:23.751 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 68 16 100511 100495 92.3423 69.0625 0.015815 0.0108244 2024-09-21T10:09:23.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:23.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:23.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:23.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:23.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:23.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:23.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:24.084 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:24.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:23 smithi076 ceph-mon[110963]: pgmap v1661: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:09:24.126 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:23 smithi076 ceph-mon[106482]: pgmap v1661: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:09:24.127 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:24.144 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:24.203 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:23 smithi191 ceph-mon[102206]: pgmap v1661: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-21T10:09:24.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:24.270 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:24.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:24.372 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:24.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:24.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:24.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:24.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:24.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:24.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:24.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:09:24.755 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 69 16 101524 101508 91.9216 63.3125 0.0132985 0.0108727 2024-09-21T10:09:24.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:24.804 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:24.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:24.885 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:09:24.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:24.973 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:24.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:24.989 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1258950063' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:24.990 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:24 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:24.993 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1258950063' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:24.993 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:24 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:25.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:25.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:25.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:25.253 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1258950063' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:25.253 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:24 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:09:25.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:25.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:25.388 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:25.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:25.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:25.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:25.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:25.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:25.681 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:25.751 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 70 16 102570 102554 91.5417 65.375 0.00927223 0.0109189 2024-09-21T10:09:25.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:25.825 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:25.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:09:25.927 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:25.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:25.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:25.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:26.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:26.093 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[110963]: pgmap v1662: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:26.093 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:26.093 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:26.093 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[110963]: osdmap e1217: 8 total, 8 up, 5 in 2024-09-21T10:09:26.093 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:26.095 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[106482]: pgmap v1662: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:26.095 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:26.095 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:26.095 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[106482]: osdmap e1217: 8 total, 8 up, 5 in 2024-09-21T10:09:26.095 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:26.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:26.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:26.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:26.258 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:26.360 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:26.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:25 smithi191 ceph-mon[102206]: pgmap v1662: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:25 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:09:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:25 smithi191 ceph-mon[102206]: osdmap e1217: 8 total, 8 up, 5 in 2024-09-21T10:09:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:26.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:26.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:26.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:26.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:26.664 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:26.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:26.700 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:26.752 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 71 16 103595 103579 91.1543 64.0625 0.0188944 0.0109644 2024-09-21T10:09:26.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.798 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:26.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:26.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:26.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:26.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:27.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:27.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:27.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:27.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:27.367 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:27.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:27.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:27.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:27.648 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:27.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:27.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 72 16 104595 104579 90.7563 62.5 0.008509 0.0110136 2024-09-21T10:09:27.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:27.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:27.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:27.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:28.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:28.057 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:28.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:09:28.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:28.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:28.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:28.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:28.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:28.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:28.434 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:28.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:28.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:28.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:28.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:28 smithi076 ceph-mon[106482]: pgmap v1664: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:09:28.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:28.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:28.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:28 smithi076 ceph-mon[110963]: pgmap v1664: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:09:28.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:28.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:28.662 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:28.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:28.752 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 73 16 105433 105417 90.2302 52.375 0.0203578 0.0110777 2024-09-21T10:09:28.815 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:28.858 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:28.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:28 smithi191 ceph-mon[102206]: pgmap v1664: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-21T10:09:28.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:28.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:28.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:28.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:28.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:28.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:29.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:29.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:29.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:29.235 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:29.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:29.418 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:29.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:29.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:29.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:29.526 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:09:29.531 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:09:29.549 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:29.651 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:29.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:29.752 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 74 16 106361 106345 89.7947 58 0.0128804 0.0111316 2024-09-21T10:09:29.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:29.804 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:29.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:29.949 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:29.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:30.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:30.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:30.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:30.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:30.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:30.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:30.332 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:30.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:30.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:30.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:30.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:30.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:30.602 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:30 smithi076 ceph-mon[106482]: pgmap v1665: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:30.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:30 smithi076 ceph-mon[110963]: pgmap v1665: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:30.633 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:30.736 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:30.753 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 75 16 107469 107453 89.52 69.25 0.0247379 0.0111657 2024-09-21T10:09:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:30.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:30 smithi191 ceph-mon[102206]: pgmap v1665: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:30.888 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:30.915 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:30.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:30.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:31.017 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:31.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:31.161 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:31.198 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:31.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:31.342 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:31.347 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1665,"stamp":"2024-09-21T10:09:28.706356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6854809573,"num_objects":104613,"num_object_clones":0,"num_object_copies":313839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104613,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":104866,"num_write_kb":6696017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":2963020,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86095,"ondisk_log_size":86095,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":22216380,"kb_used_data":19934088,"kb_used_omap":119,"kb_used_meta":2282120,"kb_avail":446509380,"statfs":{"total":479975178240,"available":457225605120,"internally_reserved":0,"allocated":20412506112,"data_stored":20399475829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":2336891080},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,4,7,12,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":682819584,"num_objects":10419,"num_object_clones":0,"num_object_copies":31257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10419,"num_write_kb":666816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006379"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":272,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896037+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:09:24.896037+0000","last_peered":"2024-09-21T10:09:24.896037+0000","last_clean":"2024-09-21T10:09:24.896037+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:09:24.896037+0000","last_undegraded":"2024-09-21T10:09:24.896037+0000","last_fullsized":"2024-09-21T10:09:24.896037+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":74,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915242+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:09:10.915242+0000","last_peered":"2024-09-21T10:09:10.915242+0000","last_clean":"2024-09-21T10:09:10.915242+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:09:10.915242+0000","last_undegraded":"2024-09-21T10:09:10.915242+0000","last_fullsized":"2024-09-21T10:09:10.915242+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1257,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896738+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:09:24.896738+0000","last_peered":"2024-09-21T10:09:24.896738+0000","last_clean":"2024-09-21T10:09:24.896738+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:09:24.896738+0000","last_undegraded":"2024-09-21T10:09:24.896738+0000","last_fullsized":"2024-09-21T10:09:24.896738+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1588,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.900535+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:09:24.900535+0000","last_peered":"2024-09-21T10:09:24.900535+0000","last_clean":"2024-09-21T10:09:24.900535+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:09:24.900535+0000","last_undegraded":"2024-09-21T10:09:24.900535+0000","last_fullsized":"2024-09-21T10:09:24.900535+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1588,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.895787+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:09:24.895787+0000","last_peered":"2024-09-21T10:09:24.895787+0000","last_clean":"2024-09-21T10:09:24.895787+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:09:24.895787+0000","last_undegraded":"2024-09-21T10:09:24.895787+0000","last_fullsized":"2024-09-21T10:09:24.895787+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915040+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:09:10.915040+0000","last_peered":"2024-09-21T10:09:10.915040+0000","last_clean":"2024-09-21T10:09:10.915040+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:09:10.915040+0000","last_undegraded":"2024-09-21T10:09:10.915040+0000","last_fullsized":"2024-09-21T10:09:10.915040+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":272,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897835+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:09:24.897835+0000","last_peered":"2024-09-21T10:09:24.897835+0000","last_clean":"2024-09-21T10:09:24.897835+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:09:24.897835+0000","last_undegraded":"2024-09-21T10:09:24.897835+0000","last_fullsized":"2024-09-21T10:09:24.897835+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":97,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.914651+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:09:10.914651+0000","last_peered":"2024-09-21T10:09:10.914651+0000","last_clean":"2024-09-21T10:09:10.914651+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:09:10.914651+0000","last_undegraded":"2024-09-21T10:09:10.914651+0000","last_fullsized":"2024-09-21T10:09:10.914651+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1217'6522","reported_seq":6540,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:28.179302+0000","last_change":"2024-09-21T10:08:03.311661+0000","last_active":"2024-09-21T10:09:28.179302+0000","last_peered":"2024-09-21T10:09:28.179302+0000","last_clean":"2024-09-21T10:09:28.179302+0000","last_became_active":"2024-09-21T10:08:03.311320+0000","last_became_peered":"2024-09-21T10:08:03.311320+0000","last_unstale":"2024-09-21T10:09:28.179302+0000","last_undegraded":"2024-09-21T10:09:28.179302+0000","last_fullsized":"2024-09-21T10:09:28.179302+0000","mapping_epoch":1208,"log_start":"1211'500","ondisk_log_start":"1211'500","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":6022,"log_dups_size":0,"ondisk_log_size":6022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:35.793126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427425792,"num_objects":6522,"num_object_clones":0,"num_object_copies":19566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6522,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":127,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908168+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:09:24.908168+0000","last_peered":"2024-09-21T10:09:24.908168+0000","last_clean":"2024-09-21T10:09:24.908168+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:09:24.908168+0000","last_undegraded":"2024-09-21T10:09:24.908168+0000","last_fullsized":"2024-09-21T10:09:24.908168+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":69,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:13.188139+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:09:13.188139+0000","last_peered":"2024-09-21T10:09:13.188139+0000","last_clean":"2024-09-21T10:09:13.188139+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:09:13.188139+0000","last_undegraded":"2024-09-21T10:09:13.188139+0000","last_fullsized":"2024-09-21T10:09:13.188139+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"1217'6645","reported_seq":6663,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:28.187014+0000","last_change":"2024-09-21T10:08:03.023459+0000","last_active":"2024-09-21T10:09:28.187014+0000","last_peered":"2024-09-21T10:09:28.187014+0000","last_clean":"2024-09-21T10:09:28.187014+0000","last_became_active":"2024-09-21T10:08:03.023101+0000","last_became_peered":"2024-09-21T10:08:03.023101+0000","last_unstale":"2024-09-21T10:09:28.187014+0000","last_undegraded":"2024-09-21T10:09:28.187014+0000","last_fullsized":"2024-09-21T10:09:28.187014+0000","mapping_epoch":1208,"log_start":"1211'600","ondisk_log_start":"1211'600","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:25.398099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435486720,"num_objects":6645,"num_object_clones":0,"num_object_copies":19935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6645,"num_write_kb":425280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":171,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907548+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:09:24.907548+0000","last_peered":"2024-09-21T10:09:24.907548+0000","last_clean":"2024-09-21T10:09:24.907548+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:09:24.907548+0000","last_undegraded":"2024-09-21T10:09:24.907548+0000","last_fullsized":"2024-09-21T10:09:24.907548+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1212,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.906925+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:09:24.906925+0000","last_peered":"2024-09-21T10:09:24.906925+0000","last_clean":"2024-09-21T10:09:24.906925+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:09:24.906925+0000","last_undegraded":"2024-09-21T10:09:24.906925+0000","last_fullsized":"2024-09-21T10:09:24.906925+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"1217'6437","reported_seq":6455,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:28.152224+0000","last_change":"2024-09-21T10:08:03.312515+0000","last_active":"2024-09-21T10:09:28.152224+0000","last_peered":"2024-09-21T10:09:28.152224+0000","last_clean":"2024-09-21T10:09:28.152224+0000","last_became_active":"2024-09-21T10:08:03.312164+0000","last_became_peered":"2024-09-21T10:08:03.312164+0000","last_unstale":"2024-09-21T10:09:28.152224+0000","last_undegraded":"2024-09-21T10:09:28.152224+0000","last_fullsized":"2024-09-21T10:09:28.152224+0000","mapping_epoch":1208,"log_start":"1211'400","ondisk_log_start":"1211'400","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":6037,"log_dups_size":0,"ondisk_log_size":6037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:23:04.283316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421855232,"num_objects":6437,"num_object_clones":0,"num_object_copies":19311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6437,"num_write_kb":411968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":86,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897908+0000","last_change":"2024-09-21T10:08:43.398923+0000","last_active":"2024-09-21T10:09:24.897908+0000","last_peered":"2024-09-21T10:09:24.897908+0000","last_clean":"2024-09-21T10:09:24.897908+0000","last_became_active":"2024-09-21T10:08:43.383598+0000","last_became_peered":"2024-09-21T10:08:43.383598+0000","last_unstale":"2024-09-21T10:09:24.897908+0000","last_undegraded":"2024-09-21T10:09:24.897908+0000","last_fullsized":"2024-09-21T10:09:24.897908+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:09.939781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:13.188197+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:09:13.188197+0000","last_peered":"2024-09-21T10:09:13.188197+0000","last_clean":"2024-09-21T10:09:13.188197+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:09:13.188197+0000","last_undegraded":"2024-09-21T10:09:13.188197+0000","last_fullsized":"2024-09-21T10:09:13.188197+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1217'6453","reported_seq":6482,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.500330+0000","last_change":"2024-09-21T10:08:02.953184+0000","last_active":"2024-09-21T10:09:27.500330+0000","last_peered":"2024-09-21T10:09:27.500330+0000","last_clean":"2024-09-21T10:09:27.500330+0000","last_became_active":"2024-09-21T10:08:02.952901+0000","last_became_peered":"2024-09-21T10:08:02.952901+0000","last_unstale":"2024-09-21T10:09:27.500330+0000","last_undegraded":"2024-09-21T10:09:27.500330+0000","last_fullsized":"2024-09-21T10:09:27.500330+0000","mapping_epoch":1208,"log_start":"1212'1500","ondisk_log_start":"1212'1500","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4953,"log_dups_size":0,"ondisk_log_size":4953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:27:48.410002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":422903808,"num_objects":6453,"num_object_clones":0,"num_object_copies":19359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6453,"num_write_kb":412992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1242,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.895843+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:09:24.895843+0000","last_peered":"2024-09-21T10:09:24.895843+0000","last_clean":"2024-09-21T10:09:24.895843+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:09:24.895843+0000","last_undegraded":"2024-09-21T10:09:24.895843+0000","last_fullsized":"2024-09-21T10:09:24.895843+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1598,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908470+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:09:24.908470+0000","last_peered":"2024-09-21T10:09:24.908470+0000","last_clean":"2024-09-21T10:09:24.908470+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:09:24.908470+0000","last_undegraded":"2024-09-21T10:09:24.908470+0000","last_fullsized":"2024-09-21T10:09:24.908470+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"1217'6557","reported_seq":6586,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.556457+0000","last_change":"2024-09-21T10:08:03.024378+0000","last_active":"2024-09-21T10:09:27.556457+0000","last_peered":"2024-09-21T10:09:27.556457+0000","last_clean":"2024-09-21T10:09:27.556457+0000","last_became_active":"2024-09-21T10:08:03.023958+0000","last_became_peered":"2024-09-21T10:08:03.023958+0000","last_unstale":"2024-09-21T10:09:27.556457+0000","last_undegraded":"2024-09-21T10:09:27.556457+0000","last_fullsized":"2024-09-21T10:09:27.556457+0000","mapping_epoch":1208,"log_start":"1211'1400","ondisk_log_start":"1211'1400","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5157,"log_dups_size":0,"ondisk_log_size":5157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:40:07.425504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429719552,"num_objects":6557,"num_object_clones":0,"num_object_copies":19671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6557,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":280,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897356+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:09:24.897356+0000","last_peered":"2024-09-21T10:09:24.897356+0000","last_clean":"2024-09-21T10:09:24.897356+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:09:24.897356+0000","last_undegraded":"2024-09-21T10:09:24.897356+0000","last_fullsized":"2024-09-21T10:09:24.897356+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1257,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.895695+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:09:24.895695+0000","last_peered":"2024-09-21T10:09:24.895695+0000","last_clean":"2024-09-21T10:09:24.895695+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:09:24.895695+0000","last_undegraded":"2024-09-21T10:09:24.895695+0000","last_fullsized":"2024-09-21T10:09:24.895695+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"1217'6685","reported_seq":6711,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.625333+0000","last_change":"2024-09-21T10:08:03.312220+0000","last_active":"2024-09-21T10:09:27.625333+0000","last_peered":"2024-09-21T10:09:27.625333+0000","last_clean":"2024-09-21T10:09:27.625333+0000","last_became_active":"2024-09-21T10:08:03.311878+0000","last_became_peered":"2024-09-21T10:08:03.311878+0000","last_unstale":"2024-09-21T10:09:27.625333+0000","last_undegraded":"2024-09-21T10:09:27.625333+0000","last_fullsized":"2024-09-21T10:09:27.625333+0000","mapping_epoch":1208,"log_start":"1212'1700","ondisk_log_start":"1212'1700","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4985,"log_dups_size":0,"ondisk_log_size":4985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:40.204833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438108160,"num_objects":6685,"num_object_clones":0,"num_object_copies":20055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6685,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:13.188179+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:09:13.188179+0000","last_peered":"2024-09-21T10:09:13.188179+0000","last_clean":"2024-09-21T10:09:13.188179+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:09:13.188179+0000","last_undegraded":"2024-09-21T10:09:13.188179+0000","last_fullsized":"2024-09-21T10:09:13.188179+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":900,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907664+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:09:24.907664+0000","last_peered":"2024-09-21T10:09:24.907664+0000","last_clean":"2024-09-21T10:09:24.907664+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:09:24.907664+0000","last_undegraded":"2024-09-21T10:09:24.907664+0000","last_fullsized":"2024-09-21T10:09:24.907664+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"1217'6565","reported_seq":6594,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.527474+0000","last_change":"2024-09-21T10:08:03.024139+0000","last_active":"2024-09-21T10:09:27.527474+0000","last_peered":"2024-09-21T10:09:27.527474+0000","last_clean":"2024-09-21T10:09:27.527474+0000","last_became_active":"2024-09-21T10:08:03.023616+0000","last_became_peered":"2024-09-21T10:08:03.023616+0000","last_unstale":"2024-09-21T10:09:27.527474+0000","last_undegraded":"2024-09-21T10:09:27.527474+0000","last_fullsized":"2024-09-21T10:09:27.527474+0000","mapping_epoch":1208,"log_start":"1212'1400","ondisk_log_start":"1212'1400","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5165,"log_dups_size":0,"ondisk_log_size":5165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:50.949095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430243840,"num_objects":6565,"num_object_clones":0,"num_object_copies":19695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6565,"num_write_kb":420160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1243,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.906822+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:09:24.906822+0000","last_peered":"2024-09-21T10:09:24.906822+0000","last_clean":"2024-09-21T10:09:24.906822+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:09:24.906822+0000","last_undegraded":"2024-09-21T10:09:24.906822+0000","last_fullsized":"2024-09-21T10:09:24.906822+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.898292+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:09:24.898292+0000","last_peered":"2024-09-21T10:09:24.898292+0000","last_clean":"2024-09-21T10:09:24.898292+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:09:24.898292+0000","last_undegraded":"2024-09-21T10:09:24.898292+0000","last_fullsized":"2024-09-21T10:09:24.898292+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"1217'6525","reported_seq":6554,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.592050+0000","last_change":"2024-09-21T10:08:03.313040+0000","last_active":"2024-09-21T10:09:27.592050+0000","last_peered":"2024-09-21T10:09:27.592050+0000","last_clean":"2024-09-21T10:09:27.592050+0000","last_became_active":"2024-09-21T10:08:03.312772+0000","last_became_peered":"2024-09-21T10:08:03.312772+0000","last_unstale":"2024-09-21T10:09:27.592050+0000","last_undegraded":"2024-09-21T10:09:27.592050+0000","last_fullsized":"2024-09-21T10:09:27.592050+0000","mapping_epoch":1208,"log_start":"1212'1600","ondisk_log_start":"1212'1600","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4925,"log_dups_size":0,"ondisk_log_size":4925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:26.873108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427622400,"num_objects":6525,"num_object_clones":0,"num_object_copies":19575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6525,"num_write_kb":417600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":272,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896152+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:09:24.896152+0000","last_peered":"2024-09-21T10:09:24.896152+0000","last_clean":"2024-09-21T10:09:24.896152+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:09:24.896152+0000","last_undegraded":"2024-09-21T10:09:24.896152+0000","last_fullsized":"2024-09-21T10:09:24.896152+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1602,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908318+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:09:24.908318+0000","last_peered":"2024-09-21T10:09:24.908318+0000","last_clean":"2024-09-21T10:09:24.908318+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:09:24.908318+0000","last_undegraded":"2024-09-21T10:09:24.908318+0000","last_fullsized":"2024-09-21T10:09:24.908318+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1217'6680","reported_seq":6697,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:28.175333+0000","last_change":"2024-09-21T10:08:03.023634+0000","last_active":"2024-09-21T10:09:28.175333+0000","last_peered":"2024-09-21T10:09:28.175333+0000","last_clean":"2024-09-21T10:09:28.175333+0000","last_became_active":"2024-09-21T10:08:03.023286+0000","last_became_peered":"2024-09-21T10:08:03.023286+0000","last_unstale":"2024-09-21T10:09:28.175333+0000","last_undegraded":"2024-09-21T10:09:28.175333+0000","last_fullsized":"2024-09-21T10:09:28.175333+0000","mapping_epoch":1208,"log_start":"1211'600","ondisk_log_start":"1211'600","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":6080,"log_dups_size":0,"ondisk_log_size":6080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:37:40.008871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437780480,"num_objects":6680,"num_object_clones":0,"num_object_copies":20040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6680,"num_write_kb":427520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":741,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.914510+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:09:10.914510+0000","last_peered":"2024-09-21T10:09:10.914510+0000","last_clean":"2024-09-21T10:09:10.914510+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:09:10.914510+0000","last_undegraded":"2024-09-21T10:09:10.914510+0000","last_fullsized":"2024-09-21T10:09:10.914510+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1256,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897197+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:09:24.897197+0000","last_peered":"2024-09-21T10:09:24.897197+0000","last_clean":"2024-09-21T10:09:24.897197+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:09:24.897197+0000","last_undegraded":"2024-09-21T10:09:24.897197+0000","last_fullsized":"2024-09-21T10:09:24.897197+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":287,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.900678+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:09:24.900678+0000","last_peered":"2024-09-21T10:09:24.900678+0000","last_clean":"2024-09-21T10:09:24.900678+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:09:24.900678+0000","last_undegraded":"2024-09-21T10:09:24.900678+0000","last_fullsized":"2024-09-21T10:09:24.900678+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":78,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.917336+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:09:10.917336+0000","last_peered":"2024-09-21T10:09:10.917336+0000","last_clean":"2024-09-21T10:09:10.917336+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:09:10.917336+0000","last_undegraded":"2024-09-21T10:09:10.917336+0000","last_fullsized":"2024-09-21T10:09:10.917336+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1217'6546","reported_seq":6575,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.623620+0000","last_change":"2024-09-21T10:08:03.312292+0000","last_active":"2024-09-21T10:09:27.623620+0000","last_peered":"2024-09-21T10:09:27.623620+0000","last_clean":"2024-09-21T10:09:27.623620+0000","last_became_active":"2024-09-21T10:08:03.312011+0000","last_became_peered":"2024-09-21T10:08:03.312011+0000","last_unstale":"2024-09-21T10:09:27.623620+0000","last_undegraded":"2024-09-21T10:09:27.623620+0000","last_fullsized":"2024-09-21T10:09:27.623620+0000","mapping_epoch":1208,"log_start":"1212'1600","ondisk_log_start":"1212'1600","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:15:26.321126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428998656,"num_objects":6546,"num_object_clones":0,"num_object_copies":19638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6546,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":886,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907082+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:09:24.907082+0000","last_peered":"2024-09-21T10:09:24.907082+0000","last_clean":"2024-09-21T10:09:24.907082+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:09:24.907082+0000","last_undegraded":"2024-09-21T10:09:24.907082+0000","last_fullsized":"2024-09-21T10:09:24.907082+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1475,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.903092+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:09:24.903092+0000","last_peered":"2024-09-21T10:09:24.903092+0000","last_clean":"2024-09-21T10:09:24.903092+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:09:24.903092+0000","last_undegraded":"2024-09-21T10:09:24.903092+0000","last_fullsized":"2024-09-21T10:09:24.903092+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":884,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907280+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:09:24.907280+0000","last_peered":"2024-09-21T10:09:24.907280+0000","last_clean":"2024-09-21T10:09:24.907280+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:09:24.907280+0000","last_undegraded":"2024-09-21T10:09:24.907280+0000","last_fullsized":"2024-09-21T10:09:24.907280+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"1217'6536","reported_seq":6562,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.522446+0000","last_change":"2024-09-21T10:08:03.024270+0000","last_active":"2024-09-21T10:09:27.522446+0000","last_peered":"2024-09-21T10:09:27.522446+0000","last_clean":"2024-09-21T10:09:27.522446+0000","last_became_active":"2024-09-21T10:08:03.023879+0000","last_became_peered":"2024-09-21T10:08:03.023879+0000","last_unstale":"2024-09-21T10:09:27.522446+0000","last_undegraded":"2024-09-21T10:09:27.522446+0000","last_fullsized":"2024-09-21T10:09:27.522446+0000","mapping_epoch":1208,"log_start":"1211'1400","ondisk_log_start":"1211'1400","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5136,"log_dups_size":0,"ondisk_log_size":5136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:09.524044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428343296,"num_objects":6536,"num_object_clones":0,"num_object_copies":19608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6536,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1516,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.903718+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:09:24.903718+0000","last_peered":"2024-09-21T10:09:24.903718+0000","last_clean":"2024-09-21T10:09:24.903718+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:09:24.903718+0000","last_undegraded":"2024-09-21T10:09:24.903718+0000","last_fullsized":"2024-09-21T10:09:24.903718+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1603,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907578+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:09:24.907578+0000","last_peered":"2024-09-21T10:09:24.907578+0000","last_clean":"2024-09-21T10:09:24.907578+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:09:24.907578+0000","last_undegraded":"2024-09-21T10:09:24.907578+0000","last_fullsized":"2024-09-21T10:09:24.907578+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1611,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908177+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:09:24.908177+0000","last_peered":"2024-09-21T10:09:24.908177+0000","last_clean":"2024-09-21T10:09:24.908177+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:09:24.908177+0000","last_undegraded":"2024-09-21T10:09:24.908177+0000","last_fullsized":"2024-09-21T10:09:24.908177+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1217'6369","reported_seq":6398,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.543498+0000","last_change":"2024-09-21T10:08:03.313299+0000","last_active":"2024-09-21T10:09:27.543498+0000","last_peered":"2024-09-21T10:09:27.543498+0000","last_clean":"2024-09-21T10:09:27.543498+0000","last_became_active":"2024-09-21T10:08:03.312642+0000","last_became_peered":"2024-09-21T10:08:03.312642+0000","last_unstale":"2024-09-21T10:09:27.543498+0000","last_undegraded":"2024-09-21T10:09:27.543498+0000","last_fullsized":"2024-09-21T10:09:27.543498+0000","mapping_epoch":1208,"log_start":"1211'1200","ondisk_log_start":"1211'1200","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5169,"log_dups_size":0,"ondisk_log_size":5169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:12:03.834580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417398784,"num_objects":6369,"num_object_clones":0,"num_object_copies":19107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6369,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":180,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907614+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:09:24.907614+0000","last_peered":"2024-09-21T10:09:24.907614+0000","last_clean":"2024-09-21T10:09:24.907614+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:09:24.907614+0000","last_undegraded":"2024-09-21T10:09:24.907614+0000","last_fullsized":"2024-09-21T10:09:24.907614+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1179,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915153+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:09:10.915153+0000","last_peered":"2024-09-21T10:09:10.915153+0000","last_clean":"2024-09-21T10:09:10.915153+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:09:10.915153+0000","last_undegraded":"2024-09-21T10:09:10.915153+0000","last_fullsized":"2024-09-21T10:09:10.915153+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1614,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.910351+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:09:24.910351+0000","last_peered":"2024-09-21T10:09:24.910351+0000","last_clean":"2024-09-21T10:09:24.910351+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:09:24.910351+0000","last_undegraded":"2024-09-21T10:09:24.910351+0000","last_fullsized":"2024-09-21T10:09:24.910351+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1217'6483","reported_seq":6511,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.632365+0000","last_change":"2024-09-21T10:08:03.313515+0000","last_active":"2024-09-21T10:09:27.632365+0000","last_peered":"2024-09-21T10:09:27.632365+0000","last_clean":"2024-09-21T10:09:27.632365+0000","last_became_active":"2024-09-21T10:08:03.313246+0000","last_became_peered":"2024-09-21T10:08:03.313246+0000","last_unstale":"2024-09-21T10:09:27.632365+0000","last_undegraded":"2024-09-21T10:09:27.632365+0000","last_fullsized":"2024-09-21T10:09:27.632365+0000","mapping_epoch":1208,"log_start":"1212'1500","ondisk_log_start":"1212'1500","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4983,"log_dups_size":0,"ondisk_log_size":4983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:03.008010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424869888,"num_objects":6483,"num_object_clones":0,"num_object_copies":19449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6483,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1204,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896592+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:09:24.896592+0000","last_peered":"2024-09-21T10:09:24.896592+0000","last_clean":"2024-09-21T10:09:24.896592+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:09:24.896592+0000","last_undegraded":"2024-09-21T10:09:24.896592+0000","last_fullsized":"2024-09-21T10:09:24.896592+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1625,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.910433+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:09:24.910433+0000","last_peered":"2024-09-21T10:09:24.910433+0000","last_clean":"2024-09-21T10:09:24.910433+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:09:24.910433+0000","last_undegraded":"2024-09-21T10:09:24.910433+0000","last_fullsized":"2024-09-21T10:09:24.910433+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":144,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907369+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:09:24.907369+0000","last_peered":"2024-09-21T10:09:24.907369+0000","last_clean":"2024-09-21T10:09:24.907369+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:09:24.907369+0000","last_undegraded":"2024-09-21T10:09:24.907369+0000","last_fullsized":"2024-09-21T10:09:24.907369+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"1217'6528","reported_seq":6557,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.554438+0000","last_change":"2024-09-21T10:08:03.313391+0000","last_active":"2024-09-21T10:09:27.554438+0000","last_peered":"2024-09-21T10:09:27.554438+0000","last_clean":"2024-09-21T10:09:27.554438+0000","last_became_active":"2024-09-21T10:08:03.312838+0000","last_became_peered":"2024-09-21T10:08:03.312838+0000","last_unstale":"2024-09-21T10:09:27.554438+0000","last_undegraded":"2024-09-21T10:09:27.554438+0000","last_fullsized":"2024-09-21T10:09:27.554438+0000","mapping_epoch":1208,"log_start":"1211'1400","ondisk_log_start":"1211'1400","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5128,"log_dups_size":0,"ondisk_log_size":5128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:34:16.978635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427819008,"num_objects":6528,"num_object_clones":0,"num_object_copies":19584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6528,"num_write_kb":417792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1602,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.910617+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:09:24.910617+0000","last_peered":"2024-09-21T10:09:24.910617+0000","last_clean":"2024-09-21T10:09:24.910617+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:09:24.910617+0000","last_undegraded":"2024-09-21T10:09:24.910617+0000","last_fullsized":"2024-09-21T10:09:24.910617+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":782,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907403+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:09:24.907403+0000","last_peered":"2024-09-21T10:09:24.907403+0000","last_clean":"2024-09-21T10:09:24.907403+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:09:24.907403+0000","last_undegraded":"2024-09-21T10:09:24.907403+0000","last_fullsized":"2024-09-21T10:09:24.907403+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1594,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908681+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:09:24.908681+0000","last_peered":"2024-09-21T10:09:24.908681+0000","last_clean":"2024-09-21T10:09:24.908681+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:09:24.908681+0000","last_undegraded":"2024-09-21T10:09:24.908681+0000","last_fullsized":"2024-09-21T10:09:24.908681+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.c","version":"1217'6491","reported_seq":6520,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:27.447646+0000","last_change":"2024-09-21T10:08:03.024073+0000","last_active":"2024-09-21T10:09:27.447646+0000","last_peered":"2024-09-21T10:09:27.447646+0000","last_clean":"2024-09-21T10:09:27.447646+0000","last_became_active":"2024-09-21T10:08:03.023647+0000","last_became_peered":"2024-09-21T10:08:03.023647+0000","last_unstale":"2024-09-21T10:09:27.447646+0000","last_undegraded":"2024-09-21T10:09:27.447646+0000","last_fullsized":"2024-09-21T10:09:27.447646+0000","mapping_epoch":1208,"log_start":"1212'1400","ondisk_log_start":"1212'1400","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:50:19.572283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425394176,"num_objects":6491,"num_object_clones":0,"num_object_copies":19473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6491,"num_write_kb":415424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":302,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897927+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:09:24.897927+0000","last_peered":"2024-09-21T10:09:24.897927+0000","last_clean":"2024-09-21T10:09:24.897927+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:09:24.897927+0000","last_undegraded":"2024-09-21T10:09:24.897927+0000","last_fullsized":"2024-09-21T10:09:24.897927+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1603,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907649+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:09:24.907649+0000","last_peered":"2024-09-21T10:09:24.907649+0000","last_clean":"2024-09-21T10:09:24.907649+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:09:24.907649+0000","last_undegraded":"2024-09-21T10:09:24.907649+0000","last_fullsized":"2024-09-21T10:09:24.907649+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":280,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896058+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:09:24.896058+0000","last_peered":"2024-09-21T10:09:24.896058+0000","last_clean":"2024-09-21T10:09:24.896058+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:09:24.896058+0000","last_undegraded":"2024-09-21T10:09:24.896058+0000","last_fullsized":"2024-09-21T10:09:24.896058+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1217'6565","reported_seq":6583,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:28.144219+0000","last_change":"2024-09-21T10:08:03.312130+0000","last_active":"2024-09-21T10:09:28.144219+0000","last_peered":"2024-09-21T10:09:28.144219+0000","last_clean":"2024-09-21T10:09:28.144219+0000","last_became_active":"2024-09-21T10:08:03.311781+0000","last_became_peered":"2024-09-21T10:08:03.311781+0000","last_unstale":"2024-09-21T10:09:28.144219+0000","last_undegraded":"2024-09-21T10:09:28.144219+0000","last_fullsized":"2024-09-21T10:09:28.144219+0000","mapping_epoch":1208,"log_start":"1211'500","ondisk_log_start":"1211'500","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:33.284319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":430243840,"num_objects":6565,"num_object_clones":0,"num_object_copies":19695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6565,"num_write_kb":420160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":143,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907376+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:09:24.907376+0000","last_peered":"2024-09-21T10:09:24.907376+0000","last_clean":"2024-09-21T10:09:24.907376+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:09:24.907376+0000","last_undegraded":"2024-09-21T10:09:24.907376+0000","last_fullsized":"2024-09-21T10:09:24.907376+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":69,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:13.190624+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:09:13.190624+0000","last_peered":"2024-09-21T10:09:13.190624+0000","last_clean":"2024-09-21T10:09:13.190624+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:09:13.190624+0000","last_undegraded":"2024-09-21T10:09:13.190624+0000","last_fullsized":"2024-09-21T10:09:13.190624+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915104+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:09:10.915104+0000","last_peered":"2024-09-21T10:09:10.915104+0000","last_clean":"2024-09-21T10:09:10.915104+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:09:10.915104+0000","last_undegraded":"2024-09-21T10:09:10.915104+0000","last_fullsized":"2024-09-21T10:09:10.915104+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915201+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:09:10.915201+0000","last_peered":"2024-09-21T10:09:10.915201+0000","last_clean":"2024-09-21T10:09:10.915201+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:09:10.915201+0000","last_undegraded":"2024-09-21T10:09:10.915201+0000","last_fullsized":"2024-09-21T10:09:10.915201+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":78,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.916872+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:09:10.916872+0000","last_peered":"2024-09-21T10:09:10.916872+0000","last_clean":"2024-09-21T10:09:10.916872+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:09:10.916872+0000","last_undegraded":"2024-09-21T10:09:10.916872+0000","last_fullsized":"2024-09-21T10:09:10.916872+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1602,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907704+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:09:24.907704+0000","last_peered":"2024-09-21T10:09:24.907704+0000","last_clean":"2024-09-21T10:09:24.907704+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:09:24.907704+0000","last_undegraded":"2024-09-21T10:09:24.907704+0000","last_fullsized":"2024-09-21T10:09:24.907704+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1215,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896795+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:09:24.896795+0000","last_peered":"2024-09-21T10:09:24.896795+0000","last_clean":"2024-09-21T10:09:24.896795+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:09:24.896795+0000","last_undegraded":"2024-09-21T10:09:24.896795+0000","last_fullsized":"2024-09-21T10:09:24.896795+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1226,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908241+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:09:24.908241+0000","last_peered":"2024-09-21T10:09:24.908241+0000","last_clean":"2024-09-21T10:09:24.908241+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:09:24.908241+0000","last_undegraded":"2024-09-21T10:09:24.908241+0000","last_fullsized":"2024-09-21T10:09:24.908241+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":300,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.903712+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:09:24.903712+0000","last_peered":"2024-09-21T10:09:24.903712+0000","last_clean":"2024-09-21T10:09:24.903712+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:09:24.903712+0000","last_undegraded":"2024-09-21T10:09:24.903712+0000","last_fullsized":"2024-09-21T10:09:24.903712+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1628,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897647+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:09:24.897647+0000","last_peered":"2024-09-21T10:09:24.897647+0000","last_clean":"2024-09-21T10:09:24.897647+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:09:24.897647+0000","last_undegraded":"2024-09-21T10:09:24.897647+0000","last_fullsized":"2024-09-21T10:09:24.897647+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":279,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896245+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:09:24.896245+0000","last_peered":"2024-09-21T10:09:24.896245+0000","last_clean":"2024-09-21T10:09:24.896245+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:09:24.896245+0000","last_undegraded":"2024-09-21T10:09:24.896245+0000","last_fullsized":"2024-09-21T10:09:24.896245+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915474+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:09:10.915474+0000","last_peered":"2024-09-21T10:09:10.915474+0000","last_clean":"2024-09-21T10:09:10.915474+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:09:10.915474+0000","last_undegraded":"2024-09-21T10:09:10.915474+0000","last_fullsized":"2024-09-21T10:09:10.915474+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.895949+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:09:24.895949+0000","last_peered":"2024-09-21T10:09:24.895949+0000","last_clean":"2024-09-21T10:09:24.895949+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:09:24.895949+0000","last_undegraded":"2024-09-21T10:09:24.895949+0000","last_fullsized":"2024-09-21T10:09:24.895949+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1590,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.898013+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:09:24.898013+0000","last_peered":"2024-09-21T10:09:24.898013+0000","last_clean":"2024-09-21T10:09:24.898013+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:09:24.898013+0000","last_undegraded":"2024-09-21T10:09:24.898013+0000","last_fullsized":"2024-09-21T10:09:24.898013+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1357,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.903620+0000","last_change":"2024-09-21T10:08:43.397612+0000","last_active":"2024-09-21T10:09:24.903620+0000","last_peered":"2024-09-21T10:09:24.903620+0000","last_clean":"2024-09-21T10:09:24.903620+0000","last_became_active":"2024-09-21T10:08:43.397254+0000","last_became_peered":"2024-09-21T10:08:43.397254+0000","last_unstale":"2024-09-21T10:09:24.903620+0000","last_undegraded":"2024-09-21T10:09:24.903620+0000","last_fullsized":"2024-09-21T10:09:24.903620+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:40.293786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":265,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.896896+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:09:24.896896+0000","last_peered":"2024-09-21T10:09:24.896896+0000","last_clean":"2024-09-21T10:09:24.896896+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:09:24.896896+0000","last_undegraded":"2024-09-21T10:09:24.896896+0000","last_fullsized":"2024-09-21T10:09:24.896896+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":149,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.907857+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:09:24.907857+0000","last_peered":"2024-09-21T10:09:24.907857+0000","last_clean":"2024-09-21T10:09:24.907857+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:09:24.907857+0000","last_undegraded":"2024-09-21T10:09:24.907857+0000","last_fullsized":"2024-09-21T10:09:24.907857+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1485,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.910546+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:09:24.910546+0000","last_peered":"2024-09-21T10:09:24.910546+0000","last_clean":"2024-09-21T10:09:24.910546+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:09:24.910546+0000","last_undegraded":"2024-09-21T10:09:24.910546+0000","last_fullsized":"2024-09-21T10:09:24.910546+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1566,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.908587+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:09:24.908587+0000","last_peered":"2024-09-21T10:09:24.908587+0000","last_clean":"2024-09-21T10:09:24.908587+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:09:24.908587+0000","last_undegraded":"2024-09-21T10:09:24.908587+0000","last_fullsized":"2024-09-21T10:09:24.908587+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1211,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897280+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:09:24.897280+0000","last_peered":"2024-09-21T10:09:24.897280+0000","last_clean":"2024-09-21T10:09:24.897280+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:09:24.897280+0000","last_undegraded":"2024-09-21T10:09:24.897280+0000","last_fullsized":"2024-09-21T10:09:24.897280+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":164,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.910694+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:09:24.910694+0000","last_peered":"2024-09-21T10:09:24.910694+0000","last_clean":"2024-09-21T10:09:24.910694+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:09:24.910694+0000","last_undegraded":"2024-09-21T10:09:24.910694+0000","last_fullsized":"2024-09-21T10:09:24.910694+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1218,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.900439+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:09:24.900439+0000","last_peered":"2024-09-21T10:09:24.900439+0000","last_clean":"2024-09-21T10:09:24.900439+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:09:24.900439+0000","last_undegraded":"2024-09-21T10:09:24.900439+0000","last_fullsized":"2024-09-21T10:09:24.900439+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":72,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.917318+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:09:10.917318+0000","last_peered":"2024-09-21T10:09:10.917318+0000","last_clean":"2024-09-21T10:09:10.917318+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:09:10.917318+0000","last_undegraded":"2024-09-21T10:09:10.917318+0000","last_fullsized":"2024-09-21T10:09:10.917318+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":272,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.903798+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:09:24.903798+0000","last_peered":"2024-09-21T10:09:24.903798+0000","last_clean":"2024-09-21T10:09:24.903798+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:09:24.903798+0000","last_undegraded":"2024-09-21T10:09:24.903798+0000","last_fullsized":"2024-09-21T10:09:24.903798+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":280,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-09-21T10:09:24.897825+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:09:24.897825+0000","last_peered":"2024-09-21T10:09:24.897825+0000","last_clean":"2024-09-21T10:09:24.897825+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:09:24.897825+0000","last_undegraded":"2024-09-21T10:09:24.897825+0000","last_fullsized":"2024-09-21T10:09:24.897825+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.914782+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:09:10.914782+0000","last_peered":"2024-09-21T10:09:10.914782+0000","last_clean":"2024-09-21T10:09:10.914782+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:09:10.914782+0000","last_undegraded":"2024-09-21T10:09:10.914782+0000","last_fullsized":"2024-09-21T10:09:10.914782+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1587,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-09-21T10:09:10.915461+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:09:10.915461+0000","last_peered":"2024-09-21T10:09:10.915461+0000","last_clean":"2024-09-21T10:09:10.915461+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:09:10.915461+0000","last_undegraded":"2024-09-21T10:09:10.915461+0000","last_fullsized":"2024-09-21T10:09:10.915461+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6854213632,"num_objects":104587,"num_object_clones":0,"num_object_copies":313761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":104587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":104587,"num_write_kb":6693568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20433338368,"data_stored":20433338368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85887,"ondisk_log_size":85887,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":11142,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234403,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496039,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4226940,"kb_used_data":3772596,"kb_used_omap":1,"kb_used_meta":454334,"kb_avail":89518212,"statfs":{"total":95995035648,"available":91666649088,"internally_reserved":0,"allocated":3863138304,"data_stored":3860522377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":465238471},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757678,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019314,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4058300,"kb_used_data":3702320,"kb_used_omap":1,"kb_used_meta":355966,"kb_avail":89686852,"statfs":{"total":95995035648,"available":91839336448,"internally_reserved":0,"allocated":3791175680,"data_stored":3788563308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":364509638},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888610,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5194408,"kb_used_data":4589020,"kb_used_omap":34,"kb_used_meta":605341,"kb_avail":88550744,"statfs":{"total":95995035648,"available":90675961856,"internally_reserved":0,"allocated":4699156480,"data_stored":4696559177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":619870195},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,7,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967849,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4121656,"kb_used_data":3674476,"kb_used_omap":41,"kb_used_meta":447126,"kb_avail":89623496,"statfs":{"total":95995035648,"available":91774459904,"internally_reserved":0,"allocated":3762663424,"data_stored":3760055592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":457857439},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047088,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4615076,"kb_used_data":4195676,"kb_used_omap":41,"kb_used_meta":419350,"kb_avail":89130076,"statfs":{"total":95995035648,"available":91269197824,"internally_reserved":0,"allocated":4296372224,"data_stored":4293775375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":429415337},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126326,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4291428352,"data_stored":4291428352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3757637632,"data_stored":3757637632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4702339072,"data_stored":4702339072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3813343232,"data_stored":3813343232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3868590080,"data_stored":3868590080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:09:31.353 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:09:31.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set unique_pool_3 pg_num 26 2024-09-21T10:09:31.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:31.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.492 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:31.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:31.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:31.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:31.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:31 smithi076 ceph-mon[106482]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:09:31.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:31 smithi076 ceph-mon[110963]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:09:31.717 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:31.753 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 76 16 108535 108519 89.2188 66.625 0.0160386 0.0112033 2024-09-21T10:09:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.760 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:31.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:31.861 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:31.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:31 smithi191 ceph-mon[102206]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:09:31.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:31.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:31.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:31.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:32.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:32.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:32.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:32.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:32.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:32.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:32.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:32.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:32.594 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:32.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:32 smithi076 ceph-mon[106482]: pgmap v1666: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:32 smithi076 ceph-mon[110963]: pgmap v1666: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:32.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:32.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:32.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 77 16 109582 109566 88.9099 65.4375 0.0131309 0.0112425 2024-09-21T10:09:32.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:32.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:32 smithi191 ceph-mon[102206]: pgmap v1666: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-21T10:09:32.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:32.971 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:32.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:32.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:33.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:33.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:33.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:33.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:33.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:33.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.369 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:33.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:33.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:33.518 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:33.619 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.45332.0:108869 8.0 e1217/1208 8:0c76ca06:::benchmark_data_smithi076_241159_object108868:head v 1217'6846, mlcod=1217'6844)", 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.087844+0000", 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.001418189, 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014446179999999999, 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.620 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087844+0000", 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087844+0000", 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087845+0000", 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.02e-07 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087878+0000", 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3484e-05 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087890+0000", 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1853e-05 2024-09-21T10:09:33.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087897+0000", 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4169999999999997e-06 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087956+0000", 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8967e-05 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087975+0000", 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.965e-05 2024-09-21T10:09:33.623 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:108858 8.3 8:cbace272:::benchmark_data_smithi076_241159_object108857:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1217)", 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.066217+0000", 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.023045419000000001, 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.023135551000000001, 2024-09-21T10:09:33.624 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 108858 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.066217+0000", 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.066217+0000", 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.066218+0000", 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.093e-06 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.066264+0000", 2024-09-21T10:09:33.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5676000000000001e-05 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.066275+0000", 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1260000000000001e-05 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.066282+0000", 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9539999999999998e-06 2024-09-21T10:09:33.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.073213+0000", 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069308060000000003 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.073340+0000", 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012712699999999999 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.073479+0000", 2024-09-21T10:09:33.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013931499999999999 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076886+0000", 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003407424 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.077315+0000", 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00042830100000000002 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088576+0000", 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011260801000000001 2024-09-21T10:09:33.629 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:108861 8.a 8:5bf96223:::benchmark_data_smithi076_241159_object108860:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1217)", 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.071957+0000", 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.017305692000000001, 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017484077000000001, 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:33.630 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 108861 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.071957+0000", 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.071957+0000", 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.071958+0000", 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.314e-06 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.072007+0000", 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9546999999999998e-05 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.072017+0000", 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3090000000000001e-06 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.072023+0000", 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9699999999999996e-06 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.075078+0000", 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030557090000000002 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.075227+0000", 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000148696 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.075363+0000", 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013547799999999999 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.082114+0000", 2024-09-21T10:09:33.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0067518049999999996 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087539+0000", 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054243099999999999 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:108862 8.a 8:573ed5ce:::benchmark_data_smithi076_241159_object108861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1217)", 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.076417+0000", 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.01284507, 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013093110999999999, 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.636 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 108862 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076417+0000", 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076417+0000", 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076419+0000", 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3519999999999999e-06 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076468+0000", 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9073000000000003e-05 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076478+0000", 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0806e-05 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076486+0000", 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4139999999999997e-06 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.078122+0000", 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0016358740000000001 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.078236+0000", 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000113878 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.078366+0000", 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013067999999999999 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.082119+0000", 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037524640000000001 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087580+0000", 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054611140000000004 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.641 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:108864 8.1 8:87a52c81:::benchmark_data_smithi076_241159_object108863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1217)", 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.076741+0000", 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.012521042, 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012836359, 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:33.642 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 108864 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076741+0000", 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076741+0000", 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076742+0000", 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6000000000000002e-07 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076791+0000", 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9360999999999997e-05 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076801+0000", 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8879999999999992e-06 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076808+0000", 2024-09-21T10:09:33.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8680000000000001e-06 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.076973+0000", 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016446500000000001 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.077080+0000", 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010699 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.077206+0000", 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000126778 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080803+0000", 2024-09-21T10:09:33.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0035970020000000002 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088070+0000", 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072662320000000001 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:108867 8.3 8:ca6208ed:::benchmark_data_smithi076_241159_object108866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1217)", 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.080795+0000", 2024-09-21T10:09:33.648 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0084667979999999993, 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0088497960000000001, 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 108867 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080795+0000", 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080795+0000", 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080797+0000", 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.274e-06 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080856+0000", 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9589000000000002e-05 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080867+0000", 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0474e-05 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080874+0000", 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3830000000000003e-06 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.081670+0000", 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00079588800000000002 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.081870+0000", 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020002999999999999 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,6", 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.082095+0000", 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000225174 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088537+0000", 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0064418280000000001 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088549+0000", 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1515000000000001e-05 2024-09-21T10:09:33.653 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.45332.0:108866 8.d e1217/1208 8:b764b411:::benchmark_data_smithi076_241159_object108865:head v 1217'6794, mlcod=1217'6792)", 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.080703+0000", 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.008559723, 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0090083279999999995, 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:09:33.654 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080703+0000", 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080703+0000", 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080703+0000", 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1699999999999997e-07 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080788+0000", 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5079000000000003e-05 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080892+0000", 2024-09-21T10:09:33.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103284 2024-09-21T10:09:33.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080900+0000", 2024-09-21T10:09:33.658 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1960000000000008e-06 2024-09-21T10:09:33.658 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.658 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.080987+0000", 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7311999999999993e-05 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.081005+0000", 2024-09-21T10:09:33.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.736e-05 2024-09-21T10:09:33.660 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.660 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.660 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.660 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.45332.0:108870 8.d e1217/1208 8:b0fc87cc:::benchmark_data_smithi076_241159_object108869:head v 1217'6795, mlcod=1217'6792)", 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.085216+0000", 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0040465969999999999, 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045411679999999999, 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:09:33.661 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.662 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.662 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.662 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.085216+0000", 2024-09-21T10:09:33.662 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.662 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.662 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.085216+0000", 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.663 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.085217+0000", 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.071e-06 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.085254+0000", 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6891000000000003e-05 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.085266+0000", 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2682e-05 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.085274+0000", 2024-09-21T10:09:33.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8329999999999994e-06 2024-09-21T10:09:33.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.666 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088765+0000", 2024-09-21T10:09:33.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034907150000000001 2024-09-21T10:09:33.667 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.667 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.667 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.667 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088783+0000", 2024-09-21T10:09:33.667 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7858000000000001e-05 2024-09-21T10:09:33.667 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.668 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.668 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.668 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.668 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.668 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:108853 8.a 8:5e575847:::benchmark_data_smithi076_241159_object108852:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1217)", 2024-09-21T10:09:33.668 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.065227+0000", 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.024035594, 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.024578408, 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 108853 2024-09-21T10:09:33.669 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.065227+0000", 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.065227+0000", 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.065227+0000", 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7800000000000002e-07 2024-09-21T10:09:33.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.672 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.672 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.065267+0000", 2024-09-21T10:09:33.672 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0295e-05 2024-09-21T10:09:33.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.065277+0000", 2024-09-21T10:09:33.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4210000000000001e-06 2024-09-21T10:09:33.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.065282+0000", 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6879999999999998e-06 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.070143+0000", 2024-09-21T10:09:33.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0048611100000000001 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.070278+0000", 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013502699999999999 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3", 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.070418+0000", 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014060099999999999 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.081892+0000", 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011474094000000001 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.082110+0000", 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021742800000000001 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop_reply(client.45332.0:108858 8.3 e1217/1208 ondisk, result = 0)", 2024-09-21T10:09:33.676 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:09:32.087469+0000", 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.001792882, 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023968829999999998, 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087469+0000", 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:09:33.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087469+0000", 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087470+0000", 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1100000000000002e-07 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087472+0000", 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1639999999999999e-06 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087478+0000", 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0789999999999997e-06 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.087482+0000", 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5600000000000004e-06 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088566+0000", 2024-09-21T10:09:33.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010834779999999999 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:09:32.088571+0000", 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0560000000000004e-06 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 10 2024-09-21T10:09:33.680 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:09:33.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:33.682 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:09:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:09:33] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-21T10:09:33.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:33.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:33.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 78 16 110656 110640 88.6306 67.125 0.00470695 0.0112744 2024-09-21T10:09:33.807 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:33.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:33.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:33.976 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:33.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:34.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.102 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:34.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:34.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:34.243 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:34.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.366 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:34.468 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:34.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:34.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:34.638 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:34.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:34.753 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 79 16 111621 111605 88.2721 60.3125 0.00342145 0.0113226 2024-09-21T10:09:34.762 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:34.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:34 smithi191 ceph-mon[102206]: pgmap v1667: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:09:34.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:34 smithi076 ceph-mon[110963]: pgmap v1667: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:09:34.876 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:34 smithi076 ceph-mon[106482]: pgmap v1667: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:09:34.903 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:34.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:35.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:35.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:35.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:35.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:35.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:35.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:35.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:35.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:35.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:35.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:35.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:35.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:35.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:35.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:35.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:35.753 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:09:35.752617+0000 min lat: 0.00119425 max lat: 0.193843 avg lat: 0.0113579 2024-09-21T10:09:35.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:09:35.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 80 16 112528 112512 87.8773 56.6875 0.00618274 0.0113579 2024-09-21T10:09:35.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:35.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:35.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:35.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:35.981 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:35.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:35.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:36.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:36.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:36.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:36.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.233 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:36.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:36.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:36.361 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:36.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:36.494 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:36.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:36.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:36.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 81 16 113534 113518 87.5683 62.875 0.0164405 0.0114146 2024-09-21T10:09:36.775 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:36 smithi191 ceph-mon[102206]: pgmap v1668: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:09:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:36 smithi076 ceph-mon[106482]: pgmap v1668: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:09:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:36 smithi076 ceph-mon[110963]: pgmap v1668: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:09:36.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:36.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:36.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:37.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:37.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:37.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:37.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:37.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:37.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:37.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.346 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:37.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:37.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:37.485 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:37.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:37.587 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:37.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:37.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:37.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:37.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:37.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 82 16 114523 114507 87.2542 61.8125 0.0043228 0.0114559 2024-09-21T10:09:37.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:37.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:37.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:37.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:37.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:38.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:38.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:38.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:38.261 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:38.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:38.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:38.391 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:38.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:38.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:38.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:38.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 83 16 115564 115548 86.9868 65.0625 0.0139819 0.0114911 2024-09-21T10:09:38.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:38.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:38 smithi191 ceph-mon[102206]: pgmap v1669: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 54 MiB/s wr, 862 op/s 2024-09-21T10:09:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:38 smithi076 ceph-mon[106482]: pgmap v1669: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 54 MiB/s wr, 862 op/s 2024-09-21T10:09:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:38 smithi076 ceph-mon[110963]: pgmap v1669: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 54 MiB/s wr, 862 op/s 2024-09-21T10:09:38.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:38.895 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:38.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:38.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:38.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:39.022 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:39.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:39.123 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.152 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:39.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:39.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:39.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:39.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:39.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:39.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:39.548 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:39.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:39.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:39.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 84 16 116530 116514 86.67 60.375 0.00757533 0.0115332 2024-09-21T10:09:39.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:39.819 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:39.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:09:39.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:39.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:39.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:40.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:40.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:40.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:40.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:40.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:40.366 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:40.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:40.492 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:40.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:40.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:40.621 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:40.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:40.722 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:40.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 85 16 117588 117572 86.4282 66.125 0.0204246 0.0115652 2024-09-21T10:09:40.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:40.765 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:40 smithi076 ceph-mon[106482]: pgmap v1670: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:09:40.765 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:40 smithi076 ceph-mon[110963]: pgmap v1670: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:09:40.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:40 smithi191 ceph-mon[102206]: pgmap v1670: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:09:40.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:40.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:41.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:41.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:41.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:41.134 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:41.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:41.279 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:41.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.339 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:41.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:41.440 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:41.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:41.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:41.512 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:41.512 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:41.525 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:41.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:41.605 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:09:41.605 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:41.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:41.639 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:09:41.639 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:41.678 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:09:41.678 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:41.716 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:09:41.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:41.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:41.745 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:09:41.745 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:41.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 86 16 118621 118605 86.174 64.5625 0.00854577 0.0115994 2024-09-21T10:09:41.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:41.763 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:41.776 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:09:41.776 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:41.809 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:09:41.810 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:41.841 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:09:41.842 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:41.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:41.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:41.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:41.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:41.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:42.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:42.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:42.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:42.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:42.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:42.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:42.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:42.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:42.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:42.416 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:42.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:42.482 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:42 smithi191 ceph-mon[102206]: pgmap v1671: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 52 MiB/s wr, 830 op/s 2024-09-21T10:09:42.482 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/901513478' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:42.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:42.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:42.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:42.671 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:42.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 87 16 119696 119680 85.9557 67.1875 0.00688704 0.0116288 2024-09-21T10:09:42.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:42.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:42.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:42 smithi076 ceph-mon[106482]: pgmap v1671: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 52 MiB/s wr, 830 op/s 2024-09-21T10:09:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/901513478' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:42 smithi076 ceph-mon[110963]: pgmap v1671: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 52 MiB/s wr, 830 op/s 2024-09-21T10:09:42.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/901513478' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:42.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:42.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:42.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:43.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:43.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:43.172 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:43.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:43.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.352 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:43.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:43.436 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:09:43.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:43.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:43.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:43.519 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3398555118' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:43.519 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:43 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:43.520 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:09:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:09:43] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:09:43.520 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3398555118' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:43.520 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:43 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:43.599 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:43.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:43.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:43.754 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 88 16 120751 120735 85.7281 65.9375 0.0118514 0.0116594 2024-09-21T10:09:43.766 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3398555118' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:43 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:09:43.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:43.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:43.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:43.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:44.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:44.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:44.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:44.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:44.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:44.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:44.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:44.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:44.460 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:44.562 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:44.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:44.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.745 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:44.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:44.755 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 89 16 121842 121826 85.5309 68.1875 0.00994964 0.0116862 2024-09-21T10:09:44.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:44 smithi191 ceph-mon[102206]: pgmap v1672: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-21T10:09:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:44 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:44 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:09:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:44 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:44 smithi191 ceph-mon[102206]: osdmap e1218: 8 total, 8 up, 5 in 2024-09-21T10:09:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[106482]: pgmap v1672: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-21T10:09:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:09:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[106482]: osdmap e1218: 8 total, 8 up, 5 in 2024-09-21T10:09:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[110963]: pgmap v1672: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-21T10:09:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:09:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:09:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:09:44.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:44 smithi076 ceph-mon[110963]: osdmap e1218: 8 total, 8 up, 5 in 2024-09-21T10:09:44.884 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:44.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:44.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:44.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:45.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:45.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:45.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:45.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:45.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:45.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:45.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:45.405 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:45.423 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:45.522 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:09:45.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:45.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:45.650 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:45.673 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:45.755 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total time run: 90.0082 2024-09-21T10:09:45.755 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total writes made: 122915 2024-09-21T10:09:45.755 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Write size: 65536 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object size: 65536 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Bandwidth (MB/sec): 85.3498 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Bandwidth: 32.5069 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max bandwidth (MB/sec): 217.062 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min bandwidth (MB/sec): 48.875 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average IOPS: 1365 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev IOPS: 520.111 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max IOPS: 3473 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min IOPS: 782 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average Latency(s): 0.0117122 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Latency(s): 0.00917912 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max latency(s): 0.193843 2024-09-21T10:09:45.756 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min latency(s): 0.00119425 2024-09-21T10:09:45.761 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:09:45.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:45.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:45.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:45.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:45 smithi191 ceph-mon[102206]: pgmap v1674: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:09:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:45 smithi076 ceph-mon[110963]: pgmap v1674: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:09:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:45 smithi076 ceph-mon[106482]: pgmap v1674: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:09:45.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:45.961 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:45.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:46.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:46.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:46.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:46.357 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.394 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:46.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:46.537 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:46.639 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.693 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:46.795 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:46.830 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:46.932 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:46.970 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:46.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:46.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:47.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:47.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:47.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.134 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:09:47.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:47.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:47.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:47.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:47.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:47.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:47.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.517 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:47.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:47.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:47.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:47.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:47.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:47.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:47.929 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:47.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:48.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:48.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:48.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:48.094 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:47 smithi076 ceph-mon[106482]: pgmap v1675: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:09:48.094 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:47 smithi076 ceph-mon[110963]: pgmap v1675: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:09:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:47 smithi191 ceph-mon[102206]: pgmap v1675: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:09:48.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:48.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:48.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:48.329 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:48.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:48.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:48.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:48.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:48.581 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:48.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:48.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:48.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:48.849 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:48.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:48.988 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:48.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:49.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:49.115 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:49.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:49.239 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:49.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:49.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:49.365 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:09:49.365 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:09:49.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:49.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:49.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:49.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:49.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:49.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:49.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:49.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:49.920 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:49.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:50.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:50.043 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:49 smithi191 ceph-mon[102206]: pgmap v1676: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 102 B/s rd, 52 MiB/s wr, 1.07k op/s 2024-09-21T10:09:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:49 smithi076 ceph-mon[106482]: pgmap v1676: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 102 B/s rd, 52 MiB/s wr, 1.07k op/s 2024-09-21T10:09:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:49 smithi076 ceph-mon[110963]: pgmap v1676: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 102 B/s rd, 52 MiB/s wr, 1.07k op/s 2024-09-21T10:09:50.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:50.169 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:50.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:50.313 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:50.328 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:50.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:09:50.375 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:50.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:50.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:50.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:50.591 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:50.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:50.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:50.716 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:50.801 INFO:teuthology.orchestra.run.smithi076.stderr:set pool 8 pg_num to 26 2024-09-21T10:09:50.818 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:50.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:50.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:50.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:50.933 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/164311301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:09:50.933 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:50 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:09:50.933 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/164311301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:09:50.933 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:50 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:09:50.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:51.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:51.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/164311301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:09:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:50 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"} : dispatch 2024-09-21T10:09:51.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:51.142 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:51.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:51.244 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.278 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:51.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:51.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.417 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:51.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:51.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:51.553 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:51.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:51.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:51.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:51.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:51.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:51.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:51.913 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:51.940 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:51.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:52.041 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:52.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:52.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:51 smithi191 ceph-mon[102206]: pgmap v1677: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 102 B/s rd, 52 MiB/s wr, 1.07k op/s 2024-09-21T10:09:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:09:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:51 smithi191 ceph-mon[102206]: osdmap e1219: 8 total, 8 up, 5 in 2024-09-21T10:09:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:51 smithi076 ceph-mon[106482]: pgmap v1677: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 102 B/s rd, 52 MiB/s wr, 1.07k op/s 2024-09-21T10:09:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:09:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:51 smithi076 ceph-mon[106482]: osdmap e1219: 8 total, 8 up, 5 in 2024-09-21T10:09:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:51 smithi076 ceph-mon[110963]: pgmap v1677: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 102 B/s rd, 52 MiB/s wr, 1.07k op/s 2024-09-21T10:09:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2024-09-21T10:09:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:51 smithi076 ceph-mon[110963]: osdmap e1219: 8 total, 8 up, 5 in 2024-09-21T10:09:52.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:52.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:52.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:52.332 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:52.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:52.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:52.471 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:52.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:52.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:52.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:52.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:52.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:52.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:52.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:53.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:53.080 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:53.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:53.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:53.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:53.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.343 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:53.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:53.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:53.487 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:53.501 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:09:53] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:09:53.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:09:53.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:53.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:53.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:09:53.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:53.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:53.503 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.590 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:53.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-21T10:09:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:09:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:09:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:53.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:53.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:53.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:53.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:53.860 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:53.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:53.962 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:53.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:53.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:53.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:54.085 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:54.106 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:54.161 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:54.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:54.229 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:54.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:54.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:54.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:54.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:54.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:54.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:54.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:54 smithi191 ceph-mon[102206]: pgmap v1679: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 110 B/s rd, 21 MiB/s wr, 814 op/s 2024-09-21T10:09:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:54 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:09:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:54 smithi191 ceph-mon[102206]: osdmap e1220: 8 total, 8 up, 5 in 2024-09-21T10:09:54.621 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:54.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:54 smithi076 ceph-mon[106482]: pgmap v1679: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 110 B/s rd, 21 MiB/s wr, 814 op/s 2024-09-21T10:09:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:54 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:09:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:54 smithi076 ceph-mon[106482]: osdmap e1220: 8 total, 8 up, 5 in 2024-09-21T10:09:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:54 smithi076 ceph-mon[110963]: pgmap v1679: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 110 B/s rd, 21 MiB/s wr, 814 op/s 2024-09-21T10:09:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:54 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-21T10:09:54.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:54 smithi076 ceph-mon[110963]: osdmap e1220: 8 total, 8 up, 5 in 2024-09-21T10:09:54.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:54.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:54.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:54.874 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:54.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:54.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:55.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:55.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:55.122 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:55.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:55.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:55.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:55.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:55.385 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:55.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:55.526 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:55.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:55.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:55.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:55 smithi191 ceph-mon[102206]: osdmap e1221: 8 total, 8 up, 5 in 2024-09-21T10:09:55.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:55 smithi076 ceph-mon[106482]: osdmap e1221: 8 total, 8 up, 5 in 2024-09-21T10:09:55.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:55 smithi076 ceph-mon[110963]: osdmap e1221: 8 total, 8 up, 5 in 2024-09-21T10:09:55.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.709 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:55.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:55.810 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:55.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:55.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:55.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:56.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:56.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:56.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:56.129 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:56.173 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:09:56.174 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:09:56.175 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:09:56.175 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:09:56.175 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:09:56.175 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:09:56.181 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:09:56.181 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:09:56.216 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:09:56.216 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:09:56.231 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:56.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:56.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:56.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:56.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:56.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:56.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:56.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:56 smithi191 ceph-mon[102206]: pgmap v1682: 99 pgs: 10 unknown, 89 active+clean; 7.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 928 op/s 2024-09-21T10:09:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:56 smithi191 ceph-mon[102206]: 8.1 scrub starts 2024-09-21T10:09:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3122935193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[106482]: pgmap v1682: 99 pgs: 10 unknown, 89 active+clean; 7.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 928 op/s 2024-09-21T10:09:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[106482]: 8.1 scrub starts 2024-09-21T10:09:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3122935193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[110963]: pgmap v1682: 99 pgs: 10 unknown, 89 active+clean; 7.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 928 op/s 2024-09-21T10:09:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[110963]: 8.1 scrub starts 2024-09-21T10:09:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:09:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:09:56.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3122935193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:09:56.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:56.671 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:56.681 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 5, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:09:56.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:09:56.773 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:56.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:56.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:56.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:56.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:57.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:57.095 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:57.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:57.197 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:09:57.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:09:57.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:57.331 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:09:57.369 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:57.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:57.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:57.600 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:57.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:57 smithi191 ceph-mon[102206]: 8.5 scrub starts 2024-09-21T10:09:57.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:57 smithi191 ceph-mon[102206]: 8.1 scrub ok 2024-09-21T10:09:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:57 smithi191 ceph-mon[102206]: 8.0 scrub starts 2024-09-21T10:09:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:57 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/987757541' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:09:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:57 smithi191 ceph-mon[102206]: 8.5 scrub ok 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[106482]: 8.5 scrub starts 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[106482]: 8.1 scrub ok 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[106482]: 8.0 scrub starts 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/987757541' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[106482]: 8.5 scrub ok 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[110963]: 8.5 scrub starts 2024-09-21T10:09:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[110963]: 8.1 scrub ok 2024-09-21T10:09:57.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[110963]: 8.0 scrub starts 2024-09-21T10:09:57.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/987757541' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:09:57.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:57 smithi076 ceph-mon[110963]: 8.5 scrub ok 2024-09-21T10:09:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:57.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:57.741 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:57.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:57.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:57.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:57.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:57.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:57.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.015 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:09:58.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:09:58.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.302 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:58.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:58.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:09:58.428 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:58.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:09:58.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:58.555 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:58 smithi191 ceph-mon[102206]: pgmap v1683: 99 pgs: 10 unknown, 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 928 op/s 2024-09-21T10:09:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:58 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/987757541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:09:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:58 smithi191 ceph-mon[102206]: osdmap e1222: 8 total, 8 up, 5 in 2024-09-21T10:09:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[106482]: pgmap v1683: 99 pgs: 10 unknown, 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 928 op/s 2024-09-21T10:09:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/987757541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:09:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[106482]: osdmap e1222: 8 total, 8 up, 5 in 2024-09-21T10:09:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[110963]: pgmap v1683: 99 pgs: 10 unknown, 89 active+clean; 7.0 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 928 op/s 2024-09-21T10:09:58.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:09:58.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/987757541' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:09:58.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:58 smithi076 ceph-mon[110963]: osdmap e1222: 8 total, 8 up, 5 in 2024-09-21T10:09:58.657 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:58.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:58.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:58.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:09:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.819 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:58.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:09:58.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:58.956 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:58.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:59.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:59.099 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:59.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:59.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:09:59.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:09:59.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:59.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:09:59.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:59.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:59.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:09:59.516 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:09:59.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:09:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:09:59.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:59.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:09:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:09:59.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:09:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-21T10:09:59.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:59.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:09:59.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:09:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:09:59.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:09:59.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:09:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:09:59.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:09:59.950 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:09:59.995 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.019 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:00.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:00.089 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:00.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:00.142 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:00.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:10:00.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:00.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:00.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:00.367 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:00.388 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:00.491 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.513 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:00.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[106482]: pgmap v1685: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[106482]: osdmap e1223: 8 total, 8 up, 5 in 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[106482]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[106482]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[110963]: pgmap v1685: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:00.563 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:10:00.564 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[110963]: osdmap e1223: 8 total, 8 up, 5 in 2024-09-21T10:10:00.564 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[110963]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-21T10:10:00.564 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:00 smithi076 ceph-mon[110963]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:10:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:00 smithi191 ceph-mon[102206]: pgmap v1685: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:00 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-21T10:10:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:00 smithi191 ceph-mon[102206]: osdmap e1223: 8 total, 8 up, 5 in 2024-09-21T10:10:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:00 smithi191 ceph-mon[102206]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-21T10:10:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:00 smithi191 ceph-mon[102206]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-21T10:10:00.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:00.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:00.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:00.760 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:00.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:00.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:00.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:00.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:00.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:01.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:01.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.131 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:01.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:01.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:01.255 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:01.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:01.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:01.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:01.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:01 smithi191 ceph-mon[102206]: osdmap e1224: 8 total, 8 up, 5 in 2024-09-21T10:10:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:10:01.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:01.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:01 smithi076 ceph-mon[106482]: osdmap e1224: 8 total, 8 up, 5 in 2024-09-21T10:10:01.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:10:01.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:01 smithi076 ceph-mon[110963]: osdmap e1224: 8 total, 8 up, 5 in 2024-09-21T10:10:01.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-21T10:10:01.644 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:01.746 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:01.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:01.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:01.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:01.895 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:01.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:01.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:02.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:02.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:02.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:02.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:02.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:02.266 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:02.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:02.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:02.392 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:02.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:02.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:02.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[106482]: pgmap v1688: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[106482]: osdmap e1225: 8 total, 8 up, 5 in 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[106482]: 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 1222'8435 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[110963]: pgmap v1688: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[110963]: osdmap e1225: 8 total, 8 up, 5 in 2024-09-21T10:10:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:02 smithi076 ceph-mon[110963]: 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 1222'8435 2024-09-21T10:10:02.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:02.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:02.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:02.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:02.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:02 smithi191 ceph-mon[102206]: pgmap v1688: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:10:02.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-21T10:10:02.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:02 smithi191 ceph-mon[102206]: osdmap e1225: 8 total, 8 up, 5 in 2024-09-21T10:10:02.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:02 smithi191 ceph-mon[102206]: 8.10 starting backfill to osd.6 from (0'0,0'0] MAX to 1222'8435 2024-09-21T10:10:02.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:02.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:02.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:03.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:03.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:03.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:03.130 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:03.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:03.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:03.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:03.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:03.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:03.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:03.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:03.480 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:03.513 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:03.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:03.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:03 smithi076 ceph-mon[110963]: osdmap e1226: 8 total, 8 up, 5 in 2024-09-21T10:10:03.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:03 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:10:03.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:03 smithi076 ceph-mon[106482]: osdmap e1226: 8 total, 8 up, 5 in 2024-09-21T10:10:03.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:03 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:10:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:10:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:10:03] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:10:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.637 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:03.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:03.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:03.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:03 smithi191 ceph-mon[102206]: osdmap e1226: 8 total, 8 up, 5 in 2024-09-21T10:10:03.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:03 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-21T10:10:03.870 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:03.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:03.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:03.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:03.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:04.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:04.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:04.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.142 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:04.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:04.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:04.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:04.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:04.388 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:04.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:04.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:04.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:04.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:04.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:04.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:04.759 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:04.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:04.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:04.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:04 smithi191 ceph-mon[102206]: pgmap v1691: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 301 op/s 2024-09-21T10:10:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:04 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:10:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:04 smithi191 ceph-mon[102206]: osdmap e1227: 8 total, 8 up, 5 in 2024-09-21T10:10:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:04 smithi191 ceph-mon[102206]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 1223'8644 2024-09-21T10:10:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:04 smithi191 ceph-mon[102206]: osdmap e1228: 8 total, 8 up, 5 in 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[106482]: pgmap v1691: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 301 op/s 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[106482]: osdmap e1227: 8 total, 8 up, 5 in 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[106482]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 1223'8644 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[106482]: osdmap e1228: 8 total, 8 up, 5 in 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[110963]: pgmap v1691: 99 pgs: 99 active+clean; 6.6 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 301 op/s 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[110963]: osdmap e1227: 8 total, 8 up, 5 in 2024-09-21T10:10:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[110963]: 8.11 starting backfill to osd.6 from (0'0,0'0] MAX to 1223'8644 2024-09-21T10:10:04.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:04 smithi076 ceph-mon[110963]: osdmap e1228: 8 total, 8 up, 5 in 2024-09-21T10:10:04.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:04.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:04.985 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:05.006 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:05.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:05.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:05.233 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:05.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:05.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:05.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:05.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:05.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:05.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:05.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:05.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:05.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:05.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:05.795 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:05.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:05.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:05.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:05.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:05.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:06.024 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:06 smithi076 ceph-mon[106482]: pgmap v1694: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 6.5 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 796 op/s; 2706/320751 objects misplaced (0.844%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:10:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:06 smithi076 ceph-mon[106482]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 1225'8440 2024-09-21T10:10:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:06 smithi076 ceph-mon[106482]: osdmap e1229: 8 total, 8 up, 5 in 2024-09-21T10:10:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:06 smithi076 ceph-mon[110963]: pgmap v1694: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 6.5 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 796 op/s; 2706/320751 objects misplaced (0.844%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:10:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:06 smithi076 ceph-mon[110963]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 1225'8440 2024-09-21T10:10:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:06 smithi076 ceph-mon[110963]: osdmap e1229: 8 total, 8 up, 5 in 2024-09-21T10:10:06.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:06.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:06.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:06.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:06 smithi191 ceph-mon[102206]: pgmap v1694: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 6.5 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 796 op/s; 2706/320751 objects misplaced (0.844%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:10:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:06 smithi191 ceph-mon[102206]: 8.12 starting backfill to osd.2 from (0'0,0'0] MAX to 1225'8440 2024-09-21T10:10:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:06 smithi191 ceph-mon[102206]: osdmap e1229: 8 total, 8 up, 5 in 2024-09-21T10:10:06.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:06.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:06.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:06.581 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:06.671 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:10:06.671 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:10:06.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:10:06.671 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:10:06.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138027 8.10 8:09da8034:::benchmark_data_smithi076_241159_object15109:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:00.354468+0000", 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 5.4205712620000002, 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0546366890000001, 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138027 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.672 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.354468+0000", 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.354468+0000", 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.354469+0000", 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.384e-06 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.354475+0000", 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0070000000000003e-06 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400462+0000", 2024-09-21T10:10:06.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.045986786000000002 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400475+0000", 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3269e-05 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400793+0000", 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031717799999999999 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400803+0000", 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0373e-05 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.405372+0000", 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0045686439999999 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.405507+0000", 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013548100000000001 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4,6", 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.405668+0000", 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016085000000000001 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.408836+0000", 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031676870000000002 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.408899+0000", 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3938999999999999e-05 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.408922+0000", 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2447999999999999e-05 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.408965+0000", 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2867999999999997e-05 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.409001+0000", 2024-09-21T10:10:06.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6708000000000002e-05 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.409105+0000", 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010306699999999999 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138321 8.10 8:0fa89c24:::benchmark_data_smithi076_241159_object15403:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:00.400858+0000", 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 5.3741807030000004, 2024-09-21T10:10:06.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0378416150000001, 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138321 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400858+0000", 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.680 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400858+0000", 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400859+0000", 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3599999999999999e-07 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400861+0000", 2024-09-21T10:10:06.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1270000000000001e-06 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400867+0000", 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2879999999999998e-06 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400872+0000", 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4920000000000004e-06 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.408808+0000", 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079362679999999998 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.408815+0000", 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.0160000000000003e-06 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.431244+0000", 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0224286680000001 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.431364+0000", 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000120752 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4,6", 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.431609+0000", 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024495799999999998 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.435859+0000", 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042490929999999998 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.437787+0000", 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019288580000000001 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.437965+0000", 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017783299999999999 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438510+0000", 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00054522800000000003 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438596+0000", 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5284999999999994e-05 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438700+0000", 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000104313 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.686 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138378 8.10 8:0ad5bad6:::benchmark_data_smithi076_241159_object15460:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:00.400907+0000", 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 5.3741323410000001, 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.041311621, 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138378 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400907+0000", 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400907+0000", 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400907+0000", 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2300000000000001e-07 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400909+0000", 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1100000000000001e-06 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400924+0000", 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.429e-05 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400930+0000", 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1360000000000002e-06 2024-09-21T10:10:06.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.408838+0000", 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0079081220000000001 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.408843+0000", 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.169e-06 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.690 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.434371+0000", 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0255281489999999 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.434488+0000", 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116762 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4,6", 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.434742+0000", 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025364099999999998 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438498+0000", 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037560789999999998 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438750+0000", 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025235200000000001 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438792+0000", 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1582000000000003e-05 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.692 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.442064+0000", 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003272545 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.442114+0000", 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9749000000000001e-05 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.442218+0000", 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000104412 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.693 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138402 8.10 8:09a89a4e:::benchmark_data_smithi076_241159_object15484:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1224)", 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:00.400955+0000", 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 5.3740843900000002, 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.050597537, 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.694 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138402 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400955+0000", 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400955+0000", 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400956+0000", 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4799999999999997e-07 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400959+0000", 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7699999999999999e-06 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400969+0000", 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0063e-05 2024-09-21T10:10:06.696 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.400977+0000", 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4789999999999999e-06 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.408872+0000", 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0078953259999999994 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:00.408878+0000", 2024-09-21T10:10:06.697 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5029999999999997e-06 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438826+0000", 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0299486 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.438949+0000", 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000122376 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4,6", 2024-09-21T10:10:06.698 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.439204+0000", 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025576599999999999 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.442229+0000", 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030241399999999998 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.451057+0000", 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0088285910000000002 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.699 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.451341+0000", 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000283341 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.451380+0000", 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9733000000000001e-05 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.451438+0000", 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.808e-05 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.700 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:01.451552+0000", 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000113921 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138404 8.11 8:8f12ce72:::benchmark_data_smithi076_241159_object15486:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367447+0000", 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4075918019999998, 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.070326288, 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.701 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138404 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367447+0000", 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.702 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367447+0000", 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367449+0000", 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.322e-06 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367454+0000", 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0989999999999998e-06 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.703 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367464+0000", 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.039e-05 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367470+0000", 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.519e-06 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367508+0000", 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8772000000000001e-05 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.704 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op must wait for map", 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367513+0000", 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2810000000000002e-06 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369222+0000", 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017089480000000001 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369231+0000", 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6420000000000002e-06 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.705 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.434005+0000", 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0647735540000001 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.434085+0000", 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0214000000000001e-05 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.434250+0000", 2024-09-21T10:10:06.706 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016467900000000001 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437503+0000", 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0032529439999999998 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437515+0000", 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2115e-05 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437548+0000", 2024-09-21T10:10:06.707 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3518e-05 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437596+0000", 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7947e-05 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437656+0000", 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9534000000000003e-05 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.708 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437774+0000", 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011781 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138476 8.11 8:8d7d3a59:::benchmark_data_smithi076_241159_object15558:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367483+0000", 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4075559019999999, 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0706738849999999, 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.709 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138476 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367483+0000", 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367483+0000", 2024-09-21T10:10:06.710 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367484+0000", 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5000000000000003e-07 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367486+0000", 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0980000000000001e-06 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367492+0000", 2024-09-21T10:10:06.711 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4359999999999999e-06 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367498+0000", 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2379999999999997e-06 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367569+0000", 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1886999999999998e-05 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.712 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367575+0000", 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5899999999999998e-06 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369747+0000", 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021723039999999999 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369755+0000", 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3540000000000002e-06 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.713 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.435415+0000", 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.065660638 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.435508+0000", 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2701000000000006e-05 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.435684+0000", 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017613199999999999 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.714 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437837+0000", 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021523900000000001 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437855+0000", 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8278999999999999e-05 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437902+0000", 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7085000000000001e-05 2024-09-21T10:10:06.715 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438004+0000", 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102277 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438063+0000", 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8780000000000003e-05 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438157+0000", 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4146000000000003e-05 2024-09-21T10:10:06.716 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138484 8.11 8:8dcd8554:::benchmark_data_smithi076_241159_object15566:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367507+0000", 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4075326189999999, 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0708633400000001, 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.717 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138484 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367507+0000", 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367507+0000", 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.718 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367507+0000", 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5499999999999999e-07 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367509+0000", 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9800000000000001e-06 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367515+0000", 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9529999999999996e-06 2024-09-21T10:10:06.719 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367521+0000", 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7060000000000001e-06 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367630+0000", 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010997699999999999 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367637+0000", 2024-09-21T10:10:06.720 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3729999999999998e-06 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369779+0000", 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021425350000000001 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369785+0000", 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0599999999999996e-06 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.721 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.436458+0000", 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0666729020000001 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.436526+0000", 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7990000000000005e-05 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.436670+0000", 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014330899999999999 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.722 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437965+0000", 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012957699999999999 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.437974+0000", 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1869999999999997e-06 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438193+0000", 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000219175 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.723 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438237+0000", 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3872000000000003e-05 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438278+0000", 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1808999999999999e-05 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438370+0000", 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1386999999999998e-05 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.724 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138509 8.11 8:881a54c6:::benchmark_data_smithi076_241159_object15591:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367527+0000", 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4075118830000002, 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.075125697, 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.725 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138509 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367527+0000", 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367527+0000", 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.726 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367528+0000", 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.79e-07 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367529+0000", 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.663e-06 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367533+0000", 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4460000000000001e-06 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.727 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367536+0000", 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4699999999999998e-06 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367673+0000", 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000136902 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367678+0000", 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0030000000000002e-06 2024-09-21T10:10:06.728 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369809+0000", 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021304900000000001 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369814+0000", 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4360000000000001e-06 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438427+0000", 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.068613042 2024-09-21T10:10:06.729 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438543+0000", 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011609599999999999 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.438697+0000", 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015382899999999999 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442321+0000", 2024-09-21T10:10:06.730 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003623589 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442331+0000", 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0084e-05 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442377+0000", 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5998000000000002e-05 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.731 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442422+0000", 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5235999999999999e-05 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442515+0000", 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2800000000000006e-05 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442653+0000", 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013833399999999999 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.732 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138514 8.11 8:8f68f197:::benchmark_data_smithi076_241159_object15596:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367560+0000", 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4074788489999999, 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0765755889999999, 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138514 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.733 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367560+0000", 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367560+0000", 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367561+0000", 2024-09-21T10:10:06.734 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3700000000000001e-07 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367563+0000", 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9829999999999999e-06 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367569+0000", 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4350000000000002e-06 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367574+0000", 2024-09-21T10:10:06.735 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3190000000000002e-06 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367705+0000", 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130247 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367707+0000", 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7819999999999998e-06 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.736 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369838+0000", 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021303699999999999 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369844+0000", 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1079999999999998e-06 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.439738+0000", 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0698935940000001 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.737 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.439847+0000", 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109913 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.440028+0000", 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018003499999999999 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442668+0000", 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002640828 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.738 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442713+0000", 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4651000000000003e-05 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442764+0000", 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0549e-05 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.443977+0000", 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001213668 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.739 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.444030+0000", 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3026999999999999e-05 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.444136+0000", 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105643 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.740 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138523 8.11 8:8a6ae6be:::benchmark_data_smithi076_241159_object15605:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367582+0000", 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4074575060000001, 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0784181450000001, 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138523 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.741 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367582+0000", 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367582+0000", 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367582+0000", 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9900000000000001e-07 2024-09-21T10:10:06.742 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367584+0000", 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.891e-06 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367590+0000", 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6359999999999999e-06 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367594+0000", 2024-09-21T10:10:06.743 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6469999999999997e-06 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367737+0000", 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000143153 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367744+0000", 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2829999999999998e-06 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.744 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369870+0000", 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021265949999999998 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369876+0000", 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9220000000000002e-06 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.441048+0000", 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.071171761 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.745 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.441164+0000", 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011600899999999999 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.441345+0000", 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000181385 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442726+0000", 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001381181 2024-09-21T10:10:06.746 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442806+0000", 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9951999999999996e-05 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442856+0000", 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9299999999999999e-05 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.445827+0000", 2024-09-21T10:10:06.747 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002970837 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.445884+0000", 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7574999999999997e-05 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.446000+0000", 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011561900000000001 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.748 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138559 8.11 8:88f4460a:::benchmark_data_smithi076_241159_object15641:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367615+0000", 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4074239240000002, 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0813958539999999, 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138559 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.749 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367615+0000", 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367615+0000", 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367616+0000", 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3700000000000001e-07 2024-09-21T10:10:06.750 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367617+0000", 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5209999999999999e-06 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367623+0000", 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.404e-06 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367626+0000", 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9579999999999999e-06 2024-09-21T10:10:06.751 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367771+0000", 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000144683 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367774+0000", 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6029999999999999e-06 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369901+0000", 2024-09-21T10:10:06.752 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002127164 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369907+0000", 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5100000000000004e-06 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442876+0000", 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.072968973 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.753 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.442966+0000", 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9936000000000001e-05 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.443120+0000", 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015374 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.445760+0000", 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026400019999999998 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.754 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.445774+0000", 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.401e-05 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.446085+0000", 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031084800000000002 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.448872+0000", 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027871380000000002 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.755 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.448908+0000", 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5831999999999998e-05 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449011+0000", 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103195 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.756 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138567 8.11 8:891e27f4:::benchmark_data_smithi076_241159_object15649:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367630+0000", 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4074090460000002, 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.081660276, 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138567 2024-09-21T10:10:06.757 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367630+0000", 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367630+0000", 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367630+0000", 2024-09-21T10:10:06.758 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.34e-07 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367631+0000", 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5499999999999996e-07 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367635+0000", 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5599999999999998e-06 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.759 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367637+0000", 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3910000000000001e-06 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367805+0000", 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000167443 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367809+0000", 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7260000000000002e-06 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.760 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369945+0000", 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021352440000000001 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369951+0000", 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8290000000000002e-06 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.444188+0000", 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.074236057 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.761 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.444300+0000", 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112847 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.444483+0000", 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018251399999999999 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.446044+0000", 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015615819999999999 2024-09-21T10:10:06.762 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449072+0000", 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030273570000000001 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449120+0000", 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8403999999999999e-05 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449150+0000", 2024-09-21T10:10:06.763 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9343e-05 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449180+0000", 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0790000000000002e-05 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449290+0000", 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.764 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138594 8.11 8:89f86662:::benchmark_data_smithi076_241159_object15676:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367642+0000", 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4073976039999998, 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0867959519999999, 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138594 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.765 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367642+0000", 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367642+0000", 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367642+0000", 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1899999999999998e-07 2024-09-21T10:10:06.766 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367643+0000", 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5519999999999999e-06 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367648+0000", 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9710000000000003e-06 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367652+0000", 2024-09-21T10:10:06.767 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.109e-06 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367839+0000", 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000187002 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367842+0000", 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5440000000000001e-06 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369975+0000", 2024-09-21T10:10:06.768 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021330139999999999 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.369980+0000", 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4299999999999997e-06 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.446114+0000", 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.07613337 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.769 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.446217+0000", 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103643 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.446388+0000", 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017096400000000001 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449106+0000", 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027171869999999998 2024-09-21T10:10:06.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449339+0000", 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023373799999999999 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454185+0000", 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0048453029999999996 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454296+0000", 2024-09-21T10:10:06.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011178799999999999 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454347+0000", 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0736999999999999e-05 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454437+0000", 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0280999999999998e-05 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138668 8.11 8:89025746:::benchmark_data_smithi076_241159_object15750:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367667+0000", 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4073720839999999, 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.086931834, 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138668 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.773 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367667+0000", 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367667+0000", 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367668+0000", 2024-09-21T10:10:06.774 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8699999999999995e-07 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367670+0000", 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9999999999999999e-06 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367675+0000", 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8629999999999996e-06 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367680+0000", 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.729e-06 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367872+0000", 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000191626 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367877+0000", 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9080000000000003e-06 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.776 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370004+0000", 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021270350000000002 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370009+0000", 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4110000000000004e-06 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449368+0000", 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.079358934 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.777 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449481+0000", 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000113446 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.449695+0000", 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021346700000000001 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454224+0000", 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045287419999999997 2024-09-21T10:10:06.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454280+0000", 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6737000000000002e-05 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454473+0000", 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0001929 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454484+0000", 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0691e-05 2024-09-21T10:10:06.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454513+0000", 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9108e-05 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454599+0000", 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5749999999999997e-05 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138679 8.11 8:8ad1348a:::benchmark_data_smithi076_241159_object15761:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367703+0000", 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4073361659999999, 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0952838540000001, 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138679 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.781 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367703+0000", 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367703+0000", 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367703+0000", 2024-09-21T10:10:06.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1899999999999998e-07 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367705+0000", 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6869999999999999e-06 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367709+0000", 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.515e-06 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367715+0000", 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3070000000000002e-06 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367903+0000", 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000188619 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367906+0000", 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4229999999999999e-06 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.784 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370035+0000", 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021296129999999998 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370042+0000", 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.268e-06 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.452653+0000", 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.082611048 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.452773+0000", 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000120037 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.453008+0000", 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000235116 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.457901+0000", 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0048934900000000003 2024-09-21T10:10:06.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.459165+0000", 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001264023 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.459221+0000", 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5655999999999997e-05 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462799+0000", 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0035777500000000002 2024-09-21T10:10:06.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462866+0000", 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6684000000000001e-05 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462987+0000", 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012129900000000001 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138708 8.11 8:89e9e5d4:::benchmark_data_smithi076_241159_object15790:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367731+0000", 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.407307861, 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.095463662, 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138708 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.789 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367731+0000", 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367731+0000", 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367732+0000", 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9099999999999999e-07 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367738+0000", 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0190000000000003e-06 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367744+0000", 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4860000000000002e-06 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367749+0000", 2024-09-21T10:10:06.791 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6589999999999998e-06 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367960+0000", 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000210669 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367962+0000", 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6740000000000001e-06 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370069+0000", 2024-09-21T10:10:06.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002106588 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370075+0000", 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5590000000000002e-06 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454634+0000", 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0845589019999999 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454750+0000", 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116068 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.454960+0000", 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020945299999999999 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.459263+0000", 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043034099999999997 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462155+0000", 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028919449999999999 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462994+0000", 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00083885399999999999 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463048+0000", 2024-09-21T10:10:06.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4444000000000003e-05 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463098+0000", 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9688000000000003e-05 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463195+0000", 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6853000000000003e-05 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.796 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138740 8.11 8:8cd59263:::benchmark_data_smithi076_241159_object15822:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367769+0000", 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4072697079999998, 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.097680142, 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138740 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367769+0000", 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367769+0000", 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367770+0000", 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.53e-07 2024-09-21T10:10:06.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367771+0000", 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.654e-06 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367776+0000", 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.685e-06 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367781+0000", 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.22e-06 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367988+0000", 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020710300000000001 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367991+0000", 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8739999999999999e-06 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370101+0000", 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021100419999999999 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370108+0000", 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7179999999999999e-06 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.456655+0000", 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.086546513 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.801 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.456758+0000", 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000103775 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.456934+0000", 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000175978 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462196+0000", 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052618229999999997 2024-09-21T10:10:06.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462999+0000", 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00080320999999999995 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463232+0000", 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023258400000000001 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465287+0000", 2024-09-21T10:10:06.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020548070000000001 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465341+0000", 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4351999999999997e-05 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465450+0000", 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000108451 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.804 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138747 8.11 8:8d6ea6e5:::benchmark_data_smithi076_241159_object15829:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367799+0000", 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4072403979999999, 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0978441480000001, 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138747 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.805 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367799+0000", 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367799+0000", 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367799+0000", 2024-09-21T10:10:06.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8099999999999998e-07 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367801+0000", 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3499999999999999e-06 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367807+0000", 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.998e-06 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367812+0000", 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7260000000000002e-06 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368015+0000", 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020280100000000001 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368017+0000", 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4889999999999998e-06 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370134+0000", 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021164640000000002 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370140+0000", 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0299999999999999e-06 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.457937+0000", 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0877973809999999 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.809 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.458037+0000", 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9908000000000006e-05 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.458190+0000", 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015302 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462747+0000", 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045566119999999998 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463004+0000", 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025758600000000002 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463266+0000", 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026159499999999999 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465488+0000", 2024-09-21T10:10:06.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022222050000000001 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465538+0000", 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0036e-05 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465643+0000", 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000104566 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138785 8.11 8:8e2f86f8:::benchmark_data_smithi076_241159_object15867:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367838+0000", 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4072016079999998, 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.099967285, 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138785 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367838+0000", 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367838+0000", 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367838+0000", 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4900000000000001e-07 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367840+0000", 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.722e-06 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367844+0000", 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1230000000000002e-06 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367848+0000", 2024-09-21T10:10:06.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4830000000000001e-06 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368038+0000", 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000190106 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368041+0000", 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4899999999999999e-06 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370165+0000", 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021241400000000001 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370172+0000", 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6449999999999999e-06 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.459303+0000", 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0891312150000001 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.459417+0000", 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011383999999999999 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.459575+0000", 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015844500000000001 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.462786+0000", 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0032114069999999999 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463009+0000", 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00022282399999999999 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465681+0000", 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026716190000000001 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.467567+0000", 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001885945 2024-09-21T10:10:06.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.467694+0000", 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000127267 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.467805+0000", 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110665 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138934 8.11 8:8bc9f568:::benchmark_data_smithi076_241159_object16016:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 3.4071732780000001, 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.101503399, 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138934 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.821 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.08e-07 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367868+0000", 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.897e-06 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367874+0000", 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8449999999999999e-06 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367879+0000", 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8080000000000004e-06 2024-09-21T10:10:06.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368055+0000", 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000176033 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368057+0000", 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.526e-06 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370195+0000", 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00213773 2024-09-21T10:10:06.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370201+0000", 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1349999999999997e-06 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.460880+0000", 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0906788519999999 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.460980+0000", 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9413000000000005e-05 2024-09-21T10:10:06.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.461096+0000", 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116951 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463014+0000", 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019179080000000001 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465731+0000", 2024-09-21T10:10:06.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027162739999999999 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.467849+0000", 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021179760000000001 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.469107+0000", 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012582139999999999 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:06.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.469259+0000", 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000152662 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.469369+0000", 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010976699999999999 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:06.828 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:10:06.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:06.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:06.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:06.985 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:06.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:06.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:07.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:07.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:07.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:07.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:07.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:07.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:07.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:07.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:07.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:07.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:07.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:07.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:07.625 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:07.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:07.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:07.754 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:07.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:07.855 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:07.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:07.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:07.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:08.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:08.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:08.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:08.158 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:08.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:08.204 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:08 smithi076 ceph-mon[106482]: pgmap v1696: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 6.5 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 731 op/s; 2706/320751 objects misplaced (0.844%); 9.1 MiB/s, 146 objects/s recovering 2024-09-21T10:10:08.204 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:08 smithi076 ceph-mon[110963]: pgmap v1696: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 6.5 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 731 op/s; 2706/320751 objects misplaced (0.844%); 9.1 MiB/s, 146 objects/s recovering 2024-09-21T10:10:08.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:08.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:08.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:08 smithi191 ceph-mon[102206]: pgmap v1696: 99 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 97 active+clean; 6.5 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 731 op/s; 2706/320751 objects misplaced (0.844%); 9.1 MiB/s, 146 objects/s recovering 2024-09-21T10:10:08.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:08.429 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:08.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:08.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:08.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:08.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:08.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:08.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:08.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:08.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:08.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:08.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:09.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:09.080 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:09.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:09.206 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:09.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:09.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:09.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:10:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:10:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-21T10:10:09.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:09.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:09.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:09.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:09.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:09.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:09.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:09.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:09.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:09.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:09.980 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:09.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:09.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:10.067 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:10:10.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:10.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:10.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:10.237 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:10.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:10.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:10.300 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:10 smithi076 ceph-mon[106482]: pgmap v1697: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 983 op/s; 6903/308985 objects misplaced (2.234%); 30 MiB/s, 483 objects/s recovering 2024-09-21T10:10:10.301 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:10:10.301 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:10 smithi076 ceph-mon[106482]: osdmap e1230: 8 total, 8 up, 5 in 2024-09-21T10:10:10.301 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:10 smithi076 ceph-mon[110963]: pgmap v1697: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 983 op/s; 6903/308985 objects misplaced (2.234%); 30 MiB/s, 483 objects/s recovering 2024-09-21T10:10:10.301 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:10:10.301 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:10 smithi076 ceph-mon[110963]: osdmap e1230: 8 total, 8 up, 5 in 2024-09-21T10:10:10.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:10.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:10 smithi191 ceph-mon[102206]: pgmap v1697: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 983 op/s; 6903/308985 objects misplaced (2.234%); 30 MiB/s, 483 objects/s recovering 2024-09-21T10:10:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-21T10:10:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:10 smithi191 ceph-mon[102206]: osdmap e1230: 8 total, 8 up, 5 in 2024-09-21T10:10:10.384 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:10.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:10.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:10.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:10.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:10.611 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:10.635 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:10.737 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:10.760 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:10.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:10.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:10.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:10.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:10.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:11.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:11.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:11.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:11.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:11.241 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:11.274 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:11.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:11.322 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:10:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:11 smithi191 ceph-mon[102206]: osdmap e1231: 8 total, 8 up, 5 in 2024-09-21T10:10:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:10:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1519428559' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:10:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[110963]: osdmap e1231: 8 total, 8 up, 5 in 2024-09-21T10:10:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:10:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1519428559' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:10:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[106482]: osdmap e1231: 8 total, 8 up, 5 in 2024-09-21T10:10:11.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:11.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:11.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:10:11.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1519428559' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:10:11.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:11.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:11.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:11.504 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:11.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:11.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:11.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:11.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:11.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:11.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:11.878 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:11.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:11.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:12.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:12.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:12.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:12.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:12.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:12.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:12.290 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:12.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:12 smithi191 ceph-mon[102206]: pgmap v1700: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 653 op/s; 6903/308985 objects misplaced (2.234%); 24 MiB/s, 377 objects/s recovering 2024-09-21T10:10:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:10:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1519428559' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:10:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:12 smithi191 ceph-mon[102206]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 1229'8589 2024-09-21T10:10:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:12 smithi191 ceph-mon[102206]: osdmap e1232: 8 total, 8 up, 5 in 2024-09-21T10:10:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:12 smithi191 ceph-mon[102206]: osdmap e1233: 8 total, 8 up, 5 in 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[106482]: pgmap v1700: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 653 op/s; 6903/308985 objects misplaced (2.234%); 24 MiB/s, 377 objects/s recovering 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1519428559' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[106482]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 1229'8589 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[106482]: osdmap e1232: 8 total, 8 up, 5 in 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[106482]: osdmap e1233: 8 total, 8 up, 5 in 2024-09-21T10:10:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[110963]: pgmap v1700: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 653 op/s; 6903/308985 objects misplaced (2.234%); 24 MiB/s, 377 objects/s recovering 2024-09-21T10:10:12.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:10:12.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1519428559' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:10:12.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[110963]: 8.13 starting backfill to osd.3 from (0'0,0'0] MAX to 1229'8589 2024-09-21T10:10:12.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[110963]: osdmap e1232: 8 total, 8 up, 5 in 2024-09-21T10:10:12.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:12 smithi076 ceph-mon[110963]: osdmap e1233: 8 total, 8 up, 5 in 2024-09-21T10:10:12.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:12.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:12.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:12.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:12.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:12.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:12.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:12.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:12.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:12.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:12.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:12.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:13.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:13.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:13.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:13.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:13.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:13.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:13.308 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:13 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:10:13.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:13 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:10:13.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:13 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-21T10:10:13.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:10:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:10:13] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T10:10:13.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:13.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:13.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:13.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:13.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:13.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:10:13.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:13.702 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:13.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:13.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:13.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:13.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:13.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:13.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:14.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:14.082 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:14.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:14.170 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:14.170 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:14.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:14.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:14.302 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[106482]: pgmap v1703: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.2 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 6903/305820 objects misplaced (2.257%) 2024-09-21T10:10:14.302 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:10:14.302 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[106482]: 8.14 starting backfill to osd.1 from (0'0,0'0] MAX to 1230'8615 2024-09-21T10:10:14.302 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[106482]: osdmap e1234: 8 total, 8 up, 5 in 2024-09-21T10:10:14.302 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[106482]: osdmap e1235: 8 total, 8 up, 5 in 2024-09-21T10:10:14.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[110963]: pgmap v1703: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.2 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 6903/305820 objects misplaced (2.257%) 2024-09-21T10:10:14.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:10:14.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[110963]: 8.14 starting backfill to osd.1 from (0'0,0'0] MAX to 1230'8615 2024-09-21T10:10:14.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[110963]: osdmap e1234: 8 total, 8 up, 5 in 2024-09-21T10:10:14.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:14 smithi076 ceph-mon[110963]: osdmap e1235: 8 total, 8 up, 5 in 2024-09-21T10:10:14.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:14.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:14.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:14 smithi191 ceph-mon[102206]: pgmap v1703: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.2 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 263 op/s; 6903/305820 objects misplaced (2.257%) 2024-09-21T10:10:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:14 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-21T10:10:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:14 smithi191 ceph-mon[102206]: 8.14 starting backfill to osd.1 from (0'0,0'0] MAX to 1230'8615 2024-09-21T10:10:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:14 smithi191 ceph-mon[102206]: osdmap e1234: 8 total, 8 up, 5 in 2024-09-21T10:10:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:14 smithi191 ceph-mon[102206]: osdmap e1235: 8 total, 8 up, 5 in 2024-09-21T10:10:14.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:14.455 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:14.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:14.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:14.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:14.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:14.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:14.831 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:14.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:14.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:14.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:15.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:15.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:15.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:15.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:15.227 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:15.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:15.329 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:15.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:15.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:10:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:15 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:10:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:15 smithi076 ceph-mon[106482]: osdmap e1236: 8 total, 8 up, 5 in 2024-09-21T10:10:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:10:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:15 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:10:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:15 smithi076 ceph-mon[110963]: osdmap e1236: 8 total, 8 up, 5 in 2024-09-21T10:10:15.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:15.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:15.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:15.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:15.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-21T10:10:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:15 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-21T10:10:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:15 smithi191 ceph-mon[102206]: osdmap e1236: 8 total, 8 up, 5 in 2024-09-21T10:10:15.615 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:15.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:15.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:15.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:15.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:15.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:15.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:15.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:15.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:15.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:16.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:16.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:16 smithi076 ceph-mon[106482]: pgmap v1706: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 762 op/s; 5758/299832 objects misplaced (1.920%); 63 MiB/s, 1.01k objects/s recovering 2024-09-21T10:10:16.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:16 smithi076 ceph-mon[106482]: 8.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1233'8777 2024-09-21T10:10:16.130 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:16 smithi076 ceph-mon[106482]: osdmap e1237: 8 total, 8 up, 5 in 2024-09-21T10:10:16.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:16 smithi076 ceph-mon[110963]: pgmap v1706: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 762 op/s; 5758/299832 objects misplaced (1.920%); 63 MiB/s, 1.01k objects/s recovering 2024-09-21T10:10:16.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:16 smithi076 ceph-mon[110963]: 8.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1233'8777 2024-09-21T10:10:16.130 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:16 smithi076 ceph-mon[110963]: osdmap e1237: 8 total, 8 up, 5 in 2024-09-21T10:10:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:16.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:16.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:16.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:16.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:16.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:16.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:16.398 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:10:16.398 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:10:16.398 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:10:16.398 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:10:16.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:16.503 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:16.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:16.604 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:16.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:16 smithi191 ceph-mon[102206]: pgmap v1706: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 762 op/s; 5758/299832 objects misplaced (1.920%); 63 MiB/s, 1.01k objects/s recovering 2024-09-21T10:10:16.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:16 smithi191 ceph-mon[102206]: 8.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1233'8777 2024-09-21T10:10:16.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:16 smithi191 ceph-mon[102206]: osdmap e1237: 8 total, 8 up, 5 in 2024-09-21T10:10:16.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:16.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:16.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:16.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:16.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:16.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:16.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:16.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:16.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:17.010 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:17.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:17.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:17.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:17.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:17.276 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:17.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:17.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:17.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:17.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:17.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:17.524 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:17.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:10:17.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:17 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:10:17.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:17 smithi191 ceph-mon[102206]: osdmap e1238: 8 total, 8 up, 5 in 2024-09-21T10:10:17.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:10:17.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:17 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:10:17.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:17 smithi076 ceph-mon[110963]: osdmap e1238: 8 total, 8 up, 5 in 2024-09-21T10:10:17.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-21T10:10:17.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:17 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-21T10:10:17.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:17 smithi076 ceph-mon[106482]: osdmap e1238: 8 total, 8 up, 5 in 2024-09-21T10:10:17.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:17.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:17.753 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:17.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:17.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:10:17.880 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:17.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:17.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:17.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:17.993 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:18.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:18.026 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:18.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:18.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:18.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:18.281 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:18 smithi076 ceph-mon[106482]: pgmap v1709: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 7879/299544 objects misplaced (2.630%); 79 MiB/s, 1.26k objects/s recovering 2024-09-21T10:10:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:18 smithi076 ceph-mon[106482]: 8.16 starting backfill to osd.1 from (0'0,0'0] MAX to 1234'8489 2024-09-21T10:10:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:18 smithi076 ceph-mon[106482]: osdmap e1239: 8 total, 8 up, 5 in 2024-09-21T10:10:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:18 smithi076 ceph-mon[110963]: pgmap v1709: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 7879/299544 objects misplaced (2.630%); 79 MiB/s, 1.26k objects/s recovering 2024-09-21T10:10:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:18 smithi076 ceph-mon[110963]: 8.16 starting backfill to osd.1 from (0'0,0'0] MAX to 1234'8489 2024-09-21T10:10:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:18 smithi076 ceph-mon[110963]: osdmap e1239: 8 total, 8 up, 5 in 2024-09-21T10:10:18.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:18.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:18.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:18.530 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:18.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:18.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:18 smithi191 ceph-mon[102206]: pgmap v1709: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 6.1 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 7879/299544 objects misplaced (2.630%); 79 MiB/s, 1.26k objects/s recovering 2024-09-21T10:10:18.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:18 smithi191 ceph-mon[102206]: 8.16 starting backfill to osd.1 from (0'0,0'0] MAX to 1234'8489 2024-09-21T10:10:18.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:18 smithi191 ceph-mon[102206]: osdmap e1239: 8 total, 8 up, 5 in 2024-09-21T10:10:18.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:18.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:18.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:18.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:18.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:18.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:18.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:18.907 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:18.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:19.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:19.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:19.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:19.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:19.162 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:19.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:19.292 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:19.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:19.394 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:19.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:19.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.553 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:19.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:19.654 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:19.680 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:19.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:19.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:19.807 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:19.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:19.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:19.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:19.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:20.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:20.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:20.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:20.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:20.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:20.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:20.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:20.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:20 smithi191 ceph-mon[102206]: pgmap v1712: 99 pgs: 1 remapped+peering, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 11315/295305 objects misplaced (3.832%); 57 MiB/s, 915 objects/s recovering 2024-09-21T10:10:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:20 smithi191 ceph-mon[102206]: osdmap e1240: 8 total, 8 up, 5 in 2024-09-21T10:10:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:20 smithi191 ceph-mon[102206]: 8.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1237'8654 2024-09-21T10:10:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:20 smithi191 ceph-mon[102206]: 8.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1237'8654 2024-09-21T10:10:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[106482]: pgmap v1712: 99 pgs: 1 remapped+peering, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 11315/295305 objects misplaced (3.832%); 57 MiB/s, 915 objects/s recovering 2024-09-21T10:10:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[106482]: osdmap e1240: 8 total, 8 up, 5 in 2024-09-21T10:10:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[106482]: 8.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1237'8654 2024-09-21T10:10:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[106482]: 8.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1237'8654 2024-09-21T10:10:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[110963]: pgmap v1712: 99 pgs: 1 remapped+peering, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 11315/295305 objects misplaced (3.832%); 57 MiB/s, 915 objects/s recovering 2024-09-21T10:10:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[110963]: osdmap e1240: 8 total, 8 up, 5 in 2024-09-21T10:10:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[110963]: 8.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1237'8654 2024-09-21T10:10:20.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:20 smithi076 ceph-mon[110963]: 8.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1237'8654 2024-09-21T10:10:20.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:20.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:20.571 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:20.592 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:20.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:20.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:20.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:20.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:20.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:20.851 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:20.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:20.953 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:20.981 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:20.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:21.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.123 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:21.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:21.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:21.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:21.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:21 smithi076 ceph-mon[106482]: osdmap e1241: 8 total, 8 up, 5 in 2024-09-21T10:10:21.345 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:21 smithi076 ceph-mon[110963]: osdmap e1241: 8 total, 8 up, 5 in 2024-09-21T10:10:21.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:21.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:21 smithi191 ceph-mon[102206]: osdmap e1241: 8 total, 8 up, 5 in 2024-09-21T10:10:21.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:21.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:21.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:21.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:21.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:21.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:21.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:21.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:21.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:21.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:21.908 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:21.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:22.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:22.049 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:22.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:22.109 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:10:22.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.177 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:22.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:22.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:22.305 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:22 smithi191 ceph-mon[102206]: pgmap v1715: 99 pgs: 1 remapped+peering, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 11315/295305 objects misplaced (3.832%); 42 MiB/s, 664 objects/s recovering 2024-09-21T10:10:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:22 smithi191 ceph-mon[102206]: osdmap e1242: 8 total, 8 up, 5 in 2024-09-21T10:10:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3463439792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:22.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1624978612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[106482]: pgmap v1715: 99 pgs: 1 remapped+peering, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 11315/295305 objects misplaced (3.832%); 42 MiB/s, 664 objects/s recovering 2024-09-21T10:10:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[106482]: osdmap e1242: 8 total, 8 up, 5 in 2024-09-21T10:10:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3463439792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1624978612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:22.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[110963]: pgmap v1715: 99 pgs: 1 remapped+peering, 3 active+remapped+backfill_wait, 2 active+remapped+backfilling, 93 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 353 op/s; 11315/295305 objects misplaced (3.832%); 42 MiB/s, 664 objects/s recovering 2024-09-21T10:10:22.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[110963]: osdmap e1242: 8 total, 8 up, 5 in 2024-09-21T10:10:22.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3463439792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:22.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1624978612' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:10:22.404 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:10:22.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:10:22.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:10:22.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:10:22.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:10:22.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:10:22.405 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:10:22.406 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:10:22.406 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:22.406 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:22.406 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:22.406 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:22.407 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:22.407 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:22.407 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:22.407 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:22.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:22.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:22.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:22.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:22.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:22.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:22.714 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:22.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:22.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:22.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:22.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:22.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:23.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:23.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:23.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:23.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:23.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:23.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:23 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:23.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:23 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1624978612' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:23.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:23 smithi191 ceph-mon[102206]: osdmap e1243: 8 total, 8 up, 5 in 2024-09-21T10:10:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:10:23] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T10:10:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1624978612' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-mon[106482]: osdmap e1243: 8 total, 8 up, 5 in 2024-09-21T10:10:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1624978612' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:10:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:23 smithi076 ceph-mon[110963]: osdmap e1243: 8 total, 8 up, 5 in 2024-09-21T10:10:23.374 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:23.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:23.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:23.641 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:23.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:23.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:23.871 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:23.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:23.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:23.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:23.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:24.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:24.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:24.154 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:24.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:24.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:24 smithi076 ceph-mon[106482]: pgmap v1718: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 259 op/s; 10035/292194 objects misplaced (3.434%); 19 MiB/s, 304 objects/s recovering 2024-09-21T10:10:24.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:24 smithi076 ceph-mon[110963]: pgmap v1718: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 259 op/s; 10035/292194 objects misplaced (3.434%); 19 MiB/s, 304 objects/s recovering 2024-09-21T10:10:24.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:24.408 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:24.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:24.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:24.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:24 smithi191 ceph-mon[102206]: pgmap v1718: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 259 op/s; 10035/292194 objects misplaced (3.434%); 19 MiB/s, 304 objects/s recovering 2024-09-21T10:10:24.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:24.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:24.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:24.791 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:24.892 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:24.917 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:24.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:24.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:25.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:25.022 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:25.045 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:25.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:25.147 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:25.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:25.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:25.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:25.312 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:10:25.312 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:10:25.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:25.423 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:25.458 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:25.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:25.584 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"} : dispatch 2024-09-21T10:10:25.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:25.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:25.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:25.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:25.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:25.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:25.960 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:25.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:26.062 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:26.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:10:26.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:26.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:26.131 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:10:26.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:26.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:26.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[106482]: pgmap v1719: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 10396/285960 objects misplaced (3.635%); 62 MiB/s, 992 objects/s recovering 2024-09-21T10:10:26.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:10:26.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[106482]: osdmap e1244: 8 total, 8 up, 5 in 2024-09-21T10:10:26.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:26.255 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/591164451' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:10:26.255 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[110963]: pgmap v1719: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 10396/285960 objects misplaced (3.635%); 62 MiB/s, 992 objects/s recovering 2024-09-21T10:10:26.256 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:10:26.256 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[110963]: osdmap e1244: 8 total, 8 up, 5 in 2024-09-21T10:10:26.256 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:26.256 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/591164451' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:10:26.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:26.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:26.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:26.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:26.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:10:26.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:26 smithi191 ceph-mon[102206]: pgmap v1719: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 551 op/s; 10396/285960 objects misplaced (3.635%); 62 MiB/s, 992 objects/s recovering 2024-09-21T10:10:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "25"}]': finished 2024-09-21T10:10:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:26 smithi191 ceph-mon[102206]: osdmap e1244: 8 total, 8 up, 5 in 2024-09-21T10:10:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/591164451' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:10:26.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:26.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:26.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:26.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:26.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:26.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:26.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:26.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:26.993 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:26.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:26.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:27.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:10:27.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:27.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:27.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:27.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:27.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/591164451' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:10:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:27 smithi076 ceph-mon[106482]: osdmap e1245: 8 total, 8 up, 5 in 2024-09-21T10:10:27.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:27 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:10:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/591164451' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:10:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:27 smithi076 ceph-mon[110963]: osdmap e1245: 8 total, 8 up, 5 in 2024-09-21T10:10:27.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:27 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:10:27.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:27.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:27.511 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:27.612 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/591164451' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:10:27.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:27 smithi191 ceph-mon[102206]: osdmap e1245: 8 total, 8 up, 5 in 2024-09-21T10:10:27.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:27 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-21T10:10:27.620 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:27.650 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:27.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:27.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:27.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:27.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:27.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:27.904 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:27.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:28.006 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:28.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:28.132 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:28.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:28.182 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[110963]: pgmap v1722: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 552 op/s; 10396/285960 objects misplaced (3.635%); 62 MiB/s, 994 objects/s recovering 2024-09-21T10:10:28.182 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:10:28.182 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[110963]: 8.18 starting backfill to osd.6 from (0'0,0'0] MAX to 1243'8749 2024-09-21T10:10:28.182 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[110963]: osdmap e1246: 8 total, 8 up, 5 in 2024-09-21T10:10:28.183 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[106482]: pgmap v1722: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 552 op/s; 10396/285960 objects misplaced (3.635%); 62 MiB/s, 994 objects/s recovering 2024-09-21T10:10:28.183 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:10:28.183 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[106482]: 8.18 starting backfill to osd.6 from (0'0,0'0] MAX to 1243'8749 2024-09-21T10:10:28.183 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:28 smithi076 ceph-mon[106482]: osdmap e1246: 8 total, 8 up, 5 in 2024-09-21T10:10:28.263 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:28.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:28.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:28.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:28.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:28.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:28.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:28.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:28 smithi191 ceph-mon[102206]: pgmap v1722: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 552 op/s; 10396/285960 objects misplaced (3.635%); 62 MiB/s, 994 objects/s recovering 2024-09-21T10:10:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:28 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-21T10:10:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:28 smithi191 ceph-mon[102206]: 8.18 starting backfill to osd.6 from (0'0,0'0] MAX to 1243'8749 2024-09-21T10:10:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:28 smithi191 ceph-mon[102206]: osdmap e1246: 8 total, 8 up, 5 in 2024-09-21T10:10:28.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:28.663 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:28.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:28.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:28.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:28.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:28.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:28.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:28.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:29.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:29.039 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:29.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:29.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:29.270 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:29.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:29.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:29.382 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:10:29.382 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:10:29.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:29.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:29.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:29.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:29.559 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:29.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:29 smithi191 ceph-mon[102206]: osdmap e1247: 8 total, 8 up, 5 in 2024-09-21T10:10:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:29 smithi191 ceph-mon[102206]: osdmap e1248: 8 total, 8 up, 5 in 2024-09-21T10:10:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:29 smithi076 ceph-mon[106482]: osdmap e1247: 8 total, 8 up, 5 in 2024-09-21T10:10:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:29 smithi076 ceph-mon[106482]: osdmap e1248: 8 total, 8 up, 5 in 2024-09-21T10:10:29.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:29 smithi076 ceph-mon[110963]: osdmap e1247: 8 total, 8 up, 5 in 2024-09-21T10:10:29.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:29 smithi076 ceph-mon[110963]: osdmap e1248: 8 total, 8 up, 5 in 2024-09-21T10:10:29.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:29.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:29.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:29.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:29.811 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:29.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:29.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:29.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:29.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:30.041 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:30.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:30.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.342 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:30.602 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:30 smithi191 ceph-mon[102206]: pgmap v1725: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 956 op/s; 7115/274476 objects misplaced (2.592%); 64 MiB/s, 1.03k objects/s recovering 2024-09-21T10:10:30.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:30 smithi191 ceph-mon[102206]: 8.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1244'9117 2024-09-21T10:10:30.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:30 smithi191 ceph-mon[102206]: osdmap e1249: 8 total, 8 up, 5 in 2024-09-21T10:10:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:30 smithi076 ceph-mon[106482]: pgmap v1725: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 956 op/s; 7115/274476 objects misplaced (2.592%); 64 MiB/s, 1.03k objects/s recovering 2024-09-21T10:10:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:30 smithi076 ceph-mon[106482]: 8.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1244'9117 2024-09-21T10:10:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:30 smithi076 ceph-mon[106482]: osdmap e1249: 8 total, 8 up, 5 in 2024-09-21T10:10:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:30 smithi076 ceph-mon[110963]: pgmap v1725: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 956 op/s; 7115/274476 objects misplaced (2.592%); 64 MiB/s, 1.03k objects/s recovering 2024-09-21T10:10:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:30 smithi076 ceph-mon[110963]: 8.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1244'9117 2024-09-21T10:10:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:30 smithi076 ceph-mon[110963]: osdmap e1249: 8 total, 8 up, 5 in 2024-09-21T10:10:30.705 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:30.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:30.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:30.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:30.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:30.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:31.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:31.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:31.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:31.267 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:31.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:31.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:31.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:31.525 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:31.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:31.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:31.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:31.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:31.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:31.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:31.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.922 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:31.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:32.023 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:32.064 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:32.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:32.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:32.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:32.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:32.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:32 smithi191 ceph-mon[102206]: pgmap v1728: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 956 op/s; 7115/274476 objects misplaced (2.592%); 64 MiB/s, 1.03k objects/s recovering 2024-09-21T10:10:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:32 smithi191 ceph-mon[102206]: osdmap e1250: 8 total, 8 up, 5 in 2024-09-21T10:10:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:32 smithi076 ceph-mon[106482]: pgmap v1728: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 956 op/s; 7115/274476 objects misplaced (2.592%); 64 MiB/s, 1.03k objects/s recovering 2024-09-21T10:10:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:32 smithi076 ceph-mon[106482]: osdmap e1250: 8 total, 8 up, 5 in 2024-09-21T10:10:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:32 smithi076 ceph-mon[110963]: pgmap v1728: 99 pgs: 1 remapped+peering, 2 active+remapped+backfill_wait, 2 active+remapped+backfilling, 94 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 956 op/s; 7115/274476 objects misplaced (2.592%); 64 MiB/s, 1.03k objects/s recovering 2024-09-21T10:10:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:32 smithi076 ceph-mon[110963]: osdmap e1250: 8 total, 8 up, 5 in 2024-09-21T10:10:32.427 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:32.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:32.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:32.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:32.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:32.694 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:32.738 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:32.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:32.867 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:32.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:32.992 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:33.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:33.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:33.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.246 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:33.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:33.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:33.384 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:33.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:33.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:33.507 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:33.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:33.608 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:10:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:10:33] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.51.0" 2024-09-21T10:10:33.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:33.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:33.790 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:33.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:33.892 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:33.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:33.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:33.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:34.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:34.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:34.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:34.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:34.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:34.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:34 smithi191 ceph-mon[102206]: pgmap v1730: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 491 op/s; 5467/271215 objects misplaced (2.016%); 59 MiB/s, 945 objects/s recovering 2024-09-21T10:10:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:34 smithi076 ceph-mon[106482]: pgmap v1730: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 491 op/s; 5467/271215 objects misplaced (2.016%); 59 MiB/s, 945 objects/s recovering 2024-09-21T10:10:34.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:34 smithi076 ceph-mon[110963]: pgmap v1730: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 491 op/s; 5467/271215 objects misplaced (2.016%); 59 MiB/s, 945 objects/s recovering 2024-09-21T10:10:34.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:34.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:34.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:34.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:34.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:34.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:34.693 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:34.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:34.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:34.863 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:34.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:34.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:34.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:34.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:35.093 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:35.120 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:35.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:35.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:35.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:35.386 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:35.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:35.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:35.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:35.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:35.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:35.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:35.781 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:35.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:35.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:35.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:36.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:36.033 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:36.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:36.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:36.273 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:36.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:36.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:36.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:36 smithi191 ceph-mon[102206]: pgmap v1731: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 548 op/s; 6202/264597 objects misplaced (2.344%); 35 MiB/s, 555 objects/s recovering 2024-09-21T10:10:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:36 smithi076 ceph-mon[110963]: pgmap v1731: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 548 op/s; 6202/264597 objects misplaced (2.344%); 35 MiB/s, 555 objects/s recovering 2024-09-21T10:10:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:36 smithi076 ceph-mon[106482]: pgmap v1731: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 548 op/s; 6202/264597 objects misplaced (2.344%); 35 MiB/s, 555 objects/s recovering 2024-09-21T10:10:36.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:36.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:36.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:36.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:36.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:36.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:36.722 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:36.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:36.850 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:36.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:36.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:36.978 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:36.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:36.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:37.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:37.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:37.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:37.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.241 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:37.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:37.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:37.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:37.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:37.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:37.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:37.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.619 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:37.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:37.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.761 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:37.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:37.896 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:37.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:37.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:38.036 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:38.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:38.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:38.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:38.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:38.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:38 smithi191 ceph-mon[102206]: pgmap v1732: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 6202/264597 objects misplaced (2.344%); 27 MiB/s, 435 objects/s recovering 2024-09-21T10:10:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:38 smithi076 ceph-mon[106482]: pgmap v1732: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 6202/264597 objects misplaced (2.344%); 27 MiB/s, 435 objects/s recovering 2024-09-21T10:10:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:38 smithi076 ceph-mon[110963]: pgmap v1732: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 6202/264597 objects misplaced (2.344%); 27 MiB/s, 435 objects/s recovering 2024-09-21T10:10:38.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:38.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:38.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:38.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:38.545 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:38.646 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:38.681 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:38.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:38.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:38.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:38.950 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:38.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:39.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:39.090 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:39.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:39.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:39.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:39.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:39.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:39.359 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:39.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:39.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:39.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:39.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:39.615 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:39.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:39.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:39.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:39.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:39.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:39.882 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:39.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:39.984 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.008 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:40.110 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:40 smithi076 ceph-mon[106482]: pgmap v1733: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 4239/248262 objects misplaced (1.707%); 36 MiB/s, 574 objects/s recovering 2024-09-21T10:10:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:40 smithi076 ceph-mon[110963]: pgmap v1733: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 4239/248262 objects misplaced (1.707%); 36 MiB/s, 574 objects/s recovering 2024-09-21T10:10:40.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:40.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:40.278 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:40.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:40 smithi191 ceph-mon[102206]: pgmap v1733: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 4239/248262 objects misplaced (1.707%); 36 MiB/s, 574 objects/s recovering 2024-09-21T10:10:40.380 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.402 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:40.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:40.503 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:40.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:40.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:40.663 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:40.765 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:40.798 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:40.900 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:40.938 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:40.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:41.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:41.063 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:41.132 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:10:41.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:41.317 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:41.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:41.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:41.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:41.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:41.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:41.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:41.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:41.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:41.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:41.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:41.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:42.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:42.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:42.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:42.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:42.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:42 smithi191 ceph-mon[102206]: pgmap v1734: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 873 op/s; 4239/248262 objects misplaced (1.707%); 31 MiB/s, 496 objects/s recovering 2024-09-21T10:10:42.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:42.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:42 smithi076 ceph-mon[106482]: pgmap v1734: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 873 op/s; 4239/248262 objects misplaced (1.707%); 31 MiB/s, 496 objects/s recovering 2024-09-21T10:10:42.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:42 smithi076 ceph-mon[110963]: pgmap v1734: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 873 op/s; 4239/248262 objects misplaced (1.707%); 31 MiB/s, 496 objects/s recovering 2024-09-21T10:10:42.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:42.501 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:42.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:42.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:42.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.641 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:42.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:42.742 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:42.783 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:42.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:42.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:42.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:42.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:43.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:43.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:43.172 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:43.295 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.330 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:43.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:10:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:10:43] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-21T10:10:43.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:43.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:43.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:43.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:43.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:43.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:43.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:43.860 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:10:43.860 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:43.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:43.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:43.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:43.897 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:10:43.898 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:43.953 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:10:43.953 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:43.992 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:43.999 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:10:43.999 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:44.032 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:10:44.032 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:44.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:44.052 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:10:44.053 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:44.069 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:10:44.069 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:44.096 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:10:44.096 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:44.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:44.161 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:44.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:44.296 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:44 smithi076 ceph-mon[106482]: pgmap v1735: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 939 op/s; 3472/241656 objects misplaced (1.437%); 30 MiB/s, 486 objects/s recovering 2024-09-21T10:10:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2506333548' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:44 smithi076 ceph-mon[110963]: pgmap v1735: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 939 op/s; 3472/241656 objects misplaced (1.437%); 30 MiB/s, 486 objects/s recovering 2024-09-21T10:10:44.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2506333548' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:44.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:44.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:44.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:44.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:44 smithi191 ceph-mon[102206]: pgmap v1735: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 939 op/s; 3472/241656 objects misplaced (1.437%); 30 MiB/s, 486 objects/s recovering 2024-09-21T10:10:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2506333548' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:44.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:44.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:44.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.830 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:44.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:44.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:44.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:44.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:45.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:45.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:45.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:45.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:45.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:45.335 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:45.358 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:45.461 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:45.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:45.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:45.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:45.617 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:45.719 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:45.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:45.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:45.866 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:45.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:45.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:45.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:46.005 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:46.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:46.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:46.151 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:46.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:46.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:46.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:46 smithi076 ceph-mon[106482]: pgmap v1736: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s; 2201/230637 objects misplaced (0.954%); 27 MiB/s, 438 objects/s recovering 2024-09-21T10:10:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:46 smithi076 ceph-mon[110963]: pgmap v1736: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s; 2201/230637 objects misplaced (0.954%); 27 MiB/s, 438 objects/s recovering 2024-09-21T10:10:46.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:46.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:46.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.535 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:46.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:46.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:46 smithi191 ceph-mon[102206]: pgmap v1736: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s; 2201/230637 objects misplaced (0.954%); 27 MiB/s, 438 objects/s recovering 2024-09-21T10:10:46.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:46.681 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:46.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:46.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:46.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:46.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:46.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:46.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:47.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:47.064 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:47.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:47.166 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:47.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:47.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:47.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:47.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:47.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:47.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:47.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:47.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:47.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:47.627 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:47.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:47.775 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:47.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:47.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:47.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:48.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:48.035 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:48.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:48.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:48.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:48.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:48.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:48.370 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:48 smithi076 ceph-mon[106482]: pgmap v1737: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 943 op/s; 2201/230637 objects misplaced (0.954%); 18 MiB/s, 288 objects/s recovering 2024-09-21T10:10:48.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:48 smithi076 ceph-mon[110963]: pgmap v1737: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 943 op/s; 2201/230637 objects misplaced (0.954%); 18 MiB/s, 288 objects/s recovering 2024-09-21T10:10:48.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:48.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:48.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:48.537 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:48.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:48 smithi191 ceph-mon[102206]: pgmap v1737: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 943 op/s; 2201/230637 objects misplaced (0.954%); 18 MiB/s, 288 objects/s recovering 2024-09-21T10:10:48.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:48.664 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:48.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:48.766 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:48.790 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:48.892 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:48.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:48.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:48.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:49.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:49.037 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:49.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:49.138 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.174 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:49.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:49.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:49.299 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:49.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:49.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:49 smithi076 ceph-mon[106482]: osdmap e1251: 8 total, 8 up, 5 in 2024-09-21T10:10:49.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:49 smithi076 ceph-mon[106482]: osdmap e1252: 8 total, 8 up, 5 in 2024-09-21T10:10:49.338 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:49 smithi076 ceph-mon[110963]: osdmap e1251: 8 total, 8 up, 5 in 2024-09-21T10:10:49.338 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:49 smithi076 ceph-mon[110963]: osdmap e1252: 8 total, 8 up, 5 in 2024-09-21T10:10:49.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:49.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:49.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:49.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:49.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:49 smithi191 ceph-mon[102206]: osdmap e1251: 8 total, 8 up, 5 in 2024-09-21T10:10:49.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:49 smithi191 ceph-mon[102206]: osdmap e1252: 8 total, 8 up, 5 in 2024-09-21T10:10:49.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:49.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:49.687 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:49.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:49.797 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:49.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:49.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:49.921 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:49.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:50.022 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:50.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:50.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:50.079 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:10:50.083 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:10:50.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:10:50.151 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:50.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:50.277 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:50.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: pgmap v1739: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.14k op/s; 505/213930 objects misplaced (0.236%); 22 MiB/s, 348 objects/s recovering 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2483641825' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3954212848' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3954212848' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:50.326 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[110963]: osdmap e1253: 8 total, 8 up, 5 in 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: pgmap v1739: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.14k op/s; 505/213930 objects misplaced (0.236%); 22 MiB/s, 348 objects/s recovering 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2483641825' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3954212848' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3954212848' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:50.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:50 smithi076 ceph-mon[106482]: osdmap e1253: 8 total, 8 up, 5 in 2024-09-21T10:10:50.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:50.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:50.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:50.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: pgmap v1739: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.14k op/s; 505/213930 objects misplaced (0.236%); 22 MiB/s, 348 objects/s recovering 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2483641825' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3954212848' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3954212848' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:10:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:50 smithi191 ceph-mon[102206]: osdmap e1253: 8 total, 8 up, 5 in 2024-09-21T10:10:50.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:50.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:50.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:50.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:50.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:50.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:50.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:50.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:50.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:51.101 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:51.125 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:51.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:51.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:51.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:51.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:51.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:51 smithi076 ceph-mon[110963]: 8.13 scrub starts 2024-09-21T10:10:51.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:51 smithi076 ceph-mon[106482]: 8.13 scrub starts 2024-09-21T10:10:51.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:51.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:51.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:51.512 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:51.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:51 smithi191 ceph-mon[102206]: 8.13 scrub starts 2024-09-21T10:10:51.613 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:51.639 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:51.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:51.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:51.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:51.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:51.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:10:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:51.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:51.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:52.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:52.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:52.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:52.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:52.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:52.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.330 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[106482]: 8.3 scrub starts 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[106482]: pgmap v1742: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 505/213930 objects misplaced (0.236%); 17 MiB/s, 277 objects/s recovering 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[106482]: 8.4 scrub starts 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[106482]: 8.4 scrub ok 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[106482]: 8.3 scrub ok 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[106482]: 8.13 scrub ok 2024-09-21T10:10:52.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[110963]: 8.3 scrub starts 2024-09-21T10:10:52.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[110963]: pgmap v1742: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 505/213930 objects misplaced (0.236%); 17 MiB/s, 277 objects/s recovering 2024-09-21T10:10:52.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[110963]: 8.4 scrub starts 2024-09-21T10:10:52.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[110963]: 8.4 scrub ok 2024-09-21T10:10:52.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[110963]: 8.3 scrub ok 2024-09-21T10:10:52.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:52 smithi076 ceph-mon[110963]: 8.13 scrub ok 2024-09-21T10:10:52.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.556 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:52.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.582 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:52.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:52 smithi191 ceph-mon[102206]: 8.3 scrub starts 2024-09-21T10:10:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:52 smithi191 ceph-mon[102206]: pgmap v1742: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 505/213930 objects misplaced (0.236%); 17 MiB/s, 277 objects/s recovering 2024-09-21T10:10:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:52 smithi191 ceph-mon[102206]: 8.4 scrub starts 2024-09-21T10:10:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:52 smithi191 ceph-mon[102206]: 8.4 scrub ok 2024-09-21T10:10:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:52 smithi191 ceph-mon[102206]: 8.3 scrub ok 2024-09-21T10:10:52.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:52 smithi191 ceph-mon[102206]: 8.13 scrub ok 2024-09-21T10:10:52.643 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:52.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:52.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:52.812 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:52.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:52.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:52.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:52.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:53.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:53.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:53.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:53.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:53.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:53.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:10:53] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.51.0" 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-mon[110963]: 8.7 scrub starts 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-mon[110963]: 8.7 scrub ok 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-mon[106482]: 8.7 scrub starts 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-mon[106482]: 8.7 scrub ok 2024-09-21T10:10:53.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:53.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:53.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:53.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:53.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:53.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:53 smithi191 ceph-mon[102206]: 8.7 scrub starts 2024-09-21T10:10:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:53 smithi191 ceph-mon[102206]: 8.7 scrub ok 2024-09-21T10:10:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:10:53.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:53.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:53.763 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:53.797 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:53.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:53.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:53.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:53.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:54.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:54.074 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:54.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:54.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:54.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:54.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:54.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.332 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:54.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:54 smithi191 ceph-mon[102206]: 8.8 scrub starts 2024-09-21T10:10:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:54 smithi191 ceph-mon[102206]: pgmap v1743: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 505/213930 objects misplaced (0.236%); 17 MiB/s, 277 objects/s recovering 2024-09-21T10:10:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:54 smithi191 ceph-mon[102206]: 8.8 scrub ok 2024-09-21T10:10:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:54 smithi191 ceph-mon[102206]: 8.0 scrub starts 2024-09-21T10:10:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[106482]: 8.8 scrub starts 2024-09-21T10:10:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[106482]: pgmap v1743: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 505/213930 objects misplaced (0.236%); 17 MiB/s, 277 objects/s recovering 2024-09-21T10:10:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[106482]: 8.8 scrub ok 2024-09-21T10:10:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[106482]: 8.0 scrub starts 2024-09-21T10:10:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[110963]: 8.8 scrub starts 2024-09-21T10:10:54.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[110963]: pgmap v1743: 99 pgs: 1 peering, 2 active+remapped+backfilling, 96 active+clean; 4.4 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 505/213930 objects misplaced (0.236%); 17 MiB/s, 277 objects/s recovering 2024-09-21T10:10:54.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[110963]: 8.8 scrub ok 2024-09-21T10:10:54.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:54 smithi076 ceph-mon[110963]: 8.0 scrub starts 2024-09-21T10:10:54.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:54.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:54.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:10:54.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:54.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.622 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:54.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:54.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:54.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:54.850 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:54.931 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:54.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:55.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:55.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:55.168 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:10:55.168 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:10:55.168 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:10:55.168 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:10:55.168 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.168 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:132909 7.3 e1197/1137 7:c519bedd:::benchmark_data_smithi076_190198_object60920:head v 1197'8527, mlcod=1197'8525)", 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:06:55.586892+0000", 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 239.575467506, 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.239487954, 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586892+0000", 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.169 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586892+0000", 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586893+0000", 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3099999999999997e-07 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586899+0000", 2024-09-21T10:10:55.170 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7780000000000004e-06 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586909+0000", 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0064000000000001e-05 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586924+0000", 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5508000000000001e-05 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.171 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.586999+0000", 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4976999999999997e-05 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.587019+0000", 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9171999999999999e-05 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.826308+0000", 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.23928909700000001 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.826380+0000", 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2626999999999994e-05 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:132911 7.8 e1197/1151 7:14ee1a0d:::benchmark_data_smithi076_190198_object60922:head v 1197'8279, mlcod=1197'8277)", 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:06:55.588507+0000", 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 239.57385255899999, 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.244804088, 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.173 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588507+0000", 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588507+0000", 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588508+0000", 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0640000000000001e-06 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588515+0000", 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4799999999999998e-06 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588526+0000", 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0993e-05 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588532+0000", 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6200000000000001e-06 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588590+0000", 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8057000000000003e-05 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588613+0000", 2024-09-21T10:10:55.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2149000000000001e-05 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.833224+0000", 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.24461100599999999 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.833311+0000", 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7719000000000005e-05 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:132910 7.3 e1197/1137 7:c0e296d1:::benchmark_data_smithi076_190198_object60921:head v 1197'8528, mlcod=1197'8525)", 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:06:55.588993+0000", 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 239.57336668900001, 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.23978186500000001, 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588993+0000", 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588993+0000", 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588994+0000", 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5099999999999999e-07 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.588998+0000", 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5689999999999998e-06 2024-09-21T10:10:55.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.825952+0000", 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.236953364 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.825969+0000", 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7564e-05 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.826405+0000", 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00043606999999999999 2024-09-21T10:10:55.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.826424+0000", 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8549000000000001e-05 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.828734+0000", 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023105719999999999 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.828775+0000", 2024-09-21T10:10:55.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0525999999999998e-05 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.34992.0:132913 7.b e1197/1187 7:dfdb3b95:::benchmark_data_smithi076_190198_object60924:head v 1197'8304, mlcod=1197'8302)", 2024-09-21T10:10:55.182 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:06:55.589779+0000", 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 239.572580389, 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.240586311, 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.589779+0000", 2024-09-21T10:10:55.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.589779+0000", 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.589780+0000", 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6300000000000004e-07 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.589787+0000", 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.567e-06 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.826005+0000", 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.23621856599999999 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.826015+0000", 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6059999999999992e-06 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.827919+0000", 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019043879999999999 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.827942+0000", 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2381000000000001e-05 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.830320+0000", 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023779199999999999 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:06:55.830366+0000", 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6119999999999999e-05 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145309 8.14 8:2cd4fe9c:::benchmark_data_smithi076_241159_object22391:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 43.055180006999997, 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.067794157, 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.187 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145309 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.188 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107182+0000", 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.843e-06 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107187+0000", 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8749999999999999e-06 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107196+0000", 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3200000000000006e-06 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107206+0000", 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0093e-05 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107256+0000", 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0154999999999997e-05 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107265+0000", 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2089999999999994e-06 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.170814+0000", 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06354872 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.170905+0000", 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0631999999999996e-05 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.171114+0000", 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000209796 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174576+0000", 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034615789999999998 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174585+0000", 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0680000000000003e-06 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174717+0000", 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000131817 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174784+0000", 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7360999999999994e-05 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174850+0000", 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6104000000000003e-05 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174974+0000", 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000123585 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145327 8.14 8:2bc8e99d:::benchmark_data_smithi076_241159_object22409:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 43.055134135000003, 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06996614, 2024-09-21T10:10:55.194 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145327 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0999999999999998e-07 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107229+0000", 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7199999999999998e-06 2024-09-21T10:10:55.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107238+0000", 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.755e-06 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107244+0000", 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.392e-06 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107296+0000", 2024-09-21T10:10:55.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2200999999999997e-05 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107303+0000", 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6379999999999998e-06 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172493+0000", 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.065190184 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172625+0000", 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013188800000000001 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172812+0000", 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018733899999999999 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174726+0000", 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00191332 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.199 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176604+0000", 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001878111 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176883+0000", 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027873299999999998 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177019+0000", 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013593700000000001 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177079+0000", 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0439000000000002e-05 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177192+0000", 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112873 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.201 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145341 8.14 8:2a80dff4:::benchmark_data_smithi076_241159_object22423:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 43.055107329999998, 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.070240276, 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145341 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107253+0000", 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4000000000000002e-07 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107259+0000", 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0270000000000001e-06 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107266+0000", 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.002e-06 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107273+0000", 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9380000000000003e-06 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108301+0000", 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010276829999999999 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108306+0000", 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2390000000000002e-06 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173660+0000", 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0653539890000001 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173734+0000", 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3981999999999997e-05 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173865+0000", 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013074199999999999 2024-09-21T10:10:55.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176550+0000", 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002685532 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176826+0000", 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027604099999999998 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.207 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177260+0000", 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00043419999999999998 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177334+0000", 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3385000000000003e-05 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177396+0000", 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1779999999999995e-05 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177493+0000", 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7195999999999996e-05 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145379 8.14 8:2e4b539b:::benchmark_data_smithi076_241159_object22461:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 43.055040421999998, 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0704400940000001, 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.209 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145379 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:10:55.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107320+0000", 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0800000000000004e-07 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107324+0000", 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.32e-06 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.211 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107332+0000", 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6440000000000006e-06 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107338+0000", 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8980000000000001e-06 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108338+0000", 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00099989499999999999 2024-09-21T10:10:55.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108346+0000", 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0250000000000002e-06 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175028+0000", 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06668181 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175119+0000", 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0938000000000002e-05 2024-09-21T10:10:55.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175301+0000", 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018216400000000001 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176843+0000", 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015422599999999999 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177544+0000", 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00070127500000000001 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177598+0000", 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3390000000000002e-05 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177645+0000", 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7596000000000003e-05 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177677+0000", 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1538999999999997e-05 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177759+0000", 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2532000000000002e-05 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145428 8.14 8:28ec7a73:::benchmark_data_smithi076_241159_object22510:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:10:55.216 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 43.055013313000003, 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0832395619999999, 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145428 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:10:55.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0999999999999999e-07 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107349+0000", 2024-09-21T10:10:55.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0020000000000001e-06 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107355+0000", 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9989999999999997e-06 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107360+0000", 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108360+0000", 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010003779999999999 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108364+0000", 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.506e-06 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177840+0000", 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.069476358 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177971+0000", 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130772 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.178223+0000", 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025209899999999999 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183904+0000", 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056812440000000002 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183937+0000", 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2718000000000001e-05 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.186948+0000", 2024-09-21T10:10:55.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030109640000000001 2024-09-21T10:10:55.223 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.223 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.223 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.223 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190363+0000", 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003414753 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190424+0000", 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1289000000000002e-05 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190586+0000", 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016216000000000001 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:55.226 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145565 8.14 8:2ab87a67:::benchmark_data_smithi076_241159_object22647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.226 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:10:55.226 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.938077368000002, 2024-09-21T10:10:55.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96663625200000003, 2024-09-21T10:10:55.226 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.226 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145565 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.227 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224284+0000", 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.812e-06 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224290+0000", 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.874e-06 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224308+0000", 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8093000000000001e-05 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224317+0000", 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7960000000000001e-06 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224386+0000", 2024-09-21T10:10:55.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8879000000000002e-05 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224395+0000", 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6079999999999996e-06 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179328+0000", 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95493383300000001 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179451+0000", 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012236699999999999 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179658+0000", 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020740600000000001 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183981+0000", 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043224200000000004 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.186993+0000", 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030123340000000002 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189973+0000", 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029805529999999999 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190741+0000", 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000767269 2024-09-21T10:10:55.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190808+0000", 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7385999999999995e-05 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190919+0000", 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110622 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145636 8.14 8:2c7856f1:::benchmark_data_smithi076_241159_object22718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.883048410000001, 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91186562000000004, 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145636 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.234 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279313+0000", 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.612e-06 2024-09-21T10:10:55.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279318+0000", 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4700000000000001e-06 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279329+0000", 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0521e-05 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.236 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279336+0000", 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4960000000000001e-06 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279390+0000", 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4639000000000003e-05 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279399+0000", 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4519999999999997e-06 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.180800+0000", 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90140108500000005 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.180917+0000", 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011759199999999999 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.181105+0000", 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000187345 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184031+0000", 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029260890000000002 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187076+0000", 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030450529999999998 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189983+0000", 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029071829999999998 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191024+0000", 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010408940000000001 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191081+0000", 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7377000000000003e-05 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191177+0000", 2024-09-21T10:10:55.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5811999999999997e-05 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145642 8.14 8:2c2c43d9:::benchmark_data_smithi076_241159_object22724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.877312252000003, 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90634175100000003, 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.241 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145642 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285049+0000", 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5689999999999999e-06 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285054+0000", 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5680000000000001e-06 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285069+0000", 2024-09-21T10:10:55.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5790999999999998e-05 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285077+0000", 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1380000000000001e-06 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285131+0000", 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4712000000000001e-05 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285140+0000", 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5930000000000006e-06 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182422+0000", 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89728208899999995 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182525+0000", 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102975 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182730+0000", 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020512300000000001 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184080+0000", 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0013496820000000001 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187121+0000", 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030413240000000002 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189992+0000", 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028713139999999998 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191253+0000", 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012601559999999999 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191297+0000", 2024-09-21T10:10:55.247 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4267000000000002e-05 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191389+0000", 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2449999999999997e-05 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145647 8.14 8:2cdee3d5:::benchmark_data_smithi076_241159_object22729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.872726706999998, 2024-09-21T10:10:55.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90206547199999998, 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145647 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289635+0000", 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.629e-06 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289640+0000", 2024-09-21T10:10:55.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5609999999999998e-06 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289653+0000", 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2500999999999999e-05 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289660+0000", 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7909999999999999e-06 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289716+0000", 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6434999999999998e-05 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289725+0000", 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.568e-06 2024-09-21T10:10:55.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184119+0000", 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89439432600000002 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184251+0000", 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013181299999999999 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184465+0000", 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021390800000000001 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187033+0000", 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025682499999999998 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190155+0000", 2024-09-21T10:10:55.255 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031220950000000001 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190600+0000", 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00044475 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191502+0000", 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00090238500000000001 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.256 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191592+0000", 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9889999999999995e-05 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191699+0000", 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010657 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145668 8.14 8:2c4067c8:::benchmark_data_smithi076_241159_object22750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.257 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.852874657000001, 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88250502399999997, 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145668 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309487+0000", 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7969999999999999e-06 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309492+0000", 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0799999999999996e-06 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309506+0000", 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4337e-05 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309537+0000", 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1062000000000002e-05 2024-09-21T10:10:55.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309583+0000", 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5954000000000001e-05 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309592+0000", 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8950000000000004e-06 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185570+0000", 2024-09-21T10:10:55.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87597768499999995 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185659+0000", 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8683000000000005e-05 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185852+0000", 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019369 2024-09-21T10:10:55.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190093+0000", 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042406789999999998 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190606+0000", 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00051294199999999996 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190699+0000", 2024-09-21T10:10:55.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2977000000000001e-05 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191748+0000", 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00104901 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191835+0000", 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6559999999999998e-05 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.264 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191990+0000", 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000155673 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145696 8.14 8:2d97a104:::benchmark_data_smithi076_241159_object22778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.828121318000001, 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.863836041, 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.265 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145696 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:10:55.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334240+0000", 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.654e-06 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334246+0000", 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.755e-06 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334262+0000", 2024-09-21T10:10:55.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6543000000000001e-05 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334271+0000", 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1829999999999994e-06 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334339+0000", 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8495000000000001e-05 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334349+0000", 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8679999999999994e-06 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187155+0000", 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85280550300000002 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187267+0000", 2024-09-21T10:10:55.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011253500000000001 2024-09-21T10:10:55.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.270 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187487+0000", 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021978000000000001 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190612+0000", 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031255240000000002 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190657+0000", 2024-09-21T10:10:55.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4929999999999998e-05 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191219+0000", 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056208899999999999 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197900+0000", 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066807790000000004 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.272 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197963+0000", 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2599000000000002e-05 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.198075+0000", 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000111804 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145737 8.14 8:2e5266a3:::benchmark_data_smithi076_241159_object22819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.273 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.790807012999998, 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.82838703400000002, 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145737 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.274 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371554+0000", 2024-09-21T10:10:55.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5710000000000001e-06 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371559+0000", 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.101e-06 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371573+0000", 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3237999999999999e-05 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371580+0000", 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1770000000000001e-06 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371638+0000", 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8353999999999997e-05 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371644+0000", 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5829999999999997e-06 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188522+0000", 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.81687860199999995 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188653+0000", 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013063699999999999 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188863+0000", 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020946300000000001 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190934+0000", 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020711869999999999 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190975+0000", 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1170000000000001e-05 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191443+0000", 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00046765700000000001 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.279 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199763+0000", 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0083201890000000004 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199822+0000", 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8839999999999999e-05 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199940+0000", 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011826499999999999 2024-09-21T10:10:55.280 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145790 8.14 8:29323519:::benchmark_data_smithi076_241159_object22872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.730823229000002, 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76945549899999999, 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.281 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145790 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431538+0000", 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.821e-06 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431544+0000", 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3539999999999999e-06 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431558+0000", 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4664999999999999e-05 2024-09-21T10:10:55.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431566+0000", 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1099999999999997e-06 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431630+0000", 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4051000000000001e-05 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431644+0000", 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4489e-05 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192021+0000", 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76037687799999998 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192123+0000", 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102094 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192373+0000", 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024965499999999999 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197733+0000", 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053602290000000002 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197778+0000", 2024-09-21T10:10:55.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4932999999999998e-05 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199979+0000", 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022014389999999999 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200021+0000", 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2040999999999999e-05 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.287 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200873+0000", 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00085203000000000004 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200992+0000", 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011870999999999999 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.288 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145960 8.14 8:2d57b90e:::benchmark_data_smithi076_241159_object23042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.533208782999999, 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.58195981299999999, 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145960 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629153+0000", 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8870000000000001e-06 2024-09-21T10:10:55.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629158+0000", 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0730000000000004e-06 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629172+0000", 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4076e-05 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629181+0000", 2024-09-21T10:10:55.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0329999999999997e-06 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629231+0000", 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0161000000000003e-05 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629239+0000", 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0839999999999992e-06 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193615+0000", 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.56437558099999996 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193705+0000", 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0105999999999994e-05 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193902+0000", 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019732800000000001 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197795+0000", 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038931259999999998 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197850+0000", 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4941000000000001e-05 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201036+0000", 2024-09-21T10:10:55.294 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031852819999999998 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210835+0000", 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097991099999999998 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210934+0000", 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9510000000000001e-05 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.295 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211111+0000", 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000176515 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145998 8.14 8:292bcf47:::benchmark_data_smithi076_241159_object23080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.443461482000004, 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.49255907199999999, 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.296 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145998 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:10:55.297 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718900+0000", 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9690000000000001e-06 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718906+0000", 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4670000000000003e-06 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.298 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718925+0000", 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9351000000000002e-05 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718931+0000", 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6620000000000002e-06 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718991+0000", 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0074000000000001e-05 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.299 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.719002+0000", 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0701e-05 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196200+0000", 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.477198858 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196319+0000", 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118368 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.300 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196556+0000", 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023754200000000001 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201345+0000", 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0047884770000000002 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210791+0000", 2024-09-21T10:10:55.301 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0094463680000000001 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211125+0000", 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00033422200000000001 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211296+0000", 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000170445 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.302 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211355+0000", 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9215999999999997e-05 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211457+0000", 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102352 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146042 8.14 8:28c57b04:::benchmark_data_smithi076_241159_object23124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:10:55.303 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 42.315583986, 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.36551782100000002, 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146042 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:10:55.304 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846778+0000", 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7630000000000001e-06 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:10:55.305 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846783+0000", 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1340000000000003e-06 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846794+0000", 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1486e-05 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846800+0000", 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6969999999999998e-06 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.306 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846850+0000", 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0513999999999998e-05 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846856+0000", 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201389+0000", 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.35453316699999998 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.307 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201518+0000", 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012838199999999999 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201830+0000", 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031166100000000002 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211170+0000", 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.009340447 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.308 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211211+0000", 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1372000000000003e-05 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211254+0000", 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2997999999999997e-05 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211964+0000", 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00070925999999999997 2024-09-21T10:10:55.309 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.212193+0000", 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000229161 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.212294+0000", 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010087899999999999 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:10:55.310 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:10:55.311 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:10:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.312 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:55.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:55 smithi076 ceph-mon[106482]: 8.0 scrub ok 2024-09-21T10:10:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:55 smithi076 ceph-mon[106482]: 8.16 scrub starts 2024-09-21T10:10:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:55 smithi076 ceph-mon[110963]: 8.0 scrub ok 2024-09-21T10:10:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:55 smithi076 ceph-mon[110963]: 8.16 scrub starts 2024-09-21T10:10:55.413 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:55.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:55.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:55.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:55.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:55 smithi191 ceph-mon[102206]: 8.0 scrub ok 2024-09-21T10:10:55.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:55 smithi191 ceph-mon[102206]: 8.16 scrub starts 2024-09-21T10:10:55.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:55.727 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:55.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:55.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:55.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:55.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:55.996 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:55.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:56.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:56.125 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:56.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:56.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:56.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:56.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:56.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:56.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:56.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:56.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:10:56.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:56 smithi191 ceph-mon[102206]: 8.14 scrub starts 2024-09-21T10:10:56.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:56 smithi191 ceph-mon[102206]: pgmap v1744: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail 2024-09-21T10:10:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:56 smithi191 ceph-mon[102206]: 8.14 scrub ok 2024-09-21T10:10:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:56 smithi191 ceph-mon[102206]: 8.16 scrub ok 2024-09-21T10:10:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[106482]: 8.14 scrub starts 2024-09-21T10:10:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[106482]: pgmap v1744: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail 2024-09-21T10:10:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[106482]: 8.14 scrub ok 2024-09-21T10:10:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[106482]: 8.16 scrub ok 2024-09-21T10:10:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[110963]: 8.14 scrub starts 2024-09-21T10:10:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[110963]: pgmap v1744: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail 2024-09-21T10:10:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[110963]: 8.14 scrub ok 2024-09-21T10:10:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[110963]: 8.16 scrub ok 2024-09-21T10:10:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:10:56.637 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:10:56.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:56.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:56.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:10:56.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:56.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:56.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:56.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:57.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:57.074 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:57.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:57.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.206 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:57.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:57.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:10:57.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:10:57.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:57 smithi076 ceph-mon[110963]: 8.15 scrub starts 2024-09-21T10:10:57.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:57 smithi076 ceph-mon[110963]: 8.15 scrub ok 2024-09-21T10:10:57.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:57 smithi076 ceph-mon[106482]: 8.15 scrub starts 2024-09-21T10:10:57.432 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:57 smithi076 ceph-mon[106482]: 8.15 scrub ok 2024-09-21T10:10:57.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:57.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:57.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:57.605 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:57.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:57 smithi191 ceph-mon[102206]: 8.15 scrub starts 2024-09-21T10:10:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:57 smithi191 ceph-mon[102206]: 8.15 scrub ok 2024-09-21T10:10:57.621 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:10:57.621 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:10:57.654 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:10:57.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:10:57.689 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:10:57.690 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:10:57.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:57.722 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:10:57.722 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:10:57.732 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:57.755 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:10:57.755 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:10:57.784 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:10:57.784 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:10:57.817 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:10:57.818 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:10:57.834 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:10:57.846 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:10:57.847 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:10:57.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:57.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:10:57.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:58.029 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:58.131 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:58.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:58.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:58.315 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:58.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:58.397 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:58.404 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:10:58.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:58.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:58.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:58.556 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[106482]: pgmap v1745: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[106482]: 8.9 scrub starts 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[106482]: 8.9 scrub ok 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3150621843' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[110963]: pgmap v1745: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[110963]: 8.9 scrub starts 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[110963]: 8.9 scrub ok 2024-09-21T10:10:58.557 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3150621843' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:58.600 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:58.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:58 smithi191 ceph-mon[102206]: pgmap v1745: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail 2024-09-21T10:10:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:58 smithi191 ceph-mon[102206]: 8.9 scrub starts 2024-09-21T10:10:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:58 smithi191 ceph-mon[102206]: 8.9 scrub ok 2024-09-21T10:10:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3150621843' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:10:58.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.644 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:58.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:58.745 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:58.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:58.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:58.919 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:58.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:58.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:59.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:59.047 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:59.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:59.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:10:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.189 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:59.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:10:59.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:10:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:59.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:10:59.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:59.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:59.521 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:10:59.564 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:59.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:10:59.699 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:10:59.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:10:59.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:10:59.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:10:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:10:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:59 smithi191 ceph-mon[102206]: 8.18 scrub starts 2024-09-21T10:10:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:59 smithi191 ceph-mon[102206]: pgmap v1746: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail 2024-09-21T10:10:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:59 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3177021373' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:10:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:10:59 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:10:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[106482]: 8.18 scrub starts 2024-09-21T10:10:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[106482]: pgmap v1746: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail 2024-09-21T10:10:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3177021373' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:10:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:10:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[110963]: 8.18 scrub starts 2024-09-21T10:10:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:10:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[110963]: pgmap v1746: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail 2024-09-21T10:10:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3177021373' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:10:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:10:59 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:10:59.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:10:59.991 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:10:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:59.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:00.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:00.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:00.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:00.255 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:00.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:00.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:00.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:00.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:11:00.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:00.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:00.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:00.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:00.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:00.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:00.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:00.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:00 smithi191 ceph-mon[102206]: 8.18 scrub ok 2024-09-21T10:11:00.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:00 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:11:00.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:00 smithi191 ceph-mon[102206]: osdmap e1254: 8 total, 8 up, 5 in 2024-09-21T10:11:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:00 smithi076 ceph-mon[106482]: 8.18 scrub ok 2024-09-21T10:11:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:00 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:11:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:00 smithi076 ceph-mon[106482]: osdmap e1254: 8 total, 8 up, 5 in 2024-09-21T10:11:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:00 smithi076 ceph-mon[110963]: 8.18 scrub ok 2024-09-21T10:11:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:00 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:11:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:00 smithi076 ceph-mon[110963]: osdmap e1254: 8 total, 8 up, 5 in 2024-09-21T10:11:00.911 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:00.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:00.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:00.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:00.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:11:01.041 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:01.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.091 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:01.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:01.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:01.242 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:01.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:01.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:01.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:01.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:01.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:01.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:01.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:01.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:01.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:01.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:01.839 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:01.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:01 smithi191 ceph-mon[102206]: pgmap v1748: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail 2024-09-21T10:11:01.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:01 smithi076 ceph-mon[106482]: pgmap v1748: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail 2024-09-21T10:11:01.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:01 smithi076 ceph-mon[110963]: pgmap v1748: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail 2024-09-21T10:11:01.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:01.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:01.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:02.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:02.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:02.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:02.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:02.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:02.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:02.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:02.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:02.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:02.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:02.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:02.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:02.733 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:02 smithi076 ceph-mon[106482]: 8.6 scrub starts 2024-09-21T10:11:02.733 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:02 smithi076 ceph-mon[106482]: 8.6 scrub ok 2024-09-21T10:11:02.733 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:02 smithi076 ceph-mon[110963]: 8.6 scrub starts 2024-09-21T10:11:02.733 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:02 smithi076 ceph-mon[110963]: 8.6 scrub ok 2024-09-21T10:11:02.769 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:02.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:02.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:02 smithi191 ceph-mon[102206]: 8.6 scrub starts 2024-09-21T10:11:02.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:02 smithi191 ceph-mon[102206]: 8.6 scrub ok 2024-09-21T10:11:02.870 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:02.904 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:02.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:03.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:03.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:03.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:03.182 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:03.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:03.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:03.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:03.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:11:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:11:03] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:11:03.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:03.438 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:03.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:03.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:03.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:03.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:03.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:03.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:03.849 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:03.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:03.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:03 smithi191 ceph-mon[102206]: pgmap v1749: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail 2024-09-21T10:11:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:03 smithi076 ceph-mon[106482]: pgmap v1749: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail 2024-09-21T10:11:03.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:03 smithi076 ceph-mon[110963]: pgmap v1749: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail 2024-09-21T10:11:03.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:04.004 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:04.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:04.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:04.130 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:04.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:04.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:04.278 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:04.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:04.380 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:04.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:04.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:04.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:04.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:04.581 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:04.581 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:04.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:04.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:04.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:04.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:04 smithi076 ceph-mon[106482]: 8.10 scrub starts 2024-09-21T10:11:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:04 smithi076 ceph-mon[106482]: 8.10 scrub ok 2024-09-21T10:11:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:04 smithi076 ceph-mon[110963]: 8.10 scrub starts 2024-09-21T10:11:04.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:04 smithi076 ceph-mon[110963]: 8.10 scrub ok 2024-09-21T10:11:04.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:04.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:04.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:05.048 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:05.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:05.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:04 smithi191 ceph-mon[102206]: 8.10 scrub starts 2024-09-21T10:11:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:04 smithi191 ceph-mon[102206]: 8.10 scrub ok 2024-09-21T10:11:05.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:05.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:05.309 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:05.311 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:05.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:05.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:05.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:05.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:05.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:05.612 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:05.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:05.642 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:11:05.714 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:05.748 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:05.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:05.876 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:05.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:05.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:05.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:05 smithi076 ceph-mon[106482]: pgmap v1750: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail 2024-09-21T10:11:05.904 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:05 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1758069273' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:05.904 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:05 smithi076 ceph-mon[110963]: pgmap v1750: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail 2024-09-21T10:11:05.904 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:05 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1758069273' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:05.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:06.015 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:06.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:05 smithi191 ceph-mon[102206]: pgmap v1750: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail 2024-09-21T10:11:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:05 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1758069273' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:06.117 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:06.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:06.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:06.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:06.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:06.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:06.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:06.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:06.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:06.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:06.630 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:06.667 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:06.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:06.769 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:06.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1758069273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[106482]: osdmap e1255: 8 total, 8 up, 5 in 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[106482]: 8.12 scrub starts 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[106482]: 8.12 scrub ok 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1758069273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:06.841 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[110963]: osdmap e1255: 8 total, 8 up, 5 in 2024-09-21T10:11:06.842 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[110963]: 8.12 scrub starts 2024-09-21T10:11:06.842 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:06 smithi076 ceph-mon[110963]: 8.12 scrub ok 2024-09-21T10:11:06.911 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:06.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:06.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:07.048 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:07.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.083 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:07.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:06 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:11:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:06 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:11:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1758069273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:06 smithi191 ceph-mon[102206]: osdmap e1255: 8 total, 8 up, 5 in 2024-09-21T10:11:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:06 smithi191 ceph-mon[102206]: 8.12 scrub starts 2024-09-21T10:11:07.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:06 smithi191 ceph-mon[102206]: 8.12 scrub ok 2024-09-21T10:11:07.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:07.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:07.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:07.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:07.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:07.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:07.574 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:07.612 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:07.714 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:07.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:07.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:07 smithi076 ceph-mon[106482]: pgmap v1752: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail 2024-09-21T10:11:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:07 smithi076 ceph-mon[106482]: 8.11 scrub starts 2024-09-21T10:11:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:07 smithi076 ceph-mon[106482]: 8.11 scrub ok 2024-09-21T10:11:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:07 smithi076 ceph-mon[110963]: pgmap v1752: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail 2024-09-21T10:11:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:07 smithi076 ceph-mon[110963]: 8.11 scrub starts 2024-09-21T10:11:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:07 smithi076 ceph-mon[110963]: 8.11 scrub ok 2024-09-21T10:11:07.878 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:07.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:07.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:07.979 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.017 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:08.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:07 smithi191 ceph-mon[102206]: pgmap v1752: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 434 GiB / 447 GiB avail 2024-09-21T10:11:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:07 smithi191 ceph-mon[102206]: 8.11 scrub starts 2024-09-21T10:11:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:07 smithi191 ceph-mon[102206]: 8.11 scrub ok 2024-09-21T10:11:08.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:08.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:08.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:08.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:08.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:08.369 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:08.410 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:08.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:08.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:08.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:08.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:08.664 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:08.766 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:08.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:08.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:08.920 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:08.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:08.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:09.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:09.044 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:08 smithi191 ceph-mon[102206]: 8.17 scrub starts 2024-09-21T10:11:09.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:08 smithi191 ceph-mon[102206]: 8.17 scrub ok 2024-09-21T10:11:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:08 smithi076 ceph-mon[106482]: 8.17 scrub starts 2024-09-21T10:11:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:08 smithi076 ceph-mon[106482]: 8.17 scrub ok 2024-09-21T10:11:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:08 smithi076 ceph-mon[110963]: 8.17 scrub starts 2024-09-21T10:11:09.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:08 smithi076 ceph-mon[110963]: 8.17 scrub ok 2024-09-21T10:11:09.146 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:09.174 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:09.276 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:09.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:09.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:09.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:09.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:09.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:09.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:09.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:09.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:09.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:09.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:09.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:09.851 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:09.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:09.953 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:09.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:09.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:09.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:10.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:09 smithi191 ceph-mon[102206]: pgmap v1753: 99 pgs: 99 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail 2024-09-21T10:11:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:09 smithi191 ceph-mon[102206]: 8.19 scrub starts 2024-09-21T10:11:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:09 smithi191 ceph-mon[102206]: 8.19 scrub ok 2024-09-21T10:11:10.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:09 smithi076 ceph-mon[106482]: pgmap v1753: 99 pgs: 99 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail 2024-09-21T10:11:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:09 smithi076 ceph-mon[106482]: 8.19 scrub starts 2024-09-21T10:11:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:09 smithi076 ceph-mon[106482]: 8.19 scrub ok 2024-09-21T10:11:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:09 smithi076 ceph-mon[110963]: pgmap v1753: 99 pgs: 99 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail 2024-09-21T10:11:10.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:09 smithi076 ceph-mon[110963]: 8.19 scrub starts 2024-09-21T10:11:10.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:09 smithi076 ceph-mon[110963]: 8.19 scrub ok 2024-09-21T10:11:10.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:10.247 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:10.349 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:10.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:10.492 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:10.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:10.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:10.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:10.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:10.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:10.782 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:10.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:10.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:10.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:10.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:11:11.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:11.069 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:11.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:11.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:11.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:11.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:11.333 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:11.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.460 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:11.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:11.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:11.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:11.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:11.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.738 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:11.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:11.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:11.860 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:11.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:11 smithi191 ceph-mon[102206]: pgmap v1754: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:11 smithi076 ceph-mon[110963]: pgmap v1754: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:11.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:11 smithi076 ceph-mon[106482]: pgmap v1754: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:11.962 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:12.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:12.102 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:12.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.138 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:12.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:12.239 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.262 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:12.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:12.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:12.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:12.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:12.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:12.526 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:12.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:12.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:12.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.667 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:12.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:12.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:12.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:12.906 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:12.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:12.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:12.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:13.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:13.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:13.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:13.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:13.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:13.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:13.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:13.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:13.473 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:13.575 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:13.613 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:13.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:11:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:11:13] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:11:13.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:13.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:13.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:13.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:13.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:13.985 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:14.010 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:14.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:13 smithi191 ceph-mon[102206]: pgmap v1755: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:13 smithi076 ceph-mon[106482]: pgmap v1755: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:13 smithi076 ceph-mon[110963]: pgmap v1755: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:14.158 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:14.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:14.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:14.300 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:14.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:14.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:14.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:14.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:14.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:14.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:14.665 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:14.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:14.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:14.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:14.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:14.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:14.966 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:14.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:14.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:15.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:15.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:15.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:15.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:15.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:15.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:15.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:15.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:15.461 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:15.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:15.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:15.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:15.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:15.712 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:15.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:15.815 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:15.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:15.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:15.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:15.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:15.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:16.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:16.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:15 smithi191 ceph-mon[102206]: pgmap v1756: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:15 smithi076 ceph-mon[106482]: pgmap v1756: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:15 smithi076 ceph-mon[110963]: pgmap v1756: 99 pgs: 99 active+clean; 2.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail 2024-09-21T10:11:16.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.238 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:16.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:16.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:16.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:16.488 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:16.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:16.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:16.659 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:16.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:16.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:16.790 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:16.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:16.920 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:16.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:16.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:17.021 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:17.057 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:17.159 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:17.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:17.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:17.324 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:17.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:17.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:17.551 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:17.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.590 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:17.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:17.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:17.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:17.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:17.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:17.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:17.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:17.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:17.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:18.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:18.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:18.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:17 smithi191 ceph-mon[102206]: pgmap v1757: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail 2024-09-21T10:11:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:17 smithi076 ceph-mon[106482]: pgmap v1757: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail 2024-09-21T10:11:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:17 smithi076 ceph-mon[110963]: pgmap v1757: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 436 GiB / 447 GiB avail 2024-09-21T10:11:18.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:18.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:18.339 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:18.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:18.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.497 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:18.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:18.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:18.627 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:18.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:18.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:18.761 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:18.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:18.890 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:18.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:18.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:18.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:19.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:19.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:19.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:19.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:19.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:19.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:19.271 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:19.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:19.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:19.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:19.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.540 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:19.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:19.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:19.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:19.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:19.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:19.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:19.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:19.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:19.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:19.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:20.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:20.063 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:19 smithi076 ceph-mon[106482]: pgmap v1758: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail 2024-09-21T10:11:20.063 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:19 smithi076 ceph-mon[110963]: pgmap v1758: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail 2024-09-21T10:11:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:20.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:19 smithi191 ceph-mon[102206]: pgmap v1758: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail 2024-09-21T10:11:20.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:20.218 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:20.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:20.370 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:20.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:20.472 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:20.515 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:20.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:20.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:20.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:20.783 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:20.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:20.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:20.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:21.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:21.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:21.353 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:21.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:21.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:21.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:21.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:21.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:21.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:21.740 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:21.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:21.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:21.907 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:21.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:21.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:22.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:22.035 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:22.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:21 smithi191 ceph-mon[102206]: pgmap v1759: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 389 op/s 2024-09-21T10:11:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:21 smithi076 ceph-mon[106482]: pgmap v1759: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 389 op/s 2024-09-21T10:11:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:21 smithi076 ceph-mon[110963]: pgmap v1759: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 389 op/s 2024-09-21T10:11:22.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:22.157 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:22.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:22.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:22.297 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:22.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:11:22.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:22.524 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:22.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.676 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:22.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:22.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:22.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:22.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:22.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:22.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:23.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:23.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:23.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:23.196 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:23.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:23.325 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:23.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:23.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:23.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:23.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:23.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:11:23] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.51.0" 2024-09-21T10:11:23.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:23.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:23.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:23.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:23.862 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:23.957 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:11:23.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:23.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:23.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:24.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:23 smithi191 ceph-mon[102206]: pgmap v1760: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 389 op/s 2024-09-21T10:11:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:23 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/609110226' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:11:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:23 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:11:24.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-mon[106482]: pgmap v1760: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 389 op/s 2024-09-21T10:11:24.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/609110226' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:11:24.125 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:11:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-mon[110963]: pgmap v1760: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 389 op/s 2024-09-21T10:11:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/609110226' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:11:24.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:23 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:11:24.132 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:24.196 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:24.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:24.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:24.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:24.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:24.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:24.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:24.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:24.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:24.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:24.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:24.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:24.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:24.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:24.796 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:24.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:24.892 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:11:24.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:11:24.893 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:24.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:24.923 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:24.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:25.024 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:25.078 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:25.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:24 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:11:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:24 smithi191 ceph-mon[102206]: osdmap e1256: 8 total, 8 up, 5 in 2024-09-21T10:11:25.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:24 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:11:25.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:24 smithi076 ceph-mon[106482]: osdmap e1256: 8 total, 8 up, 5 in 2024-09-21T10:11:25.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:24 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:11:25.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:24 smithi076 ceph-mon[110963]: osdmap e1256: 8 total, 8 up, 5 in 2024-09-21T10:11:25.180 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:25.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:25.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:25.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:25.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:25.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:25.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:25.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:25.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.637 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:25.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:25.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:25.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:25.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:25.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:25.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:25.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:26.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:26.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:26.082 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:25 smithi076 ceph-mon[106482]: pgmap v1762: 99 pgs: 99 active+clean; 1.9 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:11:26.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:26.083 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:25 smithi076 ceph-mon[110963]: pgmap v1762: 99 pgs: 99 active+clean; 1.9 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:11:26.083 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:25 smithi191 ceph-mon[102206]: pgmap v1762: 99 pgs: 99 active+clean; 1.9 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:11:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:26.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:26.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:26.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:26.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:26.320 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:26.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:26.455 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:26.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.584 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:26.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:26.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:26.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:26.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:26.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:26.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:26.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:26.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:27.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:27.071 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1464960396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:27.071 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1464960396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:27.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:27.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1464960396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:27.129 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:27.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:27.230 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:27.272 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:27.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:27.374 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:27.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:27.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:27.508 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:27.541 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:27.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:27.643 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:27.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:27.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:27.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:27.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:27.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:27.947 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:27.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:28.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:28.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:28.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:28.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:27 smithi191 ceph-mon[102206]: pgmap v1763: 99 pgs: 99 active+clean; 1.9 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:11:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:27 smithi076 ceph-mon[106482]: pgmap v1763: 99 pgs: 99 active+clean; 1.9 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:11:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:27 smithi076 ceph-mon[110963]: pgmap v1763: 99 pgs: 99 active+clean; 1.9 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s 2024-09-21T10:11:28.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:28.218 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:28.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:28.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:28.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:28.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:28.469 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:28.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:28.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:28.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:28.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:28.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:28.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:28.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:28.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:28.606 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:28.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:28.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:28.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:28.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:28.870 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:28.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:28.971 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:29.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:29.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:29.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:29.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:29.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:29.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:29.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:29.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:29.387 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:29.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:29.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:29.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:29.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:29.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:29.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:29.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:29.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:29.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:30.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:30.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:29 smithi191 ceph-mon[102206]: pgmap v1764: 99 pgs: 99 active+clean; 1.5 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:11:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:29 smithi076 ceph-mon[106482]: pgmap v1764: 99 pgs: 99 active+clean; 1.5 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:11:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:29 smithi076 ceph-mon[110963]: pgmap v1764: 99 pgs: 99 active+clean; 1.5 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:11:30.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:30.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:30.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.348 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:30.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:30.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:30.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:30.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:30.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:30.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:30.769 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:30.871 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:30.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:30.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:31.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:31.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:31.148 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:31.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:31.214 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:31.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:31.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:31.418 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:11:31.418 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:11:31.418 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:11:31.419 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:11:31.419 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:11:31.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:31.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:31.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:31.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:31.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:31.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:31.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:31.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:31.867 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:11:31.883 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:31 smithi076 ceph-mon[106482]: pgmap v1765: 99 pgs: 99 active+clean; 1.5 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:11:31.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:31.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:31.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:31.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:31.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:32.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:32.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:32.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:32.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:32.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:31 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/398912844' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:32.251 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:31 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:32.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:31 smithi076 ceph-mon[110963]: pgmap v1765: 99 pgs: 99 active+clean; 1.5 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:11:32.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:31 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/398912844' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:32.252 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:31 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:32.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:32.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:32.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:31 smithi191 ceph-mon[102206]: pgmap v1765: 99 pgs: 99 active+clean; 1.5 GiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:11:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:31 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/398912844' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:31 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:11:32.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:32.432 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:32.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:32.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:32.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:32.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.708 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:32.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:32.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:32.832 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:32.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:32.956 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:32.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:33.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:33.093 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:33.154 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:32 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:33.154 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:32 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:33.154 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:32 smithi076 ceph-mon[106482]: osdmap e1257: 8 total, 8 up, 5 in 2024-09-21T10:11:33.154 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:32 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:33.154 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:32 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:33.154 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:32 smithi076 ceph-mon[110963]: osdmap e1257: 8 total, 8 up, 5 in 2024-09-21T10:11:33.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:11:33.194 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:33.242 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:33.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:33.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:33.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:32 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:33.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:32 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:11:33.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:32 smithi191 ceph-mon[102206]: osdmap e1257: 8 total, 8 up, 5 in 2024-09-21T10:11:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:33.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:33.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:33.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:33.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:11:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:11:33] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.51.0" 2024-09-21T10:11:33.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:33.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:33.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:33.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:33.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:33.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:33.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:34.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.079 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:34.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:33 smithi076 ceph-mon[106482]: pgmap v1767: 99 pgs: 99 active+clean; 1.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 675 op/s 2024-09-21T10:11:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:33 smithi076 ceph-mon[110963]: pgmap v1767: 99 pgs: 99 active+clean; 1.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 675 op/s 2024-09-21T10:11:34.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:34.230 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:34.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:34.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:34.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:33 smithi191 ceph-mon[102206]: pgmap v1767: 99 pgs: 99 active+clean; 1.5 GiB data, 7.7 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 675 op/s 2024-09-21T10:11:34.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:34.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:34.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:34.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.633 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:34.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:34.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:34.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:34.819 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:34.825 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:34.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:34.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:34.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:35.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.057 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.106 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:11:35.109 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:11:35.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:35.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:35.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:35.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:35.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:35.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:35.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:35.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:35.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:35.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:35.582 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:35.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:35.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:35.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:35.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:35.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:35.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:35.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:35.984 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:35.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:36.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:36.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:36.158 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:35 smithi076 ceph-mon[106482]: pgmap v1768: 99 pgs: 99 active+clean; 1.2 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:11:36.158 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:35 smithi076 ceph-mon[106482]: from='client.55177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:36.158 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:35 smithi076 ceph-mon[110963]: pgmap v1768: 99 pgs: 99 active+clean; 1.2 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:11:36.158 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:35 smithi076 ceph-mon[110963]: from='client.55177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:36.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:36.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:11:36.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:36.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:36.340 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:35 smithi191 ceph-mon[102206]: pgmap v1768: 99 pgs: 99 active+clean; 1.2 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:11:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:35 smithi191 ceph-mon[102206]: from='client.55177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:36.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:36.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:36.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:36.512 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:36.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:36.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:36.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:36.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:36.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:36.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:36.920 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:36.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:37.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:37.055 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:37.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:37.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:37.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:37.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.705 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:37.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:37.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:37.833 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:37.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:37.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:37.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:38.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:38.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:38.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.143 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:38.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:38.212 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:37 smithi076 ceph-mon[106482]: pgmap v1769: 99 pgs: 99 active+clean; 1.2 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:11:38.213 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:37 smithi076 ceph-mon[110963]: pgmap v1769: 99 pgs: 99 active+clean; 1.2 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:11:38.244 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:38.283 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:37 smithi191 ceph-mon[102206]: pgmap v1769: 99 pgs: 99 active+clean; 1.2 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:11:38.385 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:38.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:38.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:38.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:38.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:38.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:38.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:38.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:38.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:38.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:38.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:38.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:38.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:38.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:38.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:39.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:39.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:39.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:39.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:39.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:39.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:39.421 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:39.459 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:39.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:39.584 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:39.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:39.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:39.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:39.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:39.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:39.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:39.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:40.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:40.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:40.096 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:39 smithi076 ceph-mon[106482]: pgmap v1770: 99 pgs: 99 active+clean; 805 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:11:40.096 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:39 smithi076 ceph-mon[110963]: pgmap v1770: 99 pgs: 99 active+clean; 805 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:11:40.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:40.129 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:40.231 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:40.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:40.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:40.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:40.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:39 smithi191 ceph-mon[102206]: pgmap v1770: 99 pgs: 99 active+clean; 805 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:11:40.389 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:40.423 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1768,"stamp":"2024-09-21T10:11:34.739145+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1251973117,"num_objects":19121,"num_object_clones":0,"num_object_copies":57363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19121,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":120169,"num_write_kb":3189330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":2963020,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":159858,"ondisk_log_size":159858,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":7363416,"kb_used_data":4381224,"kb_used_omap":119,"kb_used_meta":2981960,"kb_avail":461362344,"statfs":{"total":479975178240,"available":472435040256,"internally_reserved":0,"allocated":4486373376,"data_stored":4472437425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":3053527240},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,1,6,3,6,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-793772032,"num_objects":-12112,"num_object_clones":0,"num_object_copies":-36336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-12112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12112,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002174"},"pg_stats":[{"pgid":"8.18","version":"1257'10984","reported_seq":16126,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.771070+0000","last_change":"2024-09-21T10:10:59.104488+0000","last_active":"2024-09-21T10:11:32.771070+0000","last_peered":"2024-09-21T10:11:32.771070+0000","last_clean":"2024-09-21T10:11:32.771070+0000","last_became_active":"2024-09-21T10:10:49.084472+0000","last_became_peered":"2024-09-21T10:10:49.084472+0000","last_unstale":"2024-09-21T10:11:32.771070+0000","last_undegraded":"2024-09-21T10:11:32.771070+0000","last_fullsized":"2024-09-21T10:11:32.771070+0000","mapping_epoch":1251,"log_start":"1215'4323","ondisk_log_start":"1215'4323","created":1220,"last_epoch_clean":1252,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9782","last_scrub_stamp":"2024-09-21T10:10:59.104281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:59.104281+0000","objects_scrubbed":1801,"log_size":6661,"log_dups_size":0,"ondisk_log_size":6661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:51.831876+0000","scrub_duration":650,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38535168,"num_objects":588,"num_object_clones":0,"num_object_copies":1764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1213,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876047+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:11:31.876047+0000","last_peered":"2024-09-21T10:11:31.876047+0000","last_clean":"2024-09-21T10:11:31.876047+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:11:31.876047+0000","last_undegraded":"2024-09-21T10:11:31.876047+0000","last_fullsized":"2024-09-21T10:11:31.876047+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":115,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615580+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:11:24.615580+0000","last_peered":"2024-09-21T10:11:24.615580+0000","last_clean":"2024-09-21T10:11:24.615580+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:11:24.615580+0000","last_undegraded":"2024-09-21T10:11:24.615580+0000","last_fullsized":"2024-09-21T10:11:24.615580+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"1257'11473","reported_seq":16896,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.207262+0000","last_change":"2024-09-21T10:11:09.179304+0000","last_active":"2024-09-21T10:11:33.207262+0000","last_peered":"2024-09-21T10:11:33.207262+0000","last_clean":"2024-09-21T10:11:33.207262+0000","last_became_active":"2024-09-21T10:10:49.085004+0000","last_became_peered":"2024-09-21T10:10:49.085004+0000","last_unstale":"2024-09-21T10:11:33.207262+0000","last_undegraded":"2024-09-21T10:11:33.207262+0000","last_fullsized":"2024-09-21T10:11:33.207262+0000","mapping_epoch":1251,"log_start":"1215'4630","ondisk_log_start":"1215'4630","created":1220,"last_epoch_clean":1252,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10556","last_scrub_stamp":"2024-09-21T10:11:09.179143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:09.179143+0000","objects_scrubbed":1525,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:45:29.481256+0000","scrub_duration":0.41433468099999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39649280,"num_objects":605,"num_object_clones":0,"num_object_copies":1815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":920,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1337,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875391+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:11:31.875391+0000","last_peered":"2024-09-21T10:11:31.875391+0000","last_clean":"2024-09-21T10:11:31.875391+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:11:31.875391+0000","last_undegraded":"2024-09-21T10:11:31.875391+0000","last_fullsized":"2024-09-21T10:11:31.875391+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1669,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875525+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:11:31.875525+0000","last_peered":"2024-09-21T10:11:31.875525+0000","last_clean":"2024-09-21T10:11:31.875525+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:11:31.875525+0000","last_undegraded":"2024-09-21T10:11:31.875525+0000","last_fullsized":"2024-09-21T10:11:31.875525+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1669,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877479+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:11:31.877479+0000","last_peered":"2024-09-21T10:11:31.877479+0000","last_clean":"2024-09-21T10:11:31.877479+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:11:31.877479+0000","last_undegraded":"2024-09-21T10:11:31.877479+0000","last_fullsized":"2024-09-21T10:11:31.877479+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615689+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:11:24.615689+0000","last_peered":"2024-09-21T10:11:24.615689+0000","last_clean":"2024-09-21T10:11:24.615689+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:11:24.615689+0000","last_undegraded":"2024-09-21T10:11:24.615689+0000","last_fullsized":"2024-09-21T10:11:24.615689+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876685+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:11:31.876685+0000","last_peered":"2024-09-21T10:11:31.876685+0000","last_clean":"2024-09-21T10:11:31.876685+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:11:31.876685+0000","last_undegraded":"2024-09-21T10:11:31.876685+0000","last_fullsized":"2024-09-21T10:11:31.876685+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":138,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615310+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:11:24.615310+0000","last_peered":"2024-09-21T10:11:24.615310+0000","last_clean":"2024-09-21T10:11:24.615310+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:11:24.615310+0000","last_undegraded":"2024-09-21T10:11:24.615310+0000","last_fullsized":"2024-09-21T10:11:24.615310+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1257'11152","reported_seq":11274,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.208139+0000","last_change":"2024-09-21T10:10:51.107946+0000","last_active":"2024-09-21T10:11:33.208139+0000","last_peered":"2024-09-21T10:11:33.208139+0000","last_clean":"2024-09-21T10:11:33.208139+0000","last_became_active":"2024-09-21T10:09:54.319190+0000","last_became_peered":"2024-09-21T10:09:54.319190+0000","last_unstale":"2024-09-21T10:11:33.208139+0000","last_undegraded":"2024-09-21T10:11:33.208139+0000","last_fullsized":"2024-09-21T10:11:33.208139+0000","mapping_epoch":1208,"log_start":"1214'4028","ondisk_log_start":"1214'4028","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9591","last_scrub_stamp":"2024-09-21T10:10:51.107785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:51.107785+0000","objects_scrubbed":2178,"log_size":7124,"log_dups_size":0,"ondisk_log_size":7124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:43.774199+0000","scrub_duration":0.351714576,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39845888,"num_objects":608,"num_object_clones":0,"num_object_copies":1824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1570,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":208,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890330+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:11:31.890330+0000","last_peered":"2024-09-21T10:11:31.890330+0000","last_clean":"2024-09-21T10:11:31.890330+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:11:31.890330+0000","last_undegraded":"2024-09-21T10:11:31.890330+0000","last_fullsized":"2024-09-21T10:11:31.890330+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615574+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:11:24.615574+0000","last_peered":"2024-09-21T10:11:24.615574+0000","last_clean":"2024-09-21T10:11:24.615574+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:11:24.615574+0000","last_undegraded":"2024-09-21T10:11:24.615574+0000","last_fullsized":"2024-09-21T10:11:24.615574+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"1257'11412","reported_seq":11563,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.228729+0000","last_change":"2024-09-21T10:09:56.519177+0000","last_active":"2024-09-21T10:11:33.228729+0000","last_peered":"2024-09-21T10:11:33.228729+0000","last_clean":"2024-09-21T10:11:33.228729+0000","last_became_active":"2024-09-21T10:09:54.326338+0000","last_became_peered":"2024-09-21T10:09:54.326338+0000","last_unstale":"2024-09-21T10:11:33.228729+0000","last_undegraded":"2024-09-21T10:11:33.228729+0000","last_fullsized":"2024-09-21T10:11:33.228729+0000","mapping_epoch":1208,"log_start":"1215'4464","ondisk_log_start":"1215'4464","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1221'8437","last_scrub_stamp":"2024-09-21T10:09:56.518403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:09:56.518403+0000","objects_scrubbed":3599,"log_size":6948,"log_dups_size":0,"ondisk_log_size":6948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:23.788407+0000","scrub_duration":0.71259566799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40108032,"num_objects":612,"num_object_clones":0,"num_object_copies":1836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2987,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":252,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883646+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:11:31.883646+0000","last_peered":"2024-09-21T10:11:31.883646+0000","last_clean":"2024-09-21T10:11:31.883646+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:11:31.883646+0000","last_undegraded":"2024-09-21T10:11:31.883646+0000","last_fullsized":"2024-09-21T10:11:31.883646+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1293,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890605+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:11:31.890605+0000","last_peered":"2024-09-21T10:11:31.890605+0000","last_clean":"2024-09-21T10:11:31.890605+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:11:31.890605+0000","last_undegraded":"2024-09-21T10:11:31.890605+0000","last_fullsized":"2024-09-21T10:11:31.890605+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"1257'11028","reported_seq":11144,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.252327+0000","last_change":"2024-09-21T10:11:01.235231+0000","last_active":"2024-09-21T10:11:33.252327+0000","last_peered":"2024-09-21T10:11:33.252327+0000","last_clean":"2024-09-21T10:11:33.252327+0000","last_became_active":"2024-09-21T10:09:54.327583+0000","last_became_peered":"2024-09-21T10:09:54.327583+0000","last_unstale":"2024-09-21T10:11:33.252327+0000","last_undegraded":"2024-09-21T10:11:33.252327+0000","last_fullsized":"2024-09-21T10:11:33.252327+0000","mapping_epoch":1208,"log_start":"1214'4068","ondisk_log_start":"1214'4068","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1254'9787","last_scrub_stamp":"2024-09-21T10:11:01.235092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:01.235092+0000","objects_scrubbed":1814,"log_size":6960,"log_dups_size":0,"ondisk_log_size":6960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:36.650925+0000","scrub_duration":0.39794432699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1246,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":126,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.616102+0000","last_change":"2024-09-21T10:08:43.398923+0000","last_active":"2024-09-21T10:11:24.616102+0000","last_peered":"2024-09-21T10:11:24.616102+0000","last_clean":"2024-09-21T10:11:24.616102+0000","last_became_active":"2024-09-21T10:08:43.383598+0000","last_became_peered":"2024-09-21T10:08:43.383598+0000","last_unstale":"2024-09-21T10:11:24.616102+0000","last_undegraded":"2024-09-21T10:11:24.616102+0000","last_fullsized":"2024-09-21T10:11:24.616102+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:09.939781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615486+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:11:24.615486+0000","last_peered":"2024-09-21T10:11:24.615486+0000","last_clean":"2024-09-21T10:11:24.615486+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:11:24.615486+0000","last_undegraded":"2024-09-21T10:11:24.615486+0000","last_fullsized":"2024-09-21T10:11:24.615486+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1257'11115","reported_seq":11276,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.823615+0000","last_change":"2024-09-21T10:10:52.094294+0000","last_active":"2024-09-21T10:11:32.823615+0000","last_peered":"2024-09-21T10:11:32.823615+0000","last_clean":"2024-09-21T10:11:32.823615+0000","last_became_active":"2024-09-21T10:09:54.329468+0000","last_became_peered":"2024-09-21T10:09:54.329468+0000","last_unstale":"2024-09-21T10:11:32.823615+0000","last_undegraded":"2024-09-21T10:11:32.823615+0000","last_fullsized":"2024-09-21T10:11:32.823615+0000","mapping_epoch":1208,"log_start":"1215'5179","ondisk_log_start":"1215'5179","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9610","last_scrub_stamp":"2024-09-21T10:10:52.094143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:52.094143+0000","objects_scrubbed":2145,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:03.544577+0000","scrub_duration":665,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41680896,"num_objects":636,"num_object_clones":0,"num_object_copies":1908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1509,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1323,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875905+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:11:31.875905+0000","last_peered":"2024-09-21T10:11:31.875905+0000","last_clean":"2024-09-21T10:11:31.875905+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:11:31.875905+0000","last_undegraded":"2024-09-21T10:11:31.875905+0000","last_fullsized":"2024-09-21T10:11:31.875905+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1679,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.889966+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:11:31.889966+0000","last_peered":"2024-09-21T10:11:31.889966+0000","last_clean":"2024-09-21T10:11:31.889966+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:11:31.889966+0000","last_undegraded":"2024-09-21T10:11:31.889966+0000","last_fullsized":"2024-09-21T10:11:31.889966+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"1257'11206","reported_seq":11395,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.764784+0000","last_change":"2024-09-21T10:10:54.158428+0000","last_active":"2024-09-21T10:11:32.764784+0000","last_peered":"2024-09-21T10:11:32.764784+0000","last_clean":"2024-09-21T10:11:32.764784+0000","last_became_active":"2024-09-21T10:09:54.318768+0000","last_became_peered":"2024-09-21T10:09:54.318768+0000","last_unstale":"2024-09-21T10:11:32.764784+0000","last_undegraded":"2024-09-21T10:11:32.764784+0000","last_fullsized":"2024-09-21T10:11:32.764784+0000","mapping_epoch":1208,"log_start":"1215'4836","ondisk_log_start":"1215'4836","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9764","last_scrub_stamp":"2024-09-21T10:10:54.158201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:54.158201+0000","objects_scrubbed":2066,"log_size":6370,"log_dups_size":0,"ondisk_log_size":6370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:24:47.620753+0000","scrub_duration":676,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39780352,"num_objects":607,"num_object_clones":0,"num_object_copies":1821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1459,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":361,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876889+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:11:31.876889+0000","last_peered":"2024-09-21T10:11:31.876889+0000","last_clean":"2024-09-21T10:11:31.876889+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:11:31.876889+0000","last_undegraded":"2024-09-21T10:11:31.876889+0000","last_fullsized":"2024-09-21T10:11:31.876889+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1338,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875817+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:11:31.875817+0000","last_peered":"2024-09-21T10:11:31.875817+0000","last_clean":"2024-09-21T10:11:31.875817+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:11:31.875817+0000","last_undegraded":"2024-09-21T10:11:31.875817+0000","last_fullsized":"2024-09-21T10:11:31.875817+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"1257'11395","reported_seq":11567,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.868619+0000","last_change":"2024-09-21T10:09:55.880497+0000","last_active":"2024-09-21T10:11:32.868619+0000","last_peered":"2024-09-21T10:11:32.868619+0000","last_clean":"2024-09-21T10:11:32.868619+0000","last_became_active":"2024-09-21T10:09:54.328846+0000","last_became_peered":"2024-09-21T10:09:54.328846+0000","last_unstale":"2024-09-21T10:11:32.868619+0000","last_undegraded":"2024-09-21T10:11:32.868619+0000","last_fullsized":"2024-09-21T10:11:32.868619+0000","mapping_epoch":1208,"log_start":"1215'5400","ondisk_log_start":"1215'5400","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1221'8475","last_scrub_stamp":"2024-09-21T10:09:55.880305+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:09:55.880305+0000","objects_scrubbed":3550,"log_size":5995,"log_dups_size":0,"ondisk_log_size":5995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:00.793347+0000","scrub_duration":1167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39452672,"num_objects":602,"num_object_clones":0,"num_object_copies":1806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2948,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615536+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:11:24.615536+0000","last_peered":"2024-09-21T10:11:24.615536+0000","last_clean":"2024-09-21T10:11:24.615536+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:11:24.615536+0000","last_undegraded":"2024-09-21T10:11:24.615536+0000","last_fullsized":"2024-09-21T10:11:24.615536+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":981,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.887233+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:11:31.887233+0000","last_peered":"2024-09-21T10:11:31.887233+0000","last_clean":"2024-09-21T10:11:31.887233+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:11:31.887233+0000","last_undegraded":"2024-09-21T10:11:31.887233+0000","last_fullsized":"2024-09-21T10:11:31.887233+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"1257'11291","reported_seq":11422,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.753540+0000","last_change":"2024-09-21T10:09:54.371352+0000","last_active":"2024-09-21T10:11:32.753540+0000","last_peered":"2024-09-21T10:11:32.753540+0000","last_clean":"2024-09-21T10:11:32.753540+0000","last_became_active":"2024-09-21T10:09:54.340794+0000","last_became_peered":"2024-09-21T10:09:54.340794+0000","last_unstale":"2024-09-21T10:11:32.753540+0000","last_undegraded":"2024-09-21T10:11:32.753540+0000","last_fullsized":"2024-09-21T10:11:32.753540+0000","mapping_epoch":1208,"log_start":"1215'5079","ondisk_log_start":"1215'5079","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":6212,"log_dups_size":0,"ondisk_log_size":6212,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:11:48.458801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34177024,"num_objects":521,"num_object_clones":0,"num_object_copies":1563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7188,"num_write_kb":246720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1324,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890336+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:11:31.890336+0000","last_peered":"2024-09-21T10:11:31.890336+0000","last_clean":"2024-09-21T10:11:31.890336+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:11:31.890336+0000","last_undegraded":"2024-09-21T10:11:31.890336+0000","last_fullsized":"2024-09-21T10:11:31.890336+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":109,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615815+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:11:24.615815+0000","last_peered":"2024-09-21T10:11:24.615815+0000","last_clean":"2024-09-21T10:11:24.615815+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:11:24.615815+0000","last_undegraded":"2024-09-21T10:11:24.615815+0000","last_fullsized":"2024-09-21T10:11:24.615815+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"1257'11094","reported_seq":11255,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.846267+0000","last_change":"2024-09-21T10:10:51.164053+0000","last_active":"2024-09-21T10:11:32.846267+0000","last_peered":"2024-09-21T10:11:32.846267+0000","last_clean":"2024-09-21T10:11:32.846267+0000","last_became_active":"2024-09-21T10:09:54.326478+0000","last_became_peered":"2024-09-21T10:09:54.326478+0000","last_unstale":"2024-09-21T10:11:32.846267+0000","last_undegraded":"2024-09-21T10:11:32.846267+0000","last_fullsized":"2024-09-21T10:11:32.846267+0000","mapping_epoch":1208,"log_start":"1215'5123","ondisk_log_start":"1215'5123","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9542","last_scrub_stamp":"2024-09-21T10:10:51.163756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:51.163756+0000","objects_scrubbed":2162,"log_size":5971,"log_dups_size":0,"ondisk_log_size":5971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:13:36.714601+0000","scrub_duration":728,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876056+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:11:31.876056+0000","last_peered":"2024-09-21T10:11:31.876056+0000","last_clean":"2024-09-21T10:11:31.876056+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:11:31.876056+0000","last_undegraded":"2024-09-21T10:11:31.876056+0000","last_fullsized":"2024-09-21T10:11:31.876056+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1683,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890642+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:11:31.890642+0000","last_peered":"2024-09-21T10:11:31.890642+0000","last_clean":"2024-09-21T10:11:31.890642+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:11:31.890642+0000","last_undegraded":"2024-09-21T10:11:31.890642+0000","last_fullsized":"2024-09-21T10:11:31.890642+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1257'11321","reported_seq":11438,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.155634+0000","last_change":"2024-09-21T10:10:57.217425+0000","last_active":"2024-09-21T10:11:33.155634+0000","last_peered":"2024-09-21T10:11:33.155634+0000","last_clean":"2024-09-21T10:11:33.155634+0000","last_became_active":"2024-09-21T10:09:54.339432+0000","last_became_peered":"2024-09-21T10:09:54.339432+0000","last_unstale":"2024-09-21T10:11:33.155634+0000","last_undegraded":"2024-09-21T10:11:33.155634+0000","last_fullsized":"2024-09-21T10:11:33.155634+0000","mapping_epoch":1208,"log_start":"1214'4215","ondisk_log_start":"1214'4215","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'10025","last_scrub_stamp":"2024-09-21T10:10:57.217250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:57.217250+0000","objects_scrubbed":1929,"log_size":7106,"log_dups_size":0,"ondisk_log_size":7106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:17.043980+0000","scrub_duration":0.43705535000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40435712,"num_objects":617,"num_object_clones":0,"num_object_copies":1851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1312,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":782,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615148+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:11:24.615148+0000","last_peered":"2024-09-21T10:11:24.615148+0000","last_clean":"2024-09-21T10:11:24.615148+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:11:24.615148+0000","last_undegraded":"2024-09-21T10:11:24.615148+0000","last_fullsized":"2024-09-21T10:11:24.615148+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1337,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876565+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:11:31.876565+0000","last_peered":"2024-09-21T10:11:31.876565+0000","last_clean":"2024-09-21T10:11:31.876565+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:11:31.876565+0000","last_undegraded":"2024-09-21T10:11:31.876565+0000","last_fullsized":"2024-09-21T10:11:31.876565+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":368,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877151+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:11:31.877151+0000","last_peered":"2024-09-21T10:11:31.877151+0000","last_clean":"2024-09-21T10:11:31.877151+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:11:31.877151+0000","last_undegraded":"2024-09-21T10:11:31.877151+0000","last_fullsized":"2024-09-21T10:11:31.877151+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.616027+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:11:24.616027+0000","last_peered":"2024-09-21T10:11:24.616027+0000","last_clean":"2024-09-21T10:11:24.616027+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:11:24.616027+0000","last_undegraded":"2024-09-21T10:11:24.616027+0000","last_fullsized":"2024-09-21T10:11:24.616027+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1257'14204","reported_seq":14326,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.902275+0000","last_change":"2024-09-21T10:08:03.312292+0000","last_active":"2024-09-21T10:11:32.902275+0000","last_peered":"2024-09-21T10:11:32.902275+0000","last_clean":"2024-09-21T10:11:32.902275+0000","last_became_active":"2024-09-21T10:08:03.312011+0000","last_became_peered":"2024-09-21T10:08:03.312011+0000","last_unstale":"2024-09-21T10:11:32.902275+0000","last_undegraded":"2024-09-21T10:11:32.902275+0000","last_fullsized":"2024-09-21T10:11:32.902275+0000","mapping_epoch":1208,"log_start":"1246'9725","ondisk_log_start":"1246'9725","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4479,"log_dups_size":0,"ondisk_log_size":4479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:45.654425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14204,"num_write_kb":492928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":967,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.881909+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:11:31.881909+0000","last_peered":"2024-09-21T10:11:31.881909+0000","last_clean":"2024-09-21T10:11:31.881909+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:11:31.881909+0000","last_undegraded":"2024-09-21T10:11:31.881909+0000","last_fullsized":"2024-09-21T10:11:31.881909+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1556,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876578+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:11:31.876578+0000","last_peered":"2024-09-21T10:11:31.876578+0000","last_clean":"2024-09-21T10:11:31.876578+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:11:31.876578+0000","last_undegraded":"2024-09-21T10:11:31.876578+0000","last_fullsized":"2024-09-21T10:11:31.876578+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":965,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883437+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:11:31.883437+0000","last_peered":"2024-09-21T10:11:31.883437+0000","last_clean":"2024-09-21T10:11:31.883437+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:11:31.883437+0000","last_undegraded":"2024-09-21T10:11:31.883437+0000","last_fullsized":"2024-09-21T10:11:31.883437+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"1257'14130","reported_seq":14252,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.760892+0000","last_change":"2024-09-21T10:08:03.024270+0000","last_active":"2024-09-21T10:11:32.760892+0000","last_peered":"2024-09-21T10:11:32.760892+0000","last_clean":"2024-09-21T10:11:32.760892+0000","last_became_active":"2024-09-21T10:08:03.023879+0000","last_became_peered":"2024-09-21T10:08:03.023879+0000","last_unstale":"2024-09-21T10:11:32.760892+0000","last_undegraded":"2024-09-21T10:11:32.760892+0000","last_fullsized":"2024-09-21T10:11:32.760892+0000","mapping_epoch":1208,"log_start":"1243'9403","ondisk_log_start":"1243'9403","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4727,"log_dups_size":0,"ondisk_log_size":4727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:58:43.728824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77594624,"num_objects":1184,"num_object_clones":0,"num_object_copies":3552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14130,"num_write_kb":490048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1597,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876892+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:11:31.876892+0000","last_peered":"2024-09-21T10:11:31.876892+0000","last_clean":"2024-09-21T10:11:31.876892+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:11:31.876892+0000","last_undegraded":"2024-09-21T10:11:31.876892+0000","last_fullsized":"2024-09-21T10:11:31.876892+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1684,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882225+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:11:31.882225+0000","last_peered":"2024-09-21T10:11:31.882225+0000","last_clean":"2024-09-21T10:11:31.882225+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:11:31.882225+0000","last_undegraded":"2024-09-21T10:11:31.882225+0000","last_fullsized":"2024-09-21T10:11:31.882225+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1692,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890478+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:11:31.890478+0000","last_peered":"2024-09-21T10:11:31.890478+0000","last_clean":"2024-09-21T10:11:31.890478+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:11:31.890478+0000","last_undegraded":"2024-09-21T10:11:31.890478+0000","last_fullsized":"2024-09-21T10:11:31.890478+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1257'10998","reported_seq":11155,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.805480+0000","last_change":"2024-09-21T10:10:53.152556+0000","last_active":"2024-09-21T10:11:32.805480+0000","last_peered":"2024-09-21T10:11:32.805480+0000","last_clean":"2024-09-21T10:11:32.805480+0000","last_became_active":"2024-09-21T10:09:54.333306+0000","last_became_peered":"2024-09-21T10:09:54.333306+0000","last_unstale":"2024-09-21T10:11:32.805480+0000","last_undegraded":"2024-09-21T10:11:32.805480+0000","last_fullsized":"2024-09-21T10:11:32.805480+0000","mapping_epoch":1208,"log_start":"1215'4698","ondisk_log_start":"1215'4698","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9546","last_scrub_stamp":"2024-09-21T10:10:53.152277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:53.152277+0000","objects_scrubbed":2127,"log_size":6300,"log_dups_size":0,"ondisk_log_size":6300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:43:36.899463+0000","scrub_duration":652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43515904,"num_objects":664,"num_object_clones":0,"num_object_copies":1992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1463,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":261,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882942+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:11:31.882942+0000","last_peered":"2024-09-21T10:11:31.882942+0000","last_clean":"2024-09-21T10:11:31.882942+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:11:31.882942+0000","last_undegraded":"2024-09-21T10:11:31.882942+0000","last_fullsized":"2024-09-21T10:11:31.882942+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1220,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615137+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:11:24.615137+0000","last_peered":"2024-09-21T10:11:24.615137+0000","last_clean":"2024-09-21T10:11:24.615137+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:11:24.615137+0000","last_undegraded":"2024-09-21T10:11:24.615137+0000","last_fullsized":"2024-09-21T10:11:24.615137+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1695,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886194+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:11:31.886194+0000","last_peered":"2024-09-21T10:11:31.886194+0000","last_clean":"2024-09-21T10:11:31.886194+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:11:31.886194+0000","last_undegraded":"2024-09-21T10:11:31.886194+0000","last_fullsized":"2024-09-21T10:11:31.886194+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1257'14052","reported_seq":14174,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.872294+0000","last_change":"2024-09-21T10:08:03.313515+0000","last_active":"2024-09-21T10:11:32.872294+0000","last_peered":"2024-09-21T10:11:32.872294+0000","last_clean":"2024-09-21T10:11:32.872294+0000","last_became_active":"2024-09-21T10:08:03.313246+0000","last_became_peered":"2024-09-21T10:08:03.313246+0000","last_unstale":"2024-09-21T10:11:32.872294+0000","last_undegraded":"2024-09-21T10:11:32.872294+0000","last_fullsized":"2024-09-21T10:11:32.872294+0000","mapping_epoch":1208,"log_start":"1246'9560","ondisk_log_start":"1246'9560","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:06.835124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78774272,"num_objects":1202,"num_object_clones":0,"num_object_copies":3606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14052,"num_write_kb":488128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1285,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875286+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:11:31.875286+0000","last_peered":"2024-09-21T10:11:31.875286+0000","last_clean":"2024-09-21T10:11:31.875286+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:11:31.875286+0000","last_undegraded":"2024-09-21T10:11:31.875286+0000","last_fullsized":"2024-09-21T10:11:31.875286+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1706,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886322+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:11:31.886322+0000","last_peered":"2024-09-21T10:11:31.886322+0000","last_clean":"2024-09-21T10:11:31.886322+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:11:31.886322+0000","last_undegraded":"2024-09-21T10:11:31.886322+0000","last_fullsized":"2024-09-21T10:11:31.886322+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":225,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882034+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:11:31.882034+0000","last_peered":"2024-09-21T10:11:31.882034+0000","last_clean":"2024-09-21T10:11:31.882034+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:11:31.882034+0000","last_undegraded":"2024-09-21T10:11:31.882034+0000","last_fullsized":"2024-09-21T10:11:31.882034+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"1257'14133","reported_seq":14253,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.782822+0000","last_change":"2024-09-21T10:08:03.313391+0000","last_active":"2024-09-21T10:11:32.782822+0000","last_peered":"2024-09-21T10:11:32.782822+0000","last_clean":"2024-09-21T10:11:32.782822+0000","last_became_active":"2024-09-21T10:08:03.312838+0000","last_became_peered":"2024-09-21T10:08:03.312838+0000","last_unstale":"2024-09-21T10:11:32.782822+0000","last_undegraded":"2024-09-21T10:11:32.782822+0000","last_fullsized":"2024-09-21T10:11:32.782822+0000","mapping_epoch":1208,"log_start":"1243'9355","ondisk_log_start":"1243'9355","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:52.169702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78184448,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14133,"num_write_kb":490432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1683,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886537+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:11:31.886537+0000","last_peered":"2024-09-21T10:11:31.886537+0000","last_clean":"2024-09-21T10:11:31.886537+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:11:31.886537+0000","last_undegraded":"2024-09-21T10:11:31.886537+0000","last_fullsized":"2024-09-21T10:11:31.886537+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":863,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886851+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:11:31.886851+0000","last_peered":"2024-09-21T10:11:31.886851+0000","last_clean":"2024-09-21T10:11:31.886851+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:11:31.886851+0000","last_undegraded":"2024-09-21T10:11:31.886851+0000","last_fullsized":"2024-09-21T10:11:31.886851+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1675,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890711+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:11:31.890711+0000","last_peered":"2024-09-21T10:11:31.890711+0000","last_clean":"2024-09-21T10:11:31.890711+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:11:31.890711+0000","last_undegraded":"2024-09-21T10:11:31.890711+0000","last_fullsized":"2024-09-21T10:11:31.890711+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.c","version":"1257'14081","reported_seq":14203,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.788140+0000","last_change":"2024-09-21T10:08:03.024073+0000","last_active":"2024-09-21T10:11:32.788140+0000","last_peered":"2024-09-21T10:11:32.788140+0000","last_clean":"2024-09-21T10:11:32.788140+0000","last_became_active":"2024-09-21T10:08:03.023647+0000","last_became_peered":"2024-09-21T10:08:03.023647+0000","last_unstale":"2024-09-21T10:11:32.788140+0000","last_undegraded":"2024-09-21T10:11:32.788140+0000","last_fullsized":"2024-09-21T10:11:32.788140+0000","mapping_epoch":1208,"log_start":"1243'9343","ondisk_log_start":"1243'9343","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4738,"log_dups_size":0,"ondisk_log_size":4738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:53.088905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77660160,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14081,"num_write_kb":488512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":383,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876772+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:11:31.876772+0000","last_peered":"2024-09-21T10:11:31.876772+0000","last_clean":"2024-09-21T10:11:31.876772+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:11:31.876772+0000","last_undegraded":"2024-09-21T10:11:31.876772+0000","last_fullsized":"2024-09-21T10:11:31.876772+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1684,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883785+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:11:31.883785+0000","last_peered":"2024-09-21T10:11:31.883785+0000","last_clean":"2024-09-21T10:11:31.883785+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:11:31.883785+0000","last_undegraded":"2024-09-21T10:11:31.883785+0000","last_fullsized":"2024-09-21T10:11:31.883785+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":361,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875884+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:11:31.875884+0000","last_peered":"2024-09-21T10:11:31.875884+0000","last_clean":"2024-09-21T10:11:31.875884+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:11:31.875884+0000","last_undegraded":"2024-09-21T10:11:31.875884+0000","last_fullsized":"2024-09-21T10:11:31.875884+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1257'14179","reported_seq":14239,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.245013+0000","last_change":"2024-09-21T10:08:03.312130+0000","last_active":"2024-09-21T10:11:33.245013+0000","last_peered":"2024-09-21T10:11:33.245013+0000","last_clean":"2024-09-21T10:11:33.245013+0000","last_became_active":"2024-09-21T10:08:03.311781+0000","last_became_peered":"2024-09-21T10:08:03.311781+0000","last_unstale":"2024-09-21T10:11:33.245013+0000","last_undegraded":"2024-09-21T10:11:33.245013+0000","last_fullsized":"2024-09-21T10:11:33.245013+0000","mapping_epoch":1208,"log_start":"1234'9162","ondisk_log_start":"1234'9162","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:33.284319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74579992,"num_objects":1139,"num_object_clones":0,"num_object_copies":3417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1139,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14179,"num_write_kb":490113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":224,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883513+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:11:31.883513+0000","last_peered":"2024-09-21T10:11:31.883513+0000","last_clean":"2024-09-21T10:11:31.883513+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:11:31.883513+0000","last_undegraded":"2024-09-21T10:11:31.883513+0000","last_fullsized":"2024-09-21T10:11:31.883513+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":110,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615913+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:11:24.615913+0000","last_peered":"2024-09-21T10:11:24.615913+0000","last_clean":"2024-09-21T10:11:24.615913+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:11:24.615913+0000","last_undegraded":"2024-09-21T10:11:24.615913+0000","last_fullsized":"2024-09-21T10:11:24.615913+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615656+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:11:24.615656+0000","last_peered":"2024-09-21T10:11:24.615656+0000","last_clean":"2024-09-21T10:11:24.615656+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:11:24.615656+0000","last_undegraded":"2024-09-21T10:11:24.615656+0000","last_fullsized":"2024-09-21T10:11:24.615656+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":109,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615451+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:11:24.615451+0000","last_peered":"2024-09-21T10:11:24.615451+0000","last_clean":"2024-09-21T10:11:24.615451+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:11:24.615451+0000","last_undegraded":"2024-09-21T10:11:24.615451+0000","last_fullsized":"2024-09-21T10:11:24.615451+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615779+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:11:24.615779+0000","last_peered":"2024-09-21T10:11:24.615779+0000","last_clean":"2024-09-21T10:11:24.615779+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:11:24.615779+0000","last_undegraded":"2024-09-21T10:11:24.615779+0000","last_fullsized":"2024-09-21T10:11:24.615779+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1683,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883042+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:11:31.883042+0000","last_peered":"2024-09-21T10:11:31.883042+0000","last_clean":"2024-09-21T10:11:31.883042+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:11:31.883042+0000","last_undegraded":"2024-09-21T10:11:31.883042+0000","last_fullsized":"2024-09-21T10:11:31.883042+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1296,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875480+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:11:31.875480+0000","last_peered":"2024-09-21T10:11:31.875480+0000","last_clean":"2024-09-21T10:11:31.875480+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:11:31.875480+0000","last_undegraded":"2024-09-21T10:11:31.875480+0000","last_fullsized":"2024-09-21T10:11:31.875480+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1306,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883868+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:11:31.883868+0000","last_peered":"2024-09-21T10:11:31.883868+0000","last_clean":"2024-09-21T10:11:31.883868+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:11:31.883868+0000","last_undegraded":"2024-09-21T10:11:31.883868+0000","last_fullsized":"2024-09-21T10:11:31.883868+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":381,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876804+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:11:31.876804+0000","last_peered":"2024-09-21T10:11:31.876804+0000","last_clean":"2024-09-21T10:11:31.876804+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:11:31.876804+0000","last_undegraded":"2024-09-21T10:11:31.876804+0000","last_fullsized":"2024-09-21T10:11:31.876804+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1709,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876512+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:11:31.876512+0000","last_peered":"2024-09-21T10:11:31.876512+0000","last_clean":"2024-09-21T10:11:31.876512+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:11:31.876512+0000","last_undegraded":"2024-09-21T10:11:31.876512+0000","last_fullsized":"2024-09-21T10:11:31.876512+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":360,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876196+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:11:31.876196+0000","last_peered":"2024-09-21T10:11:31.876196+0000","last_clean":"2024-09-21T10:11:31.876196+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:11:31.876196+0000","last_undegraded":"2024-09-21T10:11:31.876196+0000","last_fullsized":"2024-09-21T10:11:31.876196+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.13","version":"1257'11137","reported_seq":17381,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.774392+0000","last_change":"2024-09-21T10:10:51.429358+0000","last_active":"2024-09-21T10:11:32.774392+0000","last_peered":"2024-09-21T10:11:32.774392+0000","last_clean":"2024-09-21T10:11:32.774392+0000","last_became_active":"2024-09-21T10:10:20.087411+0000","last_became_peered":"2024-09-21T10:10:20.087411+0000","last_unstale":"2024-09-21T10:11:32.774392+0000","last_undegraded":"2024-09-21T10:11:32.774392+0000","last_fullsized":"2024-09-21T10:11:32.774392+0000","mapping_epoch":1240,"log_start":"1215'4733","ondisk_log_start":"1215'4733","created":1220,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9635","last_scrub_stamp":"2024-09-21T10:10:51.429139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:51.429139+0000","objects_scrubbed":2092,"log_size":6404,"log_dups_size":0,"ondisk_log_size":6404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:20.720859+0000","scrub_duration":954,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1524,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615843+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:11:24.615843+0000","last_peered":"2024-09-21T10:11:24.615843+0000","last_clean":"2024-09-21T10:11:24.615843+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:11:24.615843+0000","last_undegraded":"2024-09-21T10:11:24.615843+0000","last_fullsized":"2024-09-21T10:11:24.615843+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875789+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:11:31.875789+0000","last_peered":"2024-09-21T10:11:31.875789+0000","last_clean":"2024-09-21T10:11:31.875789+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:11:31.875789+0000","last_undegraded":"2024-09-21T10:11:31.875789+0000","last_fullsized":"2024-09-21T10:11:31.875789+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"1257'11154","reported_seq":17551,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.185888+0000","last_change":"2024-09-21T10:11:06.067859+0000","last_active":"2024-09-21T10:11:33.185888+0000","last_peered":"2024-09-21T10:11:33.185888+0000","last_clean":"2024-09-21T10:11:33.185888+0000","last_became_active":"2024-09-21T10:10:16.078568+0000","last_became_peered":"2024-09-21T10:10:16.078568+0000","last_unstale":"2024-09-21T10:11:33.185888+0000","last_undegraded":"2024-09-21T10:11:33.185888+0000","last_fullsized":"2024-09-21T10:11:33.185888+0000","mapping_epoch":1236,"log_start":"1214'4005","ondisk_log_start":"1214'4005","created":1220,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10135","last_scrub_stamp":"2024-09-21T10:11:06.067704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:06.067704+0000","objects_scrubbed":1606,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:42.440109+0000","scrub_duration":0.27802364099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38076416,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1025,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1672,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877769+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:11:31.877769+0000","last_peered":"2024-09-21T10:11:31.877769+0000","last_clean":"2024-09-21T10:11:31.877769+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:11:31.877769+0000","last_undegraded":"2024-09-21T10:11:31.877769+0000","last_fullsized":"2024-09-21T10:11:31.877769+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1438,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876709+0000","last_change":"2024-09-21T10:08:43.397612+0000","last_active":"2024-09-21T10:11:31.876709+0000","last_peered":"2024-09-21T10:11:31.876709+0000","last_clean":"2024-09-21T10:11:31.876709+0000","last_became_active":"2024-09-21T10:08:43.397254+0000","last_became_peered":"2024-09-21T10:08:43.397254+0000","last_unstale":"2024-09-21T10:11:31.876709+0000","last_undegraded":"2024-09-21T10:11:31.876709+0000","last_fullsized":"2024-09-21T10:11:31.876709+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:40.293786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.11","version":"1257'11498","reported_seq":17899,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.211109+0000","last_change":"2024-09-21T10:11:07.186895+0000","last_active":"2024-09-21T10:11:33.211109+0000","last_peered":"2024-09-21T10:11:33.211109+0000","last_clean":"2024-09-21T10:11:33.211109+0000","last_became_active":"2024-09-21T10:10:28.162032+0000","last_became_peered":"2024-09-21T10:10:28.162032+0000","last_unstale":"2024-09-21T10:11:33.211109+0000","last_undegraded":"2024-09-21T10:11:33.211109+0000","last_fullsized":"2024-09-21T10:11:33.211109+0000","mapping_epoch":1246,"log_start":"1215'4619","ondisk_log_start":"1215'4619","created":1220,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10566","last_scrub_stamp":"2024-09-21T10:11:07.186675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:07.186675+0000","objects_scrubbed":1548,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:00.595388+0000","scrub_duration":0.367382598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39780352,"num_objects":607,"num_object_clones":0,"num_object_copies":1821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":941,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":345,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875583+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:11:31.875583+0000","last_peered":"2024-09-21T10:11:31.875583+0000","last_clean":"2024-09-21T10:11:31.875583+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:11:31.875583+0000","last_undegraded":"2024-09-21T10:11:31.875583+0000","last_fullsized":"2024-09-21T10:11:31.875583+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":229,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883183+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:11:31.883183+0000","last_peered":"2024-09-21T10:11:31.883183+0000","last_clean":"2024-09-21T10:11:31.883183+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:11:31.883183+0000","last_undegraded":"2024-09-21T10:11:31.883183+0000","last_fullsized":"2024-09-21T10:11:31.883183+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.10","version":"1257'11234","reported_seq":17918,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.229619+0000","last_change":"2024-09-21T10:11:04.073987+0000","last_active":"2024-09-21T10:11:33.229619+0000","last_peered":"2024-09-21T10:11:33.229619+0000","last_clean":"2024-09-21T10:11:33.229619+0000","last_became_active":"2024-09-21T10:10:12.107258+0000","last_became_peered":"2024-09-21T10:10:12.107258+0000","last_unstale":"2024-09-21T10:11:33.229619+0000","last_undegraded":"2024-09-21T10:11:33.229619+0000","last_fullsized":"2024-09-21T10:11:33.229619+0000","mapping_epoch":1232,"log_start":"1214'4167","ondisk_log_start":"1214'4167","created":1220,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":5,"last_scrub":"1254'10189","last_scrub_stamp":"2024-09-21T10:11:04.073694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:04.073694+0000","objects_scrubbed":1630,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.492179+0000","scrub_duration":0.33052306399999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37945344,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1051,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1566,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886454+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:11:31.886454+0000","last_peered":"2024-09-21T10:11:31.886454+0000","last_clean":"2024-09-21T10:11:31.886454+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:11:31.886454+0000","last_undegraded":"2024-09-21T10:11:31.886454+0000","last_fullsized":"2024-09-21T10:11:31.886454+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1647,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890613+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:11:31.890613+0000","last_peered":"2024-09-21T10:11:31.890613+0000","last_clean":"2024-09-21T10:11:31.890613+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:11:31.890613+0000","last_undegraded":"2024-09-21T10:11:31.890613+0000","last_fullsized":"2024-09-21T10:11:31.890613+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.17","version":"1257'11110","reported_seq":19925,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:33.203198+0000","last_change":"2024-09-21T10:11:08.029759+0000","last_active":"2024-09-21T10:11:33.203198+0000","last_peered":"2024-09-21T10:11:33.203198+0000","last_clean":"2024-09-21T10:11:33.203198+0000","last_became_active":"2024-09-21T10:10:30.082690+0000","last_became_peered":"2024-09-21T10:10:30.082690+0000","last_unstale":"2024-09-21T10:11:33.203198+0000","last_undegraded":"2024-09-21T10:11:33.203198+0000","last_fullsized":"2024-09-21T10:11:33.203198+0000","mapping_epoch":1248,"log_start":"1214'3983","ondisk_log_start":"1214'3983","created":1220,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10224","last_scrub_stamp":"2024-09-21T10:11:08.029553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:08.029553+0000","objects_scrubbed":1485,"log_size":7127,"log_dups_size":0,"ondisk_log_size":7127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:56.486413+0000","scrub_duration":0.23874677499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38928384,"num_objects":594,"num_object_clones":0,"num_object_copies":1782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":891,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1291,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876802+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:11:31.876802+0000","last_peered":"2024-09-21T10:11:31.876802+0000","last_clean":"2024-09-21T10:11:31.876802+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:11:31.876802+0000","last_undegraded":"2024-09-21T10:11:31.876802+0000","last_fullsized":"2024-09-21T10:11:31.876802+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":245,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886629+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:11:31.886629+0000","last_peered":"2024-09-21T10:11:31.886629+0000","last_clean":"2024-09-21T10:11:31.886629+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:11:31.886629+0000","last_undegraded":"2024-09-21T10:11:31.886629+0000","last_fullsized":"2024-09-21T10:11:31.886629+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.16","version":"1257'10927","reported_seq":16773,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.806935+0000","last_change":"2024-09-21T10:10:55.202726+0000","last_active":"2024-09-21T10:11:32.806935+0000","last_peered":"2024-09-21T10:11:32.806935+0000","last_clean":"2024-09-21T10:11:32.806935+0000","last_became_active":"2024-09-21T10:10:31.083463+0000","last_became_peered":"2024-09-21T10:10:31.083463+0000","last_unstale":"2024-09-21T10:11:32.806935+0000","last_undegraded":"2024-09-21T10:11:32.806935+0000","last_fullsized":"2024-09-21T10:11:32.806935+0000","mapping_epoch":1249,"log_start":"1215'4418","ondisk_log_start":"1215'4418","created":1220,"last_epoch_clean":1250,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9533","last_scrub_stamp":"2024-09-21T10:10:55.202568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:55.202568+0000","objects_scrubbed":2002,"log_size":6509,"log_dups_size":0,"ondisk_log_size":6509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:34.953151+0000","scrub_duration":734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39452672,"num_objects":602,"num_object_clones":0,"num_object_copies":1806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1400,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1298,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875428+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:11:31.875428+0000","last_peered":"2024-09-21T10:11:31.875428+0000","last_clean":"2024-09-21T10:11:31.875428+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:11:31.875428+0000","last_undegraded":"2024-09-21T10:11:31.875428+0000","last_fullsized":"2024-09-21T10:11:31.875428+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":113,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.616011+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:11:24.616011+0000","last_peered":"2024-09-21T10:11:24.616011+0000","last_clean":"2024-09-21T10:11:24.616011+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:11:24.616011+0000","last_undegraded":"2024-09-21T10:11:24.616011+0000","last_fullsized":"2024-09-21T10:11:24.616011+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.15","version":"1257'11339","reported_seq":16807,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.884039+0000","last_change":"2024-09-21T10:10:56.177599+0000","last_active":"2024-09-21T10:11:32.884039+0000","last_peered":"2024-09-21T10:11:32.884039+0000","last_clean":"2024-09-21T10:11:32.884039+0000","last_became_active":"2024-09-21T10:10:50.088118+0000","last_became_peered":"2024-09-21T10:10:50.088118+0000","last_unstale":"2024-09-21T10:11:32.884039+0000","last_undegraded":"2024-09-21T10:11:32.884039+0000","last_fullsized":"2024-09-21T10:11:32.884039+0000","mapping_epoch":1252,"log_start":"1215'5392","ondisk_log_start":"1215'5392","created":1220,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9907","last_scrub_stamp":"2024-09-21T10:10:56.177478+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:56.177478+0000","objects_scrubbed":2064,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:26:51.322192+0000","scrub_duration":688,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40960000,"num_objects":625,"num_object_clones":0,"num_object_copies":1875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1439,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877060+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:11:31.877060+0000","last_peered":"2024-09-21T10:11:31.877060+0000","last_clean":"2024-09-21T10:11:31.877060+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:11:31.877060+0000","last_undegraded":"2024-09-21T10:11:31.877060+0000","last_fullsized":"2024-09-21T10:11:31.877060+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":361,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877566+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:11:31.877566+0000","last_peered":"2024-09-21T10:11:31.877566+0000","last_clean":"2024-09-21T10:11:31.877566+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:11:31.877566+0000","last_undegraded":"2024-09-21T10:11:31.877566+0000","last_fullsized":"2024-09-21T10:11:31.877566+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"1257'11165","reported_seq":17546,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:32.826751+0000","last_change":"2024-09-21T10:10:55.122874+0000","last_active":"2024-09-21T10:11:32.826751+0000","last_peered":"2024-09-21T10:11:32.826751+0000","last_clean":"2024-09-21T10:11:32.826751+0000","last_became_active":"2024-09-21T10:10:21.091421+0000","last_became_peered":"2024-09-21T10:10:21.091421+0000","last_unstale":"2024-09-21T10:11:32.826751+0000","last_undegraded":"2024-09-21T10:11:32.826751+0000","last_fullsized":"2024-09-21T10:11:32.826751+0000","mapping_epoch":1241,"log_start":"1215'5254","ondisk_log_start":"1215'5254","created":1220,"last_epoch_clean":1242,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9715","last_scrub_stamp":"2024-09-21T10:10:55.122315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:55.122315+0000","objects_scrubbed":2071,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:59.171003+0000","scrub_duration":652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1457,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":109,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615418+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:11:24.615418+0000","last_peered":"2024-09-21T10:11:24.615418+0000","last_clean":"2024-09-21T10:11:24.615418+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:11:24.615418+0000","last_undegraded":"2024-09-21T10:11:24.615418+0000","last_fullsized":"2024-09-21T10:11:24.615418+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1628,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615786+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:11:24.615786+0000","last_peered":"2024-09-21T10:11:24.615786+0000","last_clean":"2024-09-21T10:11:24.615786+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:11:24.615786+0000","last_undegraded":"2024-09-21T10:11:24.615786+0000","last_fullsized":"2024-09-21T10:11:24.615786+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":1251377176,"num_objects":19095,"num_object_clones":0,"num_object_copies":57285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19095,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119890,"num_write_kb":3186881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4314574848,"data_stored":4314562632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":159650,"ondisk_log_size":159650,"up":78,"acting":78,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":11142,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234428,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496064,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485440,"kb_used_data":876852,"kb_used_omap":1,"kb_used_meta":608574,"kb_avail":92259712,"statfs":{"total":95995035648,"available":94473945088,"internally_reserved":0,"allocated":897896448,"data_stored":895096159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":623180231},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757703,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019339,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1278952,"kb_used_data":832736,"kb_used_omap":1,"kb_used_meta":446206,"kb_avail":92466200,"statfs":{"total":95995035648,"available":94685388800,"internally_reserved":0,"allocated":852721664,"data_stored":849933001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":456915398},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888635,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1539940,"kb_used_data":804056,"kb_used_omap":34,"kb_used_meta":735837,"kb_avail":92205212,"statfs":{"total":95995035648,"available":94418137088,"internally_reserved":0,"allocated":823353344,"data_stored":820575751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":753498099},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967874,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1624168,"kb_used_data":1014496,"kb_used_omap":41,"kb_used_meta":609558,"kb_avail":92120984,"statfs":{"total":95995035648,"available":94331887616,"internally_reserved":0,"allocated":1038843904,"data_stored":1036055709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":624187807},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047113,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1434916,"kb_used_data":853084,"kb_used_omap":41,"kb_used_meta":581782,"kb_avail":92310236,"statfs":{"total":95995035648,"available":94525681664,"internally_reserved":0,"allocated":873558016,"data_stored":870776805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":595745705},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126352,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":827527168,"data_stored":827523096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":985337856,"data_stored":985333784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":791150592,"data_stored":791150592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":842596352,"data_stored":842596352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":867962880,"data_stored":867958808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:11:40.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:11:40.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:40.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:40.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:40.666 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52756 7.8 7:10c9f1bf:::benchmark_data_smithi076_190198_object52755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.759 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125262+0000", 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87272951099999, 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9378176539999998, 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52756 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.760 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125262+0000", 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125262+0000", 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125263+0000", 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3790000000000001e-06 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.761 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125287+0000", 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4253e-05 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125298+0000", 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0248000000000001e-05 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125306+0000", 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5650000000000002e-06 2024-09-21T10:11:40.762 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125505+0000", 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000199271 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125518+0000", 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2286000000000001e-05 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.763 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:54.179180+0000", 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0536622609999999 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:54.182680+0000", 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034997119999999999 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:54.188088+0000", 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054078939999999999 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.764 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.051092+0000", 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86300423900000001 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.052612+0000", 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001519699 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.053801+0000", 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011891689999999999 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.765 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.055577+0000", 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017763480000000001 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.063079+0000", 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075023299999999998 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.766 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52768 7.8 7:15148440:::benchmark_data_smithi076_190198_object52767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125318+0000", 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87267279499997, 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9777066379999999, 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.767 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52768 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125318+0000", 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125318+0000", 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.768 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125319+0000", 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.82e-07 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125340+0000", 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1426000000000001e-05 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125347+0000", 2024-09-21T10:11:40.769 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5359999999999998e-06 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125352+0000", 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1549999999999998e-06 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125545+0000", 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019330899999999999 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.770 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125553+0000", 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9449999999999993e-06 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.063708+0000", 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.938154607 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.068305+0000", 2024-09-21T10:11:40.771 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004596915 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.074704+0000", 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063993690000000002 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.092571+0000", 2024-09-21T10:11:40.772 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017866988 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.093768+0000", 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0011968129999999999 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.773 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.774 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.774 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.094457+0000", 2024-09-21T10:11:40.774 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00068903300000000003 2024-09-21T10:11:40.774 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.774 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.095958+0000", 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001501346 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.103025+0000", 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.007066714 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.775 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52784 7.8 7:1ac06dd3:::benchmark_data_smithi076_190198_object52783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87263217500004, 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9892106549999999, 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.776 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52784 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.777 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125359+0000", 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9400000000000001e-07 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.778 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125381+0000", 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1469999999999999e-05 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125387+0000", 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1739999999999997e-06 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125392+0000", 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7110000000000003e-06 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.779 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125577+0000", 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018529399999999999 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125584+0000", 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6619999999999999e-06 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.078364+0000", 2024-09-21T10:11:40.780 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.952780089 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.081931+0000", 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003566657 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.087347+0000", 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0054161340000000004 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.781 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.104186+0000", 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016838907 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.105762+0000", 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015761690000000001 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.106422+0000", 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00065996800000000003 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.782 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.107987+0000", 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001565676 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.114570+0000", 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065823499999999998 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.783 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52790 7.8 7:1cf5039a:::benchmark_data_smithi076_190198_object52789:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87259334199996, 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0320771440000001, 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52790 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.784 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125398+0000", 2024-09-21T10:11:40.785 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0499999999999999e-07 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125420+0000", 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1310000000000001e-05 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125426+0000", 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0929999999999999e-06 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.786 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125430+0000", 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6879999999999998e-06 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125611+0000", 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018086400000000001 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.787 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125619+0000", 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2490000000000004e-06 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.114931+0000", 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.989312263 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.118768+0000", 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038371450000000001 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.788 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.124328+0000", 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0055604110000000003 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.145040+0000", 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.020711415 2024-09-21T10:11:40.789 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.146315+0000", 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012752219999999999 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.148584+0000", 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022692369999999999 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.790 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.150147+0000", 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001562475 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.157475+0000", 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073283669999999997 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.791 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52807 7.8 7:175fc991:::benchmark_data_smithi076_190198_object52806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125436+0000", 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87255496299997, 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.042878151, 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52807 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.792 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125436+0000", 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125436+0000", 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.793 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.794 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.794 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.794 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125437+0000", 2024-09-21T10:11:40.794 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7599999999999998e-07 2024-09-21T10:11:40.794 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.795 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.795 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125499+0000", 2024-09-21T10:11:40.795 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2487e-05 2024-09-21T10:11:40.795 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.795 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125506+0000", 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4309999999999999e-06 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125511+0000", 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9300000000000002e-06 2024-09-21T10:11:40.796 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125643+0000", 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000132325 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125650+0000", 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3529999999999997e-06 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.128543+0000", 2024-09-21T10:11:40.797 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0028929029999998 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.133053+0000", 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045100019999999999 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.798 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.139776+0000", 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0067227229999999999 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.147802+0000", 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0080258459999999997 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.158680+0000", 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010877886 2024-09-21T10:11:40.799 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.159319+0000", 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00063960700000000004 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.161032+0000", 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017127119999999999 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.800 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.168314+0000", 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072825700000000004 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52808 7.8 7:106b279b:::benchmark_data_smithi076_190198_object52807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87247443700005, 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0821514780000001, 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.801 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52808 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.802 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125517+0000", 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1800000000000001e-07 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.803 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125538+0000", 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0531999999999999e-05 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125545+0000", 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9879999999999998e-06 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125550+0000", 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1390000000000003e-06 2024-09-21T10:11:40.804 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125674+0000", 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124404 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125681+0000", 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7730000000000002e-06 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.168728+0000", 2024-09-21T10:11:40.805 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0430472690000001 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.172808+0000", 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040795359999999999 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.179117+0000", 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063089900000000004 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.806 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.197374+0000", 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018256989000000001 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.198951+0000", 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015774210000000001 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.199662+0000", 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00071113999999999997 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.807 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.201077+0000", 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014148800000000001 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.207668+0000", 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065909990000000002 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.808 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52818 7.8 7:179d769e:::benchmark_data_smithi076_190198_object52817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125556+0000", 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87243508200004, 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1853145110000001, 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.809 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52818 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125556+0000", 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125556+0000", 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.810 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125557+0000", 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0900000000000002e-07 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125576+0000", 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9735999999999999e-05 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.811 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125582+0000", 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0759999999999999e-06 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125587+0000", 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7430000000000002e-06 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125874+0000", 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00028725700000000002 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.812 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125883+0000", 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3739999999999998e-06 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.183475+0000", 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0575924579999998 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.187473+0000", 2024-09-21T10:11:40.813 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039973689999999997 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.192809+0000", 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053367349999999996 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.208062+0000", 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015252388 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.814 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.297826+0000", 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.089764627999999999 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.300098+0000", 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022719319999999999 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.302169+0000", 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020704209999999998 2024-09-21T10:11:40.815 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.310871+0000", 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0087019850000000006 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52827 7.8 7:10f0ddf6:::benchmark_data_smithi076_190198_object52826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125838+0000", 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87215337600003, 2024-09-21T10:11:40.816 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2459487029999998, 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52827 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.817 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125838+0000", 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125838+0000", 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125839+0000", 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.68e-07 2024-09-21T10:11:40.818 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125887+0000", 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8659000000000003e-05 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125897+0000", 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7210000000000006e-06 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.819 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125903+0000", 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4860000000000002e-06 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125997+0000", 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3828999999999999e-05 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.820 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.821 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.126006+0000", 2024-09-21T10:11:40.821 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.619e-06 2024-09-21T10:11:40.821 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.821 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.821 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.821 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.311618+0000", 2024-09-21T10:11:40.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.185612619 2024-09-21T10:11:40.822 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.822 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.822 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.822 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.316660+0000", 2024-09-21T10:11:40.822 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005041149 2024-09-21T10:11:40.823 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.823 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.823 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.823 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.323970+0000", 2024-09-21T10:11:40.823 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.007310024 2024-09-21T10:11:40.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.824 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.824 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.824 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.362884+0000", 2024-09-21T10:11:40.824 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.038914532000000002 2024-09-21T10:11:40.824 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.825 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.825 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.363448+0000", 2024-09-21T10:11:40.825 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056372300000000002 2024-09-21T10:11:40.825 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.825 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.826 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.826 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.364299+0000", 2024-09-21T10:11:40.826 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00085096100000000001 2024-09-21T10:11:40.826 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.826 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.827 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.366052+0000", 2024-09-21T10:11:40.827 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017532139999999999 2024-09-21T10:11:40.827 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.827 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.827 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.828 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.371787+0000", 2024-09-21T10:11:40.828 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0057344989999999997 2024-09-21T10:11:40.828 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.828 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.828 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.828 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.829 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.829 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52845 7.8 7:1187a13e:::benchmark_data_smithi076_190198_object52844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.829 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.125927+0000", 2024-09-21T10:11:40.829 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.87206407799999, 2024-09-21T10:11:40.829 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2574380139999999, 2024-09-21T10:11:40.829 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.830 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.830 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.830 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.830 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.830 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.830 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52845 2024-09-21T10:11:40.831 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.831 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.831 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.831 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.831 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125927+0000", 2024-09-21T10:11:40.831 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.832 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.832 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.832 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.832 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125927+0000", 2024-09-21T10:11:40.832 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.833 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.833 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.833 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125928+0000", 2024-09-21T10:11:40.833 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8999999999999996e-07 2024-09-21T10:11:40.833 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.834 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.834 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.834 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125974+0000", 2024-09-21T10:11:40.834 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6186999999999997e-05 2024-09-21T10:11:40.834 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.834 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.835 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125982+0000", 2024-09-21T10:11:40.835 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1950000000000003e-06 2024-09-21T10:11:40.835 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.835 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.835 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.836 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.125988+0000", 2024-09-21T10:11:40.836 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9660000000000001e-06 2024-09-21T10:11:40.836 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.836 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.836 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.126030+0000", 2024-09-21T10:11:40.837 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1625999999999998e-05 2024-09-21T10:11:40.837 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.837 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.837 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.837 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.126037+0000", 2024-09-21T10:11:40.838 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.883e-06 2024-09-21T10:11:40.838 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.838 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.838 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.838 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.328679+0000", 2024-09-21T10:11:40.839 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2026427630000001 2024-09-21T10:11:40.839 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.839 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.839 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.839 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.332779+0000", 2024-09-21T10:11:40.839 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040996399999999999 2024-09-21T10:11:40.840 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.840 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.840 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.840 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.340983+0000", 2024-09-21T10:11:40.840 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0082042360000000002 2024-09-21T10:11:40.840 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.841 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.841 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.841 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.372677+0000", 2024-09-21T10:11:40.841 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.031693547000000002 2024-09-21T10:11:40.841 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.841 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.842 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.373231+0000", 2024-09-21T10:11:40.842 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00055373499999999997 2024-09-21T10:11:40.842 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.842 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.842 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.843 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.374472+0000", 2024-09-21T10:11:40.843 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012418139999999999 2024-09-21T10:11:40.843 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.843 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.843 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.843 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.376369+0000", 2024-09-21T10:11:40.844 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0018962149999999999 2024-09-21T10:11:40.844 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.844 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.844 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.844 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.383365+0000", 2024-09-21T10:11:40.844 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069966170000000001 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52853 7.8 7:1f7ef906:::benchmark_data_smithi076_190198_object52852:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.159054+0000", 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.83893741700001, 2024-09-21T10:11:40.845 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2362974119999999, 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52853 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159054+0000", 2024-09-21T10:11:40.846 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159054+0000", 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159055+0000", 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4699999999999999e-06 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.847 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159077+0000", 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1679e-05 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159088+0000", 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1447000000000001e-05 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159096+0000", 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.374e-06 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.848 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159185+0000", 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9529000000000003e-05 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.159197+0000", 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1537e-05 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.346249+0000", 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1870516360000001 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.849 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.351584+0000", 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053351199999999996 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.357971+0000", 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063877669999999999 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.373507+0000", 2024-09-21T10:11:40.850 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.015535739999999999 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.384672+0000", 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011165059999999999 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.386260+0000", 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015877350000000001 2024-09-21T10:11:40.851 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.388538+0000", 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022780880000000002 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.395351+0000", 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00681323 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.852 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52867 7.8 7:1ff0d522:::benchmark_data_smithi076_190198_object52866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.171941+0000", 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.82605004200002, 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2690444109999999, 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52867 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.853 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171941+0000", 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171941+0000", 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.854 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171942+0000", 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4099999999999999e-07 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171975+0000", 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3419e-05 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171981+0000", 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9519999999999999e-06 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.855 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.171985+0000", 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5090000000000002e-06 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.172045+0000", 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0010999999999999e-05 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.172055+0000", 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0739e-05 2024-09-21T10:11:40.856 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.395713+0000", 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.223657384 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.400221+0000", 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045087979999999996 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.407140+0000", 2024-09-21T10:11:40.857 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0069187329999999998 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.429665+0000", 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.022524371000000001 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.430411+0000", 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00074638000000000005 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.858 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.431429+0000", 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001017712 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.433802+0000", 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023734369999999999 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.440986+0000", 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0071835249999999996 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.859 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52869 7.8 7:179415fd:::benchmark_data_smithi076_190198_object52868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.175548+0000", 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.82244328100001, 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2876107210000001, 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52869 2024-09-21T10:11:40.860 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175548+0000", 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175548+0000", 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175549+0000", 2024-09-21T10:11:40.861 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3799999999999999e-06 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175596+0000", 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6592999999999999e-05 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175604+0000", 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3829999999999992e-06 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.862 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175611+0000", 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2729999999999999e-06 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175673+0000", 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2162999999999994e-05 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.175682+0000", 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2429999999999994e-06 2024-09-21T10:11:40.863 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.412358+0000", 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2366758820000001 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.416943+0000", 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0045848190000000004 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.424233+0000", 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072906760000000003 2024-09-21T10:11:40.864 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.442258+0000", 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.018024212000000001 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.443888+0000", 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0016305110000000001 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.446498+0000", 2024-09-21T10:11:40.865 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026097339999999998 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.450291+0000", 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003793088 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.463159+0000", 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.012867764 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.866 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52884 7.8 7:106bbe8d:::benchmark_data_smithi076_190198_object52883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.196889+0000", 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.80110250200005, 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3147955009999999, 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52884 2024-09-21T10:11:40.867 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196889+0000", 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196889+0000", 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.868 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196890+0000", 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.325e-06 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196949+0000", 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8884000000000001e-05 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196960+0000", 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0795000000000001e-05 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.869 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.196967+0000", 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1110000000000002e-06 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.197033+0000", 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6381e-05 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.197044+0000", 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0789e-05 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.870 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.463653+0000", 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2666088929999999 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.470290+0000", 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066372159999999996 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.477840+0000", 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075500630000000001 2024-09-21T10:11:40.871 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.499602+0000", 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021761857999999999 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.500852+0000", 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012494400000000001 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.502175+0000", 2024-09-21T10:11:40.872 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001323418 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.504437+0000", 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022620190000000001 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.511684+0000", 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072473090000000004 2024-09-21T10:11:40.873 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52887 7.8 7:127cfd92:::benchmark_data_smithi076_190198_object52886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.199967+0000", 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.79802459699999, 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3250906430000002, 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.874 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52887 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.199967+0000", 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.875 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.876 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.876 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.199967+0000", 2024-09-21T10:11:40.876 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.876 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.876 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.199968+0000", 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4670000000000001e-06 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200013+0000", 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4697999999999998e-05 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.877 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200023+0000", 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0026e-05 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200028+0000", 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6339999999999998e-06 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200082+0000", 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3136000000000001e-05 2024-09-21T10:11:40.878 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.200090+0000", 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6659999999999997e-06 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.483007+0000", 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2829167049999999 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.879 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.487146+0000", 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004138821 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.494516+0000", 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0073705519999999998 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.512278+0000", 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017761328999999999 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.880 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.513504+0000", 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012263020000000001 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.514981+0000", 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014765399999999999 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.517827+0000", 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028469369999999999 2024-09-21T10:11:40.881 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.525057+0000", 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072298299999999996 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52908 7.8 7:19324e83:::benchmark_data_smithi076_190198_object52907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.249069+0000", 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.74892241500004, 2024-09-21T10:11:40.882 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3212883099999999, 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52908 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249069+0000", 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.883 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249069+0000", 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249070+0000", 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0210000000000001e-06 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.884 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249090+0000", 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0531e-05 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249102+0000", 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1420000000000001e-05 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249112+0000", 2024-09-21T10:11:40.885 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7829999999999993e-06 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249195+0000", 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.3673999999999999e-05 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.249207+0000", 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1392e-05 2024-09-21T10:11:40.886 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.526267+0000", 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2770600129999998 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.530238+0000", 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039713839999999997 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.887 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.537764+0000", 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0075257170000000003 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.559351+0000", 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021587614000000001 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.560460+0000", 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001108567 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.888 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.561858+0000", 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001398299 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.564078+0000", 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022197470000000002 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.570357+0000", 2024-09-21T10:11:40.889 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006279148 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.34992.0:52921 7.8 7:1be3829c:::benchmark_data_smithi076_190198_object52920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:01:53.284229+0000", 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 586.71376201400005, 2024-09-21T10:11:40.890 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.3019164519999999, 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.34992", 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3830369029", 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 52921 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284229+0000", 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.891 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284229+0000", 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284231+0000", 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2640000000000001e-06 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284251+0000", 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0055999999999999e-05 2024-09-21T10:11:40.892 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.893 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.893 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284262+0000", 2024-09-21T10:11:40.893 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1423000000000001e-05 2024-09-21T10:11:40.893 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.893 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:40.894 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.894 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284268+0000", 2024-09-21T10:11:40.894 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.913e-06 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284333+0000", 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5591000000000003e-05 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:53.284345+0000", 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1977999999999999e-05 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.895 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.541864+0000", 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2575185150000001 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.546200+0000", 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043359399999999999 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4", 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.554034+0000", 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0078339180000000005 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.896 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.571643+0000", 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.017609265999999998 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.573665+0000", 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020218509999999999 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.574919+0000", 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012535739999999999 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.897 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.577250+0000", 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023317590000000001 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:01:55.586146+0000", 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0088954040000000009 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.898 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146601 8.16 8:6aa6d95d:::benchmark_data_smithi076_241159_object23683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.155546+0000", 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 84.842444998999994, 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9885378650000001, 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146601 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.899 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155546+0000", 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155546+0000", 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.900 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155547+0000", 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8800000000000001e-07 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155552+0000", 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.549e-06 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155565+0000", 2024-09-21T10:11:40.901 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2822e-05 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155572+0000", 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5399999999999998e-06 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155674+0000", 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000101913 2024-09-21T10:11:40.902 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155684+0000", 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0051000000000001e-05 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077427+0000", 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92174305099999998 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077439+0000", 2024-09-21T10:11:40.903 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1539000000000001e-05 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.138536+0000", 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0610976110000001 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.138673+0000", 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013722100000000001 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:11:40.904 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.138919+0000", 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024506899999999999 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143738+0000", 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004819968 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143781+0000", 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2367999999999998e-05 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.905 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143842+0000", 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1582999999999996e-05 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143887+0000", 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4267999999999997e-05 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143955+0000", 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8287999999999994e-05 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.906 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144084+0000", 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012913599999999999 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146654 8.16 8:6a8aae0b:::benchmark_data_smithi076_241159_object23736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.207022+0000", 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 84.790969363000002, 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9373881310000001, 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.907 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146654 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207022+0000", 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.908 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207022+0000", 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207023+0000", 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.879999999999999e-07 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207027+0000", 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1060000000000002e-06 2024-09-21T10:11:40.909 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207037+0000", 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0183000000000001e-05 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207044+0000", 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5610000000000004e-06 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207105+0000", 2024-09-21T10:11:40.910 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1124000000000006e-05 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207115+0000", 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0583e-05 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077844+0000", 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87072874700000003 2024-09-21T10:11:40.911 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077854+0000", 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0054999999999999e-05 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.140541+0000", 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0626865750000001 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.140687+0000", 2024-09-21T10:11:40.912 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014609200000000001 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.140948+0000", 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026133599999999998 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143794+0000", 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028461609999999998 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.913 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144127+0000", 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000332765 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144222+0000", 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4636999999999995e-05 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144283+0000", 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1259000000000004e-05 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.914 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144314+0000", 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1372999999999999e-05 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144410+0000", 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5586000000000003e-05 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146670 8.16 8:6cf15343:::benchmark_data_smithi076_241159_object23752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:11:40.915 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 84.782505748999995, 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.930904897, 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146670 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:11:40.916 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4499999999999996e-07 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.917 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215490+0000", 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.946e-06 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215502+0000", 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1858999999999999e-05 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215508+0000", 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0179999999999998e-06 2024-09-21T10:11:40.918 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215566+0000", 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7441000000000001e-05 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215573+0000", 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3499999999999999e-06 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077881+0000", 2024-09-21T10:11:40.919 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86230807300000001 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077889+0000", 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1470000000000001e-06 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.142055+0000", 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.064166277 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.142132+0000", 2024-09-21T10:11:40.920 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6553999999999999e-05 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.142341+0000", 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020935499999999999 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143801+0000", 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001459922 2024-09-21T10:11:40.921 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144176+0000", 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000374366 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144260+0000", 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4666000000000006e-05 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.922 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.146221+0000", 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019611110000000002 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.146281+0000", 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9639000000000003e-05 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.146390+0000", 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109428 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.923 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146799 8.16 8:684e2f31:::benchmark_data_smithi076_241159_object23881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.321746+0000", 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 84.676245453000007, 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8305196700000002, 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146799 2024-09-21T10:11:40.924 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321746+0000", 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321746+0000", 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:11:40.925 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321747+0000", 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2950000000000001e-06 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321751+0000", 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8179999999999997e-06 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321767+0000", 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.609e-05 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.926 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321774+0000", 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2629999999999997e-06 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321834+0000", 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9308000000000002e-05 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321843+0000", 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4069999999999999e-06 2024-09-21T10:11:40.927 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077927+0000", 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75608387499999996 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077936+0000", 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9530000000000005e-06 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144466+0000", 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0665306269999999 2024-09-21T10:11:40.928 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144603+0000", 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000136425 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144840+0000", 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000237085 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.148206+0000", 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0033661810000000002 2024-09-21T10:11:40.929 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.148220+0000", 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3455000000000001e-05 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.148255+0000", 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5154000000000003e-05 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:11:40.930 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.152036+0000", 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037816870000000002 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.152095+0000", 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8462999999999999e-05 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.152265+0000", 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017058399999999999 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.931 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.932 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:11:40.932 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:11:40.932 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:11:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:40.934 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:40.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:41.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:41.071 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:41.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:41.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:41.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:41.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:41.332 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:41.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:41.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:41.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:41.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:41.582 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:41.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:41.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:41.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:41.810 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:41 smithi191 ceph-mon[102206]: pgmap v1771: 99 pgs: 99 active+clean; 805 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:11:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:41.873 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:41.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[106482]: pgmap v1771: 99 pgs: 99 active+clean; 805 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:11:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[110963]: pgmap v1771: 99 pgs: 99 active+clean; 805 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:11:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:41.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:42.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:42.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:42.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:42.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:42.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:11:42.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:42.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:42.387 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:42.395 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:42.421 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:42.446 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:42.447 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:11:42.447 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:42.493 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:11:42.493 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:42.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:42.533 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:11:42.533 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:42.567 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:11:42.567 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:42.575 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:42.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:42.597 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:11:42.597 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:42.628 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:11:42.628 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:42.661 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:11:42.661 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:42.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:42.699 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:11:42.699 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:42.700 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1154039082' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:42.700 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1154039082' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:42.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:42.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:42.831 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1154039082' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:42.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:42.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:42.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:43.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:43.097 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:43.199 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:43.232 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:43.334 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:43.369 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:43.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:43.471 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:43.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:43.580 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:11:43] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-21T10:11:43.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:43.646 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:43.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:43.775 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:11:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:11:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:43 smithi191 ceph-mon[102206]: pgmap v1772: 99 pgs: 99 active+clean; 805 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-21T10:11:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:11:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:11:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[106482]: pgmap v1772: 99 pgs: 99 active+clean; 805 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-21T10:11:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:11:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:11:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:11:43.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:43 smithi076 ceph-mon[110963]: pgmap v1772: 99 pgs: 99 active+clean; 805 MiB data, 5.4 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s 2024-09-21T10:11:43.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:43.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:43.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:43.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:44.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:44.055 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:44.157 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:44.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:44.292 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:44.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:44.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:44.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:44.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:44.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:44.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:44.607 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:44.607 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:44.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:44.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:44.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:44.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:44.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:44.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:44.973 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:44.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:45.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:45.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:45.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:45.241 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:45.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:45.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:11:45.342 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:45.380 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:45.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:45.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:45.510 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:45.597 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:45.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:45.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:45.743 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:45.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:45.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:45 smithi076 ceph-mon[110963]: pgmap v1773: 99 pgs: 99 active+clean; 429 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-09-21T10:11:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:45 smithi076 ceph-mon[106482]: pgmap v1773: 99 pgs: 99 active+clean; 429 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-09-21T10:11:45.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:45.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:45.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:46.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:46.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:45 smithi191 ceph-mon[102206]: pgmap v1773: 99 pgs: 99 active+clean; 429 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-09-21T10:11:46.148 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:46.177 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:46.279 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:46.316 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:46.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:46.442 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:46.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:46.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:46.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:46.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:11:46.671 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:46.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:46.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:46.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:46.845 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:46.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:46.970 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:46.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:47.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:47.108 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:47.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:47.239 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:47.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:47.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:47.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:47.504 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:47.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:47.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:47.635 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:47.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:47.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:47.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:47.873 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:47.891 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:47.891 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:47.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:47.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:48.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:48.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:47 smithi191 ceph-mon[102206]: pgmap v1774: 99 pgs: 99 active+clean; 429 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-21T10:11:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:47 smithi076 ceph-mon[106482]: pgmap v1774: 99 pgs: 99 active+clean; 429 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-21T10:11:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:47 smithi076 ceph-mon[110963]: pgmap v1774: 99 pgs: 99 active+clean; 429 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-21T10:11:48.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:48.168 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:48.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:48.270 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:48.303 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:48.399 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:48.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:48.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:48.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:48.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:48.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:48.694 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Removed 122915 objects 2024-09-21T10:11:48.694 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Clean up completed and total clean up time :122.933 2024-09-21T10:11:48.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:48.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:11:48.728 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:48.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:48.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:48.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:48.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:48.924 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:48.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:48.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:48.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:49.081 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-21T10:11:49.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-21T10:11:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.104 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:49.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:49.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:49.227 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:49.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:49.329 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:49.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:49.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.453 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:49.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:49.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:49.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:49.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:49.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:49.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:49.845 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:49.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:49.946 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:49.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:49.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:49.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:50.069 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:50.074 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:49 smithi191 ceph-mon[102206]: pgmap v1775: 99 pgs: 99 active+clean; 47 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-09-21T10:11:50.093 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:50.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:50.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:49 smithi076 ceph-mon[106482]: pgmap v1775: 99 pgs: 99 active+clean; 47 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-09-21T10:11:50.109 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:49 smithi076 ceph-mon[110963]: pgmap v1775: 99 pgs: 99 active+clean; 47 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 1.53k op/s 2024-09-21T10:11:50.194 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:50.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:50.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:50.340 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:50.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:50.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:50.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:50.580 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:50.580 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:50.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:50.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:50.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:50.690 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:50.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:50.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:50.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:50.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:50.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:50.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:50.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:51.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:51.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:51.188 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:51.210 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:51.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:51.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:51.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:11:51.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:51.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:51.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:51.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:51.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:51.583 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:51.605 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:11:51.606 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:11:51.613 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:11:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=cleanup t=2024-09-21T10:11:51.323452131Z level=info msg="Completed cleanup jobs" duration=1.936624ms 2024-09-21T10:11:51.613 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:11:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugins.update.checker t=2024-09-21T10:11:51.476018789Z level=info msg="Update check succeeded" duration=68.131315ms 2024-09-21T10:11:51.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:51.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:51.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.828 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:51.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:51.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:51.950 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:51.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:51.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:52.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:52.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:51 smithi191 ceph-mon[102206]: pgmap v1776: 99 pgs: 99 active+clean; 47 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:11:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:51 smithi076 ceph-mon[106482]: pgmap v1776: 99 pgs: 99 active+clean; 47 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:11:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:51 smithi076 ceph-mon[110963]: pgmap v1776: 99 pgs: 99 active+clean; 47 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:11:52.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:52.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:52.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:52.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:52.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:52.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:52.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:52.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:52.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:52.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:52.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:52.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:52.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:52.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:52.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:53.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:53.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:52 smithi191 ceph-mon[102206]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:53.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:52 smithi076 ceph-mon[106482]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:53.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:52 smithi076 ceph-mon[110963]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:11:53.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:53.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:53.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:53.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:53.324 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:53.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:53.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:53.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:53.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:53.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:11:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:11:53] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-21T10:11:53.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:53.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:53.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:53.828 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:53.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:53.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:53.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:53.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:54.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:54.077 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:54.111 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:53 smithi076 ceph-mon[106482]: pgmap v1777: 99 pgs: 99 active+clean; 47 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:11:54.111 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:53 smithi076 ceph-mon[110963]: pgmap v1777: 99 pgs: 99 active+clean; 47 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:11:54.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:53 smithi191 ceph-mon[102206]: pgmap v1777: 99 pgs: 99 active+clean; 47 MiB data, 3.2 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:11:54.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:54.202 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:54.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:54.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:54.328 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:54.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:54.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:54.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:54.480 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:54.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:54.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:54.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:54.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:54.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:54.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:54.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:54.829 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:11:54.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:54.946 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:54.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:55.018 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/56636192' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:11:55.024 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1776,"stamp":"2024-09-21T10:11:50.747091+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":48830461,"num_objects":763,"num_object_clones":0,"num_object_copies":2289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":763,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":138574,"num_write_kb":3189330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":2963020,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148315,"ondisk_log_size":148315,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":303,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":3539492,"kb_used_data":521148,"kb_used_omap":119,"kb_used_meta":3017864,"kb_avail":465186268,"statfs":{"total":479975178240,"available":476350738432,"internally_reserved":0,"allocated":533655552,"data_stored":519715675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":3090292936},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-795574272,"num_objects":-12139,"num_object_clones":0,"num_object_copies":-36417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-12139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12186,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006891"},"pg_stats":[{"pgid":"8.18","version":"1257'11539","reported_seq":16681,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.843297+0000","last_change":"2024-09-21T10:10:59.104488+0000","last_active":"2024-09-21T10:11:47.843297+0000","last_peered":"2024-09-21T10:11:47.843297+0000","last_clean":"2024-09-21T10:11:47.843297+0000","last_became_active":"2024-09-21T10:10:49.084472+0000","last_became_peered":"2024-09-21T10:10:49.084472+0000","last_unstale":"2024-09-21T10:11:47.843297+0000","last_undegraded":"2024-09-21T10:11:47.843297+0000","last_fullsized":"2024-09-21T10:11:47.843297+0000","mapping_epoch":1251,"log_start":"1216'5420","ondisk_log_start":"1216'5420","created":1220,"last_epoch_clean":1252,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9782","last_scrub_stamp":"2024-09-21T10:10:59.104281+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:59.104281+0000","objects_scrubbed":1801,"log_size":6119,"log_dups_size":0,"ondisk_log_size":6119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:51.831876+0000","scrub_duration":650,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2162688,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1768,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876047+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:11:31.876047+0000","last_peered":"2024-09-21T10:11:31.876047+0000","last_clean":"2024-09-21T10:11:31.876047+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:11:31.876047+0000","last_undegraded":"2024-09-21T10:11:31.876047+0000","last_fullsized":"2024-09-21T10:11:31.876047+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":115,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615580+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:11:24.615580+0000","last_peered":"2024-09-21T10:11:24.615580+0000","last_clean":"2024-09-21T10:11:24.615580+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:11:24.615580+0000","last_undegraded":"2024-09-21T10:11:24.615580+0000","last_fullsized":"2024-09-21T10:11:24.615580+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.19","version":"1257'12056","reported_seq":17479,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.251986+0000","last_change":"2024-09-21T10:11:09.179304+0000","last_active":"2024-09-21T10:11:48.251986+0000","last_peered":"2024-09-21T10:11:48.251986+0000","last_clean":"2024-09-21T10:11:48.251986+0000","last_became_active":"2024-09-21T10:10:49.085004+0000","last_became_peered":"2024-09-21T10:10:49.085004+0000","last_unstale":"2024-09-21T10:11:48.251986+0000","last_undegraded":"2024-09-21T10:11:48.251986+0000","last_fullsized":"2024-09-21T10:11:48.251986+0000","mapping_epoch":1251,"log_start":"1216'5810","ondisk_log_start":"1216'5810","created":1220,"last_epoch_clean":1252,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10556","last_scrub_stamp":"2024-09-21T10:11:09.179143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:09.179143+0000","objects_scrubbed":1525,"log_size":6246,"log_dups_size":0,"ondisk_log_size":6246,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:45:29.481256+0000","scrub_duration":0.41433468099999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1441792,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1503,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1337,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875391+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:11:31.875391+0000","last_peered":"2024-09-21T10:11:31.875391+0000","last_clean":"2024-09-21T10:11:31.875391+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:11:31.875391+0000","last_undegraded":"2024-09-21T10:11:31.875391+0000","last_fullsized":"2024-09-21T10:11:31.875391+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1669,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875525+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:11:31.875525+0000","last_peered":"2024-09-21T10:11:31.875525+0000","last_clean":"2024-09-21T10:11:31.875525+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:11:31.875525+0000","last_undegraded":"2024-09-21T10:11:31.875525+0000","last_fullsized":"2024-09-21T10:11:31.875525+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1669,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877479+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:11:31.877479+0000","last_peered":"2024-09-21T10:11:31.877479+0000","last_clean":"2024-09-21T10:11:31.877479+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:11:31.877479+0000","last_undegraded":"2024-09-21T10:11:31.877479+0000","last_fullsized":"2024-09-21T10:11:31.877479+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615689+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:11:24.615689+0000","last_peered":"2024-09-21T10:11:24.615689+0000","last_clean":"2024-09-21T10:11:24.615689+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:11:24.615689+0000","last_undegraded":"2024-09-21T10:11:24.615689+0000","last_fullsized":"2024-09-21T10:11:24.615689+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876685+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:11:31.876685+0000","last_peered":"2024-09-21T10:11:31.876685+0000","last_clean":"2024-09-21T10:11:31.876685+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:11:31.876685+0000","last_undegraded":"2024-09-21T10:11:31.876685+0000","last_fullsized":"2024-09-21T10:11:31.876685+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":138,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615310+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:11:24.615310+0000","last_peered":"2024-09-21T10:11:24.615310+0000","last_clean":"2024-09-21T10:11:24.615310+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:11:24.615310+0000","last_undegraded":"2024-09-21T10:11:24.615310+0000","last_fullsized":"2024-09-21T10:11:24.615310+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1257'11739","reported_seq":11861,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.181858+0000","last_change":"2024-09-21T10:10:51.107946+0000","last_active":"2024-09-21T10:11:48.181858+0000","last_peered":"2024-09-21T10:11:48.181858+0000","last_clean":"2024-09-21T10:11:48.181858+0000","last_became_active":"2024-09-21T10:09:54.319190+0000","last_became_peered":"2024-09-21T10:09:54.319190+0000","last_unstale":"2024-09-21T10:11:48.181858+0000","last_undegraded":"2024-09-21T10:11:48.181858+0000","last_fullsized":"2024-09-21T10:11:48.181858+0000","mapping_epoch":1208,"log_start":"1215'5204","ondisk_log_start":"1215'5204","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9591","last_scrub_stamp":"2024-09-21T10:10:51.107785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:51.107785+0000","objects_scrubbed":2178,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:07:43.774199+0000","scrub_duration":0.351714576,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2157,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":208,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890330+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:11:31.890330+0000","last_peered":"2024-09-21T10:11:31.890330+0000","last_clean":"2024-09-21T10:11:31.890330+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:11:31.890330+0000","last_undegraded":"2024-09-21T10:11:31.890330+0000","last_fullsized":"2024-09-21T10:11:31.890330+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":110,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615574+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:11:24.615574+0000","last_peered":"2024-09-21T10:11:24.615574+0000","last_clean":"2024-09-21T10:11:24.615574+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:11:24.615574+0000","last_undegraded":"2024-09-21T10:11:24.615574+0000","last_fullsized":"2024-09-21T10:11:24.615574+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"1257'12003","reported_seq":12154,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.255445+0000","last_change":"2024-09-21T10:09:56.519177+0000","last_active":"2024-09-21T10:11:48.255445+0000","last_peered":"2024-09-21T10:11:48.255445+0000","last_clean":"2024-09-21T10:11:48.255445+0000","last_became_active":"2024-09-21T10:09:54.326338+0000","last_became_peered":"2024-09-21T10:09:54.326338+0000","last_unstale":"2024-09-21T10:11:48.255445+0000","last_undegraded":"2024-09-21T10:11:48.255445+0000","last_fullsized":"2024-09-21T10:11:48.255445+0000","mapping_epoch":1208,"log_start":"1216'5661","ondisk_log_start":"1216'5661","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1221'8437","last_scrub_stamp":"2024-09-21T10:09:56.518403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:09:56.518403+0000","objects_scrubbed":3599,"log_size":6342,"log_dups_size":0,"ondisk_log_size":6342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:09:23.788407+0000","scrub_duration":0.71259566799999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3578,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":252,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883646+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:11:31.883646+0000","last_peered":"2024-09-21T10:11:31.883646+0000","last_clean":"2024-09-21T10:11:31.883646+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:11:31.883646+0000","last_undegraded":"2024-09-21T10:11:31.883646+0000","last_fullsized":"2024-09-21T10:11:31.883646+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1293,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890605+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:11:31.890605+0000","last_peered":"2024-09-21T10:11:31.890605+0000","last_clean":"2024-09-21T10:11:31.890605+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:11:31.890605+0000","last_undegraded":"2024-09-21T10:11:31.890605+0000","last_fullsized":"2024-09-21T10:11:31.890605+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"1257'11580","reported_seq":11696,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.154442+0000","last_change":"2024-09-21T10:11:01.235231+0000","last_active":"2024-09-21T10:11:48.154442+0000","last_peered":"2024-09-21T10:11:48.154442+0000","last_clean":"2024-09-21T10:11:48.154442+0000","last_became_active":"2024-09-21T10:09:54.327583+0000","last_became_peered":"2024-09-21T10:09:54.327583+0000","last_unstale":"2024-09-21T10:11:48.154442+0000","last_undegraded":"2024-09-21T10:11:48.154442+0000","last_fullsized":"2024-09-21T10:11:48.154442+0000","mapping_epoch":1208,"log_start":"1215'5136","ondisk_log_start":"1215'5136","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1254'9787","last_scrub_stamp":"2024-09-21T10:11:01.235092+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:01.235092+0000","objects_scrubbed":1814,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:36.650925+0000","scrub_duration":0.39794432699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1048576,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1798,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":127,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.880764+0000","last_change":"2024-09-21T10:08:43.398923+0000","last_active":"2024-09-21T10:11:31.880764+0000","last_peered":"2024-09-21T10:11:31.880764+0000","last_clean":"2024-09-21T10:11:31.880764+0000","last_became_active":"2024-09-21T10:08:43.383598+0000","last_became_peered":"2024-09-21T10:08:43.383598+0000","last_unstale":"2024-09-21T10:11:31.880764+0000","last_undegraded":"2024-09-21T10:11:31.880764+0000","last_fullsized":"2024-09-21T10:11:31.880764+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:09.939781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615486+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:11:24.615486+0000","last_peered":"2024-09-21T10:11:24.615486+0000","last_clean":"2024-09-21T10:11:24.615486+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:11:24.615486+0000","last_undegraded":"2024-09-21T10:11:24.615486+0000","last_fullsized":"2024-09-21T10:11:24.615486+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1257'11722","reported_seq":11883,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.935537+0000","last_change":"2024-09-21T10:10:52.094294+0000","last_active":"2024-09-21T10:11:47.935537+0000","last_peered":"2024-09-21T10:11:47.935537+0000","last_clean":"2024-09-21T10:11:47.935537+0000","last_became_active":"2024-09-21T10:09:54.329468+0000","last_became_peered":"2024-09-21T10:09:54.329468+0000","last_unstale":"2024-09-21T10:11:47.935537+0000","last_undegraded":"2024-09-21T10:11:47.935537+0000","last_fullsized":"2024-09-21T10:11:47.935537+0000","mapping_epoch":1208,"log_start":"1217'6370","ondisk_log_start":"1217'6370","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9610","last_scrub_stamp":"2024-09-21T10:10:52.094143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:52.094143+0000","objects_scrubbed":2145,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:04:03.544577+0000","scrub_duration":665,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1900544,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2116,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1323,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875905+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:11:31.875905+0000","last_peered":"2024-09-21T10:11:31.875905+0000","last_clean":"2024-09-21T10:11:31.875905+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:11:31.875905+0000","last_undegraded":"2024-09-21T10:11:31.875905+0000","last_fullsized":"2024-09-21T10:11:31.875905+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1679,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.889966+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:11:31.889966+0000","last_peered":"2024-09-21T10:11:31.889966+0000","last_clean":"2024-09-21T10:11:31.889966+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:11:31.889966+0000","last_undegraded":"2024-09-21T10:11:31.889966+0000","last_fullsized":"2024-09-21T10:11:31.889966+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"1257'11794","reported_seq":11983,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.838864+0000","last_change":"2024-09-21T10:10:54.158428+0000","last_active":"2024-09-21T10:11:47.838864+0000","last_peered":"2024-09-21T10:11:47.838864+0000","last_clean":"2024-09-21T10:11:47.838864+0000","last_became_active":"2024-09-21T10:09:54.318768+0000","last_became_peered":"2024-09-21T10:09:54.318768+0000","last_unstale":"2024-09-21T10:11:47.838864+0000","last_undegraded":"2024-09-21T10:11:47.838864+0000","last_fullsized":"2024-09-21T10:11:47.838864+0000","mapping_epoch":1208,"log_start":"1216'6005","ondisk_log_start":"1216'6005","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9764","last_scrub_stamp":"2024-09-21T10:10:54.158201+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:54.158201+0000","objects_scrubbed":2066,"log_size":5789,"log_dups_size":0,"ondisk_log_size":5789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:24:47.620753+0000","scrub_duration":676,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1245184,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2047,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":361,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876889+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:11:31.876889+0000","last_peered":"2024-09-21T10:11:31.876889+0000","last_clean":"2024-09-21T10:11:31.876889+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:11:31.876889+0000","last_undegraded":"2024-09-21T10:11:31.876889+0000","last_fullsized":"2024-09-21T10:11:31.876889+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1338,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875817+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:11:31.875817+0000","last_peered":"2024-09-21T10:11:31.875817+0000","last_clean":"2024-09-21T10:11:31.875817+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:11:31.875817+0000","last_undegraded":"2024-09-21T10:11:31.875817+0000","last_fullsized":"2024-09-21T10:11:31.875817+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"1257'11973","reported_seq":12145,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.864594+0000","last_change":"2024-09-21T10:09:55.880497+0000","last_active":"2024-09-21T10:11:47.864594+0000","last_peered":"2024-09-21T10:11:47.864594+0000","last_clean":"2024-09-21T10:11:47.864594+0000","last_became_active":"2024-09-21T10:09:54.328846+0000","last_became_peered":"2024-09-21T10:09:54.328846+0000","last_unstale":"2024-09-21T10:11:47.864594+0000","last_undegraded":"2024-09-21T10:11:47.864594+0000","last_fullsized":"2024-09-21T10:11:47.864594+0000","mapping_epoch":1208,"log_start":"1217'6571","ondisk_log_start":"1217'6571","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1221'8475","last_scrub_stamp":"2024-09-21T10:09:55.880305+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:09:55.880305+0000","objects_scrubbed":3550,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:00.793347+0000","scrub_duration":1167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1572864,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3526,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615536+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:11:24.615536+0000","last_peered":"2024-09-21T10:11:24.615536+0000","last_clean":"2024-09-21T10:11:24.615536+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:11:24.615536+0000","last_undegraded":"2024-09-21T10:11:24.615536+0000","last_fullsized":"2024-09-21T10:11:24.615536+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":981,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.887233+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:11:31.887233+0000","last_peered":"2024-09-21T10:11:31.887233+0000","last_clean":"2024-09-21T10:11:31.887233+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:11:31.887233+0000","last_undegraded":"2024-09-21T10:11:31.887233+0000","last_fullsized":"2024-09-21T10:11:31.887233+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"1257'11859","reported_seq":11990,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.778891+0000","last_change":"2024-09-21T10:09:54.371352+0000","last_active":"2024-09-21T10:11:47.778891+0000","last_peered":"2024-09-21T10:11:47.778891+0000","last_clean":"2024-09-21T10:11:47.778891+0000","last_became_active":"2024-09-21T10:09:54.340794+0000","last_became_peered":"2024-09-21T10:09:54.340794+0000","last_unstale":"2024-09-21T10:11:47.778891+0000","last_undegraded":"2024-09-21T10:11:47.778891+0000","last_fullsized":"2024-09-21T10:11:47.778891+0000","mapping_epoch":1208,"log_start":"1216'6165","ondisk_log_start":"1216'6165","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:11:48.458801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7756,"num_write_kb":246720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1324,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890336+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:11:31.890336+0000","last_peered":"2024-09-21T10:11:31.890336+0000","last_clean":"2024-09-21T10:11:31.890336+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:11:31.890336+0000","last_undegraded":"2024-09-21T10:11:31.890336+0000","last_fullsized":"2024-09-21T10:11:31.890336+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.880345+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:11:31.880345+0000","last_peered":"2024-09-21T10:11:31.880345+0000","last_clean":"2024-09-21T10:11:31.880345+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:11:31.880345+0000","last_undegraded":"2024-09-21T10:11:31.880345+0000","last_fullsized":"2024-09-21T10:11:31.880345+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"1257'11656","reported_seq":11817,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.790200+0000","last_change":"2024-09-21T10:10:51.164053+0000","last_active":"2024-09-21T10:11:47.790200+0000","last_peered":"2024-09-21T10:11:47.790200+0000","last_clean":"2024-09-21T10:11:47.790200+0000","last_became_active":"2024-09-21T10:09:54.326478+0000","last_became_peered":"2024-09-21T10:09:54.326478+0000","last_unstale":"2024-09-21T10:11:47.790200+0000","last_undegraded":"2024-09-21T10:11:47.790200+0000","last_fullsized":"2024-09-21T10:11:47.790200+0000","mapping_epoch":1208,"log_start":"1216'6243","ondisk_log_start":"1216'6243","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9542","last_scrub_stamp":"2024-09-21T10:10:51.163756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:51.163756+0000","objects_scrubbed":2162,"log_size":5413,"log_dups_size":0,"ondisk_log_size":5413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:13:36.714601+0000","scrub_duration":728,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2097152,"num_objects":32,"num_object_clones":0,"num_object_copies":96,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2130,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876056+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:11:31.876056+0000","last_peered":"2024-09-21T10:11:31.876056+0000","last_clean":"2024-09-21T10:11:31.876056+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:11:31.876056+0000","last_undegraded":"2024-09-21T10:11:31.876056+0000","last_fullsized":"2024-09-21T10:11:31.876056+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1683,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890642+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:11:31.890642+0000","last_peered":"2024-09-21T10:11:31.890642+0000","last_clean":"2024-09-21T10:11:31.890642+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:11:31.890642+0000","last_undegraded":"2024-09-21T10:11:31.890642+0000","last_fullsized":"2024-09-21T10:11:31.890642+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"1257'11921","reported_seq":12038,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.189666+0000","last_change":"2024-09-21T10:10:57.217425+0000","last_active":"2024-09-21T10:11:48.189666+0000","last_peered":"2024-09-21T10:11:48.189666+0000","last_clean":"2024-09-21T10:11:48.189666+0000","last_became_active":"2024-09-21T10:09:54.339432+0000","last_became_peered":"2024-09-21T10:09:54.339432+0000","last_unstale":"2024-09-21T10:11:48.189666+0000","last_undegraded":"2024-09-21T10:11:48.189666+0000","last_fullsized":"2024-09-21T10:11:48.189666+0000","mapping_epoch":1208,"log_start":"1215'5441","ondisk_log_start":"1215'5441","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'10025","last_scrub_stamp":"2024-09-21T10:10:57.217250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:57.217250+0000","objects_scrubbed":1929,"log_size":6480,"log_dups_size":0,"ondisk_log_size":6480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:17.043980+0000","scrub_duration":0.43705535000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1912,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":783,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.880253+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:11:31.880253+0000","last_peered":"2024-09-21T10:11:31.880253+0000","last_clean":"2024-09-21T10:11:31.880253+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:11:31.880253+0000","last_undegraded":"2024-09-21T10:11:31.880253+0000","last_fullsized":"2024-09-21T10:11:31.880253+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1337,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876565+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:11:31.876565+0000","last_peered":"2024-09-21T10:11:31.876565+0000","last_clean":"2024-09-21T10:11:31.876565+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:11:31.876565+0000","last_undegraded":"2024-09-21T10:11:31.876565+0000","last_fullsized":"2024-09-21T10:11:31.876565+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":368,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877151+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:11:31.877151+0000","last_peered":"2024-09-21T10:11:31.877151+0000","last_clean":"2024-09-21T10:11:31.877151+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:11:31.877151+0000","last_undegraded":"2024-09-21T10:11:31.877151+0000","last_fullsized":"2024-09-21T10:11:31.877151+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.616027+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:11:24.616027+0000","last_peered":"2024-09-21T10:11:24.616027+0000","last_clean":"2024-09-21T10:11:24.616027+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:11:24.616027+0000","last_undegraded":"2024-09-21T10:11:24.616027+0000","last_fullsized":"2024-09-21T10:11:24.616027+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1257'15346","reported_seq":15468,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.921534+0000","last_change":"2024-09-21T10:08:03.312292+0000","last_active":"2024-09-21T10:11:47.921534+0000","last_peered":"2024-09-21T10:11:47.921534+0000","last_clean":"2024-09-21T10:11:47.921534+0000","last_became_active":"2024-09-21T10:08:03.312011+0000","last_became_peered":"2024-09-21T10:08:03.312011+0000","last_unstale":"2024-09-21T10:11:47.921534+0000","last_undegraded":"2024-09-21T10:11:47.921534+0000","last_fullsized":"2024-09-21T10:11:47.921534+0000","mapping_epoch":1208,"log_start":"1250'10825","ondisk_log_start":"1250'10825","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4521,"log_dups_size":0,"ondisk_log_size":4521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:45.654425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3801088,"num_objects":58,"num_object_clones":0,"num_object_copies":174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15346,"num_write_kb":492928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":968,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882019+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:11:31.882019+0000","last_peered":"2024-09-21T10:11:31.882019+0000","last_clean":"2024-09-21T10:11:31.882019+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:11:31.882019+0000","last_undegraded":"2024-09-21T10:11:31.882019+0000","last_fullsized":"2024-09-21T10:11:31.882019+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1557,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876616+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:11:31.876616+0000","last_peered":"2024-09-21T10:11:31.876616+0000","last_clean":"2024-09-21T10:11:31.876616+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:11:31.876616+0000","last_undegraded":"2024-09-21T10:11:31.876616+0000","last_fullsized":"2024-09-21T10:11:31.876616+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":965,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883437+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:11:31.883437+0000","last_peered":"2024-09-21T10:11:31.883437+0000","last_clean":"2024-09-21T10:11:31.883437+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:11:31.883437+0000","last_undegraded":"2024-09-21T10:11:31.883437+0000","last_fullsized":"2024-09-21T10:11:31.883437+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"1257'15246","reported_seq":15368,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.825296+0000","last_change":"2024-09-21T10:08:03.024270+0000","last_active":"2024-09-21T10:11:47.825296+0000","last_peered":"2024-09-21T10:11:47.825296+0000","last_clean":"2024-09-21T10:11:47.825296+0000","last_became_active":"2024-09-21T10:08:03.023879+0000","last_became_peered":"2024-09-21T10:08:03.023879+0000","last_unstale":"2024-09-21T10:11:47.825296+0000","last_undegraded":"2024-09-21T10:11:47.825296+0000","last_fullsized":"2024-09-21T10:11:47.825296+0000","mapping_epoch":1208,"log_start":"1250'10503","ondisk_log_start":"1250'10503","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4743,"log_dups_size":0,"ondisk_log_size":4743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:58:43.728824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4456448,"num_objects":68,"num_object_clones":0,"num_object_copies":204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":68,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15246,"num_write_kb":490048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1598,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876932+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:11:31.876932+0000","last_peered":"2024-09-21T10:11:31.876932+0000","last_clean":"2024-09-21T10:11:31.876932+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:11:31.876932+0000","last_undegraded":"2024-09-21T10:11:31.876932+0000","last_fullsized":"2024-09-21T10:11:31.876932+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1685,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882260+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:11:31.882260+0000","last_peered":"2024-09-21T10:11:31.882260+0000","last_clean":"2024-09-21T10:11:31.882260+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:11:31.882260+0000","last_undegraded":"2024-09-21T10:11:31.882260+0000","last_fullsized":"2024-09-21T10:11:31.882260+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1692,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890478+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:11:31.890478+0000","last_peered":"2024-09-21T10:11:31.890478+0000","last_clean":"2024-09-21T10:11:31.890478+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:11:31.890478+0000","last_undegraded":"2024-09-21T10:11:31.890478+0000","last_fullsized":"2024-09-21T10:11:31.890478+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"1257'11636","reported_seq":11793,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.807823+0000","last_change":"2024-09-21T10:10:53.152556+0000","last_active":"2024-09-21T10:11:47.807823+0000","last_peered":"2024-09-21T10:11:47.807823+0000","last_clean":"2024-09-21T10:11:47.807823+0000","last_became_active":"2024-09-21T10:09:54.333306+0000","last_became_peered":"2024-09-21T10:09:54.333306+0000","last_unstale":"2024-09-21T10:11:47.807823+0000","last_undegraded":"2024-09-21T10:11:47.807823+0000","last_fullsized":"2024-09-21T10:11:47.807823+0000","mapping_epoch":1208,"log_start":"1216'5946","ondisk_log_start":"1216'5946","created":1208,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":0,"last_scrub":"1253'9546","last_scrub_stamp":"2024-09-21T10:10:53.152277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:53.152277+0000","objects_scrubbed":2127,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:43:36.899463+0000","scrub_duration":652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1703936,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2101,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":261,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882942+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:11:31.882942+0000","last_peered":"2024-09-21T10:11:31.882942+0000","last_clean":"2024-09-21T10:11:31.882942+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:11:31.882942+0000","last_undegraded":"2024-09-21T10:11:31.882942+0000","last_fullsized":"2024-09-21T10:11:31.882942+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1220,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615137+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:11:24.615137+0000","last_peered":"2024-09-21T10:11:24.615137+0000","last_clean":"2024-09-21T10:11:24.615137+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:11:24.615137+0000","last_undegraded":"2024-09-21T10:11:24.615137+0000","last_fullsized":"2024-09-21T10:11:24.615137+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1695,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886194+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:11:31.886194+0000","last_peered":"2024-09-21T10:11:31.886194+0000","last_clean":"2024-09-21T10:11:31.886194+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:11:31.886194+0000","last_undegraded":"2024-09-21T10:11:31.886194+0000","last_fullsized":"2024-09-21T10:11:31.886194+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"1257'15212","reported_seq":15334,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.924119+0000","last_change":"2024-09-21T10:08:03.313515+0000","last_active":"2024-09-21T10:11:47.924119+0000","last_peered":"2024-09-21T10:11:47.924119+0000","last_clean":"2024-09-21T10:11:47.924119+0000","last_became_active":"2024-09-21T10:08:03.313246+0000","last_became_peered":"2024-09-21T10:08:03.313246+0000","last_unstale":"2024-09-21T10:11:47.924119+0000","last_undegraded":"2024-09-21T10:11:47.924119+0000","last_fullsized":"2024-09-21T10:11:47.924119+0000","mapping_epoch":1208,"log_start":"1250'10660","ondisk_log_start":"1250'10660","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4552,"log_dups_size":0,"ondisk_log_size":4552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:51:06.835124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15212,"num_write_kb":488128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1285,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875286+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:11:31.875286+0000","last_peered":"2024-09-21T10:11:31.875286+0000","last_clean":"2024-09-21T10:11:31.875286+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:11:31.875286+0000","last_undegraded":"2024-09-21T10:11:31.875286+0000","last_fullsized":"2024-09-21T10:11:31.875286+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1706,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886322+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:11:31.886322+0000","last_peered":"2024-09-21T10:11:31.886322+0000","last_clean":"2024-09-21T10:11:31.886322+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:11:31.886322+0000","last_undegraded":"2024-09-21T10:11:31.886322+0000","last_fullsized":"2024-09-21T10:11:31.886322+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":225,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.882034+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:11:31.882034+0000","last_peered":"2024-09-21T10:11:31.882034+0000","last_clean":"2024-09-21T10:11:31.882034+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:11:31.882034+0000","last_undegraded":"2024-09-21T10:11:31.882034+0000","last_fullsized":"2024-09-21T10:11:31.882034+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"1257'15263","reported_seq":15385,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.837058+0000","last_change":"2024-09-21T10:08:03.313391+0000","last_active":"2024-09-21T10:11:47.837058+0000","last_peered":"2024-09-21T10:11:47.837058+0000","last_clean":"2024-09-21T10:11:47.837058+0000","last_became_active":"2024-09-21T10:08:03.312838+0000","last_became_peered":"2024-09-21T10:08:03.312838+0000","last_unstale":"2024-09-21T10:11:47.837058+0000","last_undegraded":"2024-09-21T10:11:47.837058+0000","last_fullsized":"2024-09-21T10:11:47.837058+0000","mapping_epoch":1208,"log_start":"1250'10555","ondisk_log_start":"1250'10555","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4708,"log_dups_size":0,"ondisk_log_size":4708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:41:52.169702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15263,"num_write_kb":490432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1683,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886537+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:11:31.886537+0000","last_peered":"2024-09-21T10:11:31.886537+0000","last_clean":"2024-09-21T10:11:31.886537+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:11:31.886537+0000","last_undegraded":"2024-09-21T10:11:31.886537+0000","last_fullsized":"2024-09-21T10:11:31.886537+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":863,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886851+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:11:31.886851+0000","last_peered":"2024-09-21T10:11:31.886851+0000","last_clean":"2024-09-21T10:11:31.886851+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:11:31.886851+0000","last_undegraded":"2024-09-21T10:11:31.886851+0000","last_fullsized":"2024-09-21T10:11:31.886851+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1675,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890711+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:11:31.890711+0000","last_peered":"2024-09-21T10:11:31.890711+0000","last_clean":"2024-09-21T10:11:31.890711+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:11:31.890711+0000","last_undegraded":"2024-09-21T10:11:31.890711+0000","last_fullsized":"2024-09-21T10:11:31.890711+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.c","version":"1257'15203","reported_seq":15325,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.834097+0000","last_change":"2024-09-21T10:08:03.024073+0000","last_active":"2024-09-21T10:11:47.834097+0000","last_peered":"2024-09-21T10:11:47.834097+0000","last_clean":"2024-09-21T10:11:47.834097+0000","last_became_active":"2024-09-21T10:08:03.023647+0000","last_became_peered":"2024-09-21T10:08:03.023647+0000","last_unstale":"2024-09-21T10:11:47.834097+0000","last_undegraded":"2024-09-21T10:11:47.834097+0000","last_fullsized":"2024-09-21T10:11:47.834097+0000","mapping_epoch":1208,"log_start":"1250'10443","ondisk_log_start":"1250'10443","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":4760,"log_dups_size":0,"ondisk_log_size":4760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:06:53.088905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15203,"num_write_kb":488512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":383,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876772+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:11:31.876772+0000","last_peered":"2024-09-21T10:11:31.876772+0000","last_clean":"2024-09-21T10:11:31.876772+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:11:31.876772+0000","last_undegraded":"2024-09-21T10:11:31.876772+0000","last_fullsized":"2024-09-21T10:11:31.876772+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1684,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883785+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:11:31.883785+0000","last_peered":"2024-09-21T10:11:31.883785+0000","last_clean":"2024-09-21T10:11:31.883785+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:11:31.883785+0000","last_undegraded":"2024-09-21T10:11:31.883785+0000","last_fullsized":"2024-09-21T10:11:31.883785+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":361,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875884+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:11:31.875884+0000","last_peered":"2024-09-21T10:11:31.875884+0000","last_clean":"2024-09-21T10:11:31.875884+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:11:31.875884+0000","last_undegraded":"2024-09-21T10:11:31.875884+0000","last_fullsized":"2024-09-21T10:11:31.875884+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1257'15297","reported_seq":15357,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.251995+0000","last_change":"2024-09-21T10:08:03.312130+0000","last_active":"2024-09-21T10:11:48.251995+0000","last_peered":"2024-09-21T10:11:48.251995+0000","last_clean":"2024-09-21T10:11:48.251995+0000","last_became_active":"2024-09-21T10:08:03.311781+0000","last_became_peered":"2024-09-21T10:08:03.311781+0000","last_unstale":"2024-09-21T10:11:48.251995+0000","last_undegraded":"2024-09-21T10:11:48.251995+0000","last_fullsized":"2024-09-21T10:11:48.251995+0000","mapping_epoch":1208,"log_start":"1250'10262","ondisk_log_start":"1250'10262","created":1208,"last_epoch_clean":1209,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:08:01.929359+0000","objects_scrubbed":0,"log_size":5035,"log_dups_size":0,"ondisk_log_size":5035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:33.284319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310744,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15297,"num_write_kb":490113,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":224,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883513+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:11:31.883513+0000","last_peered":"2024-09-21T10:11:31.883513+0000","last_clean":"2024-09-21T10:11:31.883513+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:11:31.883513+0000","last_undegraded":"2024-09-21T10:11:31.883513+0000","last_fullsized":"2024-09-21T10:11:31.883513+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":110,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615913+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:11:24.615913+0000","last_peered":"2024-09-21T10:11:24.615913+0000","last_clean":"2024-09-21T10:11:24.615913+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:11:24.615913+0000","last_undegraded":"2024-09-21T10:11:24.615913+0000","last_fullsized":"2024-09-21T10:11:24.615913+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615656+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:11:24.615656+0000","last_peered":"2024-09-21T10:11:24.615656+0000","last_clean":"2024-09-21T10:11:24.615656+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:11:24.615656+0000","last_undegraded":"2024-09-21T10:11:24.615656+0000","last_fullsized":"2024-09-21T10:11:24.615656+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":109,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615451+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:11:24.615451+0000","last_peered":"2024-09-21T10:11:24.615451+0000","last_clean":"2024-09-21T10:11:24.615451+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:11:24.615451+0000","last_undegraded":"2024-09-21T10:11:24.615451+0000","last_fullsized":"2024-09-21T10:11:24.615451+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615779+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:11:24.615779+0000","last_peered":"2024-09-21T10:11:24.615779+0000","last_clean":"2024-09-21T10:11:24.615779+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:11:24.615779+0000","last_undegraded":"2024-09-21T10:11:24.615779+0000","last_fullsized":"2024-09-21T10:11:24.615779+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1683,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883042+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:11:31.883042+0000","last_peered":"2024-09-21T10:11:31.883042+0000","last_clean":"2024-09-21T10:11:31.883042+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:11:31.883042+0000","last_undegraded":"2024-09-21T10:11:31.883042+0000","last_fullsized":"2024-09-21T10:11:31.883042+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1296,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875480+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:11:31.875480+0000","last_peered":"2024-09-21T10:11:31.875480+0000","last_clean":"2024-09-21T10:11:31.875480+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:11:31.875480+0000","last_undegraded":"2024-09-21T10:11:31.875480+0000","last_fullsized":"2024-09-21T10:11:31.875480+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1306,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883868+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:11:31.883868+0000","last_peered":"2024-09-21T10:11:31.883868+0000","last_clean":"2024-09-21T10:11:31.883868+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:11:31.883868+0000","last_undegraded":"2024-09-21T10:11:31.883868+0000","last_fullsized":"2024-09-21T10:11:31.883868+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":381,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876804+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:11:31.876804+0000","last_peered":"2024-09-21T10:11:31.876804+0000","last_clean":"2024-09-21T10:11:31.876804+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:11:31.876804+0000","last_undegraded":"2024-09-21T10:11:31.876804+0000","last_fullsized":"2024-09-21T10:11:31.876804+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1709,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876512+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:11:31.876512+0000","last_peered":"2024-09-21T10:11:31.876512+0000","last_clean":"2024-09-21T10:11:31.876512+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:11:31.876512+0000","last_undegraded":"2024-09-21T10:11:31.876512+0000","last_fullsized":"2024-09-21T10:11:31.876512+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":360,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876196+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:11:31.876196+0000","last_peered":"2024-09-21T10:11:31.876196+0000","last_clean":"2024-09-21T10:11:31.876196+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:11:31.876196+0000","last_undegraded":"2024-09-21T10:11:31.876196+0000","last_fullsized":"2024-09-21T10:11:31.876196+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.13","version":"1257'11685","reported_seq":17929,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.847608+0000","last_change":"2024-09-21T10:10:51.429358+0000","last_active":"2024-09-21T10:11:47.847608+0000","last_peered":"2024-09-21T10:11:47.847608+0000","last_clean":"2024-09-21T10:11:47.847608+0000","last_became_active":"2024-09-21T10:10:20.087411+0000","last_became_peered":"2024-09-21T10:10:20.087411+0000","last_unstale":"2024-09-21T10:11:47.847608+0000","last_undegraded":"2024-09-21T10:11:47.847608+0000","last_fullsized":"2024-09-21T10:11:47.847608+0000","mapping_epoch":1240,"log_start":"1216'5840","ondisk_log_start":"1216'5840","created":1220,"last_epoch_clean":1241,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9635","last_scrub_stamp":"2024-09-21T10:10:51.429139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:51.429139+0000","objects_scrubbed":2092,"log_size":5845,"log_dups_size":0,"ondisk_log_size":5845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:20.720859+0000","scrub_duration":954,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310720,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2072,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615843+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:11:24.615843+0000","last_peered":"2024-09-21T10:11:24.615843+0000","last_clean":"2024-09-21T10:11:24.615843+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:11:24.615843+0000","last_undegraded":"2024-09-21T10:11:24.615843+0000","last_fullsized":"2024-09-21T10:11:24.615843+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875789+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:11:31.875789+0000","last_peered":"2024-09-21T10:11:31.875789+0000","last_clean":"2024-09-21T10:11:31.875789+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:11:31.875789+0000","last_undegraded":"2024-09-21T10:11:31.875789+0000","last_fullsized":"2024-09-21T10:11:31.875789+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"1257'11717","reported_seq":18114,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.196249+0000","last_change":"2024-09-21T10:11:06.067859+0000","last_active":"2024-09-21T10:11:48.196249+0000","last_peered":"2024-09-21T10:11:48.196249+0000","last_clean":"2024-09-21T10:11:48.196249+0000","last_became_active":"2024-09-21T10:10:16.078568+0000","last_became_peered":"2024-09-21T10:10:16.078568+0000","last_unstale":"2024-09-21T10:11:48.196249+0000","last_undegraded":"2024-09-21T10:11:48.196249+0000","last_fullsized":"2024-09-21T10:11:48.196249+0000","mapping_epoch":1236,"log_start":"1215'5157","ondisk_log_start":"1215'5157","created":1220,"last_epoch_clean":1237,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10135","last_scrub_stamp":"2024-09-21T10:11:06.067704+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:06.067704+0000","objects_scrubbed":1606,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:23:42.440109+0000","scrub_duration":0.27802364099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1588,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1672,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877769+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:11:31.877769+0000","last_peered":"2024-09-21T10:11:31.877769+0000","last_clean":"2024-09-21T10:11:31.877769+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:11:31.877769+0000","last_undegraded":"2024-09-21T10:11:31.877769+0000","last_fullsized":"2024-09-21T10:11:31.877769+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1438,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876709+0000","last_change":"2024-09-21T10:08:43.397612+0000","last_active":"2024-09-21T10:11:31.876709+0000","last_peered":"2024-09-21T10:11:31.876709+0000","last_clean":"2024-09-21T10:11:31.876709+0000","last_became_active":"2024-09-21T10:08:43.397254+0000","last_became_peered":"2024-09-21T10:08:43.397254+0000","last_unstale":"2024-09-21T10:11:31.876709+0000","last_undegraded":"2024-09-21T10:11:31.876709+0000","last_fullsized":"2024-09-21T10:11:31.876709+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:40.293786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.11","version":"1257'12091","reported_seq":18492,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.103558+0000","last_change":"2024-09-21T10:11:07.186895+0000","last_active":"2024-09-21T10:11:48.103558+0000","last_peered":"2024-09-21T10:11:48.103558+0000","last_clean":"2024-09-21T10:11:48.103558+0000","last_became_active":"2024-09-21T10:10:28.162032+0000","last_became_peered":"2024-09-21T10:10:28.162032+0000","last_unstale":"2024-09-21T10:11:48.103558+0000","last_undegraded":"2024-09-21T10:11:48.103558+0000","last_fullsized":"2024-09-21T10:11:48.103558+0000","mapping_epoch":1246,"log_start":"1216'5820","ondisk_log_start":"1216'5820","created":1220,"last_epoch_clean":1247,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10566","last_scrub_stamp":"2024-09-21T10:11:07.186675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:07.186675+0000","objects_scrubbed":1548,"log_size":6271,"log_dups_size":0,"ondisk_log_size":6271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:31:00.595388+0000","scrub_duration":0.367382598,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1534,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":345,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875583+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:11:31.875583+0000","last_peered":"2024-09-21T10:11:31.875583+0000","last_clean":"2024-09-21T10:11:31.875583+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:11:31.875583+0000","last_undegraded":"2024-09-21T10:11:31.875583+0000","last_fullsized":"2024-09-21T10:11:31.875583+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":229,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.883183+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:11:31.883183+0000","last_peered":"2024-09-21T10:11:31.883183+0000","last_clean":"2024-09-21T10:11:31.883183+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:11:31.883183+0000","last_undegraded":"2024-09-21T10:11:31.883183+0000","last_fullsized":"2024-09-21T10:11:31.883183+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.10","version":"1257'11800","reported_seq":18484,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.114382+0000","last_change":"2024-09-21T10:11:04.073987+0000","last_active":"2024-09-21T10:11:48.114382+0000","last_peered":"2024-09-21T10:11:48.114382+0000","last_clean":"2024-09-21T10:11:48.114382+0000","last_became_active":"2024-09-21T10:10:12.107258+0000","last_became_peered":"2024-09-21T10:10:12.107258+0000","last_unstale":"2024-09-21T10:11:48.114382+0000","last_undegraded":"2024-09-21T10:11:48.114382+0000","last_fullsized":"2024-09-21T10:11:48.114382+0000","mapping_epoch":1232,"log_start":"1215'5365","ondisk_log_start":"1215'5365","created":1220,"last_epoch_clean":1233,"parent":"0.0","parent_split_bits":5,"last_scrub":"1254'10189","last_scrub_stamp":"2024-09-21T10:11:04.073694+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:04.073694+0000","objects_scrubbed":1630,"log_size":6435,"log_dups_size":0,"ondisk_log_size":6435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:06.492179+0000","scrub_duration":0.33052306399999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":851968,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1617,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1566,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886454+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:11:31.886454+0000","last_peered":"2024-09-21T10:11:31.886454+0000","last_clean":"2024-09-21T10:11:31.886454+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:11:31.886454+0000","last_undegraded":"2024-09-21T10:11:31.886454+0000","last_fullsized":"2024-09-21T10:11:31.886454+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1647,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.890613+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:11:31.890613+0000","last_peered":"2024-09-21T10:11:31.890613+0000","last_clean":"2024-09-21T10:11:31.890613+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:11:31.890613+0000","last_undegraded":"2024-09-21T10:11:31.890613+0000","last_fullsized":"2024-09-21T10:11:31.890613+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.17","version":"1257'11693","reported_seq":20508,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:48.262230+0000","last_change":"2024-09-21T10:11:08.029759+0000","last_active":"2024-09-21T10:11:48.262230+0000","last_peered":"2024-09-21T10:11:48.262230+0000","last_clean":"2024-09-21T10:11:48.262230+0000","last_became_active":"2024-09-21T10:10:30.082690+0000","last_became_peered":"2024-09-21T10:10:30.082690+0000","last_unstale":"2024-09-21T10:11:48.262230+0000","last_undegraded":"2024-09-21T10:11:48.262230+0000","last_fullsized":"2024-09-21T10:11:48.262230+0000","mapping_epoch":1248,"log_start":"1215'5167","ondisk_log_start":"1215'5167","created":1220,"last_epoch_clean":1249,"parent":"0.0","parent_split_bits":5,"last_scrub":"1255'10224","last_scrub_stamp":"2024-09-21T10:11:08.029553+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:11:08.029553+0000","objects_scrubbed":1485,"log_size":6526,"log_dups_size":0,"ondisk_log_size":6526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:56.486413+0000","scrub_duration":0.23874677499999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1474,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1291,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.876802+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:11:31.876802+0000","last_peered":"2024-09-21T10:11:31.876802+0000","last_clean":"2024-09-21T10:11:31.876802+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:11:31.876802+0000","last_undegraded":"2024-09-21T10:11:31.876802+0000","last_fullsized":"2024-09-21T10:11:31.876802+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":245,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.886629+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:11:31.886629+0000","last_peered":"2024-09-21T10:11:31.886629+0000","last_clean":"2024-09-21T10:11:31.886629+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:11:31.886629+0000","last_undegraded":"2024-09-21T10:11:31.886629+0000","last_fullsized":"2024-09-21T10:11:31.886629+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.16","version":"1257'11495","reported_seq":17342,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.805176+0000","last_change":"2024-09-21T10:10:55.202726+0000","last_active":"2024-09-21T10:11:47.805176+0000","last_peered":"2024-09-21T10:11:47.805176+0000","last_clean":"2024-09-21T10:11:47.805176+0000","last_became_active":"2024-09-21T10:10:31.083463+0000","last_became_peered":"2024-09-21T10:10:31.083463+0000","last_unstale":"2024-09-21T10:11:47.805176+0000","last_undegraded":"2024-09-21T10:11:47.805176+0000","last_fullsized":"2024-09-21T10:11:47.805176+0000","mapping_epoch":1249,"log_start":"1216'5618","ondisk_log_start":"1216'5618","created":1220,"last_epoch_clean":1250,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9533","last_scrub_stamp":"2024-09-21T10:10:55.202568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:55.202568+0000","objects_scrubbed":2002,"log_size":5877,"log_dups_size":0,"ondisk_log_size":5877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:17:34.953151+0000","scrub_duration":734,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1968,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1298,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.875428+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:11:31.875428+0000","last_peered":"2024-09-21T10:11:31.875428+0000","last_clean":"2024-09-21T10:11:31.875428+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:11:31.875428+0000","last_undegraded":"2024-09-21T10:11:31.875428+0000","last_fullsized":"2024-09-21T10:11:31.875428+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":113,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.616011+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:11:24.616011+0000","last_peered":"2024-09-21T10:11:24.616011+0000","last_clean":"2024-09-21T10:11:24.616011+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:11:24.616011+0000","last_undegraded":"2024-09-21T10:11:24.616011+0000","last_fullsized":"2024-09-21T10:11:24.616011+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.15","version":"1257'11940","reported_seq":17408,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.900973+0000","last_change":"2024-09-21T10:10:56.177599+0000","last_active":"2024-09-21T10:11:47.900973+0000","last_peered":"2024-09-21T10:11:47.900973+0000","last_clean":"2024-09-21T10:11:47.900973+0000","last_became_active":"2024-09-21T10:10:50.088118+0000","last_became_peered":"2024-09-21T10:10:50.088118+0000","last_unstale":"2024-09-21T10:11:47.900973+0000","last_undegraded":"2024-09-21T10:11:47.900973+0000","last_fullsized":"2024-09-21T10:11:47.900973+0000","mapping_epoch":1252,"log_start":"1217'6565","ondisk_log_start":"1217'6565","created":1220,"last_epoch_clean":1253,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9907","last_scrub_stamp":"2024-09-21T10:10:56.177478+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:56.177478+0000","objects_scrubbed":2064,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:26:51.322192+0000","scrub_duration":688,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1572864,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2040,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":353,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877060+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:11:31.877060+0000","last_peered":"2024-09-21T10:11:31.877060+0000","last_clean":"2024-09-21T10:11:31.877060+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:11:31.877060+0000","last_undegraded":"2024-09-21T10:11:31.877060+0000","last_fullsized":"2024-09-21T10:11:31.877060+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":361,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:31.877566+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:11:31.877566+0000","last_peered":"2024-09-21T10:11:31.877566+0000","last_clean":"2024-09-21T10:11:31.877566+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:11:31.877566+0000","last_undegraded":"2024-09-21T10:11:31.877566+0000","last_fullsized":"2024-09-21T10:11:31.877566+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"1257'11751","reported_seq":18132,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-09-21T10:11:47.874987+0000","last_change":"2024-09-21T10:10:55.122874+0000","last_active":"2024-09-21T10:11:47.874987+0000","last_peered":"2024-09-21T10:11:47.874987+0000","last_clean":"2024-09-21T10:11:47.874987+0000","last_became_active":"2024-09-21T10:10:21.091421+0000","last_became_peered":"2024-09-21T10:10:21.091421+0000","last_unstale":"2024-09-21T10:11:47.874987+0000","last_undegraded":"2024-09-21T10:11:47.874987+0000","last_fullsized":"2024-09-21T10:11:47.874987+0000","mapping_epoch":1241,"log_start":"1217'6358","ondisk_log_start":"1217'6358","created":1220,"last_epoch_clean":1242,"parent":"0.0","parent_split_bits":5,"last_scrub":"1253'9715","last_scrub_stamp":"2024-09-21T10:10:55.122315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:08:01.929359+0000","last_clean_scrub_stamp":"2024-09-21T10:10:55.122315+0000","objects_scrubbed":2071,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:59.171003+0000","scrub_duration":652,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1835008,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2043,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":109,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615418+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:11:24.615418+0000","last_peered":"2024-09-21T10:11:24.615418+0000","last_clean":"2024-09-21T10:11:24.615418+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:11:24.615418+0000","last_undegraded":"2024-09-21T10:11:24.615418+0000","last_fullsized":"2024-09-21T10:11:24.615418+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1628,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-09-21T10:11:24.615786+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:11:24.615786+0000","last_peered":"2024-09-21T10:11:24.615786+0000","last_clean":"2024-09-21T10:11:24.615786+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:11:24.615786+0000","last_undegraded":"2024-09-21T10:11:24.615786+0000","last_fullsized":"2024-09-21T10:11:24.615786+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":48234520,"num_objects":737,"num_object_clones":0,"num_object_copies":2211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":737,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138295,"num_write_kb":3186881,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":354426880,"data_stored":354418736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":148107,"ondisk_log_size":148107,"up":78,"acting":78,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":11142,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234431,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496067,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":737724,"kb_used_data":120948,"kb_used_omap":1,"kb_used_meta":616766,"kb_avail":93007428,"statfs":{"total":95995035648,"available":95239606272,"internally_reserved":0,"allocated":123850752,"data_stored":121050463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":631568839},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757706,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019342,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649012,"kb_used_data":180652,"kb_used_omap":1,"kb_used_meta":468350,"kb_avail":93096140,"statfs":{"total":95995035648,"available":95330447360,"internally_reserved":0,"allocated":184987648,"data_stored":182194986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":479590854},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888638,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":816864,"kb_used_data":76824,"kb_used_omap":34,"kb_used_meta":739997,"kb_avail":92928288,"statfs":{"total":95995035648,"available":95158566912,"internally_reserved":0,"allocated":78667776,"data_stored":75890183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":757757939},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967878,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":685488,"kb_used_data":61992,"kb_used_omap":41,"kb_used_meta":623446,"kb_avail":93059664,"statfs":{"total":95995035648,"available":95293095936,"internally_reserved":0,"allocated":63479808,"data_stored":60691686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":638409119},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047116,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650404,"kb_used_data":80732,"kb_used_omap":41,"kb_used_meta":569302,"kb_avail":93094748,"statfs":{"total":95995035648,"available":95329021952,"internally_reserved":0,"allocated":82669568,"data_stored":79888357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":582966185},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126355,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":35524608,"data_stored":35520536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":50331648,"data_stored":50331648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":38273024,"data_stored":38273024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":130088960,"data_stored":130088960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":100208640,"data_stored":100204568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:11:55.029 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:11:55.029 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:11:55.029 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:11:55.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 1000 0.05 2024-09-21T10:11:55.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:55.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:55.087 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/56636192' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:55.087 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/56636192' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:11:55.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:55.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:55.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:55.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:55.416 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:55.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:55.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:55.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:55.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:55.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:11:55.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:55.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:55.806 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:11:55.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:11:55.807 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:55.816 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:11:55.816 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:11:55.832 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:11:55.832 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:11:55.849 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:11:55.850 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:11:55.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[110963]: pgmap v1778: 99 pgs: 99 active+clean; 582 KiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-21T10:11:55.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:55.850 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/56636192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[110963]: osdmap e1258: 8 total, 8 up, 5 in 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1459318562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[106482]: pgmap v1778: 99 pgs: 99 active+clean; 582 KiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/56636192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[106482]: osdmap e1258: 8 total, 8 up, 5 in 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:55.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1459318562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:55.866 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:11:55.867 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:11:55.871 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:55.882 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:11:55.882 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:11:55.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:11:55.899 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:11:55.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:55.915 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:11:55.916 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:11:55.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:55.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:55.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:55 smithi191 ceph-mon[102206]: pgmap v1778: 99 pgs: 99 active+clean; 582 KiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-21T10:11:55.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:55 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:11:55.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/56636192' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:11:55.991 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:55 smithi191 ceph-mon[102206]: osdmap e1258: 8 total, 8 up, 5 in 2024-09-21T10:11:55.992 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:11:55.992 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1459318562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:11:56.030 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:56.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:56.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:56.174 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:56.276 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:56.297 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:56.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:56.420 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:56.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:56.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:56.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:56.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:56.665 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:56.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:56.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:56.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:11:56.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:56.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:56.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:11:57.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:57.033 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:11:57.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:57.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:57.184 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:11:57.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:57.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:57.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:57.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:57.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:57.525 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:57.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:11:57.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:57.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:57.653 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:11:57.654 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:11:57.654 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:11:57.654 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:11:57.654 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:11:57.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:57.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:57.874 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:57.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:57.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:57.986 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:57 smithi076 ceph-mon[106482]: pgmap v1780: 99 pgs: 99 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 693 op/s 2024-09-21T10:11:57.987 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:57 smithi076 ceph-mon[110963]: pgmap v1780: 99 pgs: 99 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 693 op/s 2024-09-21T10:11:57.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:58.017 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:57 smithi191 ceph-mon[102206]: pgmap v1780: 99 pgs: 99 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 693 op/s 2024-09-21T10:11:58.020 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:58.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:58.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:58.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:58.267 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:58.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:58.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:11:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.391 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:58.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:58.492 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:58.513 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:58.585 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:11:58.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.637 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:58.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:58.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:11:58.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:11:58.760 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:11:58.840 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_3' removed 2024-09-21T10:11:58.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:11:58.903 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:58.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:58.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:11:58.979 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3512101557' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:11:58.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3512101557' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:11:58.984 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3512101557' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:11:59.006 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:11:59.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:59.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:11:59.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:59.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.163 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:59.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:59.264 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:59.285 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:59.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:59.387 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:11:59.409 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:59.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:11:59.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:59.532 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:59.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:59.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:11:59.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:11:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:11:59.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:11:59.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:11:59.793 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:11:59.793 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:11:59.794 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-21T10:11:59.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_4 16 2024-09-21T10:11:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:59 smithi191 ceph-mon[102206]: pgmap v1781: 99 pgs: 99 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 76 op/s 2024-09-21T10:11:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:59 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3512101557' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:11:59.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:11:59 smithi191 ceph-mon[102206]: osdmap e1259: 8 total, 8 up, 5 in 2024-09-21T10:11:59.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:11:59.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:11:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:11:59.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:00.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:00.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:00.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:59 smithi076 ceph-mon[106482]: pgmap v1781: 99 pgs: 99 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 76 op/s 2024-09-21T10:12:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:59 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3512101557' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:12:00.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:11:59 smithi076 ceph-mon[106482]: osdmap e1259: 8 total, 8 up, 5 in 2024-09-21T10:12:00.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:59 smithi076 ceph-mon[110963]: pgmap v1781: 99 pgs: 99 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 76 op/s 2024-09-21T10:12:00.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:59 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3512101557' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:12:00.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:11:59 smithi076 ceph-mon[110963]: osdmap e1259: 8 total, 8 up, 5 in 2024-09-21T10:12:00.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:00.149 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:00.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:00.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:00.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:00.308 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:00.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:00.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:00.411 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:12:00.411 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:12:00.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:12:00.411 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [] 2024-09-21T10:12:00.411 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:12:00.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:00.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:00.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:00.640 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:00.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:00.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:00.865 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:00.887 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:00.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:00.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:00.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:01.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:01.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:12:01.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:01.133 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:01.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:01.247 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:01.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:01.308 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:01.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:01.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:01.481 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:01.503 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:01.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:01.605 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:01.626 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:01.728 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:01.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:01.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:01.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:01.959 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:01.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:01.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:01.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:02.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:01 smithi076 ceph-mon[106482]: pgmap v1783: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:02.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:01 smithi076 ceph-mon[110963]: pgmap v1783: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:02.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:01 smithi191 ceph-mon[102206]: pgmap v1783: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:02.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:02.171 INFO:teuthology.orchestra.run.smithi076.stdout:moved 0 / 219 (0%) 2024-09-21T10:12:02.171 INFO:teuthology.orchestra.run.smithi076.stdout:avg 43.8 2024-09-21T10:12:02.171 INFO:teuthology.orchestra.run.smithi076.stdout:stddev 5.26878 -> 5.26878 (expected baseline 5.91946) 2024-09-21T10:12:02.171 INFO:teuthology.orchestra.run.smithi076.stdout:min osd.2 with 36 -> 36 pgs (0.821918 -> 0.821918 * mean) 2024-09-21T10:12:02.171 INFO:teuthology.orchestra.run.smithi076.stdout:max osd.1 with 51 -> 51 pgs (1.16438 -> 1.16438 * mean) 2024-09-21T10:12:02.171 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:12:02.172 INFO:teuthology.orchestra.run.smithi076.stdout:oload 1000 2024-09-21T10:12:02.172 INFO:teuthology.orchestra.run.smithi076.stdout:max_change 0.05 2024-09-21T10:12:02.172 INFO:teuthology.orchestra.run.smithi076.stdout:max_change_osds 4 2024-09-21T10:12:02.172 INFO:teuthology.orchestra.run.smithi076.stdout:average_utilization 0.0067 2024-09-21T10:12:02.172 INFO:teuthology.orchestra.run.smithi076.stdout:overload_utilization 0.0674 2024-09-21T10:12:02.172 INFO:teuthology.orchestra.run.smithi076.stderr:no change 2024-09-21T10:12:02.220 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:02.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:02.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:02.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:02.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:02.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:02.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:02.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:02.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.712 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.733 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:02.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:02.834 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:02.858 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:02.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:02.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:02.981 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:02.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:02.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:03.059 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:12:03.059 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:12:03.059 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:12:03.059 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:12:03.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:03.104 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:02 smithi191 ceph-mon[102206]: from='client.35613 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:02 smithi076 ceph-mon[106482]: from='client.35613 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:02 smithi076 ceph-mon[110963]: from='client.35613 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:03.206 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:03.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:03.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:03.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:03.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:03.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:03.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:03.548 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:12:03] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.51.0" 2024-09-21T10:12:03.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:03.599 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:03.701 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:03.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:03.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:12:03.823 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:03.845 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:03.930 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:03.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:03.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:03.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:04.050 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:03 smithi191 ceph-mon[102206]: pgmap v1784: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:04.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:04.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:04.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:04.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:03 smithi076 ceph-mon[106482]: pgmap v1784: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:04.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:03 smithi076 ceph-mon[110963]: pgmap v1784: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:04.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:04.214 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:04.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:04.338 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:04.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:04.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:04.461 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:04.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:04.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:04.686 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:04.707 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:04.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:04.830 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:04.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:04.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:04.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:05.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:05.077 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:05.179 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:05.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:05.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:05.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:05.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:05.448 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:05.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:05.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:05.572 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:05.623 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:12:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:12:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:12:05.623 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:12:05.623 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:12:05.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:05.694 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:05.699 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-21T10:12:05.699 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 1000 1.0 2024-09-21T10:12:05.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:05.817 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:05.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:05.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:05.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:05.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:05.961 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:05 smithi191 ceph-mon[102206]: pgmap v1785: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:06.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:06.056 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:05 smithi076 ceph-mon[106482]: pgmap v1785: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:06.056 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:05 smithi076 ceph-mon[110963]: pgmap v1785: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:06.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:06.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:06.164 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:12:06.165 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:06.177 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:12:06.178 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:06.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:06.194 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:12:06.195 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:06.215 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:12:06.215 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:06.231 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:12:06.231 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:06.247 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:12:06.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:06.263 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:12:06.264 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:06.280 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:12:06.280 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:06.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:06.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:06.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:12:06.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:06.432 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:06.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:06.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:06.560 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:06.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:06.680 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:06.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:06.803 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:06.860 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_4' created 2024-09-21T10:12:06.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:06.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:06.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:06.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:06.964 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1966059451' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:12:06.964 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/721044107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:07.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:07.048 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:07.096 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1966059451' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:12:07.096 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/721044107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:07.096 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1966059451' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-21T10:12:07.096 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/721044107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:07.150 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:07.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:07.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.296 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:07.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:07.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:07.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:07.521 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:07.542 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:07.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:07.644 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:07.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:07.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:07.790 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:07.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:07.867 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:12:07.892 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:07.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:07.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:08.017 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:08.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:08.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:07 smithi191 ceph-mon[102206]: pgmap v1786: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:08.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:07 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1966059451' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:12:08.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:07 smithi191 ceph-mon[102206]: osdmap e1260: 8 total, 8 up, 5 in 2024-09-21T10:12:08.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:07 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/90428028' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:08.086 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:07 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[106482]: pgmap v1786: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1966059451' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:12:08.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[106482]: osdmap e1260: 8 total, 8 up, 5 in 2024-09-21T10:12:08.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/90428028' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:08.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[110963]: pgmap v1786: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:08.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1966059451' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-21T10:12:08.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[110963]: osdmap e1260: 8 total, 8 up, 5 in 2024-09-21T10:12:08.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/90428028' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:08.125 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:07 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:12:08.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:08.163 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:08.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:08.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:08.321 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:08.321 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:08.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:08.409 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:08.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:08.531 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:08.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:08.633 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:08.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:08.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:08.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:08.776 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:08.878 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:08.899 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:08.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:09.000 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:08 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:09.001 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:08 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:09.001 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:08 smithi191 ceph-mon[102206]: osdmap e1261: 8 total, 8 up, 5 in 2024-09-21T10:12:09.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:09.022 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:09.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:12:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:08 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:08 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:09.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:08 smithi076 ceph-mon[106482]: osdmap e1261: 8 total, 8 up, 5 in 2024-09-21T10:12:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:08 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:08 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:12:09.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:08 smithi076 ceph-mon[110963]: osdmap e1261: 8 total, 8 up, 5 in 2024-09-21T10:12:09.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:09.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:09.249 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:09.263 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:09.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:09.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:09.360 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:12:09.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:09.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:09.526 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:09.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:09.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:09.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:09.669 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:09.771 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:09.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:09.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:09.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:09.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:10.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:10.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:10.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:10.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-21T10:12:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:09 smithi191 ceph-mon[102206]: pgmap v1789: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:09 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:12:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:09 smithi191 ceph-mon[102206]: osdmap e1262: 8 total, 8 up, 5 in 2024-09-21T10:12:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:09 smithi076 ceph-mon[106482]: pgmap v1789: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:09 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:12:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:09 smithi076 ceph-mon[106482]: osdmap e1262: 8 total, 8 up, 5 in 2024-09-21T10:12:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:09 smithi076 ceph-mon[110963]: pgmap v1789: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:09 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:12:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:09 smithi076 ceph-mon[110963]: osdmap e1262: 8 total, 8 up, 5 in 2024-09-21T10:12:10.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:10.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:10.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:10.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:10.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:10.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:10.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:10.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:10.529 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:10.631 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:10.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:10.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:10.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:10.810 INFO:teuthology.orchestra.run.smithi076.stdout:moved 0 / 267 (0%) 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:avg 53.4 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:stddev 7.60526 -> 7.60526 (expected baseline 6.53605) 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:min osd.2 with 43 -> 43 pgs (0.805243 -> 0.805243 * mean) 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:max osd.1 with 64 -> 64 pgs (1.1985 -> 1.1985 * mean) 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:oload 1000 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:max_change 1 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:max_change_osds 4 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:average_utilization 0.0067 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stdout:overload_utilization 0.0673 2024-09-21T10:12:10.811 INFO:teuthology.orchestra.run.smithi076.stderr:no change 2024-09-21T10:12:10.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:10.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:10.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:11.002 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:11.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:11.085 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:11.085 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:11.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:11.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:11.249 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:11.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:11.343 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:11.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:11.381 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:11.395 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:11.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:11.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:11.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:11.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:11.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:11.731 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:11.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:11.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:11 smithi191 ceph-mon[102206]: pgmap v1791: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:11.732 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:11 smithi191 ceph-mon[102206]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:11.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[106482]: pgmap v1791: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[106482]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[110963]: pgmap v1791: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:11.748 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:11 smithi076 ceph-mon[110963]: from='client.35637 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:11.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:12:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:11.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:11.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:11.888 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:11.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:11.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:11.966 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:11.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.012 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:12.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:12.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.135 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:12.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:12.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:12.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:12.359 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.381 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:12.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:12.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:12.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:12.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:12.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:12.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:12.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:12.730 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:12.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:12.856 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:12.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:12.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:12.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:13.003 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:13.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:13.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:12:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:13.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:13.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:13.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:13.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:13.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:13.593 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:12:13] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.51.0" 2024-09-21T10:12:13.600 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:13.614 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:12:13.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 110 0.05 2024-09-21T10:12:13.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.624 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:13.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:13.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:13.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:13.870 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:13 smithi076 ceph-mon[106482]: pgmap v1792: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:13 smithi076 ceph-mon[110963]: pgmap v1792: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:13.958 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:13.958 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:13.972 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:13.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:13.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:13.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:14.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:13 smithi191 ceph-mon[102206]: pgmap v1792: 89 pgs: 3 creating+activating, 7 creating+peering, 79 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:14.117 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:14.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:14.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.240 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:14.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:14.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:14.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:14.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:14.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:14.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:14.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:14.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.615 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:14.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:14.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:12:14.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:14.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:14.840 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:14.848 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:12:14.861 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:14.863 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:12:14.864 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:14.879 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:12:14.880 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:14.895 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:12:14.896 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:14.911 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:12:14.912 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:14.928 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:12:14.928 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:14.944 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:12:14.945 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:14.960 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:12:14.960 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:14.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:14.976 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:12:14.977 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:14.984 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:14.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:15.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:15.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:15.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:15.210 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:15.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:15.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:15.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:15.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:15.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:15.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:15.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:15.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:15.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:15.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:15.810 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-21T10:12:15.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:15.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:15.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:15.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:15.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:16.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:16.094 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:15 smithi191 ceph-mon[102206]: pgmap v1793: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:16.094 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4236215076' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:16.094 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1022309398' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:12:16.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:16.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:16.113 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:16.114 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:15 smithi076 ceph-mon[106482]: pgmap v1793: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4236215076' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1022309398' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:12:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:15 smithi076 ceph-mon[110963]: pgmap v1793: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4236215076' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1022309398' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:12:16.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:16.223 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:16.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:16.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:16.346 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:16.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:16.469 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:16.571 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:16.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.594 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:16.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:16.639 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:12:16.639 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:12:16.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:12:16.640 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [] 2024-09-21T10:12:16.640 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:12:16.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:16.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:16.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:16.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:16.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:16.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:16.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:16.986 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1022309398' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:12:16.986 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:16 smithi191 ceph-mon[102206]: osdmap e1263: 8 total, 8 up, 5 in 2024-09-21T10:12:17.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:17.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1022309398' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:12:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:16 smithi076 ceph-mon[106482]: osdmap e1263: 8 total, 8 up, 5 in 2024-09-21T10:12:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1022309398' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:12:17.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:16 smithi076 ceph-mon[110963]: osdmap e1263: 8 total, 8 up, 5 in 2024-09-21T10:12:17.187 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:17.209 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:17.298 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:12:17.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:17.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:17.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:17.454 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:17.556 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:17.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:17.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:17.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:17.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:17.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:17.801 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:17.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:17.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:12:17.917 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:17 smithi076 ceph-mon[106482]: pgmap v1795: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:17.917 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:17 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:12:17.917 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:17 smithi076 ceph-mon[110963]: pgmap v1795: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:17.918 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:17 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:12:17.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:17.945 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:17.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:18.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:18.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:18.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:17 smithi191 ceph-mon[102206]: pgmap v1795: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:17 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:12:18.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:18.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:18.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:18.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:18.315 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:18.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:18.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:18.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:18.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.563 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:18.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:18.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:18.688 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:18.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:18.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:18.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:18.937 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:18.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:19.038 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:19.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:19.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:19.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:19.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:19.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:19.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:19.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:19.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:19.432 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:19.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:19.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:19.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:19.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:19.608 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:19.627 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:19.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:19.682 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:19.766 INFO:teuthology.orchestra.run.smithi076.stdout:moved 3 / 267 (1.1236%) 2024-09-21T10:12:19.766 INFO:teuthology.orchestra.run.smithi076.stdout:avg 53.4 2024-09-21T10:12:19.766 INFO:teuthology.orchestra.run.smithi076.stdout:stddev 7.60526 -> 7.76144 (expected baseline 6.53605) 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:min osd.2 with 43 -> 43 pgs (0.805243 -> 0.805243 * mean) 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:max osd.1 with 64 -> 66 pgs (1.1985 -> 1.23595 * mean) 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:oload 110 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:max_change 0.05 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:max_change_osds 4 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:average_utilization 0.0067 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:overload_utilization 0.0074 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 weight 1.0000 -> 0.9500 2024-09-21T10:12:19.767 INFO:teuthology.orchestra.run.smithi076.stderr:no change 2024-09-21T10:12:19.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:19.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:19.823 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:19 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T10:12:19.763+0000 7f86bc22c640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-21T10:12:19.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:19.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:19.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:20.023 DEBUG:teuthology.orchestra.run.smithi076:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-21T10:12:20.025 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:12:20.025 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:12:20.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:20.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:20.075 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:hints = 1 2024-09-21T10:12:20.075 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:12:20.075 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object prefix: benchmark_data_smithi076_269622 2024-09-21T10:12:20.076 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:12:20.076 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:12:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:19 smithi191 ceph-mon[102206]: pgmap v1796: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:19 smithi076 ceph-mon[106482]: pgmap v1796: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:19 smithi076 ceph-mon[110963]: pgmap v1796: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:20.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:20.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:20.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:20.317 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:20.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:20.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:20.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:20.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:20.672 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:20.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:20.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:20.834 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:20.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:20.936 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:20.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:20.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:20.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:21.073 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:21.075 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 1 16 2996 2980 186.24 186.25 0.00788789 0.00535081 2024-09-21T10:12:21.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:21.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:20 smithi191 ceph-mon[102206]: from='client.45575 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:21.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:20 smithi076 ceph-mon[106482]: from='client.45575 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:21.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:20 smithi076 ceph-mon[110963]: from='client.45575 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:21.199 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:21.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:21.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:21.366 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:21.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:21.468 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:21.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:21.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.633 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:21.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:21.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:21.772 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:21.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:21.850 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:12:21.874 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:21.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:21.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:22.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:22.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:22.075 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 2 16 5948 5932 185.362 184.5 0.00282585 0.00538284 2024-09-21T10:12:22.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:21 smithi076 ceph-mon[106482]: pgmap v1797: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:22.110 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4071986663' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:22.111 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:21 smithi076 ceph-mon[110963]: pgmap v1797: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:22.111 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4071986663' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:21 smithi191 ceph-mon[102206]: pgmap v1797: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4071986663' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:12:22.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:22.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:22.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:22.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:22.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:22.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:22.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:22.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:22.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:22.822 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-21T10:12:22.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight-by-utilization 1000 3.0 2024-09-21T10:12:22.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:22.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:22.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:22.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:23.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:23.076 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 3 16 8776 8760 182.446 176.75 0.0100297 0.00547159 2024-09-21T10:12:23.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:23.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:23.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:22 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:22 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:12:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4071986663' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:22 smithi191 ceph-mon[102206]: osdmap e1264: 8 total, 8 up, 5 in 2024-09-21T10:12:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:12:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4071986663' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:23.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[106482]: osdmap e1264: 8 total, 8 up, 5 in 2024-09-21T10:12:23.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:12:23.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:12:23.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4071986663' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:12:23.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:22 smithi076 ceph-mon[110963]: osdmap e1264: 8 total, 8 up, 5 in 2024-09-21T10:12:23.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:23.232 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:23.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:23.334 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:23.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:23.461 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:23.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:23.521 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:12:23] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.51.0" 2024-09-21T10:12:23.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:23.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:23.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:23.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:23.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:23.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:23.905 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:23.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:23.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:24.006 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:24.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:24.076 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 4 16 11448 11432 178.582 167 0.00744146 0.00559105 2024-09-21T10:12:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:23 smithi191 ceph-mon[102206]: pgmap v1799: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:24.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:23 smithi076 ceph-mon[106482]: pgmap v1799: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:24.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:23 smithi076 ceph-mon[110963]: pgmap v1799: 89 pgs: 89 active+clean; 582 KiB data, 3.1 GiB used, 444 GiB / 447 GiB avail 2024-09-21T10:12:24.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:24.180 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:24.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:24.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:24.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:24.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:24.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:24.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:24.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:24.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:24.708 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:24.810 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:24.847 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:24.949 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:24.985 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:24.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:25.076 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 5 16 13973 13957 174.426 157.812 0.00458189 0.00572577 2024-09-21T10:12:25.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:25.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:25.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:25.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:25.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:25.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:12:25.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:25.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:25.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:25.503 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:25.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:25.605 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:25.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:25.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:25.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:25.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:25.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:25.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:25.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:25.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:25.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:26.013 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:26.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 6 16 16197 16181 168.52 139 0.00504457 0.00592684 2024-09-21T10:12:26.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:25 smithi076 ceph-mon[106482]: pgmap v1800: 89 pgs: 89 active+clean; 526 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 59 MiB/s wr, 939 op/s 2024-09-21T10:12:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:25 smithi076 ceph-mon[110963]: pgmap v1800: 89 pgs: 89 active+clean; 526 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 59 MiB/s wr, 939 op/s 2024-09-21T10:12:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:26.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.141 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:26.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:26.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:26.276 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:26.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:25 smithi191 ceph-mon[102206]: pgmap v1800: 89 pgs: 89 active+clean; 526 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 59 MiB/s wr, 939 op/s 2024-09-21T10:12:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:26.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:26.399 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:26.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:26.526 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:26.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:26.649 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:26.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:26.786 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:26.888 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:26.908 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:26.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:27.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:27.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:27.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 7 16 18000 17984 160.543 112.688 0.00569187 0.00622214 2024-09-21T10:12:27.147 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:27.183 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:27.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:27.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:27.408 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:27.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:27.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:27.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:27.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:27.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:27.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:27.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:27.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:27.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:27.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:27.966 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:27.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:27.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:28.067 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:28.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 8 16 19834 19818 154.802 114.625 0.00854893 0.0064523 2024-09-21T10:12:28.091 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:27 smithi076 ceph-mon[106482]: pgmap v1801: 89 pgs: 89 active+clean; 526 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-21T10:12:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:27 smithi076 ceph-mon[110963]: pgmap v1801: 89 pgs: 89 active+clean; 526 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-21T10:12:28.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:28.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.229 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:28.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:28.281 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:12:28.281 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:12:28.281 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:12:28.281 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:12:28.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:28.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:27 smithi191 ceph-mon[102206]: pgmap v1801: 89 pgs: 89 active+clean; 526 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-21T10:12:28.366 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:28.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:28.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:28.504 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:28.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:28.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:28.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:28.745 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:28.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:28.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:28.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:28.905 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:28.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:28.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:29.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:29.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:29.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 9 16 21642 21626 150.157 113 0.0787419 0.00660815 2024-09-21T10:12:29.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:29.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:29.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:29.281 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:29.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:29.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:29.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:29.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:29.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.659 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:29.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:29.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:29.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:29.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:30.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:30.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 10 16 23702 23686 148.015 128.75 0.00621433 0.00674919 2024-09-21T10:12:30.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:30.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:30.277 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:30.311 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:29 smithi191 ceph-mon[102206]: pgmap v1802: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:12:30.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:29 smithi076 ceph-mon[110963]: pgmap v1802: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:12:30.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:29 smithi076 ceph-mon[106482]: pgmap v1802: 89 pgs: 89 active+clean; 1.2 GiB data, 6.4 GiB used, 441 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:12:30.413 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:30.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:30.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:30.581 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:30.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:30.683 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:30.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:30.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:30.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:30.856 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:30.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:30.980 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:30.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:31.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 11 16 25399 25383 144.201 106.062 0.00672904 0.00692883 2024-09-21T10:12:31.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:31.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:31.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:31.233 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:31.335 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:31.372 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:31.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:31.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:31.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:31.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:31.670 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:31.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:31.772 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.806 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:31.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:31.907 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:31.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:31.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:32.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:32.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:32.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 12 16 26944 26928 140.23 96.5625 0.0111527 0.00712434 2024-09-21T10:12:32.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:32.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:32.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:32.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:32.323 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:32.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:31 smithi191 ceph-mon[102206]: pgmap v1803: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:12:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:31 smithi076 ceph-mon[106482]: pgmap v1803: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:12:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:31 smithi076 ceph-mon[110963]: pgmap v1803: 89 pgs: 89 active+clean; 1.2 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:12:32.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:32.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:32.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:32.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:32.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:32.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:32.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:32.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.856 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:32.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:32.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:32.980 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:32.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:33.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 13 16 28712 28696 137.943 110.5 0.00593754 0.00724326 2024-09-21T10:12:33.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:33.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:33.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:33.244 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:33.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:33.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:33.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:33.468 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:33.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:33.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:33.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:33.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:12:33] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.51.0" 2024-09-21T10:12:33.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:33.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:33.754 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:33.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:33.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:33.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:33.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:34.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:34.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 14 16 30200 30184 134.732 93 0.0103156 0.0074167 2024-09-21T10:12:34.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:34.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:34.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:34.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:34.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:33 smithi191 ceph-mon[102206]: pgmap v1804: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:12:34.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:33 smithi076 ceph-mon[106482]: pgmap v1804: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:12:34.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:33 smithi076 ceph-mon[110963]: pgmap v1804: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-21T10:12:34.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:34.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:34.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:34.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:34.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:34.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:34.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:34.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:34.877 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:34.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:34.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:34.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:35.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:35.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:35.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 15 16 31601 31585 131.587 87.5625 0.0142259 0.00759341 2024-09-21T10:12:35.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.167 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:35.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:35.305 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:35.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:35.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:35.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:35.493 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:35.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:12:35.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:35.549 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:35.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:35.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:35.671 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:35.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:35.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.850 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:35.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:35.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:35.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:35.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:36.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:36.078 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 16 16 33325 33309 130.097 107.75 0.010824 0.00768097 2024-09-21T10:12:36.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.104 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:36.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:36.108 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:35 smithi076 ceph-mon[106482]: pgmap v1805: 89 pgs: 89 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-21T10:12:36.108 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:35 smithi076 ceph-mon[110963]: pgmap v1805: 89 pgs: 89 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-21T10:12:36.205 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:36.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:36.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:36.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:36.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:35 smithi191 ceph-mon[102206]: pgmap v1805: 89 pgs: 89 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-21T10:12:36.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:36.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:36.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:36.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:36.627 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:36.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:36.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:36.767 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:36.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:36.908 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:36.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:36.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:37.009 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:37.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:37.077 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 17 16 35135 35119 129.098 113.125 0.00294058 0.0077411 2024-09-21T10:12:37.145 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:37.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:37.302 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:37.325 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:37.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:37.427 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:37.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:37.492 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:12:37.493 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:37.531 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:12:37.531 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:37.565 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:12:37.566 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:37.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:37.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:37.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:37.604 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:12:37.604 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:37.634 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:12:37.635 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:37.666 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:12:37.667 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:37.699 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:12:37.699 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:37.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:37.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:37.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:37.736 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:12:37.736 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:37.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:37.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:37.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:37.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:37.992 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:37.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:37.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:38.006 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:37 smithi076 ceph-mon[106482]: pgmap v1806: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:12:38.006 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/155203433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:38.007 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:37 smithi076 ceph-mon[110963]: pgmap v1806: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:12:38.007 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/155203433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:38.078 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 18 16 36684 36668 127.304 96.8125 0.00528235 0.00785006 2024-09-21T10:12:38.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:38.131 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:38.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:38.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:38.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:37 smithi191 ceph-mon[102206]: pgmap v1806: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:12:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/155203433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:38.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:38.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:38.404 INFO:teuthology.orchestra.run.smithi076.stdout:moved 0 / 267 (0%) 2024-09-21T10:12:38.404 INFO:teuthology.orchestra.run.smithi076.stdout:avg 53.4 2024-09-21T10:12:38.404 INFO:teuthology.orchestra.run.smithi076.stdout:stddev 7.60526 -> 7.60526 (expected baseline 6.53605) 2024-09-21T10:12:38.404 INFO:teuthology.orchestra.run.smithi076.stdout:min osd.2 with 43 -> 43 pgs (0.805243 -> 0.805243 * mean) 2024-09-21T10:12:38.404 INFO:teuthology.orchestra.run.smithi076.stdout:max osd.1 with 64 -> 64 pgs (1.1985 -> 1.1985 * mean) 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stdout:oload 1000 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stdout:max_change 3 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stdout:max_change_osds 4 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stdout:average_utilization 0.0189 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stdout:overload_utilization 0.1888 2024-09-21T10:12:38.405 INFO:teuthology.orchestra.run.smithi076.stderr:no change 2024-09-21T10:12:38.496 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:38.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:38.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:38.671 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:38.773 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:38.827 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:38.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:38.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:38.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:38.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:39.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:39.078 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 19 16 38270 38254 125.82 99.125 0.0126491 0.00794214 2024-09-21T10:12:39.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:39.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:39.100 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:38 smithi076 ceph-mon[106482]: from='client.35682 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:39.100 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:38 smithi076 ceph-mon[110963]: from='client.35682 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:39.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:39.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:39.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:39.307 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:39.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:38 smithi191 ceph-mon[102206]: from='client.35682 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:39.449 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:39.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:39.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:39.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:39.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:39.754 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:39.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:39.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:39.872 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:39.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:39.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:39.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:39.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:40.006 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:40.078 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:12:40.076924+0000 min lat: 0.00130278 max lat: 0.192312 avg lat: 0.00800971 2024-09-21T10:12:40.078 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:12:40.078 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 20 16 39944 39928 124.76 104.625 0.00994119 0.00800971 2024-09-21T10:12:40.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:40.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:40.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:40.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:40.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:39 smithi191 ceph-mon[102206]: pgmap v1807: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:12:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:39 smithi076 ceph-mon[106482]: pgmap v1807: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:12:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:39 smithi076 ceph-mon[110963]: pgmap v1807: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-21T10:12:40.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:40.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:40.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:40.446 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-21T10:12:40.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-21T10:12:40.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:40.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:40.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:40.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:40.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:40.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:40.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:40.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:40.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:41.061 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:41.079 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 21 16 41500 41484 123.443 97.25 0.00969254 0.00809531 2024-09-21T10:12:41.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:41.137 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:12:41.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:41.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.247 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:41.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:41.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:41.348 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:41.388 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:41.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:41.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:41.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:41.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:41.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:41.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:41.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:41.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:41.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.028 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:42.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:42.072 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:41 smithi191 ceph-mon[102206]: pgmap v1808: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:12:42.079 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 22 16 42991 42975 122.067 93.1875 0.0144491 0.00818497 2024-09-21T10:12:42.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:42.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:42.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:42.203 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:41 smithi076 ceph-mon[106482]: pgmap v1808: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:12:42.204 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:41 smithi076 ceph-mon[110963]: pgmap v1808: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:12:42.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:42.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:42.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:42.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:42.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:42.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:42.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:42.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:42.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:42.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:42.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:42.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:43.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:43.079 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 23 16 44433 44417 120.678 90.125 0.0154264 0.00828062 2024-09-21T10:12:43.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:12:43.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:12:43.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:43.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:43.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:43.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:43.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:12:43.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:43.460 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:43.516 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:12:43] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.51.0" 2024-09-21T10:12:43.562 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:43.569 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:12:43.569 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:12:43.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:43.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:43.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:43.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:43.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:43.946 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:43.965 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:12:43.984 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:43.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:43.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:44.080 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 24 16 45796 45780 119.199 85.1875 0.0175709 0.00838371 2024-09-21T10:12:44.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:44.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:44.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:44.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:44.261 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:44.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:44.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:43 smithi191 ceph-mon[102206]: pgmap v1809: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:12:44.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/365387173' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:44.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:43 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-mon[110963]: pgmap v1809: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:12:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/365387173' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:44.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-mon[106482]: pgmap v1809: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:12:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/365387173' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:44.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:43 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:12:44.390 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:44.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:12:44.492 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:44.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:44.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:44.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:44.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:44.802 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:44.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:44.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:44.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:44.960 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:44.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:45.062 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:45.079 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 25 16 47232 47216 118.021 89.75 0.0109416 0.00846703 2024-09-21T10:12:45.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:45.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:45.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:45.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:45.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:44 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:45.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:44 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:12:45.318 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:44 smithi076 ceph-mon[106482]: osdmap e1265: 8 total, 8 up, 5 in 2024-09-21T10:12:45.322 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:44 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:45.322 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:44 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:12:45.322 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:44 smithi076 ceph-mon[110963]: osdmap e1265: 8 total, 8 up, 5 in 2024-09-21T10:12:45.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:45.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:44 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:12:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:44 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:12:45.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:44 smithi191 ceph-mon[102206]: osdmap e1265: 8 total, 8 up, 5 in 2024-09-21T10:12:45.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.365 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:45.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:45.418 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:45.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:45.493 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:45.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:45.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:45.632 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:45.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:45.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:45.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:45.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:45.898 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:45.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.000 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:46.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:46.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:46.079 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 26 16 48625 48609 116.83 87.0625 0.00652053 0.00855374 2024-09-21T10:12:46.133 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:46.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:46.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.172 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:45 smithi076 ceph-mon[106482]: pgmap v1811: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:12:46.172 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:45 smithi076 ceph-mon[110963]: pgmap v1811: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:12:46.273 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:46.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:46.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:46.360 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:45 smithi191 ceph-mon[102206]: pgmap v1811: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-21T10:12:46.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:46.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:46.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:46.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:46.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:46.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:46.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:46.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:46.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:46.992 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:46.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:47.080 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 27 16 50074 50058 115.857 90.5625 0.00354075 0.00862525 2024-09-21T10:12:47.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:47.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:47.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:47.231 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:47.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:47.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:47.359 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:12:47.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:47.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:47.483 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:47.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:47.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:47.667 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:47.770 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:47.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:47.911 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:47.934 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:47.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:48.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:48.080 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 28 16 51421 51405 114.725 84.1875 0.00953248 0.0087108 2024-09-21T10:12:48.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:48.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:47 smithi076 ceph-mon[106482]: pgmap v1812: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-21T10:12:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:47 smithi076 ceph-mon[110963]: pgmap v1812: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-21T10:12:48.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:48.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:48.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:48.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:48.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:48.352 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:48.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:47 smithi191 ceph-mon[102206]: pgmap v1812: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 100 MiB/s wr, 1.61k op/s 2024-09-21T10:12:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:48.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:48.491 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:48.593 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:48.618 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:48.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:48.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:48.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:48.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:48.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:48.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:48.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:48.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:49.080 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 29 16 52626 52610 113.366 75.3125 0.00888053 0.00881533 2024-09-21T10:12:49.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:49.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:49.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.271 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:49.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:49.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:49.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:49.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:49.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:49.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.665 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:49.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:49.766 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:49.790 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:moved 30 / 267 (11.236%) 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:avg 53.4 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:stddev 7.60526 -> 5.4626 (expected baseline 6.53605) 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:min osd.2 with 43 -> 49 pgs (0.805243 -> 0.917603 * mean) 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:max osd.1 with 64 -> 52 pgs (1.1985 -> 0.973783 * mean) 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:oload 110 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:max_change 3 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:max_change_osds 4 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:average_utilization 0.0275 2024-09-21T10:12:49.824 INFO:teuthology.orchestra.run.smithi076.stdout:overload_utilization 0.0302 2024-09-21T10:12:49.825 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1 weight 1.0000 -> 0.7861 2024-09-21T10:12:49.825 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3 weight 1.0000 -> 0.7958 2024-09-21T10:12:49.825 INFO:teuthology.orchestra.run.smithi076.stderr:no change 2024-09-21T10:12:49.891 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:49.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:49.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:50.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.039 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.080 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 30 16 53846 53830 112.129 76.25 0.0112586 0.00891305 2024-09-21T10:12:50.108 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T10:12:49.821+0000 7f86bc22c640 -1 mgr.server reply reply (2) No such file or directory no change 2024-09-21T10:12:50.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:50.144 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:12:50.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:50.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:50.312 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:50.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:50 smithi191 ceph-mon[102206]: pgmap v1813: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:12:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:50 smithi076 ceph-mon[106482]: pgmap v1813: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:12:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:50 smithi076 ceph-mon[110963]: pgmap v1813: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:12:50.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:50.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:50.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:50.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:50.833 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:50.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:50.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:50.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:51.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.081 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 31 16 55165 55149 111.168 82.4375 0.00793974 0.0089899 2024-09-21T10:12:51.101 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:51.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:51.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:51.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:51.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:51.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:51 smithi076 ceph-mon[110963]: from='client.35694 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:51.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:51 smithi076 ceph-mon[106482]: from='client.35694 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:51.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:51.492 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:51.594 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:51.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:51 smithi191 ceph-mon[102206]: from='client.35694 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:12:51.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:51.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:51.730 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:51.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:51.811 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:12:51.812 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:12:51.855 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:12:51.855 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:12:51.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:51.891 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:12:51.892 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:12:51.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:51.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:51.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:51.935 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:12:51.935 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:12:51.965 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:12:51.965 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:12:51.997 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:12:51.997 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:12:52.000 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:52.028 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:12:52.028 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:12:52.037 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:52.059 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:12:52.059 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:12:52.081 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 32 16 56440 56424 110.184 79.6875 0.0199493 0.00906936 2024-09-21T10:12:52.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:52.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:52.166 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:52 smithi076 ceph-mon[106482]: pgmap v1814: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:12:52.167 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:52 smithi076 ceph-mon[110963]: pgmap v1814: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:12:52.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:52.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:52.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:52.397 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:52 smithi191 ceph-mon[102206]: pgmap v1814: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:12:52.397 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/352593490' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:52.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:52.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:52.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:52.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:52.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/352593490' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:52.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/352593490' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:12:52.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:52.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:52.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:52.808 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:52.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:52.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:52.944 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:52.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:52.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:53.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:53.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:53.081 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 33 16 57463 57447 108.783 63.9375 0.0131965 0.00918051 2024-09-21T10:12:53.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:53.196 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:53.298 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:53.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:53.423 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:53.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:12:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:12:53] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.51.0" 2024-09-21T10:12:53.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:53.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:53.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:53.839 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:53.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:53.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:53.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:53.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:54.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:54.082 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 34 16 58500 58484 107.486 64.8125 0.0071968 0.00929776 2024-09-21T10:12:54.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:54.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:54.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:54.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:54.369 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:54.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:54.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:54.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:54.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:54.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:54 smithi191 ceph-mon[102206]: pgmap v1815: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-21T10:12:54.618 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:54 smithi076 ceph-mon[106482]: pgmap v1815: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-21T10:12:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:54 smithi076 ceph-mon[110963]: pgmap v1815: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-21T10:12:54.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:54.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:54.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:54.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:54.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:54.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:55.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:55.082 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 35 16 59539 59523 106.27 64.9375 0.0131896 0.00940497 2024-09-21T10:12:55.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:55.149 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:55.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:55.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:55.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:55.374 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:55.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:55.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:55.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:55.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:55.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:55.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.803 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:55.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:55.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:55.930 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:55.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:56.031 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:56.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:56.082 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 36 16 60520 60504 105.022 61.3125 0.00510701 0.00951644 2024-09-21T10:12:56.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:56.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:56.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:56.321 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:56.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:56.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:56.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:56.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:56.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:56.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:56 smithi191 ceph-mon[102206]: pgmap v1816: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:12:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:56.614 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:56 smithi076 ceph-mon[106482]: pgmap v1816: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:12:56.614 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:56.614 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:56.615 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:56 smithi076 ceph-mon[110963]: pgmap v1816: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:12:56.615 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:12:56.615 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:12:56.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:56.723 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:56.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:56.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:56.965 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:56.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:56.988 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:56.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:57.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 37 16 61608 61592 104.021 68 0.00969447 0.00960819 2024-09-21T10:12:57.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:57.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:57.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:57.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:57.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:57.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:57.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:57.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:57.590 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:57.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.615 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:57.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:57.716 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:57.755 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:12:57.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:57.882 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:57.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:57.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:57.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:58.024 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:58.082 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 38 15 62777 62762 103.208 73.125 0.0121548 0.00968414 2024-09-21T10:12:58.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:58.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:12:58.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:58.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:12:58.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:58.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:58.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:58.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:58.529 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:58.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:12:58 smithi191 ceph-mon[102206]: pgmap v1817: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:12:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:12:58 smithi076 ceph-mon[106482]: pgmap v1817: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:12:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:12:58 smithi076 ceph-mon[110963]: pgmap v1817: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:12:58.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:58.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:58.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:58.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:12:58.795 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:58.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:12:58.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:12:58.919 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:58.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:12:59.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.044 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 39 16 63879 63863 102.326 68.8125 0.0115783 0.00976662 2024-09-21T10:12:59.146 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:12:59.184 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:12:59.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:59.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:59.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:59.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:59.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:12:59.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:12:59.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:12:59.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:12:59.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:12:59.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:12:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:59.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:12:59.930 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:12:59.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:12:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:12:59.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:00.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:13:00.081758+0000 min lat: 0.00130278 max lat: 0.192312 avg lat: 0.00985668 2024-09-21T10:13:00.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:13:00.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 40 16 64922 64906 101.397 65.1875 0.0149257 0.00985668 2024-09-21T10:13:00.094 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:00.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:00.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:00.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:00.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:00.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:00.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.610 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:00.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:00 smithi191 ceph-mon[102206]: pgmap v1818: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:13:00.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:00 smithi076 ceph-mon[106482]: pgmap v1818: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:13:00.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:00 smithi076 ceph-mon[110963]: pgmap v1818: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:13:00.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:00.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:00.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:00.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:00.882 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:00.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:00.984 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:01.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:01.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 41 16 65824 65808 100.299 56.375 0.046728 0.0099647 2024-09-21T10:13:01.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:01.159 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:01.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:01.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:01.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:01.425 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:01.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:01.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:01.550 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:01.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:01.652 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:01.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:01.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:01.797 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:01.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:01.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:01.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:01.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:02.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:02.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:02.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 42 16 66808 66792 99.3753 61.5 0.0092804 0.0100564 2024-09-21T10:13:02.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:02.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:02.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:02.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:02.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:02.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:02 smithi076 ceph-mon[106482]: pgmap v1819: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-21T10:13:02.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:02 smithi076 ceph-mon[110963]: pgmap v1819: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-21T10:13:02.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:02.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:02.587 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:02.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:02.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:02 smithi191 ceph-mon[102206]: pgmap v1819: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-21T10:13:02.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:02.741 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:02.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:02.843 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:02.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:02.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:02.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:02.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:03.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:03.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:03.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 43 16 67995 67979 98.7894 74.1875 0.0159263 0.0101176 2024-09-21T10:13:03.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:03.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:03.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:03.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:03.273 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:03.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:03.374 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:03.408 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:03.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:03.544 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:03.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:13:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:13:03] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.51.0" 2024-09-21T10:13:03.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:03.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:03.771 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:03.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:03.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:03.934 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:03.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:04.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:04.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:04.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:04.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 44 16 69105 69089 98.1209 69.375 0.00621114 0.0101864 2024-09-21T10:13:04.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:04.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:04.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:04.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:04.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:04 smithi076 ceph-mon[106482]: pgmap v1820: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-21T10:13:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:04 smithi076 ceph-mon[110963]: pgmap v1820: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-21T10:13:04.421 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:04.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:04.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:04.547 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:04.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:04 smithi191 ceph-mon[102206]: pgmap v1820: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-21T10:13:04.613 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:13:04 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=infra.usagestats t=2024-09-21T10:13:04.335636586Z level=info msg="Usage stats are ready to report" 2024-09-21T10:13:04.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:04.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:04.818 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.855 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:04.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:04.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:04.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:04.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:04.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:05.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:05.085 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 45 16 70136 70120 97.3723 64.4375 0.0105017 0.0102645 2024-09-21T10:13:05.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:05.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:05.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:05.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:05.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:05.350 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:05.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:05.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:05.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:05.545 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:05.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:05.660 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:05.762 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:05.800 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:05.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:05.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:05.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:05.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:06.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:06.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:06.083 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 46 16 71263 71247 96.7867 70.4375 0.00573544 0.0103262 2024-09-21T10:13:06.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:06.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:06.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:06.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:06.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:06.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:06.337 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:13:06.337 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:13:06.338 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:13:06.338 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [] 2024-09-21T10:13:06.338 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:13:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:06 smithi076 ceph-mon[106482]: pgmap v1821: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-21T10:13:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:06 smithi076 ceph-mon[110963]: pgmap v1821: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-21T10:13:06.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:06.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:06.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:06.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:06.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:06.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:06 smithi191 ceph-mon[102206]: pgmap v1821: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-09-21T10:13:06.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.732 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:06.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:06.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:06.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:06.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:07.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:13:07.085 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 47 16 72377 72361 96.2055 69.625 0.0173089 0.0103886 2024-09-21T10:13:07.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.214 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:07.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:07.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:07.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:07.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:07.535 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:07.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:07.575 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:13:07.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:07.685 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:07.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:07.809 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:07.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:07.911 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:07.952 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:07.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:08.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:08.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:08.085 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 48 16 73429 73413 95.5709 65.75 0.0134329 0.010458 2024-09-21T10:13:08.180 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:08.247 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.283 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:08.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:08.320 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:08.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:08.410 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:08 smithi076 ceph-mon[110963]: pgmap v1822: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-21T10:13:08.410 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:08 smithi076 ceph-mon[106482]: pgmap v1822: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-21T10:13:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.434 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:08.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:08.535 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:08.575 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:08.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:08 smithi191 ceph-mon[102206]: pgmap v1822: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-21T10:13:08.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:08.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:08.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:08.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:08.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:08.965 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:08.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:09.007 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 4, 3, 6] out_osds: [7, 5, 0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:13:09.007 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:13:09.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:09.086 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 49 16 74448 74432 94.9186 63.6875 0.00837643 0.0105299 2024-09-21T10:13:09.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:09.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:09.236 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:09.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:09.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:09.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:09.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:09.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:09.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:09.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:09.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:09.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:09.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:09.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:09.875 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:09.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:09.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:10.009 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:10.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:10.086 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 50 16 75516 75500 94.3552 66.75 0.014129 0.0105925 2024-09-21T10:13:10.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:10.145 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:10.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:10.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:10.270 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:10.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:10.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:10.445 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:10 smithi076 ceph-mon[106482]: pgmap v1823: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:13:10.445 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:10 smithi076 ceph-mon[110963]: pgmap v1823: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:13:10.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:10.522 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:10.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:10.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:10 smithi191 ceph-mon[102206]: pgmap v1823: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:13:10.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:10.646 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:10.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:10.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:10.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:10.919 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:10.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:11.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:11.045 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:11.086 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 51 16 76649 76633 93.8935 70.8125 0.0181648 0.0106462 2024-09-21T10:13:11.146 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:11.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:11.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:11.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:11.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:11.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:11.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:11.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:11.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:11.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:11.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:11.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:11.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:11.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:11.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:11.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:11.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:11.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:11.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:12.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:12.087 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 52 16 77588 77572 93.2145 58.6875 0.0309961 0.0107213 2024-09-21T10:13:12.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:12.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:12.219 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:13:12.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:12.277 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:12 smithi191 ceph-mon[102206]: pgmap v1824: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-21T10:13:12.278 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2822750460' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:12.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:12.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:12 smithi076 ceph-mon[110963]: pgmap v1824: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-21T10:13:12.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2822750460' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:12 smithi076 ceph-mon[106482]: pgmap v1824: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-21T10:13:12.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2822750460' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:12.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:12.449 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:12.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:12.577 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:12.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:12.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:12.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:12.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:12.843 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:12.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:12.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:12.945 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:13.009 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:13.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:13.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:13:13.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:13:13.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:13:13.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:13:13.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:13:13.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:13:13.074 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:13.074 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:13.074 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:13.074 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:13.074 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:13.074 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:13.088 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 53 16 78576 78560 92.6184 61.75 0.0211263 0.0107917 2024-09-21T10:13:13.097 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:13:13.097 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:13.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:13.128 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:13:13.128 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:13.135 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:13.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:13.262 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:13.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:13:13] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.51.0" 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2822750460' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[106482]: osdmap e1266: 8 total, 8 up, 5 in 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2675134027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:13.327 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2822750460' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:13.328 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[110963]: osdmap e1266: 8 total, 8 up, 5 in 2024-09-21T10:13:13.328 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2675134027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:13.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:13.386 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:13.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:13.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:13.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:13.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:13 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2822750460' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:13 smithi191 ceph-mon[102206]: osdmap e1266: 8 total, 8 up, 5 in 2024-09-21T10:13:13.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2675134027' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:13.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:13.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:13.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:13.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:13.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:13.919 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:13.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:14.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:14.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:14.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:14.089 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 54 16 79672 79656 92.1717 68.5 0.0264943 0.0108443 2024-09-21T10:13:14.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:14.180 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:14.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:14.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:14.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:14.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:14.477 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:14 smithi076 ceph-mon[106482]: pgmap v1826: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:13:14.477 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:14 smithi076 ceph-mon[110963]: pgmap v1826: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:13:14.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:14.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:14.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:14 smithi191 ceph-mon[102206]: pgmap v1826: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-21T10:13:14.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:14.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:14.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:14.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:14.829 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:14.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:14.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:14.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:14.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:15.070 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:15.089 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 55 16 80777 80761 91.7508 69.0625 0.0117137 0.0108941 2024-09-21T10:13:15.101 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:15.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.245 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:15.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:15.346 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:15.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:15.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:15.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:15.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.662 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:15.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:15.763 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:15.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:15.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:15.964 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:15.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:15.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:16.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:16.090 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 56 16 81758 81742 91.2064 61.3125 0.00963161 0.0109592 2024-09-21T10:13:16.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:16.099 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:16.099 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:16.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:16.216 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:16.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:16.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:16.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:16.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:16.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:13:16.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:16.518 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:16 smithi076 ceph-mon[106482]: pgmap v1827: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-21T10:13:16.518 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:16 smithi076 ceph-mon[110963]: pgmap v1827: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-21T10:13:16.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:16.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:16.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:16.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:16 smithi191 ceph-mon[102206]: pgmap v1827: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-21T10:13:16.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:16.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:16.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:16.872 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:16.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:16.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:16.878 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:16.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:13:16.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:17.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:17.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:17.090 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 57 16 82781 82765 90.7276 63.9375 0.0119756 0.0110167 2024-09-21T10:13:17.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:17.171 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:17.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:17.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:17.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:17.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:17.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:17.538 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:17.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:17.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:17.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:17.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:17.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:17.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:17.927 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:17.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:13:17.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:17.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:18.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:18.090 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 58 16 83744 83728 90.2011 60.1875 0.0175064 0.0110806 2024-09-21T10:13:18.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:18.199 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:18.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:18.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:18.371 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:18 smithi076 ceph-mon[106482]: pgmap v1828: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-21T10:13:18.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:18 smithi076 ceph-mon[110963]: pgmap v1828: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-21T10:13:18.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:18.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:18.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:18.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:18.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:18 smithi191 ceph-mon[102206]: pgmap v1828: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-21T10:13:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:18.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:18.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:18.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:18.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:18.875 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:18.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:18.948 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:18.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:18.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:19.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:19.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:19.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:19.090 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 59 16 84747 84731 89.7347 62.6875 0.0132825 0.0111383 2024-09-21T10:13:19.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:19.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:19.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:19.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:19.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:19.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:19.523 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:19.625 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:19.683 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:19.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:19.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:19 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:19.721 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:19 smithi076 ceph-mon[106482]: pgmap v1829: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:19.721 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:19.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:19 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:19.722 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:19 smithi076 ceph-mon[110963]: pgmap v1829: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:19.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:19.809 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:19.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:19 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:19 smithi191 ceph-mon[102206]: pgmap v1829: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:19.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:19.949 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:19.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:19.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:20.051 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:20.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:20.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:13:20.089514+0000 min lat: 0.00130278 max lat: 0.192312 avg lat: 0.011199 2024-09-21T10:13:20.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:13:20.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 60 16 85716 85700 89.2485 60.5625 0.0138655 0.011199 2024-09-21T10:13:20.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:20.216 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:20.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:20.341 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:20.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:20.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:20.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:20.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:20.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:20.735 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:13:20.735 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:13:20.736 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:20.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:13:20.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:13:20.736 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:20.749 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:20.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:20.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:13:20.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:13:20.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:13:20.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:20.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:20.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:21.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:21.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 61 16 86732 86716 88.8259 63.5 0.0242192 0.0112519 2024-09-21T10:13:21.119 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:21.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:21.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:21.270 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:21.294 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:21.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:21.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:21.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:21.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:21.597 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:13:21.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:21.719 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:21.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:21.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:21.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:21.859 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:13:21.859 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:21.859 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:21 smithi076 ceph-mon[110963]: pgmap v1830: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:21.859 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/194436728' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:13:21.860 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:13:21.860 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:21 smithi076 ceph-mon[106482]: pgmap v1830: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:21.860 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/194436728' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:13:21.860 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:13:21.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:21 smithi191 ceph-mon[102206]: pgmap v1830: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/194436728' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:13:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:13:21.895 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:13:21.895 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:21.930 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:13:21.930 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:21.945 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:21.970 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:13:21.970 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:21.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:21.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:21.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:22.002 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:13:22.003 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:22.015 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:22.041 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:13:22.041 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:22.071 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:13:22.071 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:22.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:22.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 62 16 87707 87691 88.3761 60.9375 0.0133808 0.0113103 2024-09-21T10:13:22.102 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:13:22.103 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:22.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.139 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:22.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:22.240 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:22.278 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:22.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:22.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:22.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:22.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:22.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:22.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:13:22.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:22.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:22.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:22.780 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:22.796 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:22.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:13:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:22 smithi191 ceph-mon[102206]: osdmap e1267: 8 total, 8 up, 5 in 2024-09-21T10:13:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3986619146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:13:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:22 smithi076 ceph-mon[106482]: osdmap e1267: 8 total, 8 up, 5 in 2024-09-21T10:13:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3986619146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:13:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:22 smithi076 ceph-mon[110963]: osdmap e1267: 8 total, 8 up, 5 in 2024-09-21T10:13:22.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3986619146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:22.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:22.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:22.948 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:22.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:23.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:23.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:23.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 63 16 88667 88651 87.9256 60 0.0107913 0.0113575 2024-09-21T10:13:23.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:23.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:23.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:23.324 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:23.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:23.464 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:23.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:23.590 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:23.610 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:13:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:13:23] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.51.0" 2024-09-21T10:13:23.692 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:23.726 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:23.828 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:23 smithi191 ceph-mon[102206]: pgmap v1832: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:23.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:23.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:23 smithi076 ceph-mon[106482]: pgmap v1832: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:23.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:23 smithi076 ceph-mon[110963]: pgmap v1832: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-21T10:13:23.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:23.996 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:23.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:23.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:24.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 64 16 89549 89533 87.4131 55.125 0.0144726 0.0114351 2024-09-21T10:13:24.098 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.137 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:24.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:24.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:24.263 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:24.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:24.364 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:24.408 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:24.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:24.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:24.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:24.634 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:24.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:24.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:24.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:24.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:24.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:24.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:24.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:24.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:25.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:25.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:25.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 65 16 90384 90368 86.8712 52.1875 0.0143942 0.0115059 2024-09-21T10:13:25.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:25.170 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:25.272 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:25.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:25.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:25.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:25.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:25.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:25.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:25.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:25.727 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:25.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:25.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:25.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:25.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:25.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:25.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:26.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:26.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 66 16 91304 91288 86.4262 57.5 0.0112741 0.0115656 2024-09-21T10:13:26.104 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:26.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:26.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:26.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:26.356 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:26.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:26.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:26.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:26.606 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:26.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:26 smithi191 ceph-mon[102206]: pgmap v1833: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:13:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:26 smithi076 ceph-mon[106482]: pgmap v1833: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:13:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:26 smithi076 ceph-mon[110963]: pgmap v1833: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:13:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:26.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:26.743 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:26.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:26.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:26.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:26.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:26.993 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:26.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:26.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:27.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 67 16 92282 92266 86.0485 61.125 0.00985974 0.0116161 2024-09-21T10:13:27.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:27.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:27.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:27.263 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:27.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:27.386 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:27.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:27.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.516 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:27.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:27.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:27.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:27.743 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:27.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:27.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:27.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:27.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:27.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:28.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:28.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:28.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:28.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 68 16 93099 93083 85.534 51.0625 0.00490577 0.0116857 2024-09-21T10:13:28.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:28.187 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:28.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:28.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:28.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:28.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:28.426 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:28 smithi076 ceph-mon[106482]: pgmap v1834: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:13:28.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:28 smithi076 ceph-mon[110963]: pgmap v1834: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:13:28.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:28.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:28.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:28 smithi191 ceph-mon[102206]: pgmap v1834: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-21T10:13:28.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:28.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:28.827 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:28.849 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:28.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:28.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:28.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:29.092 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 69 16 94067 94051 85.1712 60.5 0.0158236 0.0117347 2024-09-21T10:13:29.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:29.136 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:29.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:29.260 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:29.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:29.384 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:29.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:29.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:29.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:29.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:29.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:29.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:29.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:29.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:29.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:29.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:29.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:29.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:30.085 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:30.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:30.091 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 70 16 94839 94823 84.6437 48.25 0.0125627 0.011809 2024-09-21T10:13:30.187 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:30.209 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:30.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:30.355 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:30.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:30 smithi076 ceph-mon[106482]: pgmap v1835: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-21T10:13:30.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:30 smithi076 ceph-mon[110963]: pgmap v1835: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-21T10:13:30.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:30.482 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:30.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:30 smithi191 ceph-mon[102206]: pgmap v1835: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-21T10:13:30.622 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:30.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:30.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:30.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:30.869 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:30.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:30.970 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:31.017 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:31.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:31.092 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 71 16 95666 95650 84.1796 51.6875 0.010185 0.0118742 2024-09-21T10:13:31.119 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:31.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.301 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:31.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:31.402 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:31.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:31.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:31.552 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:31.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:31.654 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:31.697 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:31.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:31.822 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:31.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:31.924 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:31.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:31.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:32.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:32.073 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:32.092 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 72 16 96550 96534 83.7778 55.25 0.00653266 0.0119308 2024-09-21T10:13:32.175 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:32.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:32.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:32.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:32.340 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:32.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:32.441 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:32.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:32.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:32.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:32 smithi191 ceph-mon[102206]: pgmap v1836: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-21T10:13:32.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:32 smithi076 ceph-mon[106482]: pgmap v1836: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-21T10:13:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:32 smithi076 ceph-mon[110963]: pgmap v1836: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 59 MiB/s wr, 943 op/s 2024-09-21T10:13:32.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:32.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:32.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:13:32.847 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.871 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:32.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:32.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:32.998 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:32.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:32.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:33.092 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 73 16 97598 97582 83.5273 65.5 0.0143131 0.0119662 2024-09-21T10:13:33.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.125 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:33.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:33.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:33.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:33.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:33.393 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:33.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:33.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:33.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:13:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:13:33] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-09-21T10:13:33.632 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:33.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:33.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:33.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:33.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:33.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:33.916 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:33.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:34.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:34.057 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:34.092 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 74 16 98689 98673 83.32 68.1875 0.00780061 0.0119974 2024-09-21T10:13:34.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:34.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:34.282 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:34.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:34.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:34.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:34 smithi076 ceph-mon[106482]: pgmap v1837: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s 2024-09-21T10:13:34.338 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:34 smithi076 ceph-mon[110963]: pgmap v1837: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s 2024-09-21T10:13:34.429 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:34.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.477 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:34.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:34.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:34.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:34 smithi191 ceph-mon[102206]: pgmap v1837: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s 2024-09-21T10:13:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:34.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:34.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:34.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:34.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:34.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:34.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:35.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.092 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 75 16 99721 99705 83.069 64.5 0.0207433 0.012033 2024-09-21T10:13:35.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.188 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:35.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:35.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:35.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.413 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:35.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:35.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:35.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:35.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:35.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:35.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:35.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:35.849 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:35.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:35.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:35.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:35.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:36.076 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.093 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 76 16 100793 100777 82.8563 67 0.0122041 0.0120634 2024-09-21T10:13:36.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:36.167 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:36.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.276 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:36.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:36 smithi076 ceph-mon[110963]: pgmap v1838: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:13:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:36 smithi076 ceph-mon[106482]: pgmap v1838: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:13:36.378 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.415 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:36.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:36.461 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:13:36.461 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:13:36.461 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:13:36.461 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:13:36.461 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:13:36.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.565 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:36.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:36.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:36 smithi191 ceph-mon[102206]: pgmap v1838: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:13:36.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:36.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:36.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:36.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:36.934 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:36.976 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:36.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:37.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:37.094 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 77 16 101783 101767 82.5831 61.875 0.0232979 0.0121037 2024-09-21T10:13:37.134 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:37.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:37.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:37.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:37.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:37.361 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:37.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:37.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:37.545 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:37.647 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:37.682 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:37.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:37.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:37.809 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:37.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:37.871 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:13:37.911 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:37.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:37.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:37.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:38.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:38.094 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 78 16 102821 102805 82.3559 64.875 0.0150874 0.0121369 2024-09-21T10:13:38.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:38.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:38.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:38.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:38.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:38 smithi076 ceph-mon[106482]: pgmap v1839: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 48 MiB/s wr, 775 op/s 2024-09-21T10:13:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:38 smithi076 ceph-mon[110963]: pgmap v1839: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 48 MiB/s wr, 775 op/s 2024-09-21T10:13:38.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:38.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:38.587 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:38.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:38.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:38.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:38 smithi191 ceph-mon[102206]: pgmap v1839: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 48 MiB/s wr, 775 op/s 2024-09-21T10:13:38.711 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:38.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:38.855 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:38.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:38.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:38.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:39.004 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:39.094 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 79 16 103786 103770 82.0769 60.3125 0.0151381 0.0121786 2024-09-21T10:13:39.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:39.130 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:39.142 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:39.195 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:13:39.231 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:39.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:39.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:39.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1971652965' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.297 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:39 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1971652965' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.297 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:39 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:39.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.536 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1971652965' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:39 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:13:39.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:39.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:39.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:39.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:39.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:39.926 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:39.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:39.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:40.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:40.071 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:40.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:40.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:13:40.094179+0000 min lat: 0.00130278 max lat: 0.193885 avg lat: 0.0122203 2024-09-21T10:13:40.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:13:40.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 80 16 104740 104724 81.7955 59.625 0.00795751 0.0122203 2024-09-21T10:13:40.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:40.191 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:13:40.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:40.302 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:40.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[106482]: pgmap v1840: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:13:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:13:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[106482]: osdmap e1268: 8 total, 8 up, 5 in 2024-09-21T10:13:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3249566585' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[110963]: pgmap v1840: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[110963]: osdmap e1268: 8 total, 8 up, 5 in 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3249566585' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:13:40.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:40 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:13:40.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:40.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:40.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:40.577 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:40 smithi191 ceph-mon[102206]: pgmap v1840: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-21T10:13:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:40 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:13:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:40 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:13:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:40 smithi191 ceph-mon[102206]: osdmap e1268: 8 total, 8 up, 5 in 2024-09-21T10:13:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3249566585' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:13:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:40 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:13:40.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:40.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:40.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:40.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:40.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:40.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:40.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:41.053 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:41.090 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:41.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 81 16 105710 105694 81.5341 60.625 0.02681 0.0122592 2024-09-21T10:13:41.192 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:41.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:41.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:41 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:13:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:41 smithi076 ceph-mon[106482]: osdmap e1269: 8 total, 8 up, 5 in 2024-09-21T10:13:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:41 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:13:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:41 smithi076 ceph-mon[110963]: osdmap e1269: 8 total, 8 up, 5 in 2024-09-21T10:13:41.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:41.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:41.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:41.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:41.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:41 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:13:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:41 smithi191 ceph-mon[102206]: osdmap e1269: 8 total, 8 up, 5 in 2024-09-21T10:13:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:41.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:41.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:41.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:41.745 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:41.766 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:41.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:41.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:41.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:41.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:41.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:42.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:42.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 82 16 106681 106665 81.2798 60.6875 0.0158221 0.0122979 2024-09-21T10:13:42.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:42.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:42.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:42.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:42.298 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:42.330 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:42 smithi076 ceph-mon[106482]: pgmap v1843: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:13:42.330 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:42 smithi076 ceph-mon[110963]: pgmap v1843: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:13:42.400 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:42.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:42.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:42.581 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:42.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:42 smithi191 ceph-mon[102206]: pgmap v1843: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-21T10:13:42.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:42.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:42.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:42.846 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:42.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:42.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:42.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:42.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:43.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:43.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 83 16 107673 107657 81.0475 62 0.00317776 0.0123304 2024-09-21T10:13:43.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:43.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:43.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.229 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:43.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:43.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:43.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:43.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:43.494 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:43.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:43.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:13:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:13:43] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-09-21T10:13:43.635 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:43.737 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:43.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:43.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:43.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:43.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:44.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:44.029 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:44.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 84 16 108495 108479 80.6943 51.375 0.0199922 0.0123876 2024-09-21T10:13:44.131 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:44.170 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:44.272 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:44.312 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:44.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:44.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:44.538 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:44.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:44.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:44 smithi191 ceph-mon[102206]: pgmap v1844: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 43 MiB/s wr, 687 op/s 2024-09-21T10:13:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:44 smithi076 ceph-mon[106482]: pgmap v1844: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 43 MiB/s wr, 687 op/s 2024-09-21T10:13:44.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:44 smithi076 ceph-mon[110963]: pgmap v1844: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 43 MiB/s wr, 687 op/s 2024-09-21T10:13:44.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:44.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:44.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:44.803 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:44.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:44.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:44.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:44.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:44.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:44.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:45.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:45.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.076 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:45.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:45.095 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 85 16 109528 109512 80.5045 64.5625 0.0151207 0.0124109 2024-09-21T10:13:45.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:45.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:45.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:45.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:45.356 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:45.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:13:45.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.482 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:45.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:45.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:45.604 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:45.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:45.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:45.743 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:45.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:45.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:45.874 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:45.887 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:45.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:45.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:46.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:46.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:46.097 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 86 16 110534 110518 80.2993 62.875 0.0175293 0.0124486 2024-09-21T10:13:46.139 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:46.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:46.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:46.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:13:46.305 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:46.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:46.408 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:46.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:46.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:46.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:46.544 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:46 smithi076 ceph-mon[106482]: pgmap v1845: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:13:46.545 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:46 smithi076 ceph-mon[110963]: pgmap v1845: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:13:46.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:46.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:46 smithi191 ceph-mon[102206]: pgmap v1845: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:13:46.683 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:46.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:46.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:46.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:46.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:46.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:46.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:46.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:47.028 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:47.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:47.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 87 16 111542 111526 80.0995 63 0.0148822 0.0124797 2024-09-21T10:13:47.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:47.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:47.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:47.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:47.372 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:47.409 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:47.511 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:47.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:47.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:47.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:47.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:47.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:47.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:47.956 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:47.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:48.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:48.077 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:13:48.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:13:48.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:48.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 88 16 112560 112544 79.9115 63.625 0.0115565 0.0125083 2024-09-21T10:13:48.115 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:13:48.116 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:13:48.151 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:13:48.152 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:13:48.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:48.194 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:13:48.195 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:13:48.223 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:13:48.224 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:13:48.229 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:48.260 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:13:48.260 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:13:48.294 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:48 smithi076 ceph-mon[106482]: pgmap v1846: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 40 MiB/s wr, 638 op/s 2024-09-21T10:13:48.294 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2524877416' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:48.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:48 smithi076 ceph-mon[110963]: pgmap v1846: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 40 MiB/s wr, 638 op/s 2024-09-21T10:13:48.294 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2524877416' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:48.295 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:13:48.295 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:13:48.326 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:13:48.326 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:13:48.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:48.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:48.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:48.480 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:48.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:48 smithi191 ceph-mon[102206]: pgmap v1846: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 40 MiB/s wr, 638 op/s 2024-09-21T10:13:48.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2524877416' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:13:48.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:48.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:48.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.770 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:48.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:48.833 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:13:48.833 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:13:48.870 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:48.895 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:48.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:48.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:49.020 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:49.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 89 16 113510 113494 79.6808 59.375 0.00957627 0.0125451 2024-09-21T10:13:49.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:49.160 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:49.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:49.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:49.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:49.387 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:49.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:49.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:49.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:49.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:49.674 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:49.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:49.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:49.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:49.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:49.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:50.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:50.079 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total time run: 90.0066 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total writes made: 114489 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Write size: 65536 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object size: 65536 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Bandwidth (MB/sec): 79.5004 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Bandwidth: 30.0641 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max bandwidth (MB/sec): 186.25 2024-09-21T10:13:50.098 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min bandwidth (MB/sec): 48.25 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average IOPS: 1272 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev IOPS: 481.025 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max IOPS: 2980 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min IOPS: 772 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average Latency(s): 0.0125744 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Latency(s): 0.0105562 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max latency(s): 0.288547 2024-09-21T10:13:50.099 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min latency(s): 0.00130278 2024-09-21T10:13:50.103 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:13:50.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:50.204 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:50.306 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:50.339 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.441 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:50.465 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:50.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:50.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:50.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:50 smithi191 ceph-mon[102206]: pgmap v1847: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:13:50.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:50 smithi076 ceph-mon[110963]: pgmap v1847: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:13:50.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:50 smithi076 ceph-mon[106482]: pgmap v1847: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-21T10:13:50.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:50.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:50.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:50.818 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:50.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:50.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:50.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:50.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:51.028 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:51.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:51.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:51.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:51.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:51.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:51.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:51.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:51.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:51.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:51.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:51.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:13:51.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:51.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:51.732 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:51.755 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:51.856 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:51.899 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:51.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:52.001 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:52.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:52.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:52.176 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:52.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:52.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:52.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:52.427 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:52.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:52.509 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:52 smithi076 ceph-mon[106482]: pgmap v1848: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 58 MiB/s wr, 928 op/s 2024-09-21T10:13:52.509 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:52 smithi076 ceph-mon[110963]: pgmap v1848: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 58 MiB/s wr, 928 op/s 2024-09-21T10:13:52.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:52.550 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:52.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:52 smithi191 ceph-mon[102206]: pgmap v1848: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 58 MiB/s wr, 928 op/s 2024-09-21T10:13:52.652 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:52.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:52.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:13:52.788 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:52.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:52.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:52.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:52.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:53.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:53.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:53.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:53.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:53.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:53.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:53.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:53.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:53.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:53.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:53.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:13:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:13:53] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-09-21T10:13:53.637 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:53.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:53.739 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:53.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:53.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:53.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:53.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:53.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:54.006 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:54.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:54.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:54.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:54.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:54.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.319 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:54.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:54.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:54.454 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:54.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:54.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:54.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:54 smithi191 ceph-mon[102206]: pgmap v1849: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 52 MiB/s wr, 843 op/s 2024-09-21T10:13:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:54 smithi076 ceph-mon[106482]: pgmap v1849: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 52 MiB/s wr, 843 op/s 2024-09-21T10:13:54.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:54 smithi076 ceph-mon[110963]: pgmap v1849: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 52 MiB/s wr, 843 op/s 2024-09-21T10:13:54.694 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:54.733 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:54.832 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:54.835 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:54.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:54.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:54.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:55.000 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:55.102 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:55.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:55.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:55.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:55.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:55.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:55.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:55.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:55.524 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:55.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:55.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:55.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:55.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:55.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:55.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:55.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:55.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:56.023 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:56.136 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:13:56.169 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:56.212 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:56.240 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:13:56.246 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:13:56.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:56.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[106482]: pgmap v1850: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 85 B/s rd, 59 MiB/s wr, 1.23k op/s 2024-09-21T10:13:56.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:56.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1830158157' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3981485796' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[110963]: pgmap v1850: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 85 B/s rd, 59 MiB/s wr, 1.23k op/s 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1830158157' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:13:56.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3981485796' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:56.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:56.462 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:56.495 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:56.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:56 smithi191 ceph-mon[102206]: pgmap v1850: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 85 B/s rd, 59 MiB/s wr, 1.23k op/s 2024-09-21T10:13:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:13:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1830158157' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:13:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:56 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:13:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3981485796' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:13:56.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:56.733 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:56.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:56.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:56.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:56.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:56.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:57.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:57.029 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:57.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:57.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:57.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:57.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:57.296 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:57.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:57.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:57.421 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:57.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:57.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:57.546 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3981485796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[106482]: osdmap e1270: 8 total, 8 up, 5 in 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:57.588 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:13:57.589 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:13:57.589 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3981485796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:57.589 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:57 smithi076 ceph-mon[110963]: osdmap e1270: 8 total, 8 up, 5 in 2024-09-21T10:13:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:57 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:13:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:57 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:13:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:57 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:13:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:57 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3981485796' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:13:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:57 smithi191 ceph-mon[102206]: osdmap e1270: 8 total, 8 up, 5 in 2024-09-21T10:13:57.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:13:57.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:57.671 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:57.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:57.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:57.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:57.896 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:57.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:57.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:58.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:58.063 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:58.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:58.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:58.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:58.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:58.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:13:58 smithi076 ceph-mon[106482]: pgmap v1852: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 102 B/s rd, 41 MiB/s wr, 1.03k op/s 2024-09-21T10:13:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:13:58 smithi076 ceph-mon[110963]: pgmap v1852: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 102 B/s rd, 41 MiB/s wr, 1.03k op/s 2024-09-21T10:13:58.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:58.453 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:58.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:58.578 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:58.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:13:58 smithi191 ceph-mon[102206]: pgmap v1852: 89 pgs: 89 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 102 B/s rd, 41 MiB/s wr, 1.03k op/s 2024-09-21T10:13:58.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:13:58.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:58.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:13:58.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:13:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.863 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:58.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:13:58.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:13:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:58.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:58.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:13:59.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:13:59.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:13:59.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:59.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:13:59.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:13:59.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:59.491 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:59.592 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:13:59.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:13:59.718 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:13:59.757 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:13:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:13:59.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:13:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:13:59.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:13:59.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:13:59.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:00.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:00.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:00.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:14:00.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:00.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:00.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:00.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:00.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:00.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:00.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:00.555 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:00.575 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:00 smithi076 ceph-mon[110963]: pgmap v1853: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 12 MiB/s wr, 1.06k op/s 2024-09-21T10:14:00.575 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:00 smithi076 ceph-mon[106482]: pgmap v1853: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 12 MiB/s wr, 1.06k op/s 2024-09-21T10:14:00.578 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:00.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:00 smithi191 ceph-mon[102206]: pgmap v1853: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 12 MiB/s wr, 1.06k op/s 2024-09-21T10:14:00.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.752 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:00.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:00.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:00.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:00.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:00.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:01.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:01.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.174 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:01.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:01.275 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:01.299 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:01.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:01.423 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:01.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:01.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:01.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:01.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:01.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:01.685 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:01.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:01.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:01.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:01.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:01.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:02.037 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:02.071 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:02.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:02.196 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:02.298 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:02.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:02.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:02.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:02.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:02.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:02.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.599 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:02.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:02 smithi191 ceph-mon[102206]: pgmap v1854: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 12 MiB/s wr, 1.06k op/s 2024-09-21T10:14:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:02 smithi076 ceph-mon[106482]: pgmap v1854: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 12 MiB/s wr, 1.06k op/s 2024-09-21T10:14:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:02 smithi076 ceph-mon[110963]: pgmap v1854: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 12 MiB/s wr, 1.06k op/s 2024-09-21T10:14:02.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:02.724 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:02.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:02.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:02.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.862 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:02.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:02.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:02.985 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:02.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:03.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:03.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:03.221 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:03.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:03.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:03.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:03.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:03.474 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:03.495 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:03.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:03.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:14:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:14:03] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:14:03.721 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:03.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:03.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:03.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:03.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:03.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:03.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:04.013 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:04.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.156 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:04.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:04.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.298 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:04.389 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:04.401 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:04.423 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:04.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:04.511 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:04 smithi076 ceph-mon[110963]: pgmap v1855: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 10 MiB/s wr, 1.06k op/s 2024-09-21T10:14:04.513 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:04 smithi076 ceph-mon[106482]: pgmap v1855: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 10 MiB/s wr, 1.06k op/s 2024-09-21T10:14:04.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:04.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:04 smithi191 ceph-mon[102206]: pgmap v1855: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 102 B/s rd, 10 MiB/s wr, 1.06k op/s 2024-09-21T10:14:04.650 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:04.691 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:04.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:04.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:04.918 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:04.973 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:04.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:05.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:05.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:05.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:05.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:05.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:05.356 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:05.381 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:05.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:05.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:05.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:05.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:05.631 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:05.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:05.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:05.755 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:05.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:05.880 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:05.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:05.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:06.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:06.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:06.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:06.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:06.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:06.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:06.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:06.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:06.389 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:06.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:06.490 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:06.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:06.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:06 smithi191 ceph-mon[102206]: pgmap v1856: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 979 op/s 2024-09-21T10:14:06.616 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:06.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:06 smithi076 ceph-mon[106482]: pgmap v1856: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 979 op/s 2024-09-21T10:14:06.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:06 smithi076 ceph-mon[110963]: pgmap v1856: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 979 op/s 2024-09-21T10:14:06.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:06.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:06.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.783 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:06.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:06.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:06.921 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:06.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:06.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:07.022 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:07.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:07.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:07.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:07.187 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:07.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:07.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:07.330 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:07.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:07.445 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.485 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:07.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:07.491 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:07.491 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:07.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:07.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:07.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:07.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:07.773 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:07.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:07.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:07.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:07.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:08.000 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:08.034 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:08.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:08.157 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:08.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:08.280 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:08.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:08.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:08.527 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:08 smithi076 ceph-mon[110963]: pgmap v1857: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 958 op/s 2024-09-21T10:14:08.527 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:08 smithi076 ceph-mon[106482]: pgmap v1857: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 958 op/s 2024-09-21T10:14:08.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:08.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:08 smithi191 ceph-mon[102206]: pgmap v1857: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 958 op/s 2024-09-21T10:14:08.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:08.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:08.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:08.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:08.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:08.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:08.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:09.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:09.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:09.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:09.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:09.232 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:09.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:09.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:09.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:09.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:09.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:09.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:09.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:09.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:09.724 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:09.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:09.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:09.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:09.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:09.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:09.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:10.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:10.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:10.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:10.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:10.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:10.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:14:10.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:10.268 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:10.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:10.369 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:10.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:10.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:10.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:10.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:10 smithi191 ceph-mon[102206]: pgmap v1858: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:14:10.614 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:10 smithi076 ceph-mon[106482]: pgmap v1858: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:14:10.616 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:10 smithi076 ceph-mon[110963]: pgmap v1858: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.21k op/s 2024-09-21T10:14:10.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:10.698 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:10.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:10.837 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:10.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:10.961 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:10.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:10.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:11.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:11.102 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:11.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:11.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.229 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:11.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:11.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:11.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:11.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:11.424 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:14:11.424 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:11.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:11.465 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:14:11.465 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:11.492 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:11.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:11.504 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:14:11.504 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:11.538 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:14:11.539 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:11.571 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:14:11.571 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:11.594 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:11.616 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:14:11.617 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:11.635 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:11.654 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:14:11.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:11.697 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:14:11.697 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:11.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:11.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:11.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:11.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[106482]: pgmap v1859: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 792 op/s 2024-09-21T10:14:11.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/858160822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:11.759 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:11.759 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:11.759 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[110963]: pgmap v1859: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 792 op/s 2024-09-21T10:14:11.760 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/858160822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:11.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.760 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:11.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:11.861 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:11 smithi191 ceph-mon[102206]: pgmap v1859: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 792 op/s 2024-09-21T10:14:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/858160822' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:11.904 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:11.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:11.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:12.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:12.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:12.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:12.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:12.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:12.268 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:12.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:12.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:12.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:12.414 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:12.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:12.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:12.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:12.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:12.599 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:14:12.605 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:14:12.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:12.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.680 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:12.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:12.781 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:12.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.809 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:12.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1660424194' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:12 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1129519693' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:14:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1660424194' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:12 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1129519693' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:14:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1660424194' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:12 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:14:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1129519693' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:14:12.911 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:12.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:12.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:12.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:13.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:13.086 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:13.187 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:13.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:13.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:13.348 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:13.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:13.473 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:13.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:13.516 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:14:13] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:14:13.575 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:13.598 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:13.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:13.699 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:13.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:13.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:13.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:13.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:13.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:13 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:13 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1129519693' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:14:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:13 smithi191 ceph-mon[102206]: osdmap e1271: 8 total, 8 up, 5 in 2024-09-21T10:14:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:13 smithi191 ceph-mon[102206]: pgmap v1861: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 955 op/s 2024-09-21T10:14:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1129519693' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:14:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[110963]: osdmap e1271: 8 total, 8 up, 5 in 2024-09-21T10:14:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[110963]: pgmap v1861: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 955 op/s 2024-09-21T10:14:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:13.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:14:13.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1129519693' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:14:13.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[106482]: osdmap e1271: 8 total, 8 up, 5 in 2024-09-21T10:14:13.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:13 smithi076 ceph-mon[106482]: pgmap v1861: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 955 op/s 2024-09-21T10:14:13.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:13.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:13.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:13.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:14.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:14.103 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:14.205 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:14.236 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:14.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:14.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:14.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:14.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:14.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:14.626 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:14.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:14.728 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:14.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.766 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:14.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:14.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:14.892 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:14.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:14.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:14.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:15.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:15.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:15.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:15.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:15.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:15.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:15.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:15.413 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:15.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:15.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:15.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:15.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:15.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:15.817 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:15.847 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:15.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:15.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:15.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:15.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:16.097 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:15 smithi191 ceph-mon[102206]: pgmap v1862: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:15 smithi076 ceph-mon[106482]: pgmap v1862: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:15 smithi076 ceph-mon[110963]: pgmap v1862: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:16.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:16.223 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:16.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:16.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:16.346 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:16.448 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:16.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:16.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:16.612 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:16.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:16.737 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:16.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:16.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:16.861 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:16.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:16.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:16.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:17.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:17.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:17.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:17.226 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:17.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:17.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:17.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:17.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:17.502 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:17.524 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:17.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.667 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:17.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:17.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:17.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:17.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:17.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:17.994 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:18.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:14:18.030 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:18.053 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:18.085 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:17 smithi076 ceph-mon[106482]: pgmap v1863: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:18.085 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:17 smithi076 ceph-mon[110963]: pgmap v1863: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:17 smithi191 ceph-mon[102206]: pgmap v1863: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:18.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:18.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:18.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:18.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:18.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:18.436 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.477 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:18.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:18.577 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:18.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:18.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:18.757 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:18.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:18.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:18.882 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:18.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:18.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:18.984 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:19.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:19.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:19.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:19.263 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:19.283 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:19.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:19.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:19.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:19.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:19.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:19.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:19.639 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:19.669 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:19.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:19.771 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:19.814 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:19.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:19.870 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:14:19.873 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:19.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:19.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:19.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:19.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:20.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:19 smithi191 ceph-mon[102206]: pgmap v1864: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:20.115 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:19 smithi076 ceph-mon[106482]: pgmap v1864: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:20.116 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:19 smithi076 ceph-mon[110963]: pgmap v1864: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:20.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:20.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:20.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.255 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:20.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:20.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:20.402 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:20.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:20.526 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:20.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:20.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:20.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:20.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:20.828 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:20.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:20.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:20.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:21.076 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:14:21.076 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.076 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:20 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:21.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:21.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:14:21.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.134 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:20 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:21.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:21.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:21.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:21.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:21.503 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:21.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:21.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:21.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:21.689 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:21.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:21.814 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:21.905 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:14:21.905 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:14:21.905 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:14:21.905 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:14:21.905 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:14:21.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:21.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:21.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:22.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:22.064 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:22.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:21 smithi191 ceph-mon[102206]: pgmap v1865: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:21 smithi076 ceph-mon[106482]: pgmap v1865: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:21 smithi076 ceph-mon[110963]: pgmap v1865: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:22.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.214 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:22.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:22.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:22.341 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:22.442 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:22.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:22.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:22.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:22.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:22.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:14:22.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:22.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:22.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:22.871 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:22.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:22.973 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.015 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:23.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:23.157 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:23.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:23.298 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:23.400 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:23.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:23.537 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:23.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:14:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:14:23] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-21T10:14:23.669 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:23.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:23.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:23.836 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:23.884 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:23.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:23.900 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:23 smithi076 ceph-mon[106482]: pgmap v1866: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:14:23.901 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:23 smithi076 ceph-mon[110963]: pgmap v1866: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:14:23.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:24.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:24.122 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:24.131 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:23 smithi191 ceph-mon[102206]: pgmap v1866: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:14:24.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:24.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:24.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:24.301 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:24.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:24.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:24.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:24.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:24.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.571 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:24.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:24.672 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:24.695 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:24.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:24.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:14:24.820 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:24.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:24.859 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:14:24.859 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:24.894 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:14:24.894 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:24.894 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3233044048' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:24.895 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3233044048' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:24.929 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:14:24.929 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:24.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:24.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:24.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:24.965 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:14:24.965 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:24.997 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:14:24.997 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:25.033 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:14:25.033 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:25.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:25.072 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:14:25.072 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:25.082 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:25.156 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3233044048' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:25.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:25.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.223 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:25.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:25.323 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:25.370 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:25.472 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:25.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:25.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:25.650 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:25.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:25.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:25.795 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:25.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:25.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:25.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:25.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:26.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:26.057 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:26.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:26.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:26.202 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:26.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:26.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:26.340 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:26.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:26.350 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:26.442 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:26.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:26.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:26.576 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:14:26.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:26.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:26.727 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:26.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:26 smithi191 ceph-mon[102206]: pgmap v1867: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.27k op/s 2024-09-21T10:14:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2768767138' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:26.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[110963]: pgmap v1867: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.27k op/s 2024-09-21T10:14:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2768767138' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:26.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[106482]: pgmap v1867: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.27k op/s 2024-09-21T10:14:26.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:14:26.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:26.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2768767138' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:14:26.892 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:26.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:26.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:27.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:27.118 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:27.165 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:27.267 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:27.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:27.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:27.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:27.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:27.572 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:27.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:27.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:27.711 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:27.751 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:27.852 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:27.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2768767138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:14:27.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:27 smithi076 ceph-mon[110963]: osdmap e1272: 8 total, 8 up, 5 in 2024-09-21T10:14:27.857 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:27 smithi076 ceph-mon[110963]: pgmap v1869: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:27.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2768767138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:14:27.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:27 smithi076 ceph-mon[106482]: osdmap e1272: 8 total, 8 up, 5 in 2024-09-21T10:14:27.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:27 smithi076 ceph-mon[106482]: pgmap v1869: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:27.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2768767138' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:14:27.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:27 smithi191 ceph-mon[102206]: osdmap e1272: 8 total, 8 up, 5 in 2024-09-21T10:14:27.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:27 smithi191 ceph-mon[102206]: pgmap v1869: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-21T10:14:27.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:27.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:27.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:28.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:28.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:28.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:28.150 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:28.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:28.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:28.273 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:28.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:28.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:28.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:28.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:28.541 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:28.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:28.643 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:28.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:28.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:28.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:28.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:28.966 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:28.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:29.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:29.091 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:29.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:29.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:29.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:29.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:29.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:29.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:29.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:29.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:29.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:29.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.706 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:29.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:29.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:29.831 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:29.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:29.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:14:29.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:29.954 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:29.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:30.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:30.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:29 smithi191 ceph-mon[102206]: pgmap v1870: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-21T10:14:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:29 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3831304244' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:29 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:29 smithi076 ceph-mon[106482]: pgmap v1870: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-21T10:14:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:29 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3831304244' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:30.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:29 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:29 smithi076 ceph-mon[110963]: pgmap v1870: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-21T10:14:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:29 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3831304244' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:30.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:29 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:14:30.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:30.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:30.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:30.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:30.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:30.433 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:30.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:30.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:30.592 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:30.630 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:14:30.694 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:30.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:30.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:30.860 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:30.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:30.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:30.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:30.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:31.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:31.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:30 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:30 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:30 smithi191 ceph-mon[102206]: osdmap e1273: 8 total, 8 up, 5 in 2024-09-21T10:14:31.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:31.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:30 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:30 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:31.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:30 smithi076 ceph-mon[106482]: osdmap e1273: 8 total, 8 up, 5 in 2024-09-21T10:14:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:30 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:14:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:30 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:14:31.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:30 smithi076 ceph-mon[110963]: osdmap e1273: 8 total, 8 up, 5 in 2024-09-21T10:14:31.212 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:14:31.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:14:31.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:31.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:31.392 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:31.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:31.551 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:31.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:31.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:31.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:31.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:31.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:31.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:31.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:31.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:31.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:32.015 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:14:32.015 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:14:32.072 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:32.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:32.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:31 smithi076 ceph-mon[110963]: pgmap v1872: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:14:32.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:31 smithi076 ceph-mon[106482]: pgmap v1872: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:14:32.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:32.242 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:32.344 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:32.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:31 smithi191 ceph-mon[102206]: pgmap v1872: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.28k op/s 2024-09-21T10:14:32.368 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:32.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:32.505 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:32.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:32.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:32.629 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:32.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:32.756 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:32.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:32.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:32.893 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:32.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:32.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:33.016 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:33.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:33.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:33.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:33.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:14:33.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:33.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:33.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:33.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:33.374 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:14:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:14:33] "GET /metrics HTTP/1.1" 200 36476 "" "Prometheus/2.51.0" 2024-09-21T10:14:33.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:33.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:33.555 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:33.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:33.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:33.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:33.741 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:33.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:33.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:33.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:33.818 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:33.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:33.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:33.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:33.960 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:34.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:34.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.115 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:34.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:33 smithi076 ceph-mon[106482]: pgmap v1873: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 700 op/s 2024-09-21T10:14:34.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:33 smithi076 ceph-mon[110963]: pgmap v1873: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 700 op/s 2024-09-21T10:14:34.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:34.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:34.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:34.340 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:34.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:33 smithi191 ceph-mon[102206]: pgmap v1873: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 700 op/s 2024-09-21T10:14:34.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:34.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:34.529 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:34.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:34.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:34.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:34.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:34.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:34.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:34.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:34.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:35.006 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:35.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:35.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:35.165 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:35.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:35.289 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:35.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:35.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:35.412 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:35.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:35.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:35.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:35.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:35.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:14:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:35.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:35.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:35.801 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:35.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:35.940 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:35.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:36.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.083 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:35 smithi076 ceph-mon[106482]: pgmap v1874: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:14:36.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:35 smithi076 ceph-mon[110963]: pgmap v1874: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:14:36.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:36.221 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:36.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:36.323 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:36.362 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:35 smithi191 ceph-mon[102206]: pgmap v1874: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:14:36.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:36.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:36.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:36.739 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:36.841 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:36.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:36.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:36.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:36.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:37.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:37.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:37.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:37.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:37.274 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:37.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:37.376 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:37.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:37.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:37.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:37.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:37.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:37.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:37.672 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:37.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:37.797 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:37.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:37.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:37.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:37.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:38.066 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:38.167 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:38.202 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:38.304 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:38.326 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:37 smithi191 ceph-mon[102206]: pgmap v1875: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:37 smithi076 ceph-mon[106482]: pgmap v1875: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:37 smithi076 ceph-mon[110963]: pgmap v1875: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-21T10:14:38.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:38.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:38.564 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:38.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:38.687 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:38.728 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:38.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:38.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:38.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:38.995 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:38.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:39.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:39.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:39.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:39.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:39.257 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:39.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:39.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:39.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:39.507 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:39.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:39.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:39.632 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:39.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:39.734 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:39.755 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:39.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:39.857 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:39.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:39.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:39.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:40.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.122 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:39 smithi076 ceph-mon[106482]: pgmap v1876: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:14:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:39 smithi076 ceph-mon[110963]: pgmap v1876: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:14:40.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:40.263 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:40.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:40.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:39 smithi191 ceph-mon[102206]: pgmap v1876: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-21T10:14:40.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:40.408 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:40.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:40.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:40.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:40.657 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:40.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:40.782 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:40.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:40.884 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:40.906 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:40.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:40.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:41.008 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:41.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:41.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:41.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:41.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:41.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:41.225 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:41.225 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:41.263 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:41.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:41.302 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:41.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:41.425 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:41.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:41.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:41.552 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:41.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:41.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:41.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:41.779 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:41.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:41.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:41.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:41.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:41.965 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:41 smithi191 ceph-mon[102206]: pgmap v1877: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 924 op/s 2024-09-21T10:14:42.041 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:42.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:42.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:42.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:42.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:42.219 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:41 smithi076 ceph-mon[106482]: pgmap v1877: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 924 op/s 2024-09-21T10:14:42.219 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:41 smithi076 ceph-mon[110963]: pgmap v1877: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 924 op/s 2024-09-21T10:14:42.295 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:42.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:42.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:42.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:42.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:42.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:42.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:42.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:42.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:42.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:42.834 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:42.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:42.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:42.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:42.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:42.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:43.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:43.114 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:43.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:43.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:43.340 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:43.380 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:43.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:43.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:43.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:43.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:43.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:43.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:14:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:14:43] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:14:43.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:43.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:43.758 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:43.797 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:43.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:43.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:43.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:43.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:44.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:44.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.298 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.332 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:43 smithi191 ceph-mon[102206]: pgmap v1878: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 865 op/s 2024-09-21T10:14:44.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:43 smithi076 ceph-mon[106482]: pgmap v1878: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 865 op/s 2024-09-21T10:14:44.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:43 smithi076 ceph-mon[110963]: pgmap v1878: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 865 op/s 2024-09-21T10:14:44.438 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.473 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:44.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.575 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:44.612 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:44.714 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:44.753 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:44.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:44.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:44.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:44.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:45.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:45.107 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:45.131 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:45.233 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:45.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:45.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:45.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:45.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:45.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:45.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:45.665 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:45.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:45.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:45.842 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:45.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:45.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:45.965 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:45.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:45.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:46.067 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.228 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.237 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:45 smithi076 ceph-mon[106482]: pgmap v1879: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:14:46.237 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:45 smithi076 ceph-mon[110963]: pgmap v1879: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:14:46.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:46.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:46.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:45 smithi191 ceph-mon[102206]: pgmap v1879: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s 2024-09-21T10:14:46.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:46.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:46.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:46.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:46.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:46.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:46.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:46.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:47.007 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:47.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:47.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:47.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:47.184 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:14:47.187 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:14:47.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:47.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:47.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:47.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:47.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:47.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:47.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:47.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:47.599 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:47.701 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.745 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:47.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:47.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:47.868 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:47.970 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:47.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:47.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:48.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:48.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:48.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:47 smithi076 ceph-mon[106482]: pgmap v1880: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 854 op/s 2024-09-21T10:14:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:47 smithi076 ceph-mon[106482]: from='client.45662 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:14:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:47 smithi076 ceph-mon[110963]: pgmap v1880: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 854 op/s 2024-09-21T10:14:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:47 smithi076 ceph-mon[110963]: from='client.45662 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:14:48.207 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1880,"stamp":"2024-09-21T10:14:46.804790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3929282557,"num_objects":59974,"num_object_clones":0,"num_object_copies":179922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59974,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":169311,"num_write_kb":7329746,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":2963020,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79340,"ondisk_log_size":79340,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":15265084,"kb_used_data":11438996,"kb_used_omap":119,"kb_used_meta":3825928,"kb_avail":453460676,"statfs":{"total":479975178240,"available":464343732224,"internally_reserved":0,"allocated":11713531904,"data_stored":11699252915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":3917750472},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-672333824,"num_objects":-10259,"num_object_clones":0,"num_object_copies":-30777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10259,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003143"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885299+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:14:29.885299+0000","last_peered":"2024-09-21T10:14:29.885299+0000","last_clean":"2024-09-21T10:14:29.885299+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:14:29.885299+0000","last_undegraded":"2024-09-21T10:14:29.885299+0000","last_fullsized":"2024-09-21T10:14:29.885299+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":133,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587668+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:14:26.587668+0000","last_peered":"2024-09-21T10:14:26.587668+0000","last_clean":"2024-09-21T10:14:26.587668+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:14:26.587668+0000","last_undegraded":"2024-09-21T10:14:26.587668+0000","last_fullsized":"2024-09-21T10:14:26.587668+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1371,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.883944+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:14:29.883944+0000","last_peered":"2024-09-21T10:14:29.883944+0000","last_clean":"2024-09-21T10:14:29.883944+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:14:29.883944+0000","last_undegraded":"2024-09-21T10:14:29.883944+0000","last_fullsized":"2024-09-21T10:14:29.883944+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1702,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884425+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:14:29.884425+0000","last_peered":"2024-09-21T10:14:29.884425+0000","last_clean":"2024-09-21T10:14:29.884425+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:14:29.884425+0000","last_undegraded":"2024-09-21T10:14:29.884425+0000","last_fullsized":"2024-09-21T10:14:29.884425+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1702,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884342+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:14:29.884342+0000","last_peered":"2024-09-21T10:14:29.884342+0000","last_clean":"2024-09-21T10:14:29.884342+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:14:29.884342+0000","last_undegraded":"2024-09-21T10:14:29.884342+0000","last_fullsized":"2024-09-21T10:14:29.884342+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588488+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:14:26.588488+0000","last_peered":"2024-09-21T10:14:26.588488+0000","last_clean":"2024-09-21T10:14:26.588488+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:14:26.588488+0000","last_undegraded":"2024-09-21T10:14:26.588488+0000","last_fullsized":"2024-09-21T10:14:26.588488+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.893508+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:14:29.893508+0000","last_peered":"2024-09-21T10:14:29.893508+0000","last_clean":"2024-09-21T10:14:29.893508+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:14:29.893508+0000","last_undegraded":"2024-09-21T10:14:29.893508+0000","last_fullsized":"2024-09-21T10:14:29.893508+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":156,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587264+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:14:26.587264+0000","last_peered":"2024-09-21T10:14:26.587264+0000","last_clean":"2024-09-21T10:14:26.587264+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:14:26.587264+0000","last_undegraded":"2024-09-21T10:14:26.587264+0000","last_fullsized":"2024-09-21T10:14:26.587264+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1273'10640","reported_seq":10678,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.350605+0000","last_change":"2024-09-21T10:12:07.875001+0000","last_active":"2024-09-21T10:14:43.350605+0000","last_peered":"2024-09-21T10:14:43.350605+0000","last_clean":"2024-09-21T10:14:43.350605+0000","last_became_active":"2024-09-21T10:12:07.874267+0000","last_became_peered":"2024-09-21T10:12:07.874267+0000","last_unstale":"2024-09-21T10:14:43.350605+0000","last_undegraded":"2024-09-21T10:14:43.350605+0000","last_fullsized":"2024-09-21T10:14:43.350605+0000","mapping_epoch":1260,"log_start":"1267'5900","ondisk_log_start":"1267'5900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4740,"log_dups_size":0,"ondisk_log_size":4740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:24:27.524428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240779264,"num_objects":3674,"num_object_clones":0,"num_object_copies":11022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10640,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":241,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877207+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:14:29.877207+0000","last_peered":"2024-09-21T10:14:29.877207+0000","last_clean":"2024-09-21T10:14:29.877207+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:14:29.877207+0000","last_undegraded":"2024-09-21T10:14:29.877207+0000","last_fullsized":"2024-09-21T10:14:29.877207+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587766+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:14:26.587766+0000","last_peered":"2024-09-21T10:14:26.587766+0000","last_clean":"2024-09-21T10:14:26.587766+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:14:26.587766+0000","last_undegraded":"2024-09-21T10:14:26.587766+0000","last_fullsized":"2024-09-21T10:14:26.587766+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"1273'10506","reported_seq":10544,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.364147+0000","last_change":"2024-09-21T10:12:07.874211+0000","last_active":"2024-09-21T10:14:43.364147+0000","last_peered":"2024-09-21T10:14:43.364147+0000","last_clean":"2024-09-21T10:14:43.364147+0000","last_became_active":"2024-09-21T10:12:07.873167+0000","last_became_peered":"2024-09-21T10:12:07.873167+0000","last_unstale":"2024-09-21T10:14:43.364147+0000","last_undegraded":"2024-09-21T10:14:43.364147+0000","last_fullsized":"2024-09-21T10:14:43.364147+0000","mapping_epoch":1260,"log_start":"1267'5800","ondisk_log_start":"1267'5800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4706,"log_dups_size":0,"ondisk_log_size":4706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:21.388436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10506,"num_write_kb":456320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":285,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876421+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:14:29.876421+0000","last_peered":"2024-09-21T10:14:29.876421+0000","last_clean":"2024-09-21T10:14:29.876421+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:14:29.876421+0000","last_undegraded":"2024-09-21T10:14:29.876421+0000","last_fullsized":"2024-09-21T10:14:29.876421+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1326,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876197+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:14:29.876197+0000","last_peered":"2024-09-21T10:14:29.876197+0000","last_clean":"2024-09-21T10:14:29.876197+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:14:29.876197+0000","last_undegraded":"2024-09-21T10:14:29.876197+0000","last_fullsized":"2024-09-21T10:14:29.876197+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"1273'10610","reported_seq":10648,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.714873+0000","last_change":"2024-09-21T10:12:08.141102+0000","last_active":"2024-09-21T10:14:43.714873+0000","last_peered":"2024-09-21T10:14:43.714873+0000","last_clean":"2024-09-21T10:14:43.714873+0000","last_became_active":"2024-09-21T10:12:08.140385+0000","last_became_peered":"2024-09-21T10:12:08.140385+0000","last_unstale":"2024-09-21T10:14:43.714873+0000","last_undegraded":"2024-09-21T10:14:43.714873+0000","last_fullsized":"2024-09-21T10:14:43.714873+0000","mapping_epoch":1260,"log_start":"1267'5600","ondisk_log_start":"1267'5600","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:03:28.434076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246808576,"num_objects":3766,"num_object_clones":0,"num_object_copies":11298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10610,"num_write_kb":460032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":144,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587843+0000","last_change":"2024-09-21T10:08:43.398923+0000","last_active":"2024-09-21T10:14:26.587843+0000","last_peered":"2024-09-21T10:14:26.587843+0000","last_clean":"2024-09-21T10:14:26.587843+0000","last_became_active":"2024-09-21T10:08:43.383598+0000","last_became_peered":"2024-09-21T10:08:43.383598+0000","last_unstale":"2024-09-21T10:14:26.587843+0000","last_undegraded":"2024-09-21T10:14:26.587843+0000","last_fullsized":"2024-09-21T10:14:26.587843+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:09.939781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588250+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:14:26.588250+0000","last_peered":"2024-09-21T10:14:26.588250+0000","last_clean":"2024-09-21T10:14:26.588250+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:14:26.588250+0000","last_undegraded":"2024-09-21T10:14:26.588250+0000","last_fullsized":"2024-09-21T10:14:26.588250+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1273'10643","reported_seq":10679,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.375756+0000","last_change":"2024-09-21T10:12:08.139354+0000","last_active":"2024-09-21T10:14:43.375756+0000","last_peered":"2024-09-21T10:14:43.375756+0000","last_clean":"2024-09-21T10:14:43.375756+0000","last_became_active":"2024-09-21T10:12:08.139034+0000","last_became_peered":"2024-09-21T10:12:08.139034+0000","last_unstale":"2024-09-21T10:14:43.375756+0000","last_undegraded":"2024-09-21T10:14:43.375756+0000","last_fullsized":"2024-09-21T10:14:43.375756+0000","mapping_epoch":1260,"log_start":"1267'5900","ondisk_log_start":"1267'5900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4743,"log_dups_size":0,"ondisk_log_size":4743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:50.864652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252903424,"num_objects":3859,"num_object_clones":0,"num_object_copies":11577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10643,"num_write_kb":464064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1356,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885171+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:14:29.885171+0000","last_peered":"2024-09-21T10:14:29.885171+0000","last_clean":"2024-09-21T10:14:29.885171+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:14:29.885171+0000","last_undegraded":"2024-09-21T10:14:29.885171+0000","last_fullsized":"2024-09-21T10:14:29.885171+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1712,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875263+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:14:29.875263+0000","last_peered":"2024-09-21T10:14:29.875263+0000","last_clean":"2024-09-21T10:14:29.875263+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:14:29.875263+0000","last_undegraded":"2024-09-21T10:14:29.875263+0000","last_fullsized":"2024-09-21T10:14:29.875263+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"1273'10269","reported_seq":10307,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.705341+0000","last_change":"2024-09-21T10:12:07.870762+0000","last_active":"2024-09-21T10:14:43.705341+0000","last_peered":"2024-09-21T10:14:43.705341+0000","last_clean":"2024-09-21T10:14:43.705341+0000","last_became_active":"2024-09-21T10:12:07.870495+0000","last_became_peered":"2024-09-21T10:12:07.870495+0000","last_unstale":"2024-09-21T10:14:43.705341+0000","last_undegraded":"2024-09-21T10:14:43.705341+0000","last_fullsized":"2024-09-21T10:14:43.705341+0000","mapping_epoch":1260,"log_start":"1266'5200","ondisk_log_start":"1266'5200","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5069,"log_dups_size":0,"ondisk_log_size":5069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:24.114095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237568000,"num_objects":3625,"num_object_clones":0,"num_object_copies":10875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10269,"num_write_kb":444608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":394,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884693+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:14:29.884693+0000","last_peered":"2024-09-21T10:14:29.884693+0000","last_clean":"2024-09-21T10:14:29.884693+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:14:29.884693+0000","last_undegraded":"2024-09-21T10:14:29.884693+0000","last_fullsized":"2024-09-21T10:14:29.884693+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1371,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885094+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:14:29.885094+0000","last_peered":"2024-09-21T10:14:29.885094+0000","last_clean":"2024-09-21T10:14:29.885094+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:14:29.885094+0000","last_undegraded":"2024-09-21T10:14:29.885094+0000","last_fullsized":"2024-09-21T10:14:29.885094+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"1273'10581","reported_seq":10619,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.376526+0000","last_change":"2024-09-21T10:12:08.140058+0000","last_active":"2024-09-21T10:14:43.376526+0000","last_peered":"2024-09-21T10:14:43.376526+0000","last_clean":"2024-09-21T10:14:43.376526+0000","last_became_active":"2024-09-21T10:12:08.139769+0000","last_became_peered":"2024-09-21T10:12:08.139769+0000","last_unstale":"2024-09-21T10:14:43.376526+0000","last_undegraded":"2024-09-21T10:14:43.376526+0000","last_fullsized":"2024-09-21T10:14:43.376526+0000","mapping_epoch":1260,"log_start":"1267'5800","ondisk_log_start":"1267'5800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4781,"log_dups_size":0,"ondisk_log_size":4781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:01.181046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251985920,"num_objects":3845,"num_object_clones":0,"num_object_copies":11535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10581,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588179+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:14:26.588179+0000","last_peered":"2024-09-21T10:14:26.588179+0000","last_clean":"2024-09-21T10:14:26.588179+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:14:26.588179+0000","last_undegraded":"2024-09-21T10:14:26.588179+0000","last_fullsized":"2024-09-21T10:14:26.588179+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1014,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.878842+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:14:29.878842+0000","last_peered":"2024-09-21T10:14:29.878842+0000","last_clean":"2024-09-21T10:14:29.878842+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:14:29.878842+0000","last_undegraded":"2024-09-21T10:14:29.878842+0000","last_fullsized":"2024-09-21T10:14:29.878842+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"1273'10536","reported_seq":10574,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.706700+0000","last_change":"2024-09-21T10:12:07.873752+0000","last_active":"2024-09-21T10:14:43.706700+0000","last_peered":"2024-09-21T10:14:43.706700+0000","last_clean":"2024-09-21T10:14:43.706700+0000","last_became_active":"2024-09-21T10:12:07.873212+0000","last_became_peered":"2024-09-21T10:12:07.873212+0000","last_unstale":"2024-09-21T10:14:43.706700+0000","last_undegraded":"2024-09-21T10:14:43.706700+0000","last_fullsized":"2024-09-21T10:14:43.706700+0000","mapping_epoch":1260,"log_start":"1267'5500","ondisk_log_start":"1267'5500","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:51:28.706753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10536,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1357,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876119+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:14:29.876119+0000","last_peered":"2024-09-21T10:14:29.876119+0000","last_clean":"2024-09-21T10:14:29.876119+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:14:29.876119+0000","last_undegraded":"2024-09-21T10:14:29.876119+0000","last_fullsized":"2024-09-21T10:14:29.876119+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587900+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:14:26.587900+0000","last_peered":"2024-09-21T10:14:26.587900+0000","last_clean":"2024-09-21T10:14:26.587900+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:14:26.587900+0000","last_undegraded":"2024-09-21T10:14:26.587900+0000","last_fullsized":"2024-09-21T10:14:26.587900+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.2","version":"1273'10449","reported_seq":10487,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.716369+0000","last_change":"2024-09-21T10:12:08.139925+0000","last_active":"2024-09-21T10:14:43.716369+0000","last_peered":"2024-09-21T10:14:43.716369+0000","last_clean":"2024-09-21T10:14:43.716369+0000","last_became_active":"2024-09-21T10:12:08.139616+0000","last_became_peered":"2024-09-21T10:12:08.139616+0000","last_unstale":"2024-09-21T10:14:43.716369+0000","last_undegraded":"2024-09-21T10:14:43.716369+0000","last_fullsized":"2024-09-21T10:14:43.716369+0000","mapping_epoch":1260,"log_start":"1266'5400","ondisk_log_start":"1266'5400","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5049,"log_dups_size":0,"ondisk_log_size":5049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:59:28.474814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248184832,"num_objects":3787,"num_object_clones":0,"num_object_copies":11361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10449,"num_write_kb":455552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":387,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880816+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:14:29.880816+0000","last_peered":"2024-09-21T10:14:29.880816+0000","last_clean":"2024-09-21T10:14:29.880816+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:14:29.880816+0000","last_undegraded":"2024-09-21T10:14:29.880816+0000","last_fullsized":"2024-09-21T10:14:29.880816+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1717,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877352+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:14:29.877352+0000","last_peered":"2024-09-21T10:14:29.877352+0000","last_clean":"2024-09-21T10:14:29.877352+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:14:29.877352+0000","last_undegraded":"2024-09-21T10:14:29.877352+0000","last_fullsized":"2024-09-21T10:14:29.877352+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1273'10872","reported_seq":10910,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.699919+0000","last_change":"2024-09-21T10:12:08.140296+0000","last_active":"2024-09-21T10:14:43.699919+0000","last_peered":"2024-09-21T10:14:43.699919+0000","last_clean":"2024-09-21T10:14:43.699919+0000","last_became_active":"2024-09-21T10:12:08.140008+0000","last_became_peered":"2024-09-21T10:12:08.140008+0000","last_unstale":"2024-09-21T10:14:43.699919+0000","last_undegraded":"2024-09-21T10:14:43.699919+0000","last_fullsized":"2024-09-21T10:14:43.699919+0000","mapping_epoch":1260,"log_start":"1267'5800","ondisk_log_start":"1267'5800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:23.289834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249036800,"num_objects":3800,"num_object_clones":0,"num_object_copies":11400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10872,"num_write_kb":469504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":800,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587168+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:14:26.587168+0000","last_peered":"2024-09-21T10:14:26.587168+0000","last_clean":"2024-09-21T10:14:26.587168+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:14:26.587168+0000","last_undegraded":"2024-09-21T10:14:26.587168+0000","last_fullsized":"2024-09-21T10:14:26.587168+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1370,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884526+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:14:29.884526+0000","last_peered":"2024-09-21T10:14:29.884526+0000","last_clean":"2024-09-21T10:14:29.884526+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:14:29.884526+0000","last_undegraded":"2024-09-21T10:14:29.884526+0000","last_fullsized":"2024-09-21T10:14:29.884526+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":401,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884580+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:14:29.884580+0000","last_peered":"2024-09-21T10:14:29.884580+0000","last_clean":"2024-09-21T10:14:29.884580+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:14:29.884580+0000","last_undegraded":"2024-09-21T10:14:29.884580+0000","last_fullsized":"2024-09-21T10:14:29.884580+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587787+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:14:26.587787+0000","last_peered":"2024-09-21T10:14:26.587787+0000","last_clean":"2024-09-21T10:14:26.587787+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:14:26.587787+0000","last_undegraded":"2024-09-21T10:14:26.587787+0000","last_fullsized":"2024-09-21T10:14:26.587787+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1273'10562","reported_seq":10585,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:46.180013+0000","last_change":"2024-09-21T10:12:08.139083+0000","last_active":"2024-09-21T10:14:46.180013+0000","last_peered":"2024-09-21T10:14:46.180013+0000","last_clean":"2024-09-21T10:14:46.180013+0000","last_became_active":"2024-09-21T10:12:08.138888+0000","last_became_peered":"2024-09-21T10:12:08.138888+0000","last_unstale":"2024-09-21T10:14:46.180013+0000","last_undegraded":"2024-09-21T10:14:46.180013+0000","last_fullsized":"2024-09-21T10:14:46.180013+0000","mapping_epoch":1260,"log_start":"1265'4600","ondisk_log_start":"1265'4600","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:53.085493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231473152,"num_objects":3532,"num_object_clones":0,"num_object_copies":10596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10562,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1001,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875463+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:14:29.875463+0000","last_peered":"2024-09-21T10:14:29.875463+0000","last_clean":"2024-09-21T10:14:29.875463+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:14:29.875463+0000","last_undegraded":"2024-09-21T10:14:29.875463+0000","last_fullsized":"2024-09-21T10:14:29.875463+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1590,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884883+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:14:29.884883+0000","last_peered":"2024-09-21T10:14:29.884883+0000","last_clean":"2024-09-21T10:14:29.884883+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:14:29.884883+0000","last_undegraded":"2024-09-21T10:14:29.884883+0000","last_fullsized":"2024-09-21T10:14:29.884883+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":999,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875939+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:14:29.875939+0000","last_peered":"2024-09-21T10:14:29.875939+0000","last_clean":"2024-09-21T10:14:29.875939+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:14:29.875939+0000","last_undegraded":"2024-09-21T10:14:29.875939+0000","last_fullsized":"2024-09-21T10:14:29.875939+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"1273'10648","reported_seq":10686,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.360960+0000","last_change":"2024-09-21T10:12:07.875082+0000","last_active":"2024-09-21T10:14:43.360960+0000","last_peered":"2024-09-21T10:14:43.360960+0000","last_clean":"2024-09-21T10:14:43.360960+0000","last_became_active":"2024-09-21T10:12:07.874700+0000","last_became_peered":"2024-09-21T10:12:07.874700+0000","last_unstale":"2024-09-21T10:14:43.360960+0000","last_undegraded":"2024-09-21T10:14:43.360960+0000","last_fullsized":"2024-09-21T10:14:43.360960+0000","mapping_epoch":1260,"log_start":"1267'5900","ondisk_log_start":"1267'5900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4748,"log_dups_size":0,"ondisk_log_size":4748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:21.481716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245366784,"num_objects":3744,"num_object_clones":0,"num_object_copies":11232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10648,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1631,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.888062+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:14:29.888062+0000","last_peered":"2024-09-21T10:14:29.888062+0000","last_clean":"2024-09-21T10:14:29.888062+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:14:29.888062+0000","last_undegraded":"2024-09-21T10:14:29.888062+0000","last_fullsized":"2024-09-21T10:14:29.888062+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1718,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875881+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:14:29.875881+0000","last_peered":"2024-09-21T10:14:29.875881+0000","last_clean":"2024-09-21T10:14:29.875881+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:14:29.875881+0000","last_undegraded":"2024-09-21T10:14:29.875881+0000","last_fullsized":"2024-09-21T10:14:29.875881+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1725,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877231+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:14:29.877231+0000","last_peered":"2024-09-21T10:14:29.877231+0000","last_clean":"2024-09-21T10:14:29.877231+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:14:29.877231+0000","last_undegraded":"2024-09-21T10:14:29.877231+0000","last_fullsized":"2024-09-21T10:14:29.877231+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1273'10663","reported_seq":10701,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.676691+0000","last_change":"2024-09-21T10:12:07.871783+0000","last_active":"2024-09-21T10:14:43.676691+0000","last_peered":"2024-09-21T10:14:43.676691+0000","last_clean":"2024-09-21T10:14:43.676691+0000","last_became_active":"2024-09-21T10:12:07.871513+0000","last_became_peered":"2024-09-21T10:12:07.871513+0000","last_unstale":"2024-09-21T10:14:43.676691+0000","last_undegraded":"2024-09-21T10:14:43.676691+0000","last_fullsized":"2024-09-21T10:14:43.676691+0000","mapping_epoch":1260,"log_start":"1267'5600","ondisk_log_start":"1267'5600","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:56.252984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246349824,"num_objects":3759,"num_object_clones":0,"num_object_copies":11277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10663,"num_write_kb":461504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":295,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875272+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:14:29.875272+0000","last_peered":"2024-09-21T10:14:29.875272+0000","last_clean":"2024-09-21T10:14:29.875272+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:14:29.875272+0000","last_undegraded":"2024-09-21T10:14:29.875272+0000","last_fullsized":"2024-09-21T10:14:29.875272+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1238,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587374+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:14:26.587374+0000","last_peered":"2024-09-21T10:14:26.587374+0000","last_clean":"2024-09-21T10:14:26.587374+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:14:26.587374+0000","last_undegraded":"2024-09-21T10:14:26.587374+0000","last_fullsized":"2024-09-21T10:14:26.587374+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1729,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875970+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:14:29.875970+0000","last_peered":"2024-09-21T10:14:29.875970+0000","last_clean":"2024-09-21T10:14:29.875970+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:14:29.875970+0000","last_undegraded":"2024-09-21T10:14:29.875970+0000","last_fullsized":"2024-09-21T10:14:29.875970+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"1273'10602","reported_seq":10639,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.375342+0000","last_change":"2024-09-21T10:12:08.139919+0000","last_active":"2024-09-21T10:14:43.375342+0000","last_peered":"2024-09-21T10:14:43.375342+0000","last_clean":"2024-09-21T10:14:43.375342+0000","last_became_active":"2024-09-21T10:12:08.139617+0000","last_became_peered":"2024-09-21T10:12:08.139617+0000","last_unstale":"2024-09-21T10:14:43.375342+0000","last_undegraded":"2024-09-21T10:14:43.375342+0000","last_fullsized":"2024-09-21T10:14:43.375342+0000","mapping_epoch":1260,"log_start":"1267'5900","ondisk_log_start":"1267'5900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4702,"log_dups_size":0,"ondisk_log_size":4702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:53.447307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252444672,"num_objects":3852,"num_object_clones":0,"num_object_copies":11556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10602,"num_write_kb":462528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1318,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.883619+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:14:29.883619+0000","last_peered":"2024-09-21T10:14:29.883619+0000","last_clean":"2024-09-21T10:14:29.883619+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:14:29.883619+0000","last_undegraded":"2024-09-21T10:14:29.883619+0000","last_fullsized":"2024-09-21T10:14:29.883619+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1740,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876029+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:14:29.876029+0000","last_peered":"2024-09-21T10:14:29.876029+0000","last_clean":"2024-09-21T10:14:29.876029+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:14:29.876029+0000","last_undegraded":"2024-09-21T10:14:29.876029+0000","last_fullsized":"2024-09-21T10:14:29.876029+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":259,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875058+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:14:29.875058+0000","last_peered":"2024-09-21T10:14:29.875058+0000","last_clean":"2024-09-21T10:14:29.875058+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:14:29.875058+0000","last_undegraded":"2024-09-21T10:14:29.875058+0000","last_fullsized":"2024-09-21T10:14:29.875058+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"1273'10434","reported_seq":10471,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.383871+0000","last_change":"2024-09-21T10:12:07.873159+0000","last_active":"2024-09-21T10:14:43.383871+0000","last_peered":"2024-09-21T10:14:43.383871+0000","last_clean":"2024-09-21T10:14:43.383871+0000","last_became_active":"2024-09-21T10:12:07.872870+0000","last_became_peered":"2024-09-21T10:12:07.872870+0000","last_unstale":"2024-09-21T10:14:43.383871+0000","last_undegraded":"2024-09-21T10:14:43.383871+0000","last_fullsized":"2024-09-21T10:14:43.383871+0000","mapping_epoch":1260,"log_start":"1267'5700","ondisk_log_start":"1267'5700","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4734,"log_dups_size":0,"ondisk_log_size":4734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:45.505674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243793920,"num_objects":3720,"num_object_clones":0,"num_object_copies":11160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10434,"num_write_kb":452928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1717,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876138+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:14:29.876138+0000","last_peered":"2024-09-21T10:14:29.876138+0000","last_clean":"2024-09-21T10:14:29.876138+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:14:29.876138+0000","last_undegraded":"2024-09-21T10:14:29.876138+0000","last_fullsized":"2024-09-21T10:14:29.876138+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":897,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877002+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:14:29.877002+0000","last_peered":"2024-09-21T10:14:29.877002+0000","last_clean":"2024-09-21T10:14:29.877002+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:14:29.877002+0000","last_undegraded":"2024-09-21T10:14:29.877002+0000","last_fullsized":"2024-09-21T10:14:29.877002+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1709,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875467+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:14:29.875467+0000","last_peered":"2024-09-21T10:14:29.875467+0000","last_clean":"2024-09-21T10:14:29.875467+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:14:29.875467+0000","last_undegraded":"2024-09-21T10:14:29.875467+0000","last_fullsized":"2024-09-21T10:14:29.875467+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"1273'10607","reported_seq":10646,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.328775+0000","last_change":"2024-09-21T10:12:07.875444+0000","last_active":"2024-09-21T10:14:43.328775+0000","last_peered":"2024-09-21T10:14:43.328775+0000","last_clean":"2024-09-21T10:14:43.328775+0000","last_became_active":"2024-09-21T10:12:07.874789+0000","last_became_peered":"2024-09-21T10:12:07.874789+0000","last_unstale":"2024-09-21T10:14:43.328775+0000","last_undegraded":"2024-09-21T10:14:43.328775+0000","last_fullsized":"2024-09-21T10:14:43.328775+0000","mapping_epoch":1260,"log_start":"1267'5900","ondisk_log_start":"1267'5900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4707,"log_dups_size":0,"ondisk_log_size":4707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:46.604054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245235736,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10607,"num_write_kb":459137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":417,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.893594+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:14:29.893594+0000","last_peered":"2024-09-21T10:14:29.893594+0000","last_clean":"2024-09-21T10:14:29.893594+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:14:29.893594+0000","last_undegraded":"2024-09-21T10:14:29.893594+0000","last_fullsized":"2024-09-21T10:14:29.893594+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1717,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876514+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:14:29.876514+0000","last_peered":"2024-09-21T10:14:29.876514+0000","last_clean":"2024-09-21T10:14:29.876514+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:14:29.876514+0000","last_undegraded":"2024-09-21T10:14:29.876514+0000","last_fullsized":"2024-09-21T10:14:29.876514+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":394,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880745+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:14:29.880745+0000","last_peered":"2024-09-21T10:14:29.880745+0000","last_clean":"2024-09-21T10:14:29.880745+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:14:29.880745+0000","last_undegraded":"2024-09-21T10:14:29.880745+0000","last_fullsized":"2024-09-21T10:14:29.880745+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1273'10410","reported_seq":10447,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:43.721028+0000","last_change":"2024-09-21T10:12:07.870698+0000","last_active":"2024-09-21T10:14:43.721028+0000","last_peered":"2024-09-21T10:14:43.721028+0000","last_clean":"2024-09-21T10:14:43.721028+0000","last_became_active":"2024-09-21T10:12:07.870409+0000","last_became_peered":"2024-09-21T10:12:07.870409+0000","last_unstale":"2024-09-21T10:14:43.721028+0000","last_undegraded":"2024-09-21T10:14:43.721028+0000","last_fullsized":"2024-09-21T10:14:43.721028+0000","mapping_epoch":1260,"log_start":"1267'5400","ondisk_log_start":"1267'5400","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:42.519565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244711424,"num_objects":3734,"num_object_clones":0,"num_object_copies":11202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10410,"num_write_kb":452608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":258,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876032+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:14:29.876032+0000","last_peered":"2024-09-21T10:14:29.876032+0000","last_clean":"2024-09-21T10:14:29.876032+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:14:29.876032+0000","last_undegraded":"2024-09-21T10:14:29.876032+0000","last_fullsized":"2024-09-21T10:14:29.876032+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587536+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:14:26.587536+0000","last_peered":"2024-09-21T10:14:26.587536+0000","last_clean":"2024-09-21T10:14:26.587536+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:14:26.587536+0000","last_undegraded":"2024-09-21T10:14:26.587536+0000","last_fullsized":"2024-09-21T10:14:26.587536+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588441+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:14:26.588441+0000","last_peered":"2024-09-21T10:14:26.588441+0000","last_clean":"2024-09-21T10:14:26.588441+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:14:26.588441+0000","last_undegraded":"2024-09-21T10:14:26.588441+0000","last_fullsized":"2024-09-21T10:14:26.588441+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587581+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:14:26.587581+0000","last_peered":"2024-09-21T10:14:26.587581+0000","last_clean":"2024-09-21T10:14:26.587581+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:14:26.587581+0000","last_undegraded":"2024-09-21T10:14:26.587581+0000","last_fullsized":"2024-09-21T10:14:26.587581+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":137,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587541+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:14:26.587541+0000","last_peered":"2024-09-21T10:14:26.587541+0000","last_clean":"2024-09-21T10:14:26.587541+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:14:26.587541+0000","last_undegraded":"2024-09-21T10:14:26.587541+0000","last_fullsized":"2024-09-21T10:14:26.587541+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1716,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875316+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:14:29.875316+0000","last_peered":"2024-09-21T10:14:29.875316+0000","last_clean":"2024-09-21T10:14:29.875316+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:14:29.875316+0000","last_undegraded":"2024-09-21T10:14:29.875316+0000","last_fullsized":"2024-09-21T10:14:29.875316+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1329,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884047+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:14:29.884047+0000","last_peered":"2024-09-21T10:14:29.884047+0000","last_clean":"2024-09-21T10:14:29.884047+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:14:29.884047+0000","last_undegraded":"2024-09-21T10:14:29.884047+0000","last_fullsized":"2024-09-21T10:14:29.884047+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1340,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876596+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:14:29.876596+0000","last_peered":"2024-09-21T10:14:29.876596+0000","last_clean":"2024-09-21T10:14:29.876596+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:14:29.876596+0000","last_undegraded":"2024-09-21T10:14:29.876596+0000","last_fullsized":"2024-09-21T10:14:29.876596+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":414,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.887967+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:14:29.887967+0000","last_peered":"2024-09-21T10:14:29.887967+0000","last_clean":"2024-09-21T10:14:29.887967+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:14:29.887967+0000","last_undegraded":"2024-09-21T10:14:29.887967+0000","last_fullsized":"2024-09-21T10:14:29.887967+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1742,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.893309+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:14:29.893309+0000","last_peered":"2024-09-21T10:14:29.893309+0000","last_clean":"2024-09-21T10:14:29.893309+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:14:29.893309+0000","last_undegraded":"2024-09-21T10:14:29.893309+0000","last_fullsized":"2024-09-21T10:14:29.893309+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880890+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:14:29.880890+0000","last_peered":"2024-09-21T10:14:29.880890+0000","last_clean":"2024-09-21T10:14:29.880890+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:14:29.880890+0000","last_undegraded":"2024-09-21T10:14:29.880890+0000","last_fullsized":"2024-09-21T10:14:29.880890+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587468+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:14:26.587468+0000","last_peered":"2024-09-21T10:14:26.587468+0000","last_clean":"2024-09-21T10:14:26.587468+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:14:26.587468+0000","last_undegraded":"2024-09-21T10:14:26.587468+0000","last_fullsized":"2024-09-21T10:14:26.587468+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880654+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:14:29.880654+0000","last_peered":"2024-09-21T10:14:29.880654+0000","last_clean":"2024-09-21T10:14:29.880654+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:14:29.880654+0000","last_undegraded":"2024-09-21T10:14:29.880654+0000","last_fullsized":"2024-09-21T10:14:29.880654+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1705,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884671+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:14:29.884671+0000","last_peered":"2024-09-21T10:14:29.884671+0000","last_clean":"2024-09-21T10:14:29.884671+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:14:29.884671+0000","last_undegraded":"2024-09-21T10:14:29.884671+0000","last_fullsized":"2024-09-21T10:14:29.884671+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1471,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885112+0000","last_change":"2024-09-21T10:08:43.397612+0000","last_active":"2024-09-21T10:14:29.885112+0000","last_peered":"2024-09-21T10:14:29.885112+0000","last_clean":"2024-09-21T10:14:29.885112+0000","last_became_active":"2024-09-21T10:08:43.397254+0000","last_became_peered":"2024-09-21T10:08:43.397254+0000","last_unstale":"2024-09-21T10:14:29.885112+0000","last_undegraded":"2024-09-21T10:14:29.885112+0000","last_fullsized":"2024-09-21T10:14:29.885112+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:40.293786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":379,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884224+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:14:29.884224+0000","last_peered":"2024-09-21T10:14:29.884224+0000","last_clean":"2024-09-21T10:14:29.884224+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:14:29.884224+0000","last_undegraded":"2024-09-21T10:14:29.884224+0000","last_fullsized":"2024-09-21T10:14:29.884224+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":263,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875459+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:14:29.875459+0000","last_peered":"2024-09-21T10:14:29.875459+0000","last_clean":"2024-09-21T10:14:29.875459+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:14:29.875459+0000","last_undegraded":"2024-09-21T10:14:29.875459+0000","last_fullsized":"2024-09-21T10:14:29.875459+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1599,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876252+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:14:29.876252+0000","last_peered":"2024-09-21T10:14:29.876252+0000","last_clean":"2024-09-21T10:14:29.876252+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:14:29.876252+0000","last_undegraded":"2024-09-21T10:14:29.876252+0000","last_fullsized":"2024-09-21T10:14:29.876252+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1680,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875356+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:14:29.875356+0000","last_peered":"2024-09-21T10:14:29.875356+0000","last_clean":"2024-09-21T10:14:29.875356+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:14:29.875356+0000","last_undegraded":"2024-09-21T10:14:29.875356+0000","last_fullsized":"2024-09-21T10:14:29.875356+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1325,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884612+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:14:29.884612+0000","last_peered":"2024-09-21T10:14:29.884612+0000","last_clean":"2024-09-21T10:14:29.884612+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:14:29.884612+0000","last_undegraded":"2024-09-21T10:14:29.884612+0000","last_fullsized":"2024-09-21T10:14:29.884612+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":279,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876229+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:14:29.876229+0000","last_peered":"2024-09-21T10:14:29.876229+0000","last_clean":"2024-09-21T10:14:29.876229+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:14:29.876229+0000","last_undegraded":"2024-09-21T10:14:29.876229+0000","last_fullsized":"2024-09-21T10:14:29.876229+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1332,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884336+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:14:29.884336+0000","last_peered":"2024-09-21T10:14:29.884336+0000","last_clean":"2024-09-21T10:14:29.884336+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:14:29.884336+0000","last_undegraded":"2024-09-21T10:14:29.884336+0000","last_fullsized":"2024-09-21T10:14:29.884336+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":130,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587711+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:14:26.587711+0000","last_peered":"2024-09-21T10:14:26.587711+0000","last_clean":"2024-09-21T10:14:26.587711+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:14:26.587711+0000","last_undegraded":"2024-09-21T10:14:26.587711+0000","last_fullsized":"2024-09-21T10:14:26.587711+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.888143+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:14:29.888143+0000","last_peered":"2024-09-21T10:14:29.888143+0000","last_clean":"2024-09-21T10:14:29.888143+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:14:29.888143+0000","last_undegraded":"2024-09-21T10:14:29.888143+0000","last_fullsized":"2024-09-21T10:14:29.888143+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":394,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884424+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:14:29.884424+0000","last_peered":"2024-09-21T10:14:29.884424+0000","last_clean":"2024-09-21T10:14:29.884424+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:14:29.884424+0000","last_undegraded":"2024-09-21T10:14:29.884424+0000","last_fullsized":"2024-09-21T10:14:29.884424+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":126,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587670+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:14:26.587670+0000","last_peered":"2024-09-21T10:14:26.587670+0000","last_clean":"2024-09-21T10:14:26.587670+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:14:26.587670+0000","last_undegraded":"2024-09-21T10:14:26.587670+0000","last_fullsized":"2024-09-21T10:14:26.587670+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1646,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587384+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:14:26.587384+0000","last_peered":"2024-09-21T10:14:26.587384+0000","last_clean":"2024-09-21T10:14:26.587384+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:14:26.587384+0000","last_undegraded":"2024-09-21T10:14:26.587384+0000","last_fullsized":"2024-09-21T10:14:26.587384+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3928686616,"num_objects":59948,"num_object_clones":0,"num_object_copies":179844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59948,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169032,"num_write_kb":7327297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11569868800,"data_stored":11569856584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79132,"ondisk_log_size":79132,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":11142,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234467,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496103,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3116224,"kb_used_data":2323064,"kb_used_omap":1,"kb_used_meta":793150,"kb_avail":90628928,"statfs":{"total":95995035648,"available":92804022272,"internally_reserved":0,"allocated":2378817536,"data_stored":2375950239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":812186055},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757742,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019378,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1974588,"kb_used_data":1379892,"kb_used_omap":1,"kb_used_meta":594686,"kb_avail":91770564,"statfs":{"total":95995035648,"available":93973057536,"internally_reserved":0,"allocated":1413009408,"data_stored":1410145666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":608958918},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888673,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3814372,"kb_used_data":2902172,"kb_used_omap":34,"kb_used_meta":912157,"kb_avail":89930780,"statfs":{"total":95995035648,"available":92089118720,"internally_reserved":0,"allocated":2971824128,"data_stored":2968979527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":934049779},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967913,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2388148,"kb_used_data":1674412,"kb_used_omap":41,"kb_used_meta":713686,"kb_avail":91357004,"statfs":{"total":95995035648,"available":93549572096,"internally_reserved":0,"allocated":1714597888,"data_stored":1711742758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":730814879},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047151,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3971752,"kb_used_data":3159456,"kb_used_omap":41,"kb_used_meta":812246,"kb_avail":89773400,"statfs":{"total":95995035648,"available":91927961600,"internally_reserved":0,"allocated":3235282944,"data_stored":3232434725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":831740841},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126390,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3216838656,"data_stored":3216834584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1680605184,"data_stored":1680605184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2939813888,"data_stored":2939813888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1395003392,"data_stored":1394999320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2337607680,"data_stored":2337603608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:14:48.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:14:48.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:48.305 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:47 smithi191 ceph-mon[102206]: pgmap v1880: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 854 op/s 2024-09-21T10:14:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:47 smithi191 ceph-mon[102206]: from='client.45662 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:14:48.406 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:48.428 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:48.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:48.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:48.581 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:14:48.582 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:14:48.582 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:14:48.582 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:14:48.602 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:14:48.603 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:14:48.642 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:14:48.642 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:14:48.655 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:48.677 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:14:48.677 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:14:48.696 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:48.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:48.709 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:14:48.710 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:14:48.739 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:14:48.739 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:14:48.772 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:14:48.772 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:14:48.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:48.820 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:48.922 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:48.945 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:48.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:49.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:49.096 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1245267332' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1245267332' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:49.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:49.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:49.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:49.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:49.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:49.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1245267332' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:14:49.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:49.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:49.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:49.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:49.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:49.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:49.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:49.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:49.850 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:49.871 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:49.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:49.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:49.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:50.097 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:50.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:50.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:50.246 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:50.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:49 smithi191 ceph-mon[102206]: pgmap v1881: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-21T10:14:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:49 smithi076 ceph-mon[106482]: pgmap v1881: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-21T10:14:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:49 smithi076 ceph-mon[110963]: pgmap v1881: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.24k op/s 2024-09-21T10:14:50.374 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:50.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.500 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:50.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:50.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:50.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:50.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:50.765 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:50.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:50.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:50.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:50.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:51.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:51.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:51.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:51.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:51.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:51.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:51.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:51.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:51.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:51.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:51.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:51.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:51.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:51.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:51.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:51.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:51.938 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:14:51.954 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:14:51.954 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.954 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:177661 9.1 9:8e6a3b5b:::benchmark_data_smithi076_269622_object63169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1273)", 2024-09-21T10:14:51.954 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:14:51.918433+0000", 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.016400669999999999, 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016430297999999999, 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 177661 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.955 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918433+0000", 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918433+0000", 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918433+0000", 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0999999999999999e-07 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.956 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918435+0000", 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2189999999999998e-06 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918441+0000", 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1340000000000001e-06 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918446+0000", 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8899999999999998e-06 2024-09-21T10:14:51.957 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.920885+0000", 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0024388510000000001 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.921002+0000", 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000117246 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:14:51.958 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.921109+0000", 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000106346 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.927612+0000", 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0065036950000000003 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:177662 9.2 9:482a685c:::benchmark_data_smithi076_269622_object63170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1273)", 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:14:51.918464+0000", 2024-09-21T10:14:51.959 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.016369517, 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.016486443, 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 177662 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:14:51.960 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918464+0000", 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918464+0000", 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918464+0000", 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0600000000000001e-07 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.961 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918466+0000", 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9599999999999999e-06 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918472+0000", 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1109999999999996e-06 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.918477+0000", 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1460000000000004e-06 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.962 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.922352+0000", 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038748609999999998 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.922456+0000", 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010392599999999999 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.922616+0000", 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015959700000000001 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.930512+0000", 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0078959349999999998 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933389+0000", 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028774450000000002 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.964 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop_reply(client.35664.0:177661 9.1 e1273/1260)", 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:14:51.925351+0000", 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0094825059999999999, 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0096711399999999999, 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.925351+0000", 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.925351+0000", 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.925352+0000", 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6649999999999999e-06 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.925365+0000", 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2836e-05 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.966 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.925385+0000", 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9465000000000001e-05 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.925395+0000", 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0553e-05 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35664.0:177668 9.4 e1273/1260 9:22e50689:::benchmark_data_smithi076_269622_object63176:head v 1273'11079, mlcod=1273'11077)", 2024-09-21T10:14:51.967 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:14:51.933261+0000", 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0015723040000000001, 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00180882, 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933261+0000", 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933261+0000", 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933262+0000", 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0330000000000001e-06 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933269+0000", 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3669999999999999e-06 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933288+0000", 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8337000000000001e-05 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933295+0000", 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3799999999999996e-06 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933393+0000", 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7456999999999999e-05 2024-09-21T10:14:51.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933415+0000", 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2152999999999999e-05 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35664.0:177671 9.f e1273/1260 9:fc7016eb:::benchmark_data_smithi076_269622_object63179:head v 1273'11174, mlcod=1273'11171)", 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:14:51.933349+0000", 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0014846250000000001, 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001773515, 2024-09-21T10:14:51.971 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933349+0000", 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933349+0000", 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933349+0000", 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9799999999999999e-07 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933353+0000", 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1740000000000002e-06 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933360+0000", 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7900000000000002e-06 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933364+0000", 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4170000000000002e-06 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933433+0000", 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8199000000000002e-05 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.933448+0000", 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5838e-05 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:14:51.974 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop_reply(client.35664.0:177661 9.1 e1273/1260)", 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:14:51.934588+0000", 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.00024514099999999999, 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00058085400000000001, 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.934588+0000", 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.975 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.934588+0000", 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.934589+0000", 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1699999999999998e-07 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.934592+0000", 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5389999999999999e-06 2024-09-21T10:14:51.976 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.934602+0000", 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3740000000000004e-06 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:14:51.934607+0000", 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6420000000000004e-06 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 6 2024-09-21T10:14:51.977 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:14:51.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:52.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:52.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:51 smithi076 ceph-mon[106482]: pgmap v1882: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 848 op/s 2024-09-21T10:14:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:51 smithi076 ceph-mon[110963]: pgmap v1882: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 848 op/s 2024-09-21T10:14:52.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:52.150 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:52.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:52.253 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:52.274 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:51 smithi191 ceph-mon[102206]: pgmap v1882: 89 pgs: 89 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 848 op/s 2024-09-21T10:14:52.284 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:52.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:52.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:52.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:52.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:52.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.682 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:52.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:52.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:14:52.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:52.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:52.928 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:52.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:52.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:53.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:53.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:53.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:53.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:53.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:53.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:14:53.245 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:53.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:53.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.352 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:53.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:53.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:53.524 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:53.581 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:14:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:14:53] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:14:53.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.676 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:53.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:53.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:53.800 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:53.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:53.902 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:53.926 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:53.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:53.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:54.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:54.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:54.173 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:54.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:54.310 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:54.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:54.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:54.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:53 smithi191 ceph-mon[102206]: pgmap v1883: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 875 op/s 2024-09-21T10:14:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:53 smithi076 ceph-mon[106482]: pgmap v1883: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 875 op/s 2024-09-21T10:14:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:53 smithi076 ceph-mon[110963]: pgmap v1883: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 875 op/s 2024-09-21T10:14:54.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:54.458 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:54.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:54.583 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:54.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:54.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:54.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:54.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:54.783 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:54.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:54.833 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:54.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:54.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:54.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:55.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:55.096 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:55.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:55.198 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:55.241 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:55.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.392 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:55.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:55.487 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:55.492 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:55.493 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:14:55.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:55.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:55.721 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:55.823 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:55.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:55.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:55.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:55.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:55.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:55.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:56.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:56.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:56.198 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:55 smithi076 ceph-mon[106482]: pgmap v1884: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s 2024-09-21T10:14:56.199 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:56.199 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:55 smithi076 ceph-mon[110963]: pgmap v1884: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s 2024-09-21T10:14:56.199 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:56.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:56.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:55 smithi191 ceph-mon[102206]: pgmap v1884: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s 2024-09-21T10:14:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:14:56.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:56.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:56.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:56.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:56.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:56.632 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:56.668 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:56.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:56.794 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:56.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:56.918 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:56.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:57.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:57.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:57.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:57.179 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:57.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:57.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:57.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:57.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:57.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:57.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:57.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:57.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:57.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:57.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:57.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:57.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:57.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:57.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:57.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:57.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:57.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:58.065 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:58.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:58.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:14:58.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:14:58.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:58.355 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:58.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:57 smithi191 ceph-mon[102206]: pgmap v1885: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 897 op/s 2024-09-21T10:14:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:57 smithi076 ceph-mon[110963]: pgmap v1885: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 897 op/s 2024-09-21T10:14:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:57 smithi076 ceph-mon[106482]: pgmap v1885: 89 pgs: 89 active+clean; 3.0 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 897 op/s 2024-09-21T10:14:58.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:14:58.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:58.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:14:58.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:58.603 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:58.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:58.728 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:58.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:14:58.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:14:58.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:58.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:14:58.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:14:58.998 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:58.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:58.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:14:59.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.226 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:59.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:14:59.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:14:59.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:14:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:59.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:14:59.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:14:59.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:14:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:14:59.905 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:14:59.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:14:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:14:59.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:00.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:00.053 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:00.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.183 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:00.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:00.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:00.306 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:00.318 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:14:59 smithi076 ceph-mon[106482]: pgmap v1886: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s 2024-09-21T10:15:00.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:14:59 smithi076 ceph-mon[110963]: pgmap v1886: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s 2024-09-21T10:15:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:14:59 smithi191 ceph-mon[102206]: pgmap v1886: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s 2024-09-21T10:15:00.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:00.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:00.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:00.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:00.665 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:00.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:00.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:00.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:00.828 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:00.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:00.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:00.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:00.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:01.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:01.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:01.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:01.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:01.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:01.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:01.465 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:01.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:01.566 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:01.589 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:01.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.691 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:01.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:01.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:01.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:01.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:01.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:01.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:01.984 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:01.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:02.085 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:02.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:02.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:02.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:02.212 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:01 smithi076 ceph-mon[110963]: pgmap v1887: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 855 op/s 2024-09-21T10:15:02.213 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:01 smithi076 ceph-mon[106482]: pgmap v1887: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 855 op/s 2024-09-21T10:15:02.232 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:02.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:01 smithi191 ceph-mon[102206]: pgmap v1887: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 855 op/s 2024-09-21T10:15:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.375 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:02.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:02.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:02.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:02.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:02.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:02.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:02.725 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:02.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:02.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:02.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:02.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:02.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:02.967 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:15:02.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:03.051 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:03.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:03.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3652761407' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:03.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3652761407' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:03.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:03.324 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:03.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3652761407' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:03.421 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:15:03.422 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:15:03.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:03.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:03.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:03.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:03.600 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:15:03] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.51.0" 2024-09-21T10:15:03.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:03.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:03.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:03.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:03.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:03.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:03.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:04.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:04.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:04.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:04.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:04.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:04.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:03 smithi191 ceph-mon[102206]: pgmap v1888: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 878 op/s 2024-09-21T10:15:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:03 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:03 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3652761407' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:03 smithi191 ceph-mon[102206]: osdmap e1274: 8 total, 8 up, 5 in 2024-09-21T10:15:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:03 smithi191 ceph-mon[102206]: from='client.35778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[106482]: pgmap v1888: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 878 op/s 2024-09-21T10:15:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3652761407' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[106482]: osdmap e1274: 8 total, 8 up, 5 in 2024-09-21T10:15:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[106482]: from='client.35778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[110963]: pgmap v1888: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 878 op/s 2024-09-21T10:15:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3652761407' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[110963]: osdmap e1274: 8 total, 8 up, 5 in 2024-09-21T10:15:04.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:03 smithi076 ceph-mon[110963]: from='client.35778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:15:04.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:04.459 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:04.561 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:04.585 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:04.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:04.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:04.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:04.839 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:04.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:04.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:04.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:04.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:05.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:05.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:05.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:05.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:05.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:05.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:05.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:05.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:05.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:05.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:05.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:05.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:05.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:05.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:05.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:05.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:05.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:06.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:06.055 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:06.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:06.156 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:06.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:06.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:06.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:06 smithi191 ceph-mon[102206]: pgmap v1890: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 967 op/s 2024-09-21T10:15:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[106482]: pgmap v1890: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 967 op/s 2024-09-21T10:15:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:06.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:06.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[110963]: pgmap v1890: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 967 op/s 2024-09-21T10:15:06.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:15:06.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:15:06.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:06.429 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:06.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:06.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:06.586 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":1889,"stamp":"2024-09-21T10:15:02.965716+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2916227069,"num_objects":44516,"num_object_clones":0,"num_object_copies":133548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44516,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":184769,"num_write_kb":7329746,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":66,"num_bytes_recovered":2963020,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79298,"ondisk_log_size":79298,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":12393916,"kb_used_data":8524692,"kb_used_omap":119,"kb_used_meta":3868808,"kb_avail":456331844,"statfs":{"total":479975178240,"available":467283808256,"internally_reserved":0,"allocated":8729284608,"data_stored":8715005619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122680,"internal_metadata":3961659592},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,3,7,5,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-669777920,"num_objects":-10220,"num_object_clones":0,"num_object_copies":-30660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10220,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.159069"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885299+0000","last_change":"2024-09-21T10:01:40.045089+0000","last_active":"2024-09-21T10:14:29.885299+0000","last_peered":"2024-09-21T10:14:29.885299+0000","last_clean":"2024-09-21T10:14:29.885299+0000","last_became_active":"2024-09-21T10:01:40.044776+0000","last_became_peered":"2024-09-21T10:01:40.044776+0000","last_unstale":"2024-09-21T10:14:29.885299+0000","last_undegraded":"2024-09-21T10:14:29.885299+0000","last_fullsized":"2024-09-21T10:14:29.885299+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:46.650503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":133,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587668+0000","last_change":"2024-09-21T10:02:39.001460+0000","last_active":"2024-09-21T10:14:26.587668+0000","last_peered":"2024-09-21T10:14:26.587668+0000","last_clean":"2024-09-21T10:14:26.587668+0000","last_became_active":"2024-09-21T10:02:38.980788+0000","last_became_peered":"2024-09-21T10:02:38.980788+0000","last_unstale":"2024-09-21T10:14:26.587668+0000","last_undegraded":"2024-09-21T10:14:26.587668+0000","last_fullsized":"2024-09-21T10:14:26.587668+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:23:44.968518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1371,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.883944+0000","last_change":"2024-09-21T10:03:28.875205+0000","last_active":"2024-09-21T10:14:29.883944+0000","last_peered":"2024-09-21T10:14:29.883944+0000","last_clean":"2024-09-21T10:14:29.883944+0000","last_became_active":"2024-09-21T10:03:28.872655+0000","last_became_peered":"2024-09-21T10:03:28.872655+0000","last_unstale":"2024-09-21T10:14:29.883944+0000","last_undegraded":"2024-09-21T10:14:29.883944+0000","last_fullsized":"2024-09-21T10:14:29.883944+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:06:08.612823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1702,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884425+0000","last_change":"2024-09-21T10:03:28.871570+0000","last_active":"2024-09-21T10:14:29.884425+0000","last_peered":"2024-09-21T10:14:29.884425+0000","last_clean":"2024-09-21T10:14:29.884425+0000","last_became_active":"2024-09-21T10:03:28.871007+0000","last_became_peered":"2024-09-21T10:03:28.871007+0000","last_unstale":"2024-09-21T10:14:29.884425+0000","last_undegraded":"2024-09-21T10:14:29.884425+0000","last_fullsized":"2024-09-21T10:14:29.884425+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:23:38.658442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1702,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884342+0000","last_change":"2024-09-21T10:03:28.867112+0000","last_active":"2024-09-21T10:14:29.884342+0000","last_peered":"2024-09-21T10:14:29.884342+0000","last_clean":"2024-09-21T10:14:29.884342+0000","last_became_active":"2024-09-21T10:03:28.866838+0000","last_became_peered":"2024-09-21T10:03:28.866838+0000","last_unstale":"2024-09-21T10:14:29.884342+0000","last_undegraded":"2024-09-21T10:14:29.884342+0000","last_fullsized":"2024-09-21T10:14:29.884342+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:52.852327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588488+0000","last_change":"2024-09-21T10:03:28.872221+0000","last_active":"2024-09-21T10:14:26.588488+0000","last_peered":"2024-09-21T10:14:26.588488+0000","last_clean":"2024-09-21T10:14:26.588488+0000","last_became_active":"2024-09-21T10:03:28.871855+0000","last_became_peered":"2024-09-21T10:03:28.871855+0000","last_unstale":"2024-09-21T10:14:26.588488+0000","last_undegraded":"2024-09-21T10:14:26.588488+0000","last_fullsized":"2024-09-21T10:14:26.588488+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:20.181687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.893508+0000","last_change":"2024-09-21T10:01:40.040699+0000","last_active":"2024-09-21T10:14:29.893508+0000","last_peered":"2024-09-21T10:14:29.893508+0000","last_clean":"2024-09-21T10:14:29.893508+0000","last_became_active":"2024-09-21T10:01:40.040026+0000","last_became_peered":"2024-09-21T10:01:40.040026+0000","last_unstale":"2024-09-21T10:14:29.893508+0000","last_undegraded":"2024-09-21T10:14:29.893508+0000","last_fullsized":"2024-09-21T10:14:29.893508+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:29.756080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":156,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587264+0000","last_change":"2024-09-21T10:03:28.923186+0000","last_active":"2024-09-21T10:14:26.587264+0000","last_peered":"2024-09-21T10:14:26.587264+0000","last_clean":"2024-09-21T10:14:26.587264+0000","last_became_active":"2024-09-21T10:03:28.891337+0000","last_became_peered":"2024-09-21T10:03:28.891337+0000","last_unstale":"2024-09-21T10:14:26.587264+0000","last_undegraded":"2024-09-21T10:14:26.587264+0000","last_fullsized":"2024-09-21T10:14:26.587264+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:10.001080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1273'11609","reported_seq":11647,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.443400+0000","last_change":"2024-09-21T10:12:07.875001+0000","last_active":"2024-09-21T10:14:58.443400+0000","last_peered":"2024-09-21T10:14:58.443400+0000","last_clean":"2024-09-21T10:14:58.443400+0000","last_became_active":"2024-09-21T10:12:07.874267+0000","last_became_peered":"2024-09-21T10:12:07.874267+0000","last_unstale":"2024-09-21T10:14:58.443400+0000","last_undegraded":"2024-09-21T10:14:58.443400+0000","last_fullsized":"2024-09-21T10:14:58.443400+0000","mapping_epoch":1260,"log_start":"1269'6900","ondisk_log_start":"1269'6900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:24:27.524428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11609,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":242,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877251+0000","last_change":"2024-09-21T10:03:28.928560+0000","last_active":"2024-09-21T10:14:29.877251+0000","last_peered":"2024-09-21T10:14:29.877251+0000","last_clean":"2024-09-21T10:14:29.877251+0000","last_became_active":"2024-09-21T10:03:28.867367+0000","last_became_peered":"2024-09-21T10:03:28.867367+0000","last_unstale":"2024-09-21T10:14:29.877251+0000","last_undegraded":"2024-09-21T10:14:29.877251+0000","last_fullsized":"2024-09-21T10:14:29.877251+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:36:59.134562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587766+0000","last_change":"2024-09-21T10:02:39.007853+0000","last_active":"2024-09-21T10:14:26.587766+0000","last_peered":"2024-09-21T10:14:26.587766+0000","last_clean":"2024-09-21T10:14:26.587766+0000","last_became_active":"2024-09-21T10:02:39.007528+0000","last_became_peered":"2024-09-21T10:02:39.007528+0000","last_unstale":"2024-09-21T10:14:26.587766+0000","last_undegraded":"2024-09-21T10:14:26.587766+0000","last_fullsized":"2024-09-21T10:14:26.587766+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:51.447024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"1273'11490","reported_seq":11528,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.369735+0000","last_change":"2024-09-21T10:12:07.874211+0000","last_active":"2024-09-21T10:14:58.369735+0000","last_peered":"2024-09-21T10:14:58.369735+0000","last_clean":"2024-09-21T10:14:58.369735+0000","last_became_active":"2024-09-21T10:12:07.873167+0000","last_became_peered":"2024-09-21T10:12:07.873167+0000","last_unstale":"2024-09-21T10:14:58.369735+0000","last_undegraded":"2024-09-21T10:14:58.369735+0000","last_fullsized":"2024-09-21T10:14:58.369735+0000","mapping_epoch":1260,"log_start":"1269'6800","ondisk_log_start":"1269'6800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4690,"log_dups_size":0,"ondisk_log_size":4690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:12:21.388436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181534720,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11490,"num_write_kb":456320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":285,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876421+0000","last_change":"2024-09-21T10:03:28.918953+0000","last_active":"2024-09-21T10:14:29.876421+0000","last_peered":"2024-09-21T10:14:29.876421+0000","last_clean":"2024-09-21T10:14:29.876421+0000","last_became_active":"2024-09-21T10:03:28.861858+0000","last_became_peered":"2024-09-21T10:03:28.861858+0000","last_unstale":"2024-09-21T10:14:29.876421+0000","last_undegraded":"2024-09-21T10:14:29.876421+0000","last_fullsized":"2024-09-21T10:14:29.876421+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:42:40.158818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1327,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876230+0000","last_change":"2024-09-21T10:01:53.145819+0000","last_active":"2024-09-21T10:14:29.876230+0000","last_peered":"2024-09-21T10:14:29.876230+0000","last_clean":"2024-09-21T10:14:29.876230+0000","last_became_active":"2024-09-21T10:01:53.145537+0000","last_became_peered":"2024-09-21T10:01:53.145537+0000","last_unstale":"2024-09-21T10:14:29.876230+0000","last_undegraded":"2024-09-21T10:14:29.876230+0000","last_fullsized":"2024-09-21T10:14:29.876230+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:58.453046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"1273'11581","reported_seq":11619,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.773594+0000","last_change":"2024-09-21T10:12:08.141102+0000","last_active":"2024-09-21T10:14:58.773594+0000","last_peered":"2024-09-21T10:14:58.773594+0000","last_clean":"2024-09-21T10:14:58.773594+0000","last_became_active":"2024-09-21T10:12:08.140385+0000","last_became_peered":"2024-09-21T10:12:08.140385+0000","last_unstale":"2024-09-21T10:14:58.773594+0000","last_undegraded":"2024-09-21T10:14:58.773594+0000","last_fullsized":"2024-09-21T10:14:58.773594+0000","mapping_epoch":1260,"log_start":"1267'6500","ondisk_log_start":"1267'6500","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5081,"log_dups_size":0,"ondisk_log_size":5081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:03:28.434076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183173120,"num_objects":2795,"num_object_clones":0,"num_object_copies":8385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11581,"num_write_kb":460032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":144,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587843+0000","last_change":"2024-09-21T10:08:43.398923+0000","last_active":"2024-09-21T10:14:26.587843+0000","last_peered":"2024-09-21T10:14:26.587843+0000","last_clean":"2024-09-21T10:14:26.587843+0000","last_became_active":"2024-09-21T10:08:43.383598+0000","last_became_peered":"2024-09-21T10:08:43.383598+0000","last_unstale":"2024-09-21T10:14:26.587843+0000","last_undegraded":"2024-09-21T10:14:26.587843+0000","last_fullsized":"2024-09-21T10:14:26.587843+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:09.939781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588250+0000","last_change":"2024-09-21T10:03:28.857039+0000","last_active":"2024-09-21T10:14:26.588250+0000","last_peered":"2024-09-21T10:14:26.588250+0000","last_clean":"2024-09-21T10:14:26.588250+0000","last_became_active":"2024-09-21T10:03:28.856729+0000","last_became_peered":"2024-09-21T10:03:28.856729+0000","last_unstale":"2024-09-21T10:14:26.588250+0000","last_undegraded":"2024-09-21T10:14:26.588250+0000","last_fullsized":"2024-09-21T10:14:26.588250+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:56.265184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1273'11625","reported_seq":11663,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.410867+0000","last_change":"2024-09-21T10:12:08.139354+0000","last_active":"2024-09-21T10:14:58.410867+0000","last_peered":"2024-09-21T10:14:58.410867+0000","last_clean":"2024-09-21T10:14:58.410867+0000","last_became_active":"2024-09-21T10:12:08.139034+0000","last_became_peered":"2024-09-21T10:12:08.139034+0000","last_unstale":"2024-09-21T10:14:58.410867+0000","last_undegraded":"2024-09-21T10:14:58.410867+0000","last_fullsized":"2024-09-21T10:14:58.410867+0000","mapping_epoch":1260,"log_start":"1269'6900","ondisk_log_start":"1269'6900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4725,"log_dups_size":0,"ondisk_log_size":4725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:50.864652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188547072,"num_objects":2877,"num_object_clones":0,"num_object_copies":8631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11625,"num_write_kb":464064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1356,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885171+0000","last_change":"2024-09-21T10:02:39.100697+0000","last_active":"2024-09-21T10:14:29.885171+0000","last_peered":"2024-09-21T10:14:29.885171+0000","last_clean":"2024-09-21T10:14:29.885171+0000","last_became_active":"2024-09-21T10:02:39.097970+0000","last_became_peered":"2024-09-21T10:02:39.097970+0000","last_unstale":"2024-09-21T10:14:29.885171+0000","last_undegraded":"2024-09-21T10:14:29.885171+0000","last_fullsized":"2024-09-21T10:14:29.885171+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:00.692188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1713,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875303+0000","last_change":"2024-09-21T10:03:28.861439+0000","last_active":"2024-09-21T10:14:29.875303+0000","last_peered":"2024-09-21T10:14:29.875303+0000","last_clean":"2024-09-21T10:14:29.875303+0000","last_became_active":"2024-09-21T10:03:28.861122+0000","last_became_peered":"2024-09-21T10:03:28.861122+0000","last_unstale":"2024-09-21T10:14:29.875303+0000","last_undegraded":"2024-09-21T10:14:29.875303+0000","last_fullsized":"2024-09-21T10:14:29.875303+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:32:39.564466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"1273'11229","reported_seq":11267,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.737533+0000","last_change":"2024-09-21T10:12:07.870762+0000","last_active":"2024-09-21T10:14:58.737533+0000","last_peered":"2024-09-21T10:14:58.737533+0000","last_clean":"2024-09-21T10:14:58.737533+0000","last_became_active":"2024-09-21T10:12:07.870495+0000","last_became_peered":"2024-09-21T10:12:07.870495+0000","last_unstale":"2024-09-21T10:14:58.737533+0000","last_undegraded":"2024-09-21T10:14:58.737533+0000","last_fullsized":"2024-09-21T10:14:58.737533+0000","mapping_epoch":1260,"log_start":"1267'6200","ondisk_log_start":"1267'6200","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5029,"log_dups_size":0,"ondisk_log_size":5029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:24.114095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174653440,"num_objects":2665,"num_object_clones":0,"num_object_copies":7995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11229,"num_write_kb":444608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":395,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884735+0000","last_change":"2024-09-21T10:03:28.920845+0000","last_active":"2024-09-21T10:14:29.884735+0000","last_peered":"2024-09-21T10:14:29.884735+0000","last_clean":"2024-09-21T10:14:29.884735+0000","last_became_active":"2024-09-21T10:03:28.865468+0000","last_became_peered":"2024-09-21T10:03:28.865468+0000","last_unstale":"2024-09-21T10:14:29.884735+0000","last_undegraded":"2024-09-21T10:14:29.884735+0000","last_fullsized":"2024-09-21T10:14:29.884735+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:54:39.041127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1371,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885094+0000","last_change":"2024-09-21T10:02:39.100381+0000","last_active":"2024-09-21T10:14:29.885094+0000","last_peered":"2024-09-21T10:14:29.885094+0000","last_clean":"2024-09-21T10:14:29.885094+0000","last_became_active":"2024-09-21T10:02:39.097650+0000","last_became_peered":"2024-09-21T10:02:39.097650+0000","last_unstale":"2024-09-21T10:14:29.885094+0000","last_undegraded":"2024-09-21T10:14:29.885094+0000","last_fullsized":"2024-09-21T10:14:29.885094+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:32:34.724562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.0","version":"1273'11589","reported_seq":11627,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.450331+0000","last_change":"2024-09-21T10:12:08.140058+0000","last_active":"2024-09-21T10:14:58.450331+0000","last_peered":"2024-09-21T10:14:58.450331+0000","last_clean":"2024-09-21T10:14:58.450331+0000","last_became_active":"2024-09-21T10:12:08.139769+0000","last_became_peered":"2024-09-21T10:12:08.139769+0000","last_unstale":"2024-09-21T10:14:58.450331+0000","last_undegraded":"2024-09-21T10:14:58.450331+0000","last_fullsized":"2024-09-21T10:14:58.450331+0000","mapping_epoch":1260,"log_start":"1269'6900","ondisk_log_start":"1269'6900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4689,"log_dups_size":0,"ondisk_log_size":4689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:01.181046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185925632,"num_objects":2837,"num_object_clones":0,"num_object_copies":8511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11589,"num_write_kb":461632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588179+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:14:26.588179+0000","last_peered":"2024-09-21T10:14:26.588179+0000","last_clean":"2024-09-21T10:14:26.588179+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:14:26.588179+0000","last_undegraded":"2024-09-21T10:14:26.588179+0000","last_fullsized":"2024-09-21T10:14:26.588179+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1014,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.878842+0000","last_change":"2024-09-21T10:01:53.134424+0000","last_active":"2024-09-21T10:14:29.878842+0000","last_peered":"2024-09-21T10:14:29.878842+0000","last_clean":"2024-09-21T10:14:29.878842+0000","last_became_active":"2024-09-21T10:01:53.133992+0000","last_became_peered":"2024-09-21T10:01:53.133992+0000","last_unstale":"2024-09-21T10:14:29.878842+0000","last_undegraded":"2024-09-21T10:14:29.878842+0000","last_fullsized":"2024-09-21T10:14:29.878842+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:54.768058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.3","version":"1273'11498","reported_seq":11536,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.754749+0000","last_change":"2024-09-21T10:12:07.873752+0000","last_active":"2024-09-21T10:14:58.754749+0000","last_peered":"2024-09-21T10:14:58.754749+0000","last_clean":"2024-09-21T10:14:58.754749+0000","last_became_active":"2024-09-21T10:12:07.873212+0000","last_became_peered":"2024-09-21T10:12:07.873212+0000","last_unstale":"2024-09-21T10:14:58.754749+0000","last_undegraded":"2024-09-21T10:14:58.754749+0000","last_fullsized":"2024-09-21T10:14:58.754749+0000","mapping_epoch":1260,"log_start":"1267'6400","ondisk_log_start":"1267'6400","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:51:28.706753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182976512,"num_objects":2792,"num_object_clones":0,"num_object_copies":8376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11498,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1358,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876157+0000","last_change":"2024-09-21T10:01:53.127268+0000","last_active":"2024-09-21T10:14:29.876157+0000","last_peered":"2024-09-21T10:14:29.876157+0000","last_clean":"2024-09-21T10:14:29.876157+0000","last_became_active":"2024-09-21T10:01:53.126984+0000","last_became_peered":"2024-09-21T10:01:53.126984+0000","last_unstale":"2024-09-21T10:14:29.876157+0000","last_undegraded":"2024-09-21T10:14:29.876157+0000","last_fullsized":"2024-09-21T10:14:29.876157+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:45:08.906378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587900+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:14:26.587900+0000","last_peered":"2024-09-21T10:14:26.587900+0000","last_clean":"2024-09-21T10:14:26.587900+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:14:26.587900+0000","last_undegraded":"2024-09-21T10:14:26.587900+0000","last_fullsized":"2024-09-21T10:14:26.587900+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.2","version":"1273'11396","reported_seq":11434,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.768213+0000","last_change":"2024-09-21T10:12:08.139925+0000","last_active":"2024-09-21T10:14:58.768213+0000","last_peered":"2024-09-21T10:14:58.768213+0000","last_clean":"2024-09-21T10:14:58.768213+0000","last_became_active":"2024-09-21T10:12:08.139616+0000","last_became_peered":"2024-09-21T10:12:08.139616+0000","last_unstale":"2024-09-21T10:14:58.768213+0000","last_undegraded":"2024-09-21T10:14:58.768213+0000","last_fullsized":"2024-09-21T10:14:58.768213+0000","mapping_epoch":1260,"log_start":"1267'6300","ondisk_log_start":"1267'6300","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:59:28.474814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186122240,"num_objects":2840,"num_object_clones":0,"num_object_copies":8520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11396,"num_write_kb":455552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":387,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880816+0000","last_change":"2024-09-21T10:01:40.045786+0000","last_active":"2024-09-21T10:14:29.880816+0000","last_peered":"2024-09-21T10:14:29.880816+0000","last_clean":"2024-09-21T10:14:29.880816+0000","last_became_active":"2024-09-21T10:01:40.044376+0000","last_became_peered":"2024-09-21T10:01:40.044376+0000","last_unstale":"2024-09-21T10:14:29.880816+0000","last_undegraded":"2024-09-21T10:14:29.880816+0000","last_fullsized":"2024-09-21T10:14:29.880816+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1145,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:14.218750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1717,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877352+0000","last_change":"2024-09-21T09:56:45.284583+0000","last_active":"2024-09-21T10:14:29.877352+0000","last_peered":"2024-09-21T10:14:29.877352+0000","last_clean":"2024-09-21T10:14:29.877352+0000","last_became_active":"2024-09-21T09:56:45.284283+0000","last_became_peered":"2024-09-21T09:56:45.284283+0000","last_unstale":"2024-09-21T10:14:29.877352+0000","last_undegraded":"2024-09-21T10:14:29.877352+0000","last_fullsized":"2024-09-21T10:14:29.877352+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:39.492529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"1273'11864","reported_seq":11900,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.775729+0000","last_change":"2024-09-21T10:12:08.140296+0000","last_active":"2024-09-21T10:14:58.775729+0000","last_peered":"2024-09-21T10:14:58.775729+0000","last_clean":"2024-09-21T10:14:58.775729+0000","last_became_active":"2024-09-21T10:12:08.140008+0000","last_became_peered":"2024-09-21T10:12:08.140008+0000","last_unstale":"2024-09-21T10:14:58.775729+0000","last_undegraded":"2024-09-21T10:14:58.775729+0000","last_fullsized":"2024-09-21T10:14:58.775729+0000","mapping_epoch":1260,"log_start":"1269'6800","ondisk_log_start":"1269'6800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:19:23.289834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184025088,"num_objects":2808,"num_object_clones":0,"num_object_copies":8424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11864,"num_write_kb":469504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":800,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587168+0000","last_change":"2024-09-21T10:05:03.306623+0000","last_active":"2024-09-21T10:14:26.587168+0000","last_peered":"2024-09-21T10:14:26.587168+0000","last_clean":"2024-09-21T10:14:26.587168+0000","last_became_active":"2024-09-21T10:03:29.921501+0000","last_became_peered":"2024-09-21T10:03:29.921501+0000","last_unstale":"2024-09-21T10:14:26.587168+0000","last_undegraded":"2024-09-21T10:14:26.587168+0000","last_fullsized":"2024-09-21T10:14:26.587168+0000","mapping_epoch":1170,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:35:09.367034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1371,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884568+0000","last_change":"2024-09-21T10:05:03.300562+0000","last_active":"2024-09-21T10:14:29.884568+0000","last_peered":"2024-09-21T10:14:29.884568+0000","last_clean":"2024-09-21T10:14:29.884568+0000","last_became_active":"2024-09-21T10:02:38.998918+0000","last_became_peered":"2024-09-21T10:02:38.998918+0000","last_unstale":"2024-09-21T10:14:29.884568+0000","last_undegraded":"2024-09-21T10:14:29.884568+0000","last_fullsized":"2024-09-21T10:14:29.884568+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:21:25.285157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":401,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884580+0000","last_change":"2024-09-21T10:02:39.002487+0000","last_active":"2024-09-21T10:14:29.884580+0000","last_peered":"2024-09-21T10:14:29.884580+0000","last_clean":"2024-09-21T10:14:29.884580+0000","last_became_active":"2024-09-21T10:02:39.002233+0000","last_became_peered":"2024-09-21T10:02:39.002233+0000","last_unstale":"2024-09-21T10:14:29.884580+0000","last_undegraded":"2024-09-21T10:14:29.884580+0000","last_fullsized":"2024-09-21T10:14:29.884580+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:50.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587787+0000","last_change":"2024-09-21T10:03:28.857611+0000","last_active":"2024-09-21T10:14:26.587787+0000","last_peered":"2024-09-21T10:14:26.587787+0000","last_clean":"2024-09-21T10:14:26.587787+0000","last_became_active":"2024-09-21T10:03:28.857204+0000","last_became_peered":"2024-09-21T10:03:28.857204+0000","last_unstale":"2024-09-21T10:14:26.587787+0000","last_undegraded":"2024-09-21T10:14:26.587787+0000","last_fullsized":"2024-09-21T10:14:26.587787+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:33.799303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1273'11495","reported_seq":11518,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:15:01.237388+0000","last_change":"2024-09-21T10:12:08.139083+0000","last_active":"2024-09-21T10:15:01.237388+0000","last_peered":"2024-09-21T10:15:01.237388+0000","last_clean":"2024-09-21T10:15:01.237388+0000","last_became_active":"2024-09-21T10:12:08.138888+0000","last_became_peered":"2024-09-21T10:12:08.138888+0000","last_unstale":"2024-09-21T10:15:01.237388+0000","last_undegraded":"2024-09-21T10:15:01.237388+0000","last_fullsized":"2024-09-21T10:15:01.237388+0000","mapping_epoch":1260,"log_start":"1267'5600","ondisk_log_start":"1267'5600","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:47:53.085493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170328064,"num_objects":2599,"num_object_clones":0,"num_object_copies":7797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11495,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1001,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875463+0000","last_change":"2024-09-21T10:03:28.896648+0000","last_active":"2024-09-21T10:14:29.875463+0000","last_peered":"2024-09-21T10:14:29.875463+0000","last_clean":"2024-09-21T10:14:29.875463+0000","last_became_active":"2024-09-21T10:03:28.896272+0000","last_became_peered":"2024-09-21T10:03:28.896272+0000","last_unstale":"2024-09-21T10:14:29.875463+0000","last_undegraded":"2024-09-21T10:14:29.875463+0000","last_fullsized":"2024-09-21T10:14:29.875463+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:20.205057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1590,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884883+0000","last_change":"2024-09-21T10:03:28.878443+0000","last_active":"2024-09-21T10:14:29.884883+0000","last_peered":"2024-09-21T10:14:29.884883+0000","last_clean":"2024-09-21T10:14:29.884883+0000","last_became_active":"2024-09-21T10:03:28.877877+0000","last_became_peered":"2024-09-21T10:03:28.877877+0000","last_unstale":"2024-09-21T10:14:29.884883+0000","last_undegraded":"2024-09-21T10:14:29.884883+0000","last_fullsized":"2024-09-21T10:14:29.884883+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:58:30.721108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":999,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875939+0000","last_change":"2024-09-21T10:03:28.848492+0000","last_active":"2024-09-21T10:14:29.875939+0000","last_peered":"2024-09-21T10:14:29.875939+0000","last_clean":"2024-09-21T10:14:29.875939+0000","last_became_active":"2024-09-21T10:03:28.848071+0000","last_became_peered":"2024-09-21T10:03:28.848071+0000","last_unstale":"2024-09-21T10:14:29.875939+0000","last_undegraded":"2024-09-21T10:14:29.875939+0000","last_fullsized":"2024-09-21T10:14:29.875939+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:51:42.196589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"1273'11615","reported_seq":11652,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.437781+0000","last_change":"2024-09-21T10:12:07.875082+0000","last_active":"2024-09-21T10:14:58.437781+0000","last_peered":"2024-09-21T10:14:58.437781+0000","last_clean":"2024-09-21T10:14:58.437781+0000","last_became_active":"2024-09-21T10:12:07.874700+0000","last_became_peered":"2024-09-21T10:12:07.874700+0000","last_unstale":"2024-09-21T10:14:58.437781+0000","last_undegraded":"2024-09-21T10:14:58.437781+0000","last_fullsized":"2024-09-21T10:14:58.437781+0000","mapping_epoch":1260,"log_start":"1269'6900","ondisk_log_start":"1269'6900","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4715,"log_dups_size":0,"ondisk_log_size":4715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:21.481716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181993472,"num_objects":2777,"num_object_clones":0,"num_object_copies":8331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11615,"num_write_kb":460544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1631,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.888062+0000","last_change":"2024-09-21T10:03:28.878730+0000","last_active":"2024-09-21T10:14:29.888062+0000","last_peered":"2024-09-21T10:14:29.888062+0000","last_clean":"2024-09-21T10:14:29.888062+0000","last_became_active":"2024-09-21T10:03:28.875506+0000","last_became_peered":"2024-09-21T10:03:28.875506+0000","last_unstale":"2024-09-21T10:14:29.888062+0000","last_undegraded":"2024-09-21T10:14:29.888062+0000","last_fullsized":"2024-09-21T10:14:29.888062+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:02:03.835042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1718,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875881+0000","last_change":"2024-09-21T10:02:38.986851+0000","last_active":"2024-09-21T10:14:29.875881+0000","last_peered":"2024-09-21T10:14:29.875881+0000","last_clean":"2024-09-21T10:14:29.875881+0000","last_became_active":"2024-09-21T10:02:38.986471+0000","last_became_peered":"2024-09-21T10:02:38.986471+0000","last_unstale":"2024-09-21T10:14:29.875881+0000","last_undegraded":"2024-09-21T10:14:29.875881+0000","last_fullsized":"2024-09-21T10:14:29.875881+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:05.480000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1725,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877231+0000","last_change":"2024-09-21T10:03:28.867671+0000","last_active":"2024-09-21T10:14:29.877231+0000","last_peered":"2024-09-21T10:14:29.877231+0000","last_clean":"2024-09-21T10:14:29.877231+0000","last_became_active":"2024-09-21T10:03:28.867002+0000","last_became_peered":"2024-09-21T10:03:28.867002+0000","last_unstale":"2024-09-21T10:14:29.877231+0000","last_undegraded":"2024-09-21T10:14:29.877231+0000","last_fullsized":"2024-09-21T10:14:29.877231+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:53.321680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"1273'11637","reported_seq":11675,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.758749+0000","last_change":"2024-09-21T10:12:07.871783+0000","last_active":"2024-09-21T10:14:58.758749+0000","last_peered":"2024-09-21T10:14:58.758749+0000","last_clean":"2024-09-21T10:14:58.758749+0000","last_became_active":"2024-09-21T10:12:07.871513+0000","last_became_peered":"2024-09-21T10:12:07.871513+0000","last_unstale":"2024-09-21T10:14:58.758749+0000","last_undegraded":"2024-09-21T10:14:58.758749+0000","last_fullsized":"2024-09-21T10:14:58.758749+0000","mapping_epoch":1260,"log_start":"1268'6600","ondisk_log_start":"1268'6600","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5037,"log_dups_size":0,"ondisk_log_size":5037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:56.252984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182517760,"num_objects":2785,"num_object_clones":0,"num_object_copies":8355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11637,"num_write_kb":461504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":295,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875272+0000","last_change":"2024-09-21T10:03:28.860071+0000","last_active":"2024-09-21T10:14:29.875272+0000","last_peered":"2024-09-21T10:14:29.875272+0000","last_clean":"2024-09-21T10:14:29.875272+0000","last_became_active":"2024-09-21T10:03:28.859720+0000","last_became_peered":"2024-09-21T10:03:28.859720+0000","last_unstale":"2024-09-21T10:14:29.875272+0000","last_undegraded":"2024-09-21T10:14:29.875272+0000","last_fullsized":"2024-09-21T10:14:29.875272+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:09.047251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1238,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587374+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:14:26.587374+0000","last_peered":"2024-09-21T10:14:26.587374+0000","last_clean":"2024-09-21T10:14:26.587374+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:14:26.587374+0000","last_undegraded":"2024-09-21T10:14:26.587374+0000","last_fullsized":"2024-09-21T10:14:26.587374+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1729,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875970+0000","last_change":"2024-09-21T09:56:45.293636+0000","last_active":"2024-09-21T10:14:29.875970+0000","last_peered":"2024-09-21T10:14:29.875970+0000","last_clean":"2024-09-21T10:14:29.875970+0000","last_became_active":"2024-09-21T09:56:45.292804+0000","last_became_peered":"2024-09-21T09:56:45.292804+0000","last_unstale":"2024-09-21T10:14:29.875970+0000","last_undegraded":"2024-09-21T10:14:29.875970+0000","last_fullsized":"2024-09-21T10:14:29.875970+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:55:35.535535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"1273'11570","reported_seq":11608,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.330871+0000","last_change":"2024-09-21T10:12:08.139919+0000","last_active":"2024-09-21T10:14:58.330871+0000","last_peered":"2024-09-21T10:14:58.330871+0000","last_clean":"2024-09-21T10:14:58.330871+0000","last_became_active":"2024-09-21T10:12:08.139617+0000","last_became_peered":"2024-09-21T10:12:08.139617+0000","last_unstale":"2024-09-21T10:14:58.330871+0000","last_undegraded":"2024-09-21T10:14:58.330871+0000","last_fullsized":"2024-09-21T10:14:58.330871+0000","mapping_epoch":1260,"log_start":"1269'6800","ondisk_log_start":"1269'6800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:53.447307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189005824,"num_objects":2884,"num_object_clones":0,"num_object_copies":8652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11570,"num_write_kb":462528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1319,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884494+0000","last_change":"2024-09-21T10:02:39.003993+0000","last_active":"2024-09-21T10:14:29.884494+0000","last_peered":"2024-09-21T10:14:29.884494+0000","last_clean":"2024-09-21T10:14:29.884494+0000","last_became_active":"2024-09-21T10:02:38.998704+0000","last_became_peered":"2024-09-21T10:02:38.998704+0000","last_unstale":"2024-09-21T10:14:29.884494+0000","last_undegraded":"2024-09-21T10:14:29.884494+0000","last_fullsized":"2024-09-21T10:14:29.884494+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:38.565779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1740,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876029+0000","last_change":"2024-09-21T10:03:28.865195+0000","last_active":"2024-09-21T10:14:29.876029+0000","last_peered":"2024-09-21T10:14:29.876029+0000","last_clean":"2024-09-21T10:14:29.876029+0000","last_became_active":"2024-09-21T10:03:28.864891+0000","last_became_peered":"2024-09-21T10:03:28.864891+0000","last_unstale":"2024-09-21T10:14:29.876029+0000","last_undegraded":"2024-09-21T10:14:29.876029+0000","last_fullsized":"2024-09-21T10:14:29.876029+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:01.603305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":259,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875058+0000","last_change":"2024-09-21T10:03:28.849530+0000","last_active":"2024-09-21T10:14:29.875058+0000","last_peered":"2024-09-21T10:14:29.875058+0000","last_clean":"2024-09-21T10:14:29.875058+0000","last_became_active":"2024-09-21T10:03:28.849188+0000","last_became_peered":"2024-09-21T10:03:28.849188+0000","last_unstale":"2024-09-21T10:14:29.875058+0000","last_undegraded":"2024-09-21T10:14:29.875058+0000","last_fullsized":"2024-09-21T10:14:29.875058+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:37.276036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"1273'11428","reported_seq":11466,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.444057+0000","last_change":"2024-09-21T10:12:07.873159+0000","last_active":"2024-09-21T10:14:58.444057+0000","last_peered":"2024-09-21T10:14:58.444057+0000","last_clean":"2024-09-21T10:14:58.444057+0000","last_became_active":"2024-09-21T10:12:07.872870+0000","last_became_peered":"2024-09-21T10:12:07.872870+0000","last_unstale":"2024-09-21T10:14:58.444057+0000","last_undegraded":"2024-09-21T10:14:58.444057+0000","last_fullsized":"2024-09-21T10:14:58.444057+0000","mapping_epoch":1260,"log_start":"1269'6700","ondisk_log_start":"1269'6700","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4728,"log_dups_size":0,"ondisk_log_size":4728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:48:45.505674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178651136,"num_objects":2726,"num_object_clones":0,"num_object_copies":8178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11428,"num_write_kb":452928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1717,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876138+0000","last_change":"2024-09-21T10:03:28.846029+0000","last_active":"2024-09-21T10:14:29.876138+0000","last_peered":"2024-09-21T10:14:29.876138+0000","last_clean":"2024-09-21T10:14:29.876138+0000","last_became_active":"2024-09-21T10:03:28.845643+0000","last_became_peered":"2024-09-21T10:03:28.845643+0000","last_unstale":"2024-09-21T10:14:29.876138+0000","last_undegraded":"2024-09-21T10:14:29.876138+0000","last_fullsized":"2024-09-21T10:14:29.876138+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:53.366966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":897,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877002+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:14:29.877002+0000","last_peered":"2024-09-21T10:14:29.877002+0000","last_clean":"2024-09-21T10:14:29.877002+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:14:29.877002+0000","last_undegraded":"2024-09-21T10:14:29.877002+0000","last_fullsized":"2024-09-21T10:14:29.877002+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1709,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875467+0000","last_change":"2024-09-21T10:03:28.852643+0000","last_active":"2024-09-21T10:14:29.875467+0000","last_peered":"2024-09-21T10:14:29.875467+0000","last_clean":"2024-09-21T10:14:29.875467+0000","last_became_active":"2024-09-21T10:03:28.852261+0000","last_became_peered":"2024-09-21T10:03:28.852261+0000","last_unstale":"2024-09-21T10:14:29.875467+0000","last_undegraded":"2024-09-21T10:14:29.875467+0000","last_fullsized":"2024-09-21T10:14:29.875467+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:14:23.916058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"1273'11530","reported_seq":11569,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.428184+0000","last_change":"2024-09-21T10:12:07.875444+0000","last_active":"2024-09-21T10:14:58.428184+0000","last_peered":"2024-09-21T10:14:58.428184+0000","last_clean":"2024-09-21T10:14:58.428184+0000","last_became_active":"2024-09-21T10:12:07.874789+0000","last_became_peered":"2024-09-21T10:12:07.874789+0000","last_unstale":"2024-09-21T10:14:58.428184+0000","last_undegraded":"2024-09-21T10:14:58.428184+0000","last_fullsized":"2024-09-21T10:14:58.428184+0000","mapping_epoch":1260,"log_start":"1269'6800","ondisk_log_start":"1269'6800","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":4730,"log_dups_size":0,"ondisk_log_size":4730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:46.604054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184746008,"num_objects":2820,"num_object_clones":0,"num_object_copies":8460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2820,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11530,"num_write_kb":459137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":417,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.893594+0000","last_change":"2024-09-21T10:03:28.923024+0000","last_active":"2024-09-21T10:14:29.893594+0000","last_peered":"2024-09-21T10:14:29.893594+0000","last_clean":"2024-09-21T10:14:29.893594+0000","last_became_active":"2024-09-21T10:03:28.870779+0000","last_became_peered":"2024-09-21T10:03:28.870779+0000","last_unstale":"2024-09-21T10:14:29.893594+0000","last_undegraded":"2024-09-21T10:14:29.893594+0000","last_fullsized":"2024-09-21T10:14:29.893594+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:09.580186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1717,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876514+0000","last_change":"2024-09-21T10:02:38.976133+0000","last_active":"2024-09-21T10:14:29.876514+0000","last_peered":"2024-09-21T10:14:29.876514+0000","last_clean":"2024-09-21T10:14:29.876514+0000","last_became_active":"2024-09-21T10:02:38.975533+0000","last_became_peered":"2024-09-21T10:02:38.975533+0000","last_unstale":"2024-09-21T10:14:29.876514+0000","last_undegraded":"2024-09-21T10:14:29.876514+0000","last_fullsized":"2024-09-21T10:14:29.876514+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:01.496529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":395,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880776+0000","last_change":"2024-09-21T10:03:28.896677+0000","last_active":"2024-09-21T10:14:29.880776+0000","last_peered":"2024-09-21T10:14:29.880776+0000","last_clean":"2024-09-21T10:14:29.880776+0000","last_became_active":"2024-09-21T10:03:28.891232+0000","last_became_peered":"2024-09-21T10:03:28.891232+0000","last_unstale":"2024-09-21T10:14:29.880776+0000","last_undegraded":"2024-09-21T10:14:29.880776+0000","last_fullsized":"2024-09-21T10:14:29.880776+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:20:55.107534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1273'11334","reported_seq":11372,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:58.767414+0000","last_change":"2024-09-21T10:12:07.870698+0000","last_active":"2024-09-21T10:14:58.767414+0000","last_peered":"2024-09-21T10:14:58.767414+0000","last_clean":"2024-09-21T10:14:58.767414+0000","last_became_active":"2024-09-21T10:12:07.870409+0000","last_became_peered":"2024-09-21T10:12:07.870409+0000","last_unstale":"2024-09-21T10:14:58.767414+0000","last_undegraded":"2024-09-21T10:14:58.767414+0000","last_fullsized":"2024-09-21T10:14:58.767414+0000","mapping_epoch":1260,"log_start":"1267'6300","ondisk_log_start":"1267'6300","created":1260,"last_epoch_clean":1261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:12:06.852508+0000","last_clean_scrub_stamp":"2024-09-21T10:12:06.852508+0000","objects_scrubbed":0,"log_size":5034,"log_dups_size":0,"ondisk_log_size":5034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:31:42.519565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184156160,"num_objects":2810,"num_object_clones":0,"num_object_copies":8430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11334,"num_write_kb":452608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":258,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876032+0000","last_change":"2024-09-21T10:03:28.851965+0000","last_active":"2024-09-21T10:14:29.876032+0000","last_peered":"2024-09-21T10:14:29.876032+0000","last_clean":"2024-09-21T10:14:29.876032+0000","last_became_active":"2024-09-21T10:03:28.848146+0000","last_became_peered":"2024-09-21T10:03:28.848146+0000","last_unstale":"2024-09-21T10:14:29.876032+0000","last_undegraded":"2024-09-21T10:14:29.876032+0000","last_fullsized":"2024-09-21T10:14:29.876032+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:56:58.128232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587536+0000","last_change":"2024-09-21T10:02:38.992276+0000","last_active":"2024-09-21T10:14:26.587536+0000","last_peered":"2024-09-21T10:14:26.587536+0000","last_clean":"2024-09-21T10:14:26.587536+0000","last_became_active":"2024-09-21T10:02:38.991989+0000","last_became_peered":"2024-09-21T10:02:38.991989+0000","last_unstale":"2024-09-21T10:14:26.587536+0000","last_undegraded":"2024-09-21T10:14:26.587536+0000","last_fullsized":"2024-09-21T10:14:26.587536+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:05.518960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.588441+0000","last_change":"2024-09-21T10:03:28.841137+0000","last_active":"2024-09-21T10:14:26.588441+0000","last_peered":"2024-09-21T10:14:26.588441+0000","last_clean":"2024-09-21T10:14:26.588441+0000","last_became_active":"2024-09-21T10:03:28.840846+0000","last_became_peered":"2024-09-21T10:03:28.840846+0000","last_unstale":"2024-09-21T10:14:26.588441+0000","last_undegraded":"2024-09-21T10:14:26.588441+0000","last_fullsized":"2024-09-21T10:14:26.588441+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:57.436029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587581+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:14:26.587581+0000","last_peered":"2024-09-21T10:14:26.587581+0000","last_clean":"2024-09-21T10:14:26.587581+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:14:26.587581+0000","last_undegraded":"2024-09-21T10:14:26.587581+0000","last_fullsized":"2024-09-21T10:14:26.587581+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":137,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587541+0000","last_change":"2024-09-21T10:02:38.990337+0000","last_active":"2024-09-21T10:14:26.587541+0000","last_peered":"2024-09-21T10:14:26.587541+0000","last_clean":"2024-09-21T10:14:26.587541+0000","last_became_active":"2024-09-21T10:02:38.980524+0000","last_became_peered":"2024-09-21T10:02:38.980524+0000","last_unstale":"2024-09-21T10:14:26.587541+0000","last_undegraded":"2024-09-21T10:14:26.587541+0000","last_fullsized":"2024-09-21T10:14:26.587541+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:00.425290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1716,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875316+0000","last_change":"2024-09-21T10:02:38.993104+0000","last_active":"2024-09-21T10:14:29.875316+0000","last_peered":"2024-09-21T10:14:29.875316+0000","last_clean":"2024-09-21T10:14:29.875316+0000","last_became_active":"2024-09-21T10:02:38.992729+0000","last_became_peered":"2024-09-21T10:02:38.992729+0000","last_unstale":"2024-09-21T10:14:29.875316+0000","last_undegraded":"2024-09-21T10:14:29.875316+0000","last_fullsized":"2024-09-21T10:14:29.875316+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:43.384192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1329,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884047+0000","last_change":"2024-09-21T10:03:28.926494+0000","last_active":"2024-09-21T10:14:29.884047+0000","last_peered":"2024-09-21T10:14:29.884047+0000","last_clean":"2024-09-21T10:14:29.884047+0000","last_became_active":"2024-09-21T10:03:28.872183+0000","last_became_peered":"2024-09-21T10:03:28.872183+0000","last_unstale":"2024-09-21T10:14:29.884047+0000","last_undegraded":"2024-09-21T10:14:29.884047+0000","last_fullsized":"2024-09-21T10:14:29.884047+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:30.295976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1340,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876596+0000","last_change":"2024-09-21T10:02:11.228381+0000","last_active":"2024-09-21T10:14:29.876596+0000","last_peered":"2024-09-21T10:14:29.876596+0000","last_clean":"2024-09-21T10:14:29.876596+0000","last_became_active":"2024-09-21T10:02:11.228078+0000","last_became_peered":"2024-09-21T10:02:11.228078+0000","last_unstale":"2024-09-21T10:14:29.876596+0000","last_undegraded":"2024-09-21T10:14:29.876596+0000","last_fullsized":"2024-09-21T10:14:29.876596+0000","mapping_epoch":1155,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:55:14.283320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":414,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.887967+0000","last_change":"2024-09-21T10:03:28.875124+0000","last_active":"2024-09-21T10:14:29.887967+0000","last_peered":"2024-09-21T10:14:29.887967+0000","last_clean":"2024-09-21T10:14:29.887967+0000","last_became_active":"2024-09-21T10:03:28.871561+0000","last_became_peered":"2024-09-21T10:03:28.871561+0000","last_unstale":"2024-09-21T10:14:29.887967+0000","last_undegraded":"2024-09-21T10:14:29.887967+0000","last_fullsized":"2024-09-21T10:14:29.887967+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:09.410507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1742,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.893309+0000","last_change":"2024-09-21T10:03:28.871812+0000","last_active":"2024-09-21T10:14:29.893309+0000","last_peered":"2024-09-21T10:14:29.893309+0000","last_clean":"2024-09-21T10:14:29.893309+0000","last_became_active":"2024-09-21T10:03:28.871216+0000","last_became_peered":"2024-09-21T10:03:28.871216+0000","last_unstale":"2024-09-21T10:14:29.893309+0000","last_undegraded":"2024-09-21T10:14:29.893309+0000","last_fullsized":"2024-09-21T10:14:29.893309+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:41:53.519784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880890+0000","last_change":"2024-09-21T10:02:39.004172+0000","last_active":"2024-09-21T10:14:29.880890+0000","last_peered":"2024-09-21T10:14:29.880890+0000","last_clean":"2024-09-21T10:14:29.880890+0000","last_became_active":"2024-09-21T10:02:39.003733+0000","last_became_peered":"2024-09-21T10:02:39.003733+0000","last_unstale":"2024-09-21T10:14:29.880890+0000","last_undegraded":"2024-09-21T10:14:29.880890+0000","last_fullsized":"2024-09-21T10:14:29.880890+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:53.324383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":136,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587468+0000","last_change":"2024-09-21T10:03:28.872474+0000","last_active":"2024-09-21T10:14:26.587468+0000","last_peered":"2024-09-21T10:14:26.587468+0000","last_clean":"2024-09-21T10:14:26.587468+0000","last_became_active":"2024-09-21T10:03:28.872199+0000","last_became_peered":"2024-09-21T10:03:28.872199+0000","last_unstale":"2024-09-21T10:14:26.587468+0000","last_undegraded":"2024-09-21T10:14:26.587468+0000","last_fullsized":"2024-09-21T10:14:26.587468+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:26.795194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.880654+0000","last_change":"2024-09-21T10:03:28.897177+0000","last_active":"2024-09-21T10:14:29.880654+0000","last_peered":"2024-09-21T10:14:29.880654+0000","last_clean":"2024-09-21T10:14:29.880654+0000","last_became_active":"2024-09-21T10:03:28.891830+0000","last_became_peered":"2024-09-21T10:03:28.891830+0000","last_unstale":"2024-09-21T10:14:29.880654+0000","last_undegraded":"2024-09-21T10:14:29.880654+0000","last_fullsized":"2024-09-21T10:14:29.880654+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:49.744281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1705,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884671+0000","last_change":"2024-09-21T10:03:28.867344+0000","last_active":"2024-09-21T10:14:29.884671+0000","last_peered":"2024-09-21T10:14:29.884671+0000","last_clean":"2024-09-21T10:14:29.884671+0000","last_became_active":"2024-09-21T10:03:28.866520+0000","last_became_peered":"2024-09-21T10:03:28.866520+0000","last_unstale":"2024-09-21T10:14:29.884671+0000","last_undegraded":"2024-09-21T10:14:29.884671+0000","last_fullsized":"2024-09-21T10:14:29.884671+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:20:22.281872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1471,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.885112+0000","last_change":"2024-09-21T10:08:43.397612+0000","last_active":"2024-09-21T10:14:29.885112+0000","last_peered":"2024-09-21T10:14:29.885112+0000","last_clean":"2024-09-21T10:14:29.885112+0000","last_became_active":"2024-09-21T10:08:43.397254+0000","last_became_peered":"2024-09-21T10:08:43.397254+0000","last_unstale":"2024-09-21T10:14:29.885112+0000","last_undegraded":"2024-09-21T10:14:29.885112+0000","last_fullsized":"2024-09-21T10:14:29.885112+0000","mapping_epoch":1213,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:40.293786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":379,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884224+0000","last_change":"2024-09-21T09:56:45.289798+0000","last_active":"2024-09-21T10:14:29.884224+0000","last_peered":"2024-09-21T10:14:29.884224+0000","last_clean":"2024-09-21T10:14:29.884224+0000","last_became_active":"2024-09-21T09:56:45.289362+0000","last_became_peered":"2024-09-21T09:56:45.289362+0000","last_unstale":"2024-09-21T10:14:29.884224+0000","last_undegraded":"2024-09-21T10:14:29.884224+0000","last_fullsized":"2024-09-21T10:14:29.884224+0000","mapping_epoch":1092,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1093,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:41.570731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":263,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875459+0000","last_change":"2024-09-21T10:01:53.129894+0000","last_active":"2024-09-21T10:14:29.875459+0000","last_peered":"2024-09-21T10:14:29.875459+0000","last_clean":"2024-09-21T10:14:29.875459+0000","last_became_active":"2024-09-21T10:01:53.129536+0000","last_became_peered":"2024-09-21T10:01:53.129536+0000","last_unstale":"2024-09-21T10:14:29.875459+0000","last_undegraded":"2024-09-21T10:14:29.875459+0000","last_fullsized":"2024-09-21T10:14:29.875459+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:16:20.426074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1599,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876252+0000","last_change":"2024-09-21T10:01:22.661328+0000","last_active":"2024-09-21T10:14:29.876252+0000","last_peered":"2024-09-21T10:14:29.876252+0000","last_clean":"2024-09-21T10:14:29.876252+0000","last_became_active":"2024-09-21T10:01:22.661013+0000","last_became_peered":"2024-09-21T10:01:22.661013+0000","last_unstale":"2024-09-21T10:14:29.876252+0000","last_undegraded":"2024-09-21T10:14:29.876252+0000","last_fullsized":"2024-09-21T10:14:29.876252+0000","mapping_epoch":1142,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1143,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:32.078027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1680,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.875356+0000","last_change":"2024-09-21T10:03:28.866625+0000","last_active":"2024-09-21T10:14:29.875356+0000","last_peered":"2024-09-21T10:14:29.875356+0000","last_clean":"2024-09-21T10:14:29.875356+0000","last_became_active":"2024-09-21T10:03:28.866255+0000","last_became_peered":"2024-09-21T10:03:28.866255+0000","last_unstale":"2024-09-21T10:14:29.875356+0000","last_undegraded":"2024-09-21T10:14:29.875356+0000","last_fullsized":"2024-09-21T10:14:29.875356+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:04.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1325,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884612+0000","last_change":"2024-09-21T10:01:53.137654+0000","last_active":"2024-09-21T10:14:29.884612+0000","last_peered":"2024-09-21T10:14:29.884612+0000","last_clean":"2024-09-21T10:14:29.884612+0000","last_became_active":"2024-09-21T10:01:53.137358+0000","last_became_peered":"2024-09-21T10:01:53.137358+0000","last_unstale":"2024-09-21T10:14:29.884612+0000","last_undegraded":"2024-09-21T10:14:29.884612+0000","last_fullsized":"2024-09-21T10:14:29.884612+0000","mapping_epoch":1150,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1151,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:52:50.469665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":279,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.876229+0000","last_change":"2024-09-21T10:02:38.996666+0000","last_active":"2024-09-21T10:14:29.876229+0000","last_peered":"2024-09-21T10:14:29.876229+0000","last_clean":"2024-09-21T10:14:29.876229+0000","last_became_active":"2024-09-21T10:02:38.980753+0000","last_became_peered":"2024-09-21T10:02:38.980753+0000","last_unstale":"2024-09-21T10:14:29.876229+0000","last_undegraded":"2024-09-21T10:14:29.876229+0000","last_fullsized":"2024-09-21T10:14:29.876229+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:13:44.825061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1332,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884336+0000","last_change":"2024-09-21T10:03:28.871301+0000","last_active":"2024-09-21T10:14:29.884336+0000","last_peered":"2024-09-21T10:14:29.884336+0000","last_clean":"2024-09-21T10:14:29.884336+0000","last_became_active":"2024-09-21T10:03:28.870469+0000","last_became_peered":"2024-09-21T10:03:28.870469+0000","last_unstale":"2024-09-21T10:14:29.884336+0000","last_undegraded":"2024-09-21T10:14:29.884336+0000","last_fullsized":"2024-09-21T10:14:29.884336+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:20:46.720248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":130,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587711+0000","last_change":"2024-09-21T10:02:39.003201+0000","last_active":"2024-09-21T10:14:26.587711+0000","last_peered":"2024-09-21T10:14:26.587711+0000","last_clean":"2024-09-21T10:14:26.587711+0000","last_became_active":"2024-09-21T10:02:38.984653+0000","last_became_peered":"2024-09-21T10:02:38.984653+0000","last_unstale":"2024-09-21T10:14:26.587711+0000","last_undegraded":"2024-09-21T10:14:26.587711+0000","last_fullsized":"2024-09-21T10:14:26.587711+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:54:25.864626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":386,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.888143+0000","last_change":"2024-09-21T10:03:28.870383+0000","last_active":"2024-09-21T10:14:29.888143+0000","last_peered":"2024-09-21T10:14:29.888143+0000","last_clean":"2024-09-21T10:14:29.888143+0000","last_became_active":"2024-09-21T10:03:28.869968+0000","last_became_peered":"2024-09-21T10:03:28.869968+0000","last_unstale":"2024-09-21T10:14:29.888143+0000","last_undegraded":"2024-09-21T10:14:29.888143+0000","last_fullsized":"2024-09-21T10:14:29.888143+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:43.691981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":395,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.884555+0000","last_change":"2024-09-21T10:02:39.005987+0000","last_active":"2024-09-21T10:14:29.884555+0000","last_peered":"2024-09-21T10:14:29.884555+0000","last_clean":"2024-09-21T10:14:29.884555+0000","last_became_active":"2024-09-21T10:02:39.004655+0000","last_became_peered":"2024-09-21T10:02:39.004655+0000","last_unstale":"2024-09-21T10:14:29.884555+0000","last_undegraded":"2024-09-21T10:14:29.884555+0000","last_fullsized":"2024-09-21T10:14:29.884555+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:48:10.272898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":127,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-09-21T10:14:29.877344+0000","last_change":"2024-09-21T10:02:38.979043+0000","last_active":"2024-09-21T10:14:29.877344+0000","last_peered":"2024-09-21T10:14:29.877344+0000","last_clean":"2024-09-21T10:14:29.877344+0000","last_became_active":"2024-09-21T10:02:38.978824+0000","last_became_peered":"2024-09-21T10:02:38.978824+0000","last_unstale":"2024-09-21T10:14:29.877344+0000","last_undegraded":"2024-09-21T10:14:29.877344+0000","last_fullsized":"2024-09-21T10:14:29.877344+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:52.548995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1646,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-09-21T10:14:26.587384+0000","last_change":"2024-09-21T10:03:28.846741+0000","last_active":"2024-09-21T10:14:26.587384+0000","last_peered":"2024-09-21T10:14:26.587384+0000","last_clean":"2024-09-21T10:14:26.587384+0000","last_became_active":"2024-09-21T10:03:28.843663+0000","last_became_peered":"2024-09-21T10:03:28.843663+0000","last_unstale":"2024-09-21T10:14:26.587384+0000","last_undegraded":"2024-09-21T10:14:26.587384+0000","last_fullsized":"2024-09-21T10:14:26.587384+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:44:14.808440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2915631128,"num_objects":44490,"num_object_clones":0,"num_object_copies":133470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44490,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":184490,"num_write_kb":7327297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8556982272,"data_stored":8556970056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79090,"ondisk_log_size":79090,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":11142,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":167936,"data_stored":27840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12558,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234470,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496106,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2532544,"kb_used_data":1717240,"kb_used_omap":1,"kb_used_meta":815294,"kb_avail":91212608,"statfs":{"total":95995035648,"available":93401710592,"internally_reserved":0,"allocated":1758453760,"data_stored":1755586463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":834861511},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757745,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019381,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1617148,"kb_used_data":1018356,"kb_used_omap":1,"kb_used_meta":598782,"kb_avail":92128004,"statfs":{"total":95995035648,"available":94339076096,"internally_reserved":0,"allocated":1042796544,"data_stored":1039932802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":613153222},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":829,"seq":3560527888676,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3072420,"kb_used_data":2155996,"kb_used_omap":34,"kb_used_meta":916317,"kb_avail":90672732,"statfs":{"total":95995035648,"available":92848877568,"internally_reserved":0,"allocated":2207739904,"data_stored":2204895303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":34829,"internal_metadata":938309619},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793967916,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1966132,"kb_used_data":1244076,"kb_used_omap":41,"kb_used_meta":722006,"kb_avail":91779020,"statfs":{"total":95995035648,"available":93981716480,"internally_reserved":0,"allocated":1273933824,"data_stored":1271078694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":739334559},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047154,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3205672,"kb_used_data":2389024,"kb_used_omap":41,"kb_used_meta":816406,"kb_avail":90539480,"statfs":{"total":95995035648,"available":92712427520,"internally_reserved":0,"allocated":2446360576,"data_stored":2443512357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":836000681},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,4,3,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":3135326126393,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3086,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2389118976,"data_stored":2389114904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1241513984,"data_stored":1241513984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2179727360,"data_stored":2179727360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1025314816,"data_stored":1025310744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1721307136,"data_stored":1721303064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:15:06.590 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:15:06.590 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:15:06.590 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 4, 3, 6] 2024-09-21T10:15:06.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 2 2024-09-21T10:15:06.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:06.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.721 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:06.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:06.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:06.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:06.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:06.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:06.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:07.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:07.112 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:07.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:07.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:07.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:07.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:07.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:07.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:07.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:07.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:07.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:07.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:07.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:07.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:07.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:07.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:07.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:07.947 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:07.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:08.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:08.072 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:08.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:08.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:08.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:08.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:08.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:08 smithi191 ceph-mon[102206]: pgmap v1891: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 961 op/s 2024-09-21T10:15:08.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:08 smithi076 ceph-mon[110963]: pgmap v1891: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 961 op/s 2024-09-21T10:15:08.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:08 smithi076 ceph-mon[106482]: pgmap v1891: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 961 op/s 2024-09-21T10:15:08.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:08.464 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:08.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:08.603 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:08.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:08.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:08.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:08.829 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:08.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:08.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:08.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:08.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:09.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:09.118 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:09.216 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:15:09.220 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:09.242 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:09.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:09.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:09.366 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:09.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:09.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:09.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:09.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:09.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:09.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:09.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:09.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:09.929 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:09.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:10.031 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:10.069 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:10.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:10.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:10.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:10.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:10.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:10 smithi191 ceph-mon[102206]: pgmap v1892: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:15:10.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:10 smithi076 ceph-mon[106482]: pgmap v1892: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:15:10.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:10 smithi076 ceph-mon[110963]: pgmap v1892: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:15:10.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:10.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:10.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:10.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:10.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:10.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:10.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:10.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:10.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:10.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:10.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:10.946 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:10.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:10.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:11.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:11.121 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:11.222 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:11.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:11.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:11.364 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:11.364 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.392 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:11.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:11.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:11.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:11.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:11.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:11.659 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:11.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:11.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:11.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:11.887 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:11.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:11.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:11.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:12.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:12.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:12.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.203 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:12.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:12.217 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:15:12.217 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:15:12.217 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:15:12.218 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:15:12.218 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:15:12.304 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:12.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:12.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:12 smithi076 ceph-mon[106482]: pgmap v1893: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:15:12.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:12 smithi076 ceph-mon[110963]: pgmap v1893: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:15:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:12 smithi191 ceph-mon[102206]: pgmap v1893: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 963 op/s 2024-09-21T10:15:12.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:12.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:12.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:12.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:12.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:12.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:12.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:12.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:12.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:12.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:12.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:12.924 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:15:12.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:12.986 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:13.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:13.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:13.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:13.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:13.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:13.315 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:13.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.462 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:13.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:13.536 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:15:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:15:13] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.51.0" 2024-09-21T10:15:13.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:13.590 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:13.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:13.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:13.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:13.773 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:13.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:13.852 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:15:13.852 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:13.863 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:13.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:13.895 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:15:13.895 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:13.936 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:15:13.937 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:13.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:13.973 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:15:13.973 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:13.987 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:13.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:13.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:14.003 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:15:14.003 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:14.041 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:15:14.041 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:14.071 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:15:14.072 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:14.089 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:14.103 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:15:14.104 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:14.104 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:14 smithi076 ceph-mon[106482]: pgmap v1894: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 968 op/s 2024-09-21T10:15:14.104 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1762691120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:14.104 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:14 smithi076 ceph-mon[110963]: pgmap v1894: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 968 op/s 2024-09-21T10:15:14.104 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1762691120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:14.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:14.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:14.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:14.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:14.352 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:14 smithi191 ceph-mon[102206]: pgmap v1894: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 968 op/s 2024-09-21T10:15:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1762691120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:14.376 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:14.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:14.500 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:14.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:14.627 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:14.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:14.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:14.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:14.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:14.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:14.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:14.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:15.022 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:15.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:15.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:15.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:15.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:15.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:15.285 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:15.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:15.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:15.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:15.535 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:15.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:15.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:15.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.677 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:15.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:15.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:15.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:15.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:15.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:15.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:15.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:16.045 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:16.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.080 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:16.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:16.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:16.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:16.307 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.344 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:16.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:16 smithi191 ceph-mon[102206]: pgmap v1895: 89 pgs: 89 active+clean; 1.8 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:15:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:16 smithi076 ceph-mon[106482]: pgmap v1895: 89 pgs: 89 active+clean; 1.8 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:15:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:16 smithi076 ceph-mon[110963]: pgmap v1895: 89 pgs: 89 active+clean; 1.8 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.22k op/s 2024-09-21T10:15:16.445 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:16.469 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:16.571 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:16.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:16.696 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:16.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:16.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:16.870 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:16.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:16.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:16.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:16.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:16.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:17.002 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:15:17.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:17.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.168 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:17.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:17.270 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:17.313 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:17.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:17.441 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:17.534 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:17.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:17.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:17.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:17 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:15:17.315+0000 7f240edc2640 -1 mon.a@0(leader).osd e1274 definitely_dead 0 2024-09-21T10:15:17.722 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:17.766 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:17.861 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:17.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:17.891 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:17.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:17.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:17.992 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:18.015 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:18.091 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.2. 2024-09-21T10:15:18.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:18.158 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:18.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:18.302 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:18 smithi191 ceph-mon[102206]: pgmap v1896: 89 pgs: 89 active+clean; 1.8 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 872 op/s 2024-09-21T10:15:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:18 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1594093692' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-21T10:15:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:18 smithi191 ceph-mon[102206]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-21T10:15:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:18 smithi076 ceph-mon[110963]: pgmap v1896: 89 pgs: 89 active+clean; 1.8 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 872 op/s 2024-09-21T10:15:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:18 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1594093692' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-21T10:15:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:18 smithi076 ceph-mon[110963]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-21T10:15:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:18 smithi076 ceph-mon[106482]: pgmap v1896: 89 pgs: 89 active+clean; 1.8 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 872 op/s 2024-09-21T10:15:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:18 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1594093692' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-21T10:15:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:18 smithi076 ceph-mon[106482]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-21T10:15:18.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:18.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:18.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:18.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:18.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:18.672 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:18.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:18.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:18.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:18.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:18.918 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:18.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:19.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:19.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:19.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:19.098 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:19 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1594093692' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-21T10:15:19.098 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:19 smithi076 ceph-mon[106482]: osdmap e1275: 8 total, 8 up, 4 in 2024-09-21T10:15:19.098 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:19 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1594093692' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-21T10:15:19.098 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:19 smithi076 ceph-mon[110963]: osdmap e1275: 8 total, 8 up, 4 in 2024-09-21T10:15:19.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:19.164 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:19.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:19.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:19.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:19.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:19.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:19 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1594093692' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-21T10:15:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:19 smithi191 ceph-mon[102206]: osdmap e1275: 8 total, 8 up, 4 in 2024-09-21T10:15:19.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:19.409 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:19.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:19.511 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:19.533 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:19.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:19.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:19.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:19.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:19.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:19.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:19.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:19.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:19.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:20.003 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:20.024 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:20.105 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:15:20.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:20.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:20.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:20.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:20.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:20.348 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[110963]: pgmap v1898: 89 pgs: 17 peering, 72 active+clean; 1.7 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 798 op/s 2024-09-21T10:15:20.348 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[110963]: osdmap e1276: 8 total, 8 up, 4 in 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1622061162' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[106482]: pgmap v1898: 89 pgs: 17 peering, 72 active+clean; 1.7 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 798 op/s 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[106482]: osdmap e1276: 8 total, 8 up, 4 in 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1622061162' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:20.349 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:20 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:20.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:20 smithi191 ceph-mon[102206]: pgmap v1898: 89 pgs: 17 peering, 72 active+clean; 1.7 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 798 op/s 2024-09-21T10:15:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:20 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:15:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:20 smithi191 ceph-mon[102206]: osdmap e1276: 8 total, 8 up, 4 in 2024-09-21T10:15:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:20 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1622061162' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:20 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:20.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:20.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:20.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:20.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:20.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:20.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:20.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:20.831 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:20.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:20.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:20.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:20.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:21.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:21.097 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:21.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:21.227 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:21.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: 9.b starting backfill to osd.3 from (0'0,0'0] MAX to 1274'12538 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: osdmap e1277: 8 total, 8 up, 4 in 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: 9.6 starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12885 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: 9.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12837 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: 9.8 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'13097 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'12804 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[106482]: 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12817 2024-09-21T10:15:21.231 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: 9.b starting backfill to osd.3 from (0'0,0'0] MAX to 1274'12538 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: osdmap e1277: 8 total, 8 up, 4 in 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: 9.6 starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12885 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: 9.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12837 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: 9.8 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'13097 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'12804 2024-09-21T10:15:21.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:21 smithi076 ceph-mon[110963]: 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12817 2024-09-21T10:15:21.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: 9.b starting backfill to osd.3 from (0'0,0'0] MAX to 1274'12538 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: osdmap e1277: 8 total, 8 up, 4 in 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: 9.6 starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12885 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: 9.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12837 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: 9.8 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'13097 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'12804 2024-09-21T10:15:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:21 smithi191 ceph-mon[102206]: 9.f starting backfill to osd.4 from (0'0,0'0] MAX to 1274'12817 2024-09-21T10:15:21.366 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:21.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:21.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:21.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:21.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:21.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:21.658 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:21.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:21.810 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:21.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:21.940 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:21.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:22.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:22.083 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:22.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:22.209 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:22.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:22.349 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:22.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:22.508 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:22.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:22.538 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:22 smithi076 ceph-mon[106482]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'12676 2024-09-21T10:15:22.538 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:22 smithi076 ceph-mon[106482]: pgmap v1901: 89 pgs: 32 peering, 57 active+clean; 1.5 GiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:15:22.538 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:22 smithi076 ceph-mon[106482]: osdmap e1278: 8 total, 8 up, 4 in 2024-09-21T10:15:22.538 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:22 smithi076 ceph-mon[110963]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'12676 2024-09-21T10:15:22.538 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:22 smithi076 ceph-mon[110963]: pgmap v1901: 89 pgs: 32 peering, 57 active+clean; 1.5 GiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:15:22.538 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:22 smithi076 ceph-mon[110963]: osdmap e1278: 8 total, 8 up, 4 in 2024-09-21T10:15:22.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:22.678 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:22 smithi191 ceph-mon[102206]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1274'12676 2024-09-21T10:15:22.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:22 smithi191 ceph-mon[102206]: pgmap v1901: 89 pgs: 32 peering, 57 active+clean; 1.5 GiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 820 op/s 2024-09-21T10:15:22.679 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:22 smithi191 ceph-mon[102206]: osdmap e1278: 8 total, 8 up, 4 in 2024-09-21T10:15:22.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:22.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:22.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:22.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:22.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:22.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:23.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:23.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:23.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:23.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:23.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:15:23.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:23.371 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:23 smithi076 ceph-mon[106482]: osdmap e1279: 8 total, 8 up, 4 in 2024-09-21T10:15:23.371 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:23 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1682933236' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:23.372 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:15:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:15:23] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.51.0" 2024-09-21T10:15:23.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:23 smithi076 ceph-mon[110963]: osdmap e1279: 8 total, 8 up, 4 in 2024-09-21T10:15:23.372 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:23 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1682933236' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:23.403 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:23.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:23.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:23 smithi191 ceph-mon[102206]: osdmap e1279: 8 total, 8 up, 4 in 2024-09-21T10:15:23.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:23 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1682933236' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:23.639 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:23.674 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:23.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:23.776 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:23.800 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:23.805 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:23.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:23.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:15:23.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:23.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:23.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:23.806 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:23.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:23.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:23.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:23.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:24.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:24.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:24.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:24.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:24.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:24.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:24.413 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:15:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.445 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:24.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:15:24.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:24.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:24.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:24 smithi191 ceph-mon[102206]: pgmap v1904: 89 pgs: 1 active+remapped+backfilling, 33 peering, 55 active+clean; 1.5 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 569 op/s; 1394/74676 objects misplaced (1.867%); 1.9 MiB/s, 2 keys/s, 31 objects/s recovering 2024-09-21T10:15:24.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:24 smithi076 ceph-mon[110963]: pgmap v1904: 89 pgs: 1 active+remapped+backfilling, 33 peering, 55 active+clean; 1.5 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 569 op/s; 1394/74676 objects misplaced (1.867%); 1.9 MiB/s, 2 keys/s, 31 objects/s recovering 2024-09-21T10:15:24.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:24 smithi076 ceph-mon[106482]: pgmap v1904: 89 pgs: 1 active+remapped+backfilling, 33 peering, 55 active+clean; 1.5 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 569 op/s; 1394/74676 objects misplaced (1.867%); 1.9 MiB/s, 2 keys/s, 31 objects/s recovering 2024-09-21T10:15:24.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:24.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:24.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:24.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:24.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:24.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:24.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:25.055 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:25.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:25.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:25.103 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6] out_osds: [7, 5, 0, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:15:25.103 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:15:25.103 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-21T10:15:25.103 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:15:25.103 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:15:25.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:25.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:25.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:25.308 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:25.352 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:25.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:25.454 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:25.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:25.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:25.579 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:25.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:25 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:15:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:25 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:15:25.619 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:25 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:15:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:25 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:15:25.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:25 smithi076 systemd[1]: Stopping Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:15:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:25 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T10:15:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:25 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:15:25.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:25.757 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:25.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:25.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:25.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:25.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:25.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:26.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:26.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:26.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:26.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:26.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:26.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:26.373 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:26.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:26.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:26.525 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:26 smithi191 ceph-mon[102206]: pgmap v1905: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 1 peering, 81 active+clean; 1.3 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 603 op/s; 8873/64848 objects misplaced (13.683%); 9.6 MiB/s, 1 keys/s, 155 objects/s recovering 2024-09-21T10:15:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:26 smithi076 ceph-mon[106482]: pgmap v1905: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 1 peering, 81 active+clean; 1.3 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 603 op/s; 8873/64848 objects misplaced (13.683%); 9.6 MiB/s, 1 keys/s, 155 objects/s recovering 2024-09-21T10:15:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:26 smithi076 ceph-mon[110963]: pgmap v1905: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 1 peering, 81 active+clean; 1.3 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 603 op/s; 8873/64848 objects misplaced (13.683%); 9.6 MiB/s, 1 keys/s, 155 objects/s recovering 2024-09-21T10:15:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:26.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:26.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:26.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:26.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:26.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:26.915 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:26.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:27.017 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:27.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:27.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:27.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:27.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:27.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:27.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:27.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:27.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:27.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:27.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:27.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:27.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:27.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:27.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:27.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:27.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:27.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:27.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:28.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.102 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:28.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:28.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:28.229 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:28.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:28.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:28.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:28 smithi076 ceph-mon[106482]: pgmap v1906: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.3 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 561 op/s; 8145/63885 objects misplaced (12.749%); 13 MiB/s, 1 keys/s, 206 objects/s recovering 2024-09-21T10:15:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:28 smithi076 ceph-mon[110963]: pgmap v1906: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.3 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 561 op/s; 8145/63885 objects misplaced (12.749%); 13 MiB/s, 1 keys/s, 206 objects/s recovering 2024-09-21T10:15:28.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:28.492 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:28.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:28.594 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146601 8.16 8:6aa6d95d:::benchmark_data_smithi076_241159_object23683:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.155546+0000", 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 312.49629927699999, 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9885378650000001, 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.612 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146601 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155546+0000", 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155546+0000", 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155547+0000", 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8800000000000001e-07 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155552+0000", 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.549e-06 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155565+0000", 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2822e-05 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155572+0000", 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5399999999999998e-06 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155674+0000", 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000101913 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.615 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155684+0000", 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0051000000000001e-05 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077427+0000", 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.92174305099999998 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077439+0000", 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1539000000000001e-05 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.138536+0000", 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0610976110000001 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.138673+0000", 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013722100000000001 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.138919+0000", 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024506899999999999 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.617 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143738+0000", 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.004819968 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143781+0000", 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2367999999999998e-05 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143842+0000", 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1582999999999996e-05 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143887+0000", 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4267999999999997e-05 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143955+0000", 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8287999999999994e-05 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144084+0000", 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012913599999999999 2024-09-21T10:15:28.619 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146654 8.16 8:6a8aae0b:::benchmark_data_smithi076_241159_object23736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.207022+0000", 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 312.44482364100003, 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9373881310000001, 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:15:28.620 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146654 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207022+0000", 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207022+0000", 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207023+0000", 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.879999999999999e-07 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207027+0000", 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1060000000000002e-06 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207037+0000", 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0183000000000001e-05 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.622 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207044+0000", 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5610000000000004e-06 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207105+0000", 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1124000000000006e-05 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.207115+0000", 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0583e-05 2024-09-21T10:15:28.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077844+0000", 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87072874700000003 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077854+0000", 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0054999999999999e-05 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.140541+0000", 2024-09-21T10:15:28.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0626865750000001 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.140687+0000", 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014609200000000001 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.140948+0000", 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00026133599999999998 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143794+0000", 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028461609999999998 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144127+0000", 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000332765 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144222+0000", 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4636999999999995e-05 2024-09-21T10:15:28.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144283+0000", 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1259000000000004e-05 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144314+0000", 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1372999999999999e-05 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144410+0000", 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5586000000000003e-05 2024-09-21T10:15:28.627 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146670 8.16 8:6cf15343:::benchmark_data_smithi076_241159_object23752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 312.43636002699998, 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.930904897, 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:15:28.628 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146670 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215486+0000", 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4499999999999996e-07 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215490+0000", 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.946e-06 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215502+0000", 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1858999999999999e-05 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215508+0000", 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0179999999999998e-06 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215566+0000", 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7441000000000001e-05 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.215573+0000", 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3499999999999999e-06 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077881+0000", 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.86230807300000001 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077889+0000", 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1470000000000001e-06 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.142055+0000", 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.064166277 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.142132+0000", 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6553999999999999e-05 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.142341+0000", 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020935499999999999 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.143801+0000", 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001459922 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144176+0000", 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000374366 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144260+0000", 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4666000000000006e-05 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.146221+0000", 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019611110000000002 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.146281+0000", 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9639000000000003e-05 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.146390+0000", 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109428 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146799 8.16 8:684e2f31:::benchmark_data_smithi076_241159_object23881:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:15.321746+0000", 2024-09-21T10:15:28.635 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 312.33009973100002, 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8305196700000002, 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146799 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321746+0000", 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321746+0000", 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321747+0000", 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2950000000000001e-06 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321751+0000", 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8179999999999997e-06 2024-09-21T10:15:28.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321767+0000", 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.609e-05 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321774+0000", 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2629999999999997e-06 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321834+0000", 2024-09-21T10:15:28.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9308000000000002e-05 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.321843+0000", 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4069999999999999e-06 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077927+0000", 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75608387499999996 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:15:28.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:16.077936+0000", 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9530000000000005e-06 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144466+0000", 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0665306269999999 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144603+0000", 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000136425 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2,6", 2024-09-21T10:15:28.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.144840+0000", 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000237085 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.148206+0000", 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0033661810000000002 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.148220+0000", 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3455000000000001e-05 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.148255+0000", 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5154000000000003e-05 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.152036+0000", 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037816870000000002 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.152095+0000", 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8462999999999999e-05 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:17.152265+0000", 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017058399999999999 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:30197 9.3 9:c42a544d:::benchmark_data_smithi076_269622_object30196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1264)", 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:12:34.076294+0000", 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 173.575551494, 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.185499682, 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.643 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 30197 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076294+0000", 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.644 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076294+0000", 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076295+0000", 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4079999999999999e-06 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076317+0000", 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1900999999999999e-05 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076329+0000", 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2098e-05 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076337+0000", 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2679999999999997e-06 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076405+0000", 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8158999999999996e-05 2024-09-21T10:15:28.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076671+0000", 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000266364 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,6", 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076890+0000", 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021898499999999999 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078182+0000", 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001292252 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257463+0000", 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.17928016899999999 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.261708+0000", 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042448679999999997 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.261756+0000", 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8210000000000001e-05 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.261794+0000", 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8000000000000002e-05 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:30201 9.c 9:34e8307c:::benchmark_data_smithi076_269622_object30200:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1264)", 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:12:34.076683+0000", 2024-09-21T10:15:28.649 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 173.575162912, 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.18085680400000001, 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 30201 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076683+0000", 2024-09-21T10:15:28.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076683+0000", 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076684+0000", 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0800000000000003e-07 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.651 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076703+0000", 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8987999999999998e-05 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076714+0000", 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1175999999999999e-05 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076721+0000", 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9430000000000003e-06 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076773+0000", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2120000000000002e-05 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076893+0000", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000120013 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077031+0000", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013808700000000001 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257217+0000", 2024-09-21T10:15:28.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.180185864 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257242+0000", 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5134e-05 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257426+0000", 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018456500000000001 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257490+0000", 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3099e-05 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257539+0000", 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9907000000000002e-05 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:30203 9.2 9:471bf1e8:::benchmark_data_smithi076_269622_object30202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1264)", 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:12:34.076985+0000", 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 173.57486029099999, 2024-09-21T10:15:28.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.183117111, 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 30203 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076985+0000", 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076985+0000", 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.076986+0000", 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.64e-07 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077007+0000", 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1180999999999999e-05 2024-09-21T10:15:28.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077018+0000", 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0793e-05 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077025+0000", 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6640000000000001e-06 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077152+0000", 2024-09-21T10:15:28.658 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000127453 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077281+0000", 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012899100000000001 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.077405+0000", 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000123497 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078671+0000", 2024-09-21T10:15:28.659 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001266508 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.079048+0000", 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00037633799999999999 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.260015+0000", 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.180967039 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.660 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.260063+0000", 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8087999999999998e-05 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.260102+0000", 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9595000000000003e-05 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:30205 9.c 9:39b36f1a:::benchmark_data_smithi076_269622_object30204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1264)", 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:12:34.078494+0000", 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 173.57335149900001, 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.17920498300000001, 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.661 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 30205 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078494+0000", 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.662 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078494+0000", 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078495+0000", 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2619999999999999e-06 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078518+0000", 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2691e-05 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.663 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078527+0000", 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9290000000000004e-06 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078532+0000", 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6310000000000002e-06 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078567+0000", 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5179000000000003e-05 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.664 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078676+0000", 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109137 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.078755+0000", 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9017000000000006e-05 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257246+0000", 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.17849122100000001 2024-09-21T10:15:28.665 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257585+0000", 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00033910100000000002 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257625+0000", 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9357999999999998e-05 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257666+0000", 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1229000000000001e-05 2024-09-21T10:15:28.666 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:12:34.257699+0000", 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3228000000000001e-05 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:94432 9.7 9:e8bff64d:::benchmark_data_smithi076_269622_object94431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:13:29.543882+0000", 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 118.107963626, 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.18991880899999999, 2024-09-21T10:15:28.667 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 94432 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543882+0000", 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.668 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543882+0000", 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543884+0000", 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8080000000000001e-06 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543922+0000", 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8439000000000002e-05 2024-09-21T10:15:28.669 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543932+0000", 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0155e-05 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543939+0000", 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7429999999999996e-06 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.543995+0000", 2024-09-21T10:15:28.670 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6224999999999998e-05 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.544123+0000", 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012733300000000001 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.544251+0000", 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012816600000000001 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.719805+0000", 2024-09-21T10:15:28.671 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.17555452599999999 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.719832+0000", 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.6427e-05 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.733614+0000", 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013781791999999999 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.672 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.733662+0000", 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8489999999999998e-05 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.733801+0000", 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013870500000000001 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:94438 9.7 9:ed99985b:::benchmark_data_smithi076_269622_object94437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:13:29.548827+0000", 2024-09-21T10:15:28.673 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 118.10301898500001, 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.185103029, 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 94438 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.548827+0000", 2024-09-21T10:15:28.674 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.548827+0000", 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.548828+0000", 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.519e-06 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.675 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.548888+0000", 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0134999999999999e-05 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.548901+0000", 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2593e-05 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.548908+0000", 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3250000000000002e-06 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.676 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.550256+0000", 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0013476919999999999 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.550389+0000", 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013324100000000001 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.717201+0000", 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.16681211400000001 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.677 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.719879+0000", 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002677469 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.720402+0000", 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00052384600000000001 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.733834+0000", 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.013431085000000001 2024-09-21T10:15:28.678 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.733887+0000", 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3554000000000003e-05 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:29.733930+0000", 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2456000000000001e-05 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.679 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:107678 9.7 9:e5f838a6:::benchmark_data_smithi076_269622_object107677:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:13:43.184097+0000", 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 104.467748665, 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.17947407200000001, 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 107678 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.680 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184097+0000", 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184097+0000", 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184098+0000", 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4839999999999999e-06 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.681 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184124+0000", 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5658000000000001e-05 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184138+0000", 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3658e-05 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184146+0000", 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5529999999999993e-06 2024-09-21T10:15:28.682 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184209+0000", 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2933000000000002e-05 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184359+0000", 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014946999999999999 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.184499+0000", 2024-09-21T10:15:28.683 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014009499999999999 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.193018+0000", 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0085192519999999997 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.196394+0000", 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0033755640000000002 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.360775+0000", 2024-09-21T10:15:28.684 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.16438185899999999 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.363527+0000", 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027511459999999999 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:13:43.363571+0000", 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4400000000000002e-05 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.685 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189108 9.8 9:14f5aea5:::benchmark_data_smithi076_269622_object74616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.767777+0000", 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.884068664000001, 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.215300619, 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189108 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.686 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767777+0000", 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767777+0000", 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767778+0000", 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6080000000000001e-06 2024-09-21T10:15:28.687 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767782+0000", 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4539999999999999e-06 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767791+0000", 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3309999999999993e-06 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767798+0000", 2024-09-21T10:15:28.688 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2369999999999997e-06 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767845+0000", 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7116000000000001e-05 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767976+0000", 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013112 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.768087+0000", 2024-09-21T10:15:28.689 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110979 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771820+0000", 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037336380000000001 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782302+0000", 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.010481638 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.690 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982570+0000", 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.200268376 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982820+0000", 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000249356 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.983077+0000", 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025776599999999998 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.691 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189111 9.7 9:e2f42055:::benchmark_data_smithi076_269622_object74619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.767943+0000", 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.883902235000001, 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.21479205900000001, 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189111 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.692 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767943+0000", 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767943+0000", 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767944+0000", 2024-09-21T10:15:28.693 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3600000000000003e-07 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767948+0000", 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5719999999999999e-06 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767959+0000", 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1260999999999999e-05 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.767965+0000", 2024-09-21T10:15:28.694 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4579999999999998e-06 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.768019+0000", 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3955000000000001e-05 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.768121+0000", 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000101761 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.695 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.768235+0000", 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000113484 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.779525+0000", 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.011290219000000001 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.785755+0000", 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0062298409999999999 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.696 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982363+0000", 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.196608061 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982686+0000", 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00032334600000000002 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982735+0000", 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9364999999999999e-05 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.697 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189113 9.7 9:e048ab8a:::benchmark_data_smithi076_269622_object74621:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.771738+0000", 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.880107751000001, 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.21848294200000001, 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189113 2024-09-21T10:15:28.698 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771738+0000", 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771738+0000", 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.699 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771739+0000", 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3209999999999999e-06 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771744+0000", 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1209999999999998e-06 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771756+0000", 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1729000000000001e-05 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.700 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771763+0000", 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.063e-06 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771850+0000", 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7173e-05 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.771971+0000", 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012047199999999999 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.701 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.772079+0000", 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000108754 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.779562+0000", 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0074823470000000003 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.788263+0000", 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0087016100000000002 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.702 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.990135+0000", 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.20187186500000001 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.990182+0000", 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6456999999999999e-05 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.990221+0000", 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9029999999999997e-05 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.703 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189115 9.8 9:1ee11031:::benchmark_data_smithi076_269622_object74623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.774271+0000", 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.877574199000001, 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.212774873, 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189115 2024-09-21T10:15:28.704 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774271+0000", 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774271+0000", 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.705 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774273+0000", 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3829999999999999e-06 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774278+0000", 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5389999999999998e-06 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774294+0000", 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5783000000000001e-05 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.706 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774302+0000", 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7829999999999999e-06 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774358+0000", 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5829999999999999e-05 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774460+0000", 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102123 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.707 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774562+0000", 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010171900000000001 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.778227+0000", 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0036655860000000002 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.783505+0000", 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052783220000000002 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.708 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.986831+0000", 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.20332521100000001 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.986996+0000", 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000165604 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987046+0000", 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9990000000000001e-05 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.709 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189116 9.7 9:e3ebc87a:::benchmark_data_smithi076_269622_object74624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.774871+0000", 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.876974573999998, 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.23151303300000001, 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.710 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189116 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774871+0000", 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774871+0000", 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.711 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774872+0000", 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5499999999999996e-07 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774877+0000", 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.651e-06 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774888+0000", 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.181e-05 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.712 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.774896+0000", 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3590000000000002e-06 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.776651+0000", 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017551209999999999 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.776755+0000", 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000104195 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.713 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.776861+0000", 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105491 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.781536+0000", 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046749540000000003 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.788297+0000", 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0067616949999999999 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.714 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:04.006186+0000", 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.217888426 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:04.006338+0000", 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015201799999999999 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:04.006384+0000", 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6357999999999999e-05 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.715 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189122 9.2 9:44aa4bbf:::benchmark_data_smithi076_269622_object74630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.782295+0000", 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.869550528000001, 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.20047944400000001, 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189122 2024-09-21T10:15:28.716 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782295+0000", 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782295+0000", 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.717 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782296+0000", 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3480000000000001e-06 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782300+0000", 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5209999999999998e-06 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782309+0000", 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5400000000000001e-06 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.718 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782316+0000", 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5060000000000001e-06 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782354+0000", 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7682000000000003e-05 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782468+0000", 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011456299999999999 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.719 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782593+0000", 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124683 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.785922+0000", 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0033291919999999999 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.787085+0000", 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00116271 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.720 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982501+0000", 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.19541630199999999 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982736+0000", 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023524700000000001 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.982774+0000", 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8149999999999999e-05 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.721 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189123 9.7 9:e66d8386:::benchmark_data_smithi076_269622_object74631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.782327+0000", 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.869518098, 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.224264091, 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.722 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189123 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782327+0000", 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782327+0000", 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.723 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782328+0000", 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3500000000000002e-07 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782330+0000", 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8330000000000001e-06 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782335+0000", 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6869999999999999e-06 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.724 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782340+0000", 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6410000000000001e-06 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782715+0000", 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00037515999999999998 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.782942+0000", 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00022634200000000001 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.725 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.783112+0000", 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000170025 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.786050+0000", 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029389260000000001 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.788923+0000", 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028728920000000002 2024-09-21T10:15:28.726 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:04.006428+0000", 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.21750445500000001 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:04.006553+0000", 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000125134 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:04.006592+0000", 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8560999999999999e-05 2024-09-21T10:15:28.727 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189133 9.8 9:1af20ca2:::benchmark_data_smithi076_269622_object74641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.791691+0000", 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.860154251000001, 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.195625406, 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.728 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189133 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791691+0000", 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791691+0000", 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.729 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791692+0000", 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1349999999999999e-06 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791696+0000", 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.591e-06 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791706+0000", 2024-09-21T10:15:28.730 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8050000000000001e-06 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791711+0000", 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2909999999999998e-06 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791765+0000", 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3792000000000003e-05 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.731 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791874+0000", 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000108853 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.791989+0000", 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000115708 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.794527+0000", 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002537858 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.732 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.794637+0000", 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000109776 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987086+0000", 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.192448745 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987279+0000", 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000192733 2024-09-21T10:15:28.733 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987317+0000", 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8118999999999999e-05 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:189142 9.8 9:17bb1189:::benchmark_data_smithi076_269622_object74650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:03.801176+0000", 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 23.850669499999999, 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.18636245400000001, 2024-09-21T10:15:28.734 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 189142 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801176+0000", 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.735 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801176+0000", 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801177+0000", 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1100000000000005e-07 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801181+0000", 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0319999999999997e-06 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.736 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801190+0000", 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.4660000000000005e-06 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801196+0000", 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3210000000000004e-06 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801245+0000", 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9221000000000002e-05 2024-09-21T10:15:28.737 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801366+0000", 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012158399999999999 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,2", 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.801491+0000", 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012446399999999999 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.803614+0000", 2024-09-21T10:15:28.738 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021232899999999999 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.803938+0000", 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00032396500000000001 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987352+0000", 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.18341344700000001 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:15:28.739 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987503+0000", 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000151034 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:03.987538+0000", 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5818999999999997e-05 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:15:28.740 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:15:28.741 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:28 smithi191 ceph-mon[102206]: pgmap v1906: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.3 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 561 op/s; 8145/63885 objects misplaced (12.749%); 13 MiB/s, 1 keys/s, 206 objects/s recovering 2024-09-21T10:15:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:28.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:28.843 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:28.877 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:28.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:28.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.000 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.102 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:29.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:29.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:29.280 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:29.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:29.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.676 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:29.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:29.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:29.803 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:29.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:29.905 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:29.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:29.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:29.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:30.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:30.065 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:30.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:30.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:30.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:30.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:30.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:30.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:30.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:30.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:30.526 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:30 smithi076 ceph-mon[106482]: pgmap v1907: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.2 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 757 op/s; 6832/57036 objects misplaced (11.978%); 15 MiB/s, 1 keys/s, 246 objects/s recovering 2024-09-21T10:15:30.526 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:30 smithi076 ceph-mon[110963]: pgmap v1907: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.2 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 757 op/s; 6832/57036 objects misplaced (11.978%); 15 MiB/s, 1 keys/s, 246 objects/s recovering 2024-09-21T10:15:30.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:30.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:30.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:30 smithi191 ceph-mon[102206]: pgmap v1907: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.2 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 757 op/s; 6832/57036 objects misplaced (11.978%); 15 MiB/s, 1 keys/s, 246 objects/s recovering 2024-09-21T10:15:30.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:30.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:30.808 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:30.834 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:30.936 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:30.968 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:30.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:30.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:31.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:31.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:31.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:31.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:31.322 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:31.356 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:31.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:31.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:31.487 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:31.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:31.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:31.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:31.740 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:31.842 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:31.867 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:31.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:31.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:31.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:31.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:32.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:32.113 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:32.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:32.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:32.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:32.338 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:32.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:32.474 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:32.482 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:32 smithi076 ceph-mon[106482]: pgmap v1908: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.0 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 828 op/s; 5617/51087 objects misplaced (10.995%); 16 MiB/s, 1 keys/s, 254 objects/s recovering 2024-09-21T10:15:32.482 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:32 smithi076 ceph-mon[106482]: osdmap e1280: 8 total, 8 up, 4 in 2024-09-21T10:15:32.483 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:32 smithi076 ceph-mon[110963]: pgmap v1908: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.0 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 828 op/s; 5617/51087 objects misplaced (10.995%); 16 MiB/s, 1 keys/s, 254 objects/s recovering 2024-09-21T10:15:32.483 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:32 smithi076 ceph-mon[110963]: osdmap e1280: 8 total, 8 up, 4 in 2024-09-21T10:15:32.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:32.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:32.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:32 smithi191 ceph-mon[102206]: pgmap v1908: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.0 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 828 op/s; 5617/51087 objects misplaced (10.995%); 16 MiB/s, 1 keys/s, 254 objects/s recovering 2024-09-21T10:15:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:32 smithi191 ceph-mon[102206]: osdmap e1280: 8 total, 8 up, 4 in 2024-09-21T10:15:32.620 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:32.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:32.722 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:32.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:32.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:32.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:32.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:32.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:32.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:33.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:33.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:33.123 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:33.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:33.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:33.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:33.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:33.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:33.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:33.503 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:33.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:33.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:33 smithi191 ceph-mon[102206]: osdmap e1281: 8 total, 8 up, 4 in 2024-09-21T10:15:33.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:33 smithi076 ceph-mon[106482]: osdmap e1281: 8 total, 8 up, 4 in 2024-09-21T10:15:33.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:33 smithi076 ceph-mon[110963]: osdmap e1281: 8 total, 8 up, 4 in 2024-09-21T10:15:33.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:15:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:15:33] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.51.0" 2024-09-21T10:15:33.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:33.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:33.732 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:33.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:33.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:33.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:33.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:33.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:34.004 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:34.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:34.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:34.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:34.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:34.231 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:34.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:34.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:34.361 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:34.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:34.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:34.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:34.521 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:34.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:34 smithi191 ceph-mon[102206]: pgmap v1911: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.0 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 608 op/s; 5039/50244 objects misplaced (10.029%); 16 MiB/s, 258 objects/s recovering 2024-09-21T10:15:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:34 smithi191 ceph-mon[102206]: osdmap e1282: 8 total, 8 up, 4 in 2024-09-21T10:15:34.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:34 smithi076 ceph-mon[106482]: pgmap v1911: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.0 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 608 op/s; 5039/50244 objects misplaced (10.029%); 16 MiB/s, 258 objects/s recovering 2024-09-21T10:15:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:34 smithi076 ceph-mon[106482]: osdmap e1282: 8 total, 8 up, 4 in 2024-09-21T10:15:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:34 smithi076 ceph-mon[110963]: pgmap v1911: 89 pgs: 4 active+remapped+backfill_wait, 3 active+remapped+backfilling, 82 active+clean; 1.0 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 608 op/s; 5039/50244 objects misplaced (10.029%); 16 MiB/s, 258 objects/s recovering 2024-09-21T10:15:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:34 smithi076 ceph-mon[110963]: osdmap e1282: 8 total, 8 up, 4 in 2024-09-21T10:15:34.644 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:34.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:34.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:34.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:34.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:34.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:34.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:35.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:35.162 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:35.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:35.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:35.535 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.575 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:35.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:35.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:35 smithi191 ceph-mon[102206]: osdmap e1283: 8 total, 8 up, 4 in 2024-09-21T10:15:35.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:35 smithi076 ceph-mon[106482]: osdmap e1283: 8 total, 8 up, 4 in 2024-09-21T10:15:35.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:35 smithi076 ceph-mon[110963]: osdmap e1283: 8 total, 8 up, 4 in 2024-09-21T10:15:35.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:35.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:35.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:35.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:35.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:35.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:35.945 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:35.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:35.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:36.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:36.123 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:36.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:36.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:36.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:36.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:36.489 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:36.511 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:36.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:36.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:36.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:36 smithi191 ceph-mon[102206]: pgmap v1914: 89 pgs: 2 peering, 3 active+remapped+backfilling, 84 active+clean; 882 MiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 1220/42372 objects misplaced (2.879%); 55 MiB/s, 887 objects/s recovering 2024-09-21T10:15:36.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:36 smithi076 ceph-mon[106482]: pgmap v1914: 89 pgs: 2 peering, 3 active+remapped+backfilling, 84 active+clean; 882 MiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 1220/42372 objects misplaced (2.879%); 55 MiB/s, 887 objects/s recovering 2024-09-21T10:15:36.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:36 smithi076 ceph-mon[110963]: pgmap v1914: 89 pgs: 2 peering, 3 active+remapped+backfilling, 84 active+clean; 882 MiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 1220/42372 objects misplaced (2.879%); 55 MiB/s, 887 objects/s recovering 2024-09-21T10:15:36.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:36.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:36.757 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:36.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:36.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:36.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:36.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:36.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:36.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:15:36.945 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:36.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:37.004 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:37.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:37.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:37.128 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:37.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:37.253 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:37.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:37.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:37 smithi076 ceph-mon[106482]: osdmap e1284: 8 total, 8 up, 4 in 2024-09-21T10:15:37.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:37 smithi076 ceph-mon[106482]: osdmap e1285: 8 total, 8 up, 4 in 2024-09-21T10:15:37.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:37 smithi076 ceph-mon[110963]: osdmap e1284: 8 total, 8 up, 4 in 2024-09-21T10:15:37.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:37 smithi076 ceph-mon[110963]: osdmap e1285: 8 total, 8 up, 4 in 2024-09-21T10:15:37.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:37.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:37.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:37.522 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:37.571 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:37 smithi191 ceph-mon[102206]: osdmap e1284: 8 total, 8 up, 4 in 2024-09-21T10:15:37.572 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:37 smithi191 ceph-mon[102206]: osdmap e1285: 8 total, 8 up, 4 in 2024-09-21T10:15:37.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:37.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:37.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:37.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:37.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:37.885 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:37.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:37.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:38.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:38.048 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:38.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:38.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:38.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:38.282 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:38.330 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:38.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:38 smithi191 ceph-mon[102206]: pgmap v1916: 89 pgs: 2 peering, 2 active+remapped+backfilling, 85 active+clean; 873 MiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 597 op/s; 1132/41964 objects misplaced (2.698%); 43 MiB/s, 682 objects/s recovering 2024-09-21T10:15:38.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:38.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.472 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:38.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:38.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:38.576 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:38 smithi076 ceph-mon[106482]: pgmap v1916: 89 pgs: 2 peering, 2 active+remapped+backfilling, 85 active+clean; 873 MiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 597 op/s; 1132/41964 objects misplaced (2.698%); 43 MiB/s, 682 objects/s recovering 2024-09-21T10:15:38.577 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:38 smithi076 ceph-mon[110963]: pgmap v1916: 89 pgs: 2 peering, 2 active+remapped+backfilling, 85 active+clean; 873 MiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 597 op/s; 1132/41964 objects misplaced (2.698%); 43 MiB/s, 682 objects/s recovering 2024-09-21T10:15:38.577 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[137810]: 2024-09-21T10:15:38.462+0000 7fd7ea4cf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:15:38.577 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[137810]: 2024-09-21T10:15:38.462+0000 7fd7ea4cf640 -1 osd.3 1285 *** Got signal Terminated *** 2024-09-21T10:15:38.577 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:38 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[137810]: 2024-09-21T10:15:38.463+0000 7fd7ea4cf640 -1 osd.3 1285 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:15:38.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:38.713 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:38.751 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:38.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:38.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:15:38.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:38.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:38.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:38.988 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:39.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:39.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:39.097 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:15:39.097 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:15:39.097 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:15:39.097 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:15:39.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:39.145 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:39.196 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:39.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:39.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:39 smithi191 ceph-mon[102206]: osd.3 marked itself down and dead 2024-09-21T10:15:39.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:39.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:39.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:39.521 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:39.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:39.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:39 smithi076 ceph-mon[106482]: osd.3 marked itself down and dead 2024-09-21T10:15:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:39 smithi076 ceph-mon[110963]: osd.3 marked itself down and dead 2024-09-21T10:15:39.644 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:39.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:39.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.768 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:39.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:39.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:15:39.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:39.894 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:39.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:39.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:40.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:40.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:40.033 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:40.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.141 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:40.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:40.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:40.263 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:40.296 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:40 smithi076 podman[291001]: 2024-09-21 10:15:40.027651181 +0000 UTC m=+2.257016796 container died 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T10:15:40.316 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:15:40.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:40.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:40.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:40 smithi191 ceph-mon[102206]: pgmap v1918: 89 pgs: 1 peering, 1 active+remapped+backfilling, 87 active+clean; 750 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 843 op/s; 464/36051 objects misplaced (1.287%); 42 MiB/s, 679 objects/s recovering 2024-09-21T10:15:40.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:40 smithi191 ceph-mon[102206]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:15:40.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:40 smithi191 ceph-mon[102206]: osdmap e1286: 8 total, 7 up, 4 in 2024-09-21T10:15:40.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2835718653' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:40.482 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:40 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:40.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:40.520 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:40.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:40.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:40 smithi076 podman[291001]: 2024-09-21 10:15:40.402224276 +0000 UTC m=+2.631589859 container cleanup 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T10:15:40.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:40 smithi076 bash[291001]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:15:40.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[106482]: pgmap v1918: 89 pgs: 1 peering, 1 active+remapped+backfilling, 87 active+clean; 750 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 843 op/s; 464/36051 objects misplaced (1.287%); 42 MiB/s, 679 objects/s recovering 2024-09-21T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[106482]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[106482]: osdmap e1286: 8 total, 7 up, 4 in 2024-09-21T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2835718653' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:40.625 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:40.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[110963]: pgmap v1918: 89 pgs: 1 peering, 1 active+remapped+backfilling, 87 active+clean; 750 MiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 843 op/s; 464/36051 objects misplaced (1.287%); 42 MiB/s, 679 objects/s recovering 2024-09-21T10:15:40.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[110963]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:15:40.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[110963]: osdmap e1286: 8 total, 7 up, 4 in 2024-09-21T10:15:40.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2835718653' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:40.626 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:40 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:15:40.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:40.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.792 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:40.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:40.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:40.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:40.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:40.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:41.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:41.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:41.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:41.157 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:41.185 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:41.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:41.299 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:40 smithi076 podman[291314]: 2024-09-21 10:15:40.907266556 +0000 UTC m=+0.905067568 container remove 82190957c6ce1b725029feae60fad89fc9d8e0c7ea8dfcccba0bd12cf4128d04 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T10:15:41.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:41.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:41.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:41.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:41.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:41.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.588 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:41.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:41.611 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:41 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:41.611 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:41 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:15:41.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:41 smithi191 ceph-mon[102206]: osdmap e1287: 8 total, 7 up, 4 in 2024-09-21T10:15:41.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:41.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:41.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:41.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:15:41.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[106482]: osdmap e1287: 8 total, 7 up, 4 in 2024-09-21T10:15:41.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:41.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:41.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:41.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:15:41.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[110963]: osdmap e1287: 8 total, 7 up, 4 in 2024-09-21T10:15:41.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:41.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:41.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:41.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:41.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:41.857 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:41.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:41.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:41.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:41.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:42.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:42.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.134 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:42.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:42.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:42.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:42.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:42 smithi191 ceph-mon[102206]: pgmap v1921: 89 pgs: 34 stale+active+clean, 55 active+clean; 628 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 889 op/s; 16 MiB/s, 263 objects/s recovering 2024-09-21T10:15:42.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:42.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:42.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.542 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:42.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:42.561 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:42 smithi076 ceph-mon[106482]: pgmap v1921: 89 pgs: 34 stale+active+clean, 55 active+clean; 628 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 889 op/s; 16 MiB/s, 263 objects/s recovering 2024-09-21T10:15:42.561 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:42 smithi076 ceph-mon[110963]: pgmap v1921: 89 pgs: 34 stale+active+clean, 55 active+clean; 628 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 889 op/s; 16 MiB/s, 263 objects/s recovering 2024-09-21T10:15:42.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:42.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:42.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:42.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:42.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:42.915 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:42.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:42.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:42.975 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:15:42.976 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:15:42.976 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:15:42.976 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:15:42.976 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:15:43.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.077 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:43.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:43.178 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:43.214 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:43.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:43.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:43.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:43.367 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:15:43.367 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:43.389 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:15:43.389 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:43.411 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:15:43.412 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:43.412 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:15:43.434 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:15:43.434 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:43.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:43.453 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:15:43.453 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:43.470 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:15:43.470 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:43.470 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:15:43] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [1, 6]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 3]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.5", "id": [1, 4]} : dispatch 2024-09-21T10:15:43.471 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.d", "id": [6, 3]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [1, 6]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 3]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.5", "id": [1, 4]} : dispatch 2024-09-21T10:15:43.472 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.d", "id": [6, 3]} : dispatch 2024-09-21T10:15:43.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:43.487 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:15:43.487 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:43.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:43.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [1, 6]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 3]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.5", "id": [1, 4]} : dispatch 2024-09-21T10:15:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.d", "id": [6, 3]} : dispatch 2024-09-21T10:15:43.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:43.708 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:15:43.726 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:43.743 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:43 smithi076 podman[291641]: 2024-09-21 10:15:43.453095593 +0000 UTC m=+0.016384140 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:15:43.743 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:43 smithi076 podman[291641]: 2024-09-21 10:15:43.619845198 +0000 UTC m=+0.183133735 container create c4141c16ad5489b4965fef22983f7d0682e05334c422b7cc091a6f434a4c3647 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:15:43.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:43.850 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:43.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:43.907 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:43.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:43.973 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:43.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:44.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:44.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:44.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:44.305 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:44 smithi076 podman[291641]: 2024-09-21 10:15:44.048607182 +0000 UTC m=+0.611895720 container init c4141c16ad5489b4965fef22983f7d0682e05334c422b7cc091a6f434a4c3647 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b) 2024-09-21T10:15:44.305 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:44 smithi076 podman[291641]: 2024-09-21 10:15:44.05357873 +0000 UTC m=+0.616867271 container start c4141c16ad5489b4965fef22983f7d0682e05334c422b7cc091a6f434a4c3647 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-62da54b) 2024-09-21T10:15:44.305 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:44 smithi076 podman[291641]: 2024-09-21 10:15:44.156770474 +0000 UTC m=+0.720059010 container attach c4141c16ad5489b4965fef22983f7d0682e05334c422b7cc091a6f434a4c3647 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T10:15:44.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:15:44.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:44.343 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:44.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.467 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:44.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:44.588 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: pgmap v1922: 89 pgs: 17 active+undersized, 19 stale+active+clean, 10 active+undersized+degraded, 43 active+clean; 584 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 1849/28083 objects degraded (6.584%); 12 MiB/s, 184 objects/s recovering 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 1849/28083 objects degraded (6.584%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [1, 3]}]': finished 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 3]}]': finished 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [1, 6]}]': finished 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 3]}]': finished 2024-09-21T10:15:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4]}]': finished 2024-09-21T10:15:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]': finished 2024-09-21T10:15:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.5", "id": [1, 4]}]': finished 2024-09-21T10:15:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.d", "id": [6, 3]}]': finished 2024-09-21T10:15:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: osdmap e1288: 8 total, 7 up, 4 in 2024-09-21T10:15:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/317962192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:44.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:44 smithi191 ceph-mon[102206]: osdmap e1289: 8 total, 7 up, 4 in 2024-09-21T10:15:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: pgmap v1922: 89 pgs: 17 active+undersized, 19 stale+active+clean, 10 active+undersized+degraded, 43 active+clean; 584 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 1849/28083 objects degraded (6.584%); 12 MiB/s, 184 objects/s recovering 2024-09-21T10:15:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 1849/28083 objects degraded (6.584%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [1, 3]}]': finished 2024-09-21T10:15:44.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 3]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [1, 6]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 3]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.5", "id": [1, 4]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.d", "id": [6, 3]}]': finished 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: osdmap e1288: 8 total, 7 up, 4 in 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/317962192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[106482]: osdmap e1289: 8 total, 7 up, 4 in 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: pgmap v1922: 89 pgs: 17 active+undersized, 19 stale+active+clean, 10 active+undersized+degraded, 43 active+clean; 584 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 770 op/s; 1849/28083 objects degraded (6.584%); 12 MiB/s, 184 objects/s recovering 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 1849/28083 objects degraded (6.584%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:44.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [1, 3]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.4", "id": [1, 3]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [1, 6]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 3]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 3]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.5", "id": [1, 4]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "9.d", "id": [6, 3]}]': finished 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: osdmap e1288: 8 total, 7 up, 4 in 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/317962192' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:44.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:44 smithi076 ceph-mon[110963]: osdmap e1289: 8 total, 7 up, 4 in 2024-09-21T10:15:44.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:44.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:44.791 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:15:44.791 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:15:44.813 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:44.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:44.843 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:44.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:44.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:44.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:44.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:45.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:45.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:45.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:45.208 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:45.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:45.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:45.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:45.489 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:45.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:45.591 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:45.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:45.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:45.752 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:45.787 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:45.787 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:45.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:45.891 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:45.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:45.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:45.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:46.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.028 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:46.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:46.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:46.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:46.261 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:46.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:46.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:46 smithi076 ceph-mon[106482]: pgmap v1925: 89 pgs: 1 peering, 44 active+undersized, 22 active+undersized+degraded, 22 active+clean; 316 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 906 op/s; 3711/15198 objects degraded (24.418%) 2024-09-21T10:15:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:46 smithi076 ceph-mon[106482]: osdmap e1290: 8 total, 7 up, 4 in 2024-09-21T10:15:46.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:46 smithi076 ceph-mon[106482]: 9.5 starting backfill to osd.4 from (0'0,0'0] MAX to 1287'14099 2024-09-21T10:15:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:46 smithi076 ceph-mon[110963]: pgmap v1925: 89 pgs: 1 peering, 44 active+undersized, 22 active+undersized+degraded, 22 active+clean; 316 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 906 op/s; 3711/15198 objects degraded (24.418%) 2024-09-21T10:15:46.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:46 smithi076 ceph-mon[110963]: osdmap e1290: 8 total, 7 up, 4 in 2024-09-21T10:15:46.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:46 smithi076 ceph-mon[110963]: 9.5 starting backfill to osd.4 from (0'0,0'0] MAX to 1287'14099 2024-09-21T10:15:46.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:46.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:46.489 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:46 smithi191 ceph-mon[102206]: pgmap v1925: 89 pgs: 1 peering, 44 active+undersized, 22 active+undersized+degraded, 22 active+clean; 316 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 906 op/s; 3711/15198 objects degraded (24.418%) 2024-09-21T10:15:46.489 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:46 smithi191 ceph-mon[102206]: osdmap e1290: 8 total, 7 up, 4 in 2024-09-21T10:15:46.489 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:46 smithi191 ceph-mon[102206]: 9.5 starting backfill to osd.4 from (0'0,0'0] MAX to 1287'14099 2024-09-21T10:15:46.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:15:46.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:46.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:46.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:46.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:46.792 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:15:46.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:46.869 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:46.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:46.872 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:46.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:47.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:47.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:47.110 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:15:46 smithi076 podman[291641]: 2024-09-21 10:15:46.796126205 +0000 UTC m=+3.359414743 container died c4141c16ad5489b4965fef22983f7d0682e05334c422b7cc091a6f434a4c3647 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS) 2024-09-21T10:15:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:47.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:47.241 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:47.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:47.364 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:47.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:47.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:47.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:47.611 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:47.631 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:47.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:47.754 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:47.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:47.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:47.877 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:47.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:47.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:15:47.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:48.000 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:48.027 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:15:48.028 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:15:48.035 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:48.101 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.124 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:48.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:48.169 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:15:48.209 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Removed 114489 objects 2024-09-21T10:15:48.209 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Clean up completed and total clean up time :118.106 2024-09-21T10:15:48.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:48.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:48.311 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:48 smithi076 ceph-mon[106482]: pgmap v1927: 89 pgs: 6 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 44 active+undersized, 22 active+undersized+degraded, 15 active+clean; 195 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 1.15k op/s; 2311/9429 objects degraded (24.509%); 94/9429 objects misplaced (0.997%); 1.3 MiB/s, 19 objects/s recovering 2024-09-21T10:15:48.311 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:48 smithi076 ceph-mon[106482]: osdmap e1291: 8 total, 7 up, 4 in 2024-09-21T10:15:48.311 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1844770962' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:48.311 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:48 smithi076 ceph-mon[110963]: pgmap v1927: 89 pgs: 6 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 44 active+undersized, 22 active+undersized+degraded, 15 active+clean; 195 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 1.15k op/s; 2311/9429 objects degraded (24.509%); 94/9429 objects misplaced (0.997%); 1.3 MiB/s, 19 objects/s recovering 2024-09-21T10:15:48.311 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:48 smithi076 ceph-mon[110963]: osdmap e1291: 8 total, 7 up, 4 in 2024-09-21T10:15:48.311 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1844770962' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:48.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:48.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:48 smithi191 ceph-mon[102206]: pgmap v1927: 89 pgs: 6 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 44 active+undersized, 22 active+undersized+degraded, 15 active+clean; 195 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 1.15k op/s; 2311/9429 objects degraded (24.509%); 94/9429 objects misplaced (0.997%); 1.3 MiB/s, 19 objects/s recovering 2024-09-21T10:15:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:48 smithi191 ceph-mon[102206]: osdmap e1291: 8 total, 7 up, 4 in 2024-09-21T10:15:48.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1844770962' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:15:48.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:48.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:48.493 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:48.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:48.595 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:48.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:48.651 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:15:48.651 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:15:48.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.740 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:48.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:48.841 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:48.865 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:48.967 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:48.989 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:48.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:49.090 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:49.113 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:49.214 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:49.218 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:49.218 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1844770962' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:49.218 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[106482]: osdmap e1292: 8 total, 7 up, 4 in 2024-09-21T10:15:49.218 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 2311/9429 objects degraded (24.509%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:49.218 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:49.219 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1844770962' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:49.219 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[110963]: osdmap e1292: 8 total, 7 up, 4 in 2024-09-21T10:15:49.219 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 2311/9429 objects degraded (24.509%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:49.236 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:49.295 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:49 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:49.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:49 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1844770962' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:15:49.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:49 smithi191 ceph-mon[102206]: osdmap e1292: 8 total, 7 up, 4 in 2024-09-21T10:15:49.296 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 2311/9429 objects degraded (24.509%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:49.317 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:15:49.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:49.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:49.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:49.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:49.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:49.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:49.519 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:49.587 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:49.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:49.710 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:49.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:49.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:49.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:49.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:15:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:49.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:49.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:15:50.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:50.102 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:50.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:50.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:50.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:50.267 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:50 smithi076 ceph-mon[106482]: pgmap v1930: 89 pgs: 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 52 active+undersized, 14 active+undersized+degraded, 15 active+clean; 33 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 967 op/s; 453/1626 objects degraded (27.860%); 94/1626 objects misplaced (5.781%); 1.6 MiB/s, 26 objects/s recovering 2024-09-21T10:15:50.267 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:50 smithi076 ceph-mon[110963]: pgmap v1930: 89 pgs: 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 52 active+undersized, 14 active+undersized+degraded, 15 active+clean; 33 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 967 op/s; 453/1626 objects degraded (27.860%); 94/1626 objects misplaced (5.781%); 1.6 MiB/s, 26 objects/s recovering 2024-09-21T10:15:50.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:50.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:50.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:50.383 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:15:50.383 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:15:50.384 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:15:50.384 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:50.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:15:50.400 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:50.400 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:15:50.416 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:15:50.416 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:50.432 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:15:50.433 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:50.448 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:15:50.449 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:50.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:50.464 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:15:50.465 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:50.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:50.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:50.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:50.595 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:50 smithi191 ceph-mon[102206]: pgmap v1930: 89 pgs: 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 52 active+undersized, 14 active+undersized+degraded, 15 active+clean; 33 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 967 op/s; 453/1626 objects degraded (27.860%); 94/1626 objects misplaced (5.781%); 1.6 MiB/s, 26 objects/s recovering 2024-09-21T10:15:50.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:50.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:50.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:50.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:50.942 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:50.963 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:50.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:50.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:50.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:15:51.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:51.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:51.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:51.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:15:51.236 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:15:51.254 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:51.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:51.316 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/23073837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:51.333 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:51.335 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:15:51.335 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:15:51.335 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:15:51.335 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:15:51.336 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:15:51.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:51.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:51.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:51.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:51.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/23073837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/23073837' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:51.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:51.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:51.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:51.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:51.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:51.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:51.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:51.947 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:51.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:15:52.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:52.102 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:52.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:52.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.238 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:15:52.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:52.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:52.358 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:52 smithi191 ceph-mon[102206]: pgmap v1931: 89 pgs: 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 52 active+undersized, 14 active+undersized+degraded, 15 active+clean; 33 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 753 op/s; 453/1626 objects degraded (27.860%); 94/1626 objects misplaced (5.781%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:15:52.371 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:52 smithi076 ceph-mon[106482]: pgmap v1931: 89 pgs: 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 52 active+undersized, 14 active+undersized+degraded, 15 active+clean; 33 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 753 op/s; 453/1626 objects degraded (27.860%); 94/1626 objects misplaced (5.781%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:15:52.371 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:52 smithi076 ceph-mon[110963]: pgmap v1931: 89 pgs: 7 active+clean+remapped, 1 active+undersized+degraded+remapped+backfilling, 52 active+undersized, 14 active+undersized+degraded, 15 active+clean; 33 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 753 op/s; 453/1626 objects degraded (27.860%); 94/1626 objects misplaced (5.781%); 1.3 MiB/s, 20 objects/s recovering 2024-09-21T10:15:52.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:52.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:52.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:52.598 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:52.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:52.720 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:52.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:52.844 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:52.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:52.967 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:52.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.029 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6] out_osds: [7, 5, 0, 2] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2024-09-21T10:15:53.029 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:15:53.029 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 4, 5, 6, 7] 2024-09-21T10:15:53.029 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:15:53.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:15:53.069 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.090 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:53.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:53.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:53.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:53.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:15:53.251 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:53 smithi076 systemd[1]: Stopping Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:15:53.313 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:53.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:53.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:53.496 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:53.517 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:53.563 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:15:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:15:53] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-21T10:15:53.569 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-21T10:15:53.569 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-21T10:15:53.591 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:53.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:53.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:53.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:53.838 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[122652]: 2024-09-21T10:15:53.560+0000 7f19986f3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:15:53.839 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[122652]: 2024-09-21T10:15:53.560+0000 7f19986f3640 -1 osd.0 1292 *** Got signal Terminated *** 2024-09-21T10:15:53.839 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[122652]: 2024-09-21T10:15:53.560+0000 7f19986f3640 -1 osd.0 1292 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:15:53.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:53.888 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:53.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:53.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:53.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:54.011 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:54.051 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:15:54.051 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:15:54.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:15:54.051 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:15:54.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:138968 8.12 8:49cd1d61:::benchmark_data_smithi076_241159_object16050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.469346+0000", 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.578123678, 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.829499169, 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138968 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469346+0000", 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469346+0000", 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469345+0000", 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469350+0000", 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3650000000000003e-06 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469361+0000", 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0767e-05 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469369+0000", 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9929999999999995e-06 2024-09-21T10:15:54.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469420+0000", 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1514999999999997e-05 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.469427+0000", 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9480000000000002e-06 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.069886+0000", 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.60045924299999998 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.069894+0000", 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9899999999999997e-06 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.295805+0000", 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2259100590000001 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.295916+0000", 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111279 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.296075+0000", 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015928399999999999 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298575+0000", 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024996250000000001 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298582+0000", 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9539999999999998e-06 2024-09-21T10:15:54.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298724+0000", 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141934 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298758+0000", 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4817000000000002e-05 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298785+0000", 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7012000000000002e-05 2024-09-21T10:15:54.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298845+0000", 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9447000000000002e-05 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:138971 8.12 8:4aff8226:::benchmark_data_smithi076_241159_object16053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.470434+0000", 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.57703550600002, 2024-09-21T10:15:54.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8286310729999999, 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138971 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470434+0000", 2024-09-21T10:15:54.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470434+0000", 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470433+0000", 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470438+0000", 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9269999999999996e-06 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470447+0000", 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9050000000000003e-06 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470453+0000", 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1759999999999998e-06 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470490+0000", 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7539000000000001e-05 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.470496+0000", 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4090000000000002e-06 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070252+0000", 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59975569299999998 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070257+0000", 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4589999999999997e-06 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.296942+0000", 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2266854979999999 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.297028+0000", 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5822999999999995e-05 2024-09-21T10:15:54.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.297160+0000", 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013141499999999999 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298730+0000", 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0015703410000000001 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298877+0000", 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000146491 2024-09-21T10:15:54.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298911+0000", 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4044999999999997e-05 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298939+0000", 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8085e-05 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298971+0000", 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2746000000000001e-05 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.299065+0000", 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3518999999999997e-05 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:138974 8.12 8:4d62b09b:::benchmark_data_smithi076_241159_object16056:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.471479+0000", 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.57599082899998, 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.828983201, 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.067 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138974 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471479+0000", 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471479+0000", 2024-09-21T10:15:54.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471478+0000", 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471482+0000", 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3470000000000001e-06 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471492+0000", 2024-09-21T10:15:54.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3819999999999993e-06 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471496+0000", 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6129999999999997e-06 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471533+0000", 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6403999999999999e-05 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.471539+0000", 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1929999999999998e-06 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070277+0000", 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.59873835900000005 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070282+0000", 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7450000000000003e-06 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.297813+0000", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.227530961 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.297916+0000", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102878 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298032+0000", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116085 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.298883+0000", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00085077099999999995 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300196+0000", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0013138830000000001 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300238+0000", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1936e-05 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300288+0000", 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9568999999999999e-05 2024-09-21T10:15:54.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300343+0000", 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4839000000000001e-05 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300462+0000", 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011895700000000001 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.074 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:138992 8.12 8:4dc52a65:::benchmark_data_smithi076_241159_object16074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.482795+0000", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.56467426500001, 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.819025441, 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138992 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482795+0000", 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482795+0000", 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482794+0000", 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482800+0000", 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8499999999999999e-06 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482811+0000", 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1535999999999999e-05 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482818+0000", 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0740000000000004e-06 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482883+0000", 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4335e-05 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.482894+0000", 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1272e-05 2024-09-21T10:15:54.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070301+0000", 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.58740682 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070305+0000", 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9559999999999999e-06 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.299095+0000", 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.228790002 2024-09-21T10:15:54.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.299164+0000", 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9604999999999995e-05 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.299278+0000", 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011322100000000001 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300244+0000", 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00096670600000000001 2024-09-21T10:15:54.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300526+0000", 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00028152500000000003 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300575+0000", 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8869999999999998e-05 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.301660+0000", 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010848629999999999 2024-09-21T10:15:54.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.301710+0000", 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0266000000000003e-05 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.301821+0000", 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011064 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:138997 8.12 8:4f2f5030:::benchmark_data_smithi076_241159_object16079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.082 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.484973+0000", 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.562496237, 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8189514760000001, 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 138997 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.484973+0000", 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.484973+0000", 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.484972+0000", 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.484977+0000", 2024-09-21T10:15:54.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3889999999999998e-06 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.484985+0000", 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3010000000000007e-06 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.484990+0000", 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5839999999999996e-06 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.485030+0000", 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0564e-05 2024-09-21T10:15:54.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.485037+0000", 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8070000000000002e-06 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070324+0000", 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.58528690400000005 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070328+0000", 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2039999999999999e-06 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300601+0000", 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.230273242 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300720+0000", 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000119165 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.300913+0000", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000192998 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.302905+0000", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0019911310000000002 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303732+0000", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00082758799999999998 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303762+0000", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9621e-05 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303802+0000", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9708e-05 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303845+0000", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3970000000000001e-05 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303925+0000", 2024-09-21T10:15:54.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9145000000000002e-05 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:139023 8.12 8:4ad63c26:::benchmark_data_smithi076_241159_object16105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.505939+0000", 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.54153049500002, 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.800293208, 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.090 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 139023 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.505939+0000", 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.505939+0000", 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.505938+0000", 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.505943+0000", 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8640000000000004e-06 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.505953+0000", 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0101999999999999e-05 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.505958+0000", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7990000000000001e-06 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.506000+0000", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1848000000000003e-05 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.506010+0000", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.005e-05 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070347+0000", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.56433713699999999 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070352+0000", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6919999999999999e-06 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.301859+0000", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2315060289999999 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.301966+0000", 2024-09-21T10:15:54.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010720199999999999 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.302148+0000", 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000181936 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303982+0000", 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0018347190000000001 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303994+0000", 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1256e-05 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306014+0000", 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020203780000000002 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306081+0000", 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6594000000000006e-05 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306125+0000", 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4589000000000003e-05 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306232+0000", 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106918 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:139072 8.12 8:4d84816a:::benchmark_data_smithi076_241159_object16154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.539001+0000", 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.508468768, 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.767508839, 2024-09-21T10:15:54.097 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 139072 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539001+0000", 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539001+0000", 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539000+0000", 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539004+0000", 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4390000000000002e-06 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539014+0000", 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3700000000000001e-06 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539018+0000", 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6210000000000003e-06 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539086+0000", 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8144000000000004e-05 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.539100+0000", 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3801000000000001e-05 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070372+0000", 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.53127173800000005 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070376+0000", 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8709999999999999e-06 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.302930+0000", 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2325541840000001 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303007+0000", 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6477999999999995e-05 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.303140+0000", 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000133103 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.305021+0000", 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001881658 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306049+0000", 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0010272040000000001 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306275+0000", 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022657800000000001 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306400+0000", 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000124645 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306436+0000", 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6502999999999999e-05 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306509+0000", 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3225000000000004e-05 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.104 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:139101 8.12 8:4830cee5:::benchmark_data_smithi076_241159_object16183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1227)", 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:03.560320+0000", 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 350.48714938400002, 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7464208829999999, 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 139101 2024-09-21T10:15:54.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560320+0000", 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560320+0000", 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560319+0000", 2024-09-21T10:15:54.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560324+0000", 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2249999999999999e-06 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560334+0000", 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7659999999999993e-06 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560339+0000", 2024-09-21T10:15:54.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1250000000000001e-06 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560383+0000", 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3823999999999999e-05 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:03.560395+0000", 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2024e-05 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070394+0000", 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.50999941800000004 2024-09-21T10:15:54.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:04.070398+0000", 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9349999999999996e-06 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.304017+0000", 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.233618801 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.304097+0000", 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9622000000000004e-05 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 2,3,4", 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.304261+0000", 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000164081 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306341+0000", 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002080134 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306350+0000", 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0140000000000004e-06 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306545+0000", 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019444500000000001 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306616+0000", 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1598000000000001e-05 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306659+0000", 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2762000000000002e-05 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:05.306741+0000", 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2035999999999999e-05 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147043 8.17 8:e96afdab:::benchmark_data_smithi076_241159_object24125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.180683+0000", 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.86678686599998, 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9437305139999999, 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.112 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147043 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180683+0000", 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180683+0000", 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180681+0000", 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180692+0000", 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1579999999999999e-05 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180719+0000", 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6656e-05 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180732+0000", 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.347e-05 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180821+0000", 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8821999999999998e-05 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.180839+0000", 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.738e-05 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079296+0000", 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89845756600000004 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079307+0000", 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.06e-05 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.121499+0000", 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0421925729999999 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.121608+0000", 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108148 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.121789+0000", 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018187699999999999 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124034+0000", 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002244673 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124076+0000", 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2079000000000001e-05 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124119+0000", 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3179000000000001e-05 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124152+0000", 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2416000000000003e-05 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124292+0000", 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014000300000000001 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124322+0000", 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0162999999999999e-05 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124413+0000", 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1145999999999998e-05 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.119 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147045 8.17 8:ebd12d7c:::benchmark_data_smithi076_241159_object24127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.181518+0000", 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.86595151500001, 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9456334320000002, 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147045 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181518+0000", 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181518+0000", 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181517+0000", 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181522+0000", 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4870000000000002e-06 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181534+0000", 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1293e-05 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181539+0000", 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1630000000000004e-06 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181565+0000", 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5840999999999999e-05 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.181569+0000", 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5990000000000004e-06 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079335+0000", 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89776545100000005 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079348+0000", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2791e-05 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.123112+0000", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.043764519 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.123204+0000", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2406999999999998e-05 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.123341+0000", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013685800000000001 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.125448+0000", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021062870000000001 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.126776+0000", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001328348 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.126813+0000", 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7129000000000003e-05 2024-09-21T10:15:54.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.126879+0000", 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6044000000000007e-05 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.126919+0000", 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9722999999999999e-05 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.126966+0000", 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6961e-05 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127151+0000", 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000185489 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147102 8.17 8:eaa86616:::benchmark_data_smithi076_241159_object24184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.222320+0000", 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.82514958000002, 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9089315340000002, 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.127 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147102 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222320+0000", 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222320+0000", 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222319+0000", 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222324+0000", 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7779999999999998e-06 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222334+0000", 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9790000000000005e-06 2024-09-21T10:15:54.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222339+0000", 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0000000000000004e-06 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222400+0000", 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1657999999999998e-05 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222417+0000", 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6758000000000001e-05 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079373+0000", 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85695617599999996 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079381+0000", 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1140000000000006e-06 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124449+0000", 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0450674259999999 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124561+0000", 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011254700000000001 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.124713+0000", 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015206400000000001 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.126823+0000", 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021096029999999998 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127199+0000", 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000375867 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127396+0000", 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019670700000000001 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127489+0000", 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3436999999999993e-05 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.131117+0000", 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036281730000000002 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.131153+0000", 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5413000000000001e-05 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.131251+0000", 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8696999999999994e-05 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147103 8.17 8:ead43706:::benchmark_data_smithi076_241159_object24185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.222423+0000", 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.825046407, 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.91485466, 2024-09-21T10:15:54.135 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147103 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222423+0000", 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222423+0000", 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222422+0000", 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222426+0000", 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1640000000000001e-06 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222431+0000", 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4349999999999996e-06 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222435+0000", 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.878e-06 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222456+0000", 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0968999999999999e-05 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.222460+0000", 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2660000000000003e-06 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079436+0000", 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85697592199999995 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079445+0000", 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2480000000000002e-06 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.125483+0000", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0460376360000001 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.125656+0000", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000173015 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.125860+0000", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020452900000000001 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127205+0000", 2024-09-21T10:15:54.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001344329 2024-09-21T10:15:54.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:54.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127448+0000", 2024-09-21T10:15:54.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000242945 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127537+0000", 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9279999999999999e-05 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127588+0000", 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0664000000000001e-05 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137007+0000", 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.009419607 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137066+0000", 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9169999999999998e-05 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137278+0000", 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021116 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.144 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147106 8.17 8:ee94c5ad:::benchmark_data_smithi076_241159_object24188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.223051+0000", 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.82441817099999, 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9145694519999998, 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147106 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223051+0000", 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223051+0000", 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223050+0000", 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223056+0000", 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2610000000000001e-06 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223065+0000", 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8549999999999997e-06 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223073+0000", 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4800000000000004e-06 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223108+0000", 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5210999999999999e-05 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.223132+0000", 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3790000000000001e-05 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079478+0000", 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.85634653400000005 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079487+0000", 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1950000000000003e-06 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127644+0000", 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0481568489999999 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127767+0000", 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012354799999999999 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.127979+0000", 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021148900000000001 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.130073+0000", 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020939859999999999 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.130077+0000", 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7700000000000001e-06 2024-09-21T10:15:54.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.136916+0000", 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068382039999999996 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137339+0000", 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00042311699999999998 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137451+0000", 2024-09-21T10:15:54.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111982 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137501+0000", 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0841000000000003e-05 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137621+0000", 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011920499999999999 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147185 8.17 8:eb1607bf:::benchmark_data_smithi076_241159_object24267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.283198+0000", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.76427121299997, 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8547205170000001, 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147185 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283198+0000", 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283198+0000", 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283197+0000", 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283202+0000", 2024-09-21T10:15:54.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1100000000000002e-06 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283216+0000", 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3227999999999999e-05 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283222+0000", 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7429999999999996e-06 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283271+0000", 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8884999999999997e-05 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.283278+0000", 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4520000000000002e-06 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079512+0000", 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.79623450699999998 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079517+0000", 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4580000000000003e-06 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.128896+0000", 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0493794590000001 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.129019+0000", 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012301400000000001 2024-09-21T10:15:54.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.129222+0000", 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000203136 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.131303+0000", 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0020811890000000002 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.131309+0000", 2024-09-21T10:15:54.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0780000000000001e-06 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137398+0000", 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060880980000000001 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137666+0000", 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00026828000000000003 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137739+0000", 2024-09-21T10:15:54.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3396e-05 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137788+0000", 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9125999999999997e-05 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137919+0000", 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000130321 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.160 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147190 8.17 8:e8e50bcc:::benchmark_data_smithi076_241159_object24272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.286792+0000", 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.76067724400002, 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.851410389, 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147190 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286792+0000", 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286792+0000", 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286791+0000", 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286797+0000", 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6450000000000002e-06 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286810+0000", 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3356999999999999e-05 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286817+0000", 2024-09-21T10:15:54.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.956e-06 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286866+0000", 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9070000000000003e-05 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.286877+0000", 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0822000000000001e-05 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079547+0000", 2024-09-21T10:15:54.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.79267021299999996 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079557+0000", 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4450000000000002e-06 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.130090+0000", 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.050533087 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.130174+0000", 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4011999999999995e-05 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.130315+0000", 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000141328 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.136858+0000", 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065432859999999997 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.136873+0000", 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4908e-05 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137705+0000", 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00083142100000000005 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.137964+0000", 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025890100000000002 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.138041+0000", 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7460999999999996e-05 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.138088+0000", 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6591000000000002e-05 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.138203+0000", 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114926 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.168 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147191 8.17 8:e93e944b:::benchmark_data_smithi076_241159_object24273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.287351+0000", 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.76011854699999, 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8560828649999999, 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147191 2024-09-21T10:15:54.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287351+0000", 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287351+0000", 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287350+0000", 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287358+0000", 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6399999999999997e-06 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287369+0000", 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1218e-05 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287375+0000", 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6790000000000002e-06 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287420+0000", 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5807999999999999e-05 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.287430+0000", 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6549999999999999e-06 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079579+0000", 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.79214936499999999 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079585+0000", 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2440000000000001e-06 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.138246+0000", 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0586609840000001 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.138371+0000", 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012568099999999999 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.138708+0000", 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00033626000000000001 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.141691+0000", 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029828490000000001 2024-09-21T10:15:54.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143081+0000", 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001390152 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143132+0000", 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0949000000000003e-05 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143161+0000", 2024-09-21T10:15:54.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.972e-05 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143221+0000", 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9410999999999998e-05 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143303+0000", 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2379000000000005e-05 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143434+0000", 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013059200000000001 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147228 8.17 8:ea11739e:::benchmark_data_smithi076_241159_object24310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.314731+0000", 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.73273791299999, 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.834369854, 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.177 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147228 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314731+0000", 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314731+0000", 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314731+0000", 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314735+0000", 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7139999999999999e-06 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314743+0000", 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6230000000000003e-06 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314748+0000", 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9069999999999998e-06 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314785+0000", 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7299999999999999e-05 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.314793+0000", 2024-09-21T10:15:54.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2849999999999997e-06 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079615+0000", 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.76482222399999999 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079623+0000", 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2740000000000007e-06 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.140233+0000", 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0606096890000001 2024-09-21T10:15:54.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.140352+0000", 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011910500000000001 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.140592+0000", 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00024053200000000001 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143088+0000", 2024-09-21T10:15:54.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002495635 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143190+0000", 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010228100000000001 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.146106+0000", 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029160219999999999 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.148773+0000", 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002666191 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.148929+0000", 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015604599999999999 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149003+0000", 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4744000000000001e-05 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149101+0000", 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7940000000000001e-05 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147297 8.17 8:e9d29e4c:::benchmark_data_smithi076_241159_object24379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.372324+0000", 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.67514574500001, 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.776942697, 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147297 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372324+0000", 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372324+0000", 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372322+0000", 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372331+0000", 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6249999999999996e-06 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372344+0000", 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3339999999999999e-05 2024-09-21T10:15:54.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372351+0000", 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9290000000000001e-06 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372405+0000", 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3916999999999999e-05 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.372420+0000", 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5102999999999999e-05 2024-09-21T10:15:54.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079648+0000", 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.70722817699999996 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079653+0000", 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1280000000000001e-06 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.141725+0000", 2024-09-21T10:15:54.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0620726570000001 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.141844+0000", 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011874 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.142077+0000", 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000232842 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.146115+0000", 2024-09-21T10:15:54.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040381920000000003 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.146140+0000", 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5202e-05 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.146166+0000", 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6251999999999999e-05 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149137+0000", 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0029700849999999999 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149173+0000", 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6159999999999999e-05 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149203+0000", 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0351000000000001e-05 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149266+0000", 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3276999999999997e-05 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147323 8.17 8:ecdadc3f:::benchmark_data_smithi076_241159_object24405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.395713+0000", 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.65175668699999, 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.753737817, 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.193 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147323 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395713+0000", 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395713+0000", 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395712+0000", 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395719+0000", 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7270000000000001e-06 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395730+0000", 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.132e-05 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395736+0000", 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3029999999999996e-06 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395795+0000", 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9190000000000001e-05 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.395807+0000", 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1056000000000001e-05 2024-09-21T10:15:54.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079684+0000", 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.68387718900000005 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079693+0000", 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2690000000000005e-06 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143474+0000", 2024-09-21T10:15:54.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.063781415 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143593+0000", 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011820999999999999 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.143804+0000", 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021169400000000001 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.146174+0000", 2024-09-21T10:15:54.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002369215 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.146201+0000", 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7838999999999999e-05 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.148867+0000", 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0026658379999999998 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149301+0000", 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00043411099999999999 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149352+0000", 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1165000000000002e-05 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149380+0000", 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7107999999999999e-05 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149451+0000", 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0964999999999996e-05 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.45332.0:147337 8.17 8:ea2b2472:::benchmark_data_smithi076_241159_object24419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1238)", 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:10:17.408592+0000", 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 336.63887763600002, 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.748040496, 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:15:54.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.45332", 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 147337 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408592+0000", 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408592+0000", 2024-09-21T10:15:54.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408591+0000", 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408596+0000", 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.215e-06 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:15:54.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408607+0000", 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0829000000000001e-05 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408612+0000", 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0699999999999997e-06 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408647+0000", 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5536e-05 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:17.408654+0000", 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.754e-06 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079717+0000", 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.67106339400000004 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:18.079721+0000", 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1400000000000002e-06 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:15:54.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.144878+0000", 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0651562569999999 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.145002+0000", 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012446199999999999 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,2,3,4", 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.145189+0000", 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018703799999999999 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:15:54.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.147312+0000", 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021226690000000002 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.148722+0000", 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014103950000000001 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.149483+0000", 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076093500000000002 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.153536+0000", 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0040529609999999999 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.156424+0000", 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002888205 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.156486+0000", 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1575999999999995e-05 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:15:54.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:10:19.156632+0000", 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014631900000000001 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:15:54.209 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:15:54.210 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:15:54.229 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:54.331 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:54.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:54 smithi191 ceph-mon[102206]: pgmap v1932: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 656 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:15:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2768412420' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:54 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:54 smithi191 ceph-mon[102206]: osd.0 marked itself down and dead 2024-09-21T10:15:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 18/78 objects degraded (23.077%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:54.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:54.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:54.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:54.599 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:54.608 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[106482]: pgmap v1932: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 656 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:15:54.608 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2768412420' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:54.608 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:54.608 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[106482]: osd.0 marked itself down and dead 2024-09-21T10:15:54.608 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 18/78 objects degraded (23.077%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:54.608 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[110963]: pgmap v1932: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 656 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 1.1 MiB/s, 17 objects/s recovering 2024-09-21T10:15:54.609 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2768412420' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:54.609 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:15:54.609 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[110963]: osd.0 marked itself down and dead 2024-09-21T10:15:54.609 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 18/78 objects degraded (23.077%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T10:15:54.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:54.721 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:54.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:54.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:54.945 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:54.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:54.966 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:54.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:55.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:15:55.068 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:55.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:55.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:55.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:55.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:55.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:55.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:55.333 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:55.356 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:55 smithi076 podman[292935]: 2024-09-21 10:15:55.060793456 +0000 UTC m=+1.632953815 container died f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T10:15:55.356 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:55 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:55.356 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:55 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:55.356 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:55 smithi076 ceph-mon[106482]: osdmap e1293: 8 total, 6 up, 4 in 2024-09-21T10:15:55.356 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:55 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:55.357 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:55 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:55.357 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:55 smithi076 ceph-mon[110963]: osdmap e1293: 8 total, 6 up, 4 in 2024-09-21T10:15:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:55 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:15:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:55 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:15:55.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:55 smithi191 ceph-mon[102206]: osdmap e1293: 8 total, 6 up, 4 in 2024-09-21T10:15:55.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:55.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:55.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:55.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:55 smithi076 podman[292935]: 2024-09-21 10:15:55.576246809 +0000 UTC m=+2.148407167 container cleanup f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T10:15:55.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:55 smithi076 bash[292935]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:15:55.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:55.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:55.707 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:55.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:55.828 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:55.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:55.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:55.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:56.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:56.074 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:56.091 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:15:55 smithi076 podman[293135]: 2024-09-21 10:15:55.836269181 +0000 UTC m=+1.099530200 container remove f0b24f5d8966ca46cf8559df9ef033ecfeff9906558b28b073d18d98c47c2038 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T10:15:56.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:56.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:56.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:56.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:56.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:56.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:56 smithi076 ceph-mon[110963]: pgmap v1934: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 406 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 92 KiB/s, 1 objects/s recovering 2024-09-21T10:15:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:56.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:56 smithi076 ceph-mon[106482]: pgmap v1934: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 406 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 92 KiB/s, 1 objects/s recovering 2024-09-21T10:15:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:56.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:56.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:56.445 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:56.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:56.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:56 smithi191 ceph-mon[102206]: pgmap v1934: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 406 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 92 KiB/s, 1 objects/s recovering 2024-09-21T10:15:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:15:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:15:56.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:56.694 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:56.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:56.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:56.820 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:56.922 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:56.946 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:56.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:57.039 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:15:57.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:57.072 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:57.132 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:15:57.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:57.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:57.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:57.323 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:57.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:57.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:57.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:57.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:57.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:57.674 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:57.697 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:57.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:57.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:57.833 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:15:57.833 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:15:57.833 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:15:57.843 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:15:57.844 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:15:57.844 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:15:57.860 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:15:57.860 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:15:57.877 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:15:57.877 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:15:57.893 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:15:57.893 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:15:57.910 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:15:57.910 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:15:57.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:57.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:57.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:58.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:58.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:58.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:58.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:58.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:58.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:58.302 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.329 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:58.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:58.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:58.455 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:58.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:58.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:58.582 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:58 smithi191 ceph-mon[102206]: pgmap v1935: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 359 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 81 KiB/s, 1 objects/s recovering 2024-09-21T10:15:58.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:15:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4242558054' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:58.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:58 smithi076 ceph-mon[106482]: pgmap v1935: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 359 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 81 KiB/s, 1 objects/s recovering 2024-09-21T10:15:58.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:15:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4242558054' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:58 smithi076 ceph-mon[110963]: pgmap v1935: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 359 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 81 KiB/s, 1 objects/s recovering 2024-09-21T10:15:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:15:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4242558054' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:15:58.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:58.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:58.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:58.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:15:58.833 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:15:58.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:58.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:58.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:59.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:15:59.084 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:15:59.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:59.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:59.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:59.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:15:59.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:15:59.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:59.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:59.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:59.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:15:59.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:15:59.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:15:59.808 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:15:59.832 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:15:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:15:59.934 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:15:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:15:59.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:15:59.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:00.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:00.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:00.187 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:00.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:00.303 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:00 smithi076 ceph-mon[106482]: pgmap v1936: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 51 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 70 KiB/s, 1 objects/s recovering 2024-09-21T10:16:00.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:00 smithi076 ceph-mon[110963]: pgmap v1936: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 51 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 70 KiB/s, 1 objects/s recovering 2024-09-21T10:16:00.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:00.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.337 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:00.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:00.461 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:00.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:00.585 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:00.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:00 smithi191 ceph-mon[102206]: pgmap v1936: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 51 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 70 KiB/s, 1 objects/s recovering 2024-09-21T10:16:00.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:00.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:00.735 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:16:00.759 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:16:00.810 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:00.833 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:00.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:00.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:00.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:01.058 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:01.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:01.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:01.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:01.203 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_4' removed 2024-09-21T10:16:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:01.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:01.328 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3226906153' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:16:01.328 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3226906153' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:16:01.331 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:01.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:01.455 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:01.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:01.557 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:01.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:01.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3226906153' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:16:01.662 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:01.663 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:01.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:01.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:01.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:01.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:01.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:01.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:01.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:01.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:02.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:02.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:02.173 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:02.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:02.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:02.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:02.320 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:02 smithi076 ceph-mon[106482]: pgmap v1937: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 51 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 70 KiB/s, 1 objects/s recovering 2024-09-21T10:16:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3226906153' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:16:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:02 smithi076 ceph-mon[106482]: osdmap e1294: 8 total, 6 up, 4 in 2024-09-21T10:16:02.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:02 smithi076 ceph-mon[110963]: pgmap v1937: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 51 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 70 KiB/s, 1 objects/s recovering 2024-09-21T10:16:02.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3226906153' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:16:02.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:02 smithi076 ceph-mon[110963]: osdmap e1294: 8 total, 6 up, 4 in 2024-09-21T10:16:02.421 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:02.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:02.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:02.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:02.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:02 smithi191 ceph-mon[102206]: pgmap v1937: 89 pgs: 8 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 51 op/s; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%); 70 KiB/s, 1 objects/s recovering 2024-09-21T10:16:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3226906153' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:16:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:02 smithi191 ceph-mon[102206]: osdmap e1294: 8 total, 6 up, 4 in 2024-09-21T10:16:02.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:02.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.693 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:02.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:02.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:02.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:02.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:02.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:02.917 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:02.940 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:02.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:02.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:02.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:16:03.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:03.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:03.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:03.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:03.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.312 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:03.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:03.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:03.416 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:16:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:16:03] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-09-21T10:16:03.418 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:16:03.418 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:16:03.418 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-21T10:16:03.418 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_5 16 2024-09-21T10:16:03.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:03.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:03.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:03.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:03.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:03.661 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:03.683 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:03.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:03.808 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:03.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:03.933 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:03.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:03.937 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:04.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:04.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:04.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:04.147 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:16:04 smithi076 podman[294248]: 2024-09-21 10:16:03.92790195 +0000 UTC m=+0.449624907 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:16:04.147 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:16:04 smithi076 podman[294248]: 2024-09-21 10:16:04.042501975 +0000 UTC m=+0.564224923 container create 8508430614c8d8d18dffc61c72c03bc2ed6dc156e474fcdaf17c0f836d945f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T10:16:04.159 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:04.182 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:04.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:04.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:04.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:04.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:04.419 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:04 smithi076 ceph-mon[106482]: pgmap v1939: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:04.420 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:04 smithi076 ceph-mon[110963]: pgmap v1939: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:04.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:04.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:04.563 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:04.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:04 smithi191 ceph-mon[102206]: pgmap v1939: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:04.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:04.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:04.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:04.716 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:16:04 smithi076 podman[294248]: 2024-09-21 10:16:04.515692879 +0000 UTC m=+1.037415819 container init 8508430614c8d8d18dffc61c72c03bc2ed6dc156e474fcdaf17c0f836d945f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T10:16:04.716 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:16:04 smithi076 podman[294248]: 2024-09-21 10:16:04.520671271 +0000 UTC m=+1.042394207 container start 8508430614c8d8d18dffc61c72c03bc2ed6dc156e474fcdaf17c0f836d945f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T10:16:04.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:04.817 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:04.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:04.919 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:04.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:04.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:05.043 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:05.044 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:16:04 smithi076 podman[294248]: 2024-09-21 10:16:04.733058903 +0000 UTC m=+1.254781855 container attach 8508430614c8d8d18dffc61c72c03bc2ed6dc156e474fcdaf17c0f836d945f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T10:16:05.044 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:16:04 smithi076 podman[294248]: 2024-09-21 10:16:04.733283904 +0000 UTC m=+1.255006851 container died 8508430614c8d8d18dffc61c72c03bc2ed6dc156e474fcdaf17c0f836d945f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T10:16:05.064 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:05.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:05.166 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:05.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:05.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.316 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:05.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:05.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:05.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:05.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:05.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:05.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:05.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:05.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:05.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:05.817 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:05.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:05.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:16:05.839 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:05.864 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:16:05.864 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:16:05.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:05.941 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:05.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:05.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:06.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:06.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:06.166 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:06.188 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:06.290 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:06.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:06.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:06.434 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:06.501 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:06 smithi076 ceph-mon[106482]: pgmap v1940: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:06.501 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:06.501 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:06 smithi076 ceph-mon[110963]: pgmap v1940: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:06.501 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:06.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:06.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:06 smithi191 ceph-mon[102206]: pgmap v1940: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:16:06.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:06.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:06.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:06.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:06.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:06.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:06.933 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:06.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:06.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:07.034 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:07.057 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:07.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:07.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:07.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:07.282 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:07.307 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:07.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:07.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:07.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:07.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:07.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:07.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:07.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:07.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:07.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:07.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:07.907 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:07.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:07.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:08.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:08.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:08.154 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:08.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.176 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:08.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:08.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.298 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:08 smithi191 ceph-mon[102206]: pgmap v1941: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:08.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:08.420 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:08.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:08.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:08 smithi076 ceph-mon[106482]: pgmap v1941: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:08 smithi076 ceph-mon[110963]: pgmap v1941: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:08.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:08.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:08.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:08.790 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:08.892 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:08.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:08.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:09.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:09.160 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:09.261 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:09.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:09.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:09.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:09.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:09.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:09.896 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:09.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:09.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:09.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:10.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:10.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:10.125 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:10.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:10.238 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:10 smithi076 ceph-mon[106482]: pgmap v1942: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:10.238 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:10 smithi076 ceph-mon[110963]: pgmap v1942: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:10.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:10.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:10.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:10.394 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:10.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:10.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:10.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:10.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:10 smithi191 ceph-mon[102206]: pgmap v1942: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:10.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:10.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:10.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:10.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:10.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:10.866 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6] out_osds: [7, 5, 0, 2] dead_osds: [3, 0] live_osds: [1, 2, 4, 5, 6, 7] 2024-09-21T10:16:10.866 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:10.866 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2024-09-21T10:16:10.866 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2024-09-21T10:16:10.866 DEBUG:teuthology.orchestra.run.smithi076:osd.3> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T10:16:10.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl start ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:16:10.887 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:10.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:10.989 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:11.010 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:11.112 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:11.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:10 smithi076 systemd[1]: Starting Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:16:11.135 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:11.146 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:16:11.146 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:16:11.220 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:16:11.237 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:11.258 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:11.360 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:11.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:11.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:11.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2289291264' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:11.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:11.383 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:11.467 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:11.467 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2289291264' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:11.468 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:11.468 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:11.468 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:11.468 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2289291264' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:11.468 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:11.468 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:11.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:11.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:11.630 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:11.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:11.755 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:11.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:11.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:16:11.856 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:11.882 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:11.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:11.984 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:12.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:12.107 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:12.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:12.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:12.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:12.251 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.353 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:12.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:12.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:12.396 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:12 smithi191 ceph-mon[102206]: pgmap v1943: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:12.396 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:12 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:12.396 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2289291264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:12.396 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:12 smithi191 ceph-mon[102206]: osdmap e1295: 8 total, 6 up, 4 in 2024-09-21T10:16:12.396 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:12.397 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:12 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:12.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:12.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:12.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:12.573 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[106482]: pgmap v1943: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:12.573 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:12.573 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2289291264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:12.573 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[106482]: osdmap e1295: 8 total, 6 up, 4 in 2024-09-21T10:16:12.573 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[110963]: pgmap v1943: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2289291264' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[110963]: osdmap e1295: 8 total, 6 up, 4 in 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:12.574 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:12 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:12.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:12.621 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:12.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:12.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:12.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:12.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:12.991 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:12.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:13.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:13.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:13.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:13.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:13.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:13.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:13.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:13.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:13.587 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:13.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:16:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:16:13] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-21T10:16:13.633 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:13.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:13.731 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:13.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:13.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:13.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:13.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:13.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:13.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:14.079 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:14.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:14.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:14.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:14.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:14.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:14 smithi076 ceph-mon[106482]: pgmap v1945: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:14 smithi076 ceph-mon[110963]: pgmap v1945: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:14.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:14.470 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:14.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:14.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:14.598 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:16:14.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:14 smithi191 ceph-mon[102206]: pgmap v1945: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:14.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:14.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:14.750 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:14 smithi076 podman[295261]: 2024-09-21 10:16:14.392653085 +0000 UTC m=+1.747615630 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:16:14.750 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:14 smithi076 podman[295261]: 2024-09-21 10:16:14.510997396 +0000 UTC m=+1.865959926 container create 686a617b9fe6d90794984fec083eb001a90ee935afabb6c9c09a1294ffb910cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:16:14.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:14.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:14.849 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:16:14.852 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:16:14.852 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:14.855 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:14.876 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:16:14.876 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:14.876 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:16:14.894 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:16:14.895 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:14.911 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:16:14.911 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:14.927 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:16:14.928 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:14.943 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:16:14.944 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:14.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:14.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:14.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:15.069 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:15.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.092 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:15.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:15.102 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:14 smithi076 podman[295261]: 2024-09-21 10:16:14.982700825 +0000 UTC m=+2.337663355 container init 686a617b9fe6d90794984fec083eb001a90ee935afabb6c9c09a1294ffb910cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T10:16:15.102 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:14 smithi076 podman[295261]: 2024-09-21 10:16:14.988620045 +0000 UTC m=+2.343582586 container start 686a617b9fe6d90794984fec083eb001a90ee935afabb6c9c09a1294ffb910cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:16:15.102 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:15 smithi076 podman[295261]: 2024-09-21 10:16:15.100072865 +0000 UTC m=+2.455035399 container attach 686a617b9fe6d90794984fec083eb001a90ee935afabb6c9c09a1294ffb910cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:16:15.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:15.216 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:15.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:15.340 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:15.348 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3910488531' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3910488531' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3910488531' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:15.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:15 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:16:15.374 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:15 smithi076 bash[295261]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:16:15.374 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:15 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:16:15.374 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:15 smithi076 bash[295261]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:16:15.442 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:15.464 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:15.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:15.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:15.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:15.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:15.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:15.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:15.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:15.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:15.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:15.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:15.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:16.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:16.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:16.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:16.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:16.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:16.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:16.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:16.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:16.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:16.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:16.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:16.551 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-mon[106482]: pgmap v1946: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:16.551 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:16.552 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:16.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-mon[110963]: pgmap v1946: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:16.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:16.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:16.552 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:16:16.552 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:16:16.552 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T10:16:16.552 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T10:16:16.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:16.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:16.695 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:16.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:16.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:16.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:16.847 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T10:16:16.847 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T10:16:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:16 smithi191 ceph-mon[102206]: pgmap v1946: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:16.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:16.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:16.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:16.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:17.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:17.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:17.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:16:17.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:16:17.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:16:17.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:16:17.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:16:17.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:16:17.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[295497]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T10:16:17.124 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:16 smithi076 bash[295261]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T10:16:17.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:17.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:17.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:17.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:17.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:17.373 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:16:17.373 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:16:17.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:17.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:17.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:17.567 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:17.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:17.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:17.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:17.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:17.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:17.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:17.917 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:17.940 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:17.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:18.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:18.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:18.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:18.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.196 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:18.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:18.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:18.323 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:18.352 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:138934 8.11 8:8bc9f568:::benchmark_data_smithi076_241159_object16016:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1226)", 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 375.98112586600001, 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.101503399, 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.353 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 138934 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.354 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367866+0000", 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.08e-07 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367868+0000", 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.897e-06 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.355 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367874+0000", 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8449999999999999e-06 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.367879+0000", 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8080000000000004e-06 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368055+0000", 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000176033 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.356 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting_for_map not empty", 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.368057+0000", 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.526e-06 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370195+0000", 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00213773 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:02.370201+0000", 2024-09-21T10:16:18.357 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1349999999999997e-06 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.460880+0000", 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0906788519999999 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.460980+0000", 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9413000000000005e-05 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.358 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,3,6", 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.461096+0000", 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000116951 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.463014+0000", 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0019179080000000001 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.465731+0000", 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0027162739999999999 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.359 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.467849+0000", 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021179760000000001 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.469107+0000", 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012582139999999999 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.469259+0000", 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000152662 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.360 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:03.469369+0000", 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010976699999999999 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146143 8.15 8:aa674412:::benchmark_data_smithi076_241159_object23225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.249046+0000", 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 365.09994599999999, 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.889343131, 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.361 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146143 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249046+0000", 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249046+0000", 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.362 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249046+0000", 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5499999999999999e-07 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249048+0000", 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.626e-06 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249053+0000", 2024-09-21T10:16:18.363 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6799999999999998e-06 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249058+0000", 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6469999999999997e-06 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.253030+0000", 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039716559999999996 2024-09-21T10:16:18.364 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.253040+0000", 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0199e-05 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075155+0000", 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.82211540000000005 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075181+0000", 2024-09-21T10:16:18.365 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5239999999999999e-05 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.133671+0000", 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0584906679999999 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.133796+0000", 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124831 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.366 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.367 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.134073+0000", 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027701199999999998 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138184+0000", 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0041105020000000003 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138199+0000", 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5492999999999999e-05 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.368 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138227+0000", 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7977999999999999e-05 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138252+0000", 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.5289000000000001e-05 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138299+0000", 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.6350000000000002e-05 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.369 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138389+0000", 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0204999999999994e-05 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146147 8.15 8:ac2df393:::benchmark_data_smithi076_241159_object23229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.249988+0000", 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 365.09900355399998, 2024-09-21T10:16:18.370 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8887060409999998, 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146147 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249988+0000", 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.371 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249988+0000", 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249989+0000", 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0780000000000001e-06 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.249994+0000", 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3810000000000001e-06 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.372 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.250005+0000", 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1229e-05 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.250012+0000", 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9090000000000003e-06 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.253062+0000", 2024-09-21T10:16:18.373 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030506959999999999 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.253069+0000", 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6470000000000001e-06 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075837+0000", 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.82276748 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.374 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075846+0000", 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2960000000000004e-06 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.135298+0000", 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0594521640000001 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.135448+0000", 2024-09-21T10:16:18.375 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014966400000000001 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.135669+0000", 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00022160999999999999 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138234+0000", 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025644449999999998 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138428+0000", 2024-09-21T10:16:18.376 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019411600000000001 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138471+0000", 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2728999999999997e-05 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138506+0000", 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.5539e-05 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.377 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138569+0000", 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3211000000000002e-05 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138694+0000", 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012484700000000001 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146162 8.15 8:adc49035:::benchmark_data_smithi076_241159_object23244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.264257+0000", 2024-09-21T10:16:18.378 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 365.08473496599998, 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.874756042, 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146162 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264257+0000", 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.379 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264257+0000", 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264258+0000", 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.671e-06 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264264+0000", 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5450000000000003e-06 2024-09-21T10:16:18.380 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264277+0000", 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3030999999999999e-05 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264285+0000", 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0090000000000007e-06 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264383+0000", 2024-09-21T10:16:18.381 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8408999999999999e-05 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.264392+0000", 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5569999999999996e-06 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075898+0000", 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.81150605799999997 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.382 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075906+0000", 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7589999999999997e-06 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.136705+0000", 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0607989280000001 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.136825+0000", 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012065000000000001 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.383 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.137031+0000", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020549399999999999 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138440+0000", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0014091290000000001 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138755+0000", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000315447 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.384 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138813+0000", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7450000000000001e-05 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138873+0000", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0587000000000001e-05 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.138941+0000", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7309000000000002e-05 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.139013+0000", 2024-09-21T10:16:18.385 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2008999999999994e-05 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146225 8.15 8:a8a6ec96:::benchmark_data_smithi076_241159_object23307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.299696+0000", 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 365.04929561199998, 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.846065442, 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.386 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146225 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299696+0000", 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299696+0000", 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.387 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299697+0000", 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2099999999999996e-07 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299699+0000", 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4399999999999999e-06 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299706+0000", 2024-09-21T10:16:18.388 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9079999999999998e-06 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299712+0000", 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5079999999999996e-06 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299767+0000", 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5174999999999999e-05 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.299775+0000", 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7260000000000008e-06 2024-09-21T10:16:18.389 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075940+0000", 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.77616432499999999 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075947+0000", 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.5610000000000001e-06 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.139067+0000", 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0631200709999999 2024-09-21T10:16:18.390 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.139418+0000", 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000350761 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.139673+0000", 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025468099999999998 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145417+0000", 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005743967 2024-09-21T10:16:18.391 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145460+0000", 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2799000000000001e-05 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145473+0000", 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3427e-05 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145575+0000", 2024-09-21T10:16:18.392 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000101925 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145635+0000", 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0476999999999997e-05 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145762+0000", 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012606999999999999 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.393 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146249 8.15 8:aa7ac04d:::benchmark_data_smithi076_241159_object23331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.316644+0000", 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 365.032347942, 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.829589581, 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146249 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.394 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316644+0000", 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316644+0000", 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.395 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316645+0000", 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.471e-06 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316650+0000", 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0880000000000002e-06 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316663+0000", 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2319e-05 2024-09-21T10:16:18.396 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316670+0000", 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1799999999999999e-06 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316770+0000", 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000100457 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.397 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.316779+0000", 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1220000000000001e-06 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075975+0000", 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.75919540500000005 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.075982+0000", 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1670000000000002e-06 2024-09-21T10:16:18.398 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.141305+0000", 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.065323424 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.141457+0000", 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000151941 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.399 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.141683+0000", 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00022535400000000001 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145493+0000", 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003810253 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.145536+0000", 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3077000000000001e-05 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.400 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146029+0000", 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000492549 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146073+0000", 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4487000000000003e-05 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.401 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146128+0000", 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4685999999999998e-05 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146233+0000", 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010560099999999999 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146272 8.15 8:a87601da:::benchmark_data_smithi076_241159_object23354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.334484+0000", 2024-09-21T10:16:18.402 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 365.01450805600001, 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.812034631, 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146272 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334484+0000", 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.403 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334484+0000", 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334484+0000", 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9299999999999998e-07 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334487+0000", 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8490000000000002e-06 2024-09-21T10:16:18.404 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334495+0000", 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6510000000000001e-06 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334500+0000", 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3750000000000002e-06 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334603+0000", 2024-09-21T10:16:18.405 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010249799999999999 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.334613+0000", 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0169e-05 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076007+0000", 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.74139381699999996 2024-09-21T10:16:18.406 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076014+0000", 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.7640000000000005e-06 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.143473+0000", 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.067458993 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.143588+0000", 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000114663 2024-09-21T10:16:18.407 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.143882+0000", 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00029370500000000001 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146041+0000", 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0021589000000000001 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146275+0000", 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023392399999999999 2024-09-21T10:16:18.408 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146313+0000", 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.8251999999999999e-05 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146350+0000", 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7475999999999998e-05 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146406+0000", 2024-09-21T10:16:18.409 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5788999999999997e-05 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146518+0000", 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112213 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.410 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146349 8.15 8:ab421b49:::benchmark_data_smithi076_241159_object23431:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.387296+0000", 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.96169523999998, 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.76440799, 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.411 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146349 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387296+0000", 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.412 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387296+0000", 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387298+0000", 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1119999999999999e-06 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387302+0000", 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5669999999999996e-06 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.413 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387310+0000", 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9689999999999994e-06 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387317+0000", 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9009999999999997e-06 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387376+0000", 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8658e-05 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.414 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.387385+0000", 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5300000000000002e-06 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076038+0000", 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.68865289900000004 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076045+0000", 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.973e-06 2024-09-21T10:16:18.415 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146550+0000", 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.070504897 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146660+0000", 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110497 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.146954+0000", 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00029386199999999999 2024-09-21T10:16:18.416 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.149814+0000", 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002859257 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.149827+0000", 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3176999999999999e-05 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151422+0000", 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015955089999999999 2024-09-21T10:16:18.417 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151465+0000", 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2948000000000003e-05 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151603+0000", 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000137634 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.418 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151704+0000", 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0001016 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146374 8.15 8:ac194182:::benchmark_data_smithi076_241159_object23456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.407920+0000", 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.94107145300001, 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.747921584, 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.419 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146374 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.407920+0000", 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.420 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.407920+0000", 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.407922+0000", 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.463e-06 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.407927+0000", 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9910000000000002e-06 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.421 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.407943+0000", 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6461e-05 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.407952+0000", 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4989999999999994e-06 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.422 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.423 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.423 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.408010+0000", 2024-09-21T10:16:18.423 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8016999999999997e-05 2024-09-21T10:16:18.423 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.423 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.423 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.408020+0000", 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0771000000000001e-05 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076069+0000", 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.66804855500000004 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076076+0000", 2024-09-21T10:16:18.425 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3490000000000003e-06 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.148096+0000", 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.072019686 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.148200+0000", 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010403000000000001 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.426 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.148442+0000", 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024181499999999999 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151435+0000", 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029926589999999999 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151740+0000", 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000305896 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.427 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155482+0000", 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0037419599999999999 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155676+0000", 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000193983 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155730+0000", 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3168000000000003e-05 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.428 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155842+0000", 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112281 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146431 8.15 8:a8b6952a:::benchmark_data_smithi076_241159_object23513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.453465+0000", 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.89552679299999, 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7045872399999999, 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.429 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146431 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453465+0000", 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453465+0000", 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.430 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453466+0000", 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.403e-06 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453469+0000", 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.9289999999999998e-06 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453477+0000", 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2069999999999995e-06 2024-09-21T10:16:18.431 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453483+0000", 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0880000000000002e-06 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453557+0000", 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4909999999999999e-05 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.453566+0000", 2024-09-21T10:16:18.432 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8400000000000001e-06 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076099+0000", 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.62253225199999995 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076105+0000", 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2190000000000001e-06 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.433 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.149851+0000", 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.073745849 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.149962+0000", 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000111661 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.150179+0000", 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000217059 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.434 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155357+0000", 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0051776779999999998 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155441+0000", 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4066000000000005e-05 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155521+0000", 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0006999999999995e-05 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.435 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155885+0000", 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00036350199999999998 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155949+0000", 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4721999999999995e-05 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.158052+0000", 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002102848 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.436 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146482 8.15 8:a8cba698:::benchmark_data_smithi076_241159_object23564:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.496870+0000", 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.85212169200003, 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.666173203, 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.437 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.438 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.438 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.438 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146482 2024-09-21T10:16:18.438 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.438 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.438 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496870+0000", 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496870+0000", 2024-09-21T10:16:18.439 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496871+0000", 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.449e-06 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.440 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496876+0000", 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7090000000000001e-06 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496890+0000", 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3535e-05 2024-09-21T10:16:18.441 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.442 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.442 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.442 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496897+0000", 2024-09-21T10:16:18.442 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1019999999999999e-06 2024-09-21T10:16:18.442 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.442 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.443 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.443 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496962+0000", 2024-09-21T10:16:18.443 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4725000000000001e-05 2024-09-21T10:16:18.443 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.443 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.443 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.496968+0000", 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9719999999999997e-06 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076126+0000", 2024-09-21T10:16:18.444 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.57915835999999998 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076134+0000", 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.8329999999999994e-06 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151766+0000", 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0756326439999999 2024-09-21T10:16:18.445 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.151864+0000", 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7403000000000002e-05 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.152149+0000", 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00028484300000000001 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.446 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155401+0000", 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0032525100000000001 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155560+0000", 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015849599999999999 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155597+0000", 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7642999999999999e-05 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.447 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.162870+0000", 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0072723299999999996 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.162936+0000", 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6365000000000006e-05 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.163043+0000", 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000107284 2024-09-21T10:16:18.448 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146516 8.15 8:a973df6e:::benchmark_data_smithi076_241159_object23598:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.528756+0000", 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.82023581300001, 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.640280121, 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.449 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146516 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528756+0000", 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528756+0000", 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.450 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528757+0000", 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0440000000000001e-06 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528761+0000", 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2139999999999998e-06 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528769+0000", 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6850000000000001e-06 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.451 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528776+0000", 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7429999999999996e-06 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528935+0000", 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000159304 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.452 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.528945+0000", 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0468999999999999e-05 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076158+0000", 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.54721223500000005 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076165+0000", 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2230000000000001e-06 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.453 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.153540+0000", 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0773747600000001 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.153640+0000", 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010073099999999999 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.153862+0000", 2024-09-21T10:16:18.454 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00022189 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.155635+0000", 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0017730300000000001 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.160397+0000", 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0047613539999999998 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.455 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.163083+0000", 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0026859729999999999 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.168839+0000", 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0057569099999999996 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.168903+0000", 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3552000000000004e-05 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.456 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169036+0000", 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013300399999999999 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146517 8.15 8:abee46a0:::benchmark_data_smithi076_241159_object23599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.529275+0000", 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.819717148, 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6401575749999999, 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.457 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146517 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529275+0000", 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.458 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529275+0000", 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529275+0000", 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4600000000000004e-07 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529279+0000", 2024-09-21T10:16:18.459 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.258e-06 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529289+0000", 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0747999999999999e-05 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529296+0000", 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2949999999999999e-06 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.460 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529352+0000", 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6446000000000001e-05 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.529360+0000", 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1689999999999992e-06 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076187+0000", 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.54682646899999998 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.461 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076195+0000", 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2060000000000007e-06 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.158094+0000", 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0818989219999999 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.158201+0000", 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010718 2024-09-21T10:16:18.462 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.158510+0000", 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00030854899999999998 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.163509+0000", 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0049991000000000002 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.463 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.166959+0000", 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034507750000000001 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169216+0000", 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022560689999999999 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169263+0000", 2024-09-21T10:16:18.464 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7324000000000003e-05 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169319+0000", 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6258000000000002e-05 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169432+0000", 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011306099999999999 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.465 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146527 8.15 8:ac9d317e:::benchmark_data_smithi076_241159_object23609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.537192+0000", 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.81179934900001, 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.636535555, 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.466 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146527 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537192+0000", 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.467 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537192+0000", 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537193+0000", 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1009999999999999e-06 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537198+0000", 2024-09-21T10:16:18.468 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.408e-06 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537209+0000", 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0696000000000001e-05 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537214+0000", 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2739999999999998e-06 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.469 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537266+0000", 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2454999999999998e-05 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.537272+0000", 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7520000000000002e-06 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076221+0000", 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.53894868200000001 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.470 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076228+0000", 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2309999999999999e-06 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.160425+0000", 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0841970869999999 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.160630+0000", 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020525299999999999 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.471 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.160903+0000", 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027259400000000003 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.168795+0000", 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0078916109999999998 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169472+0000", 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00067728999999999999 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.472 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169515+0000", 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.3080000000000001e-05 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.173568+0000", 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0040531910000000003 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.173618+0000", 2024-09-21T10:16:18.473 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9978999999999997e-05 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.173728+0000", 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010987100000000001 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146532 8.15 8:af014f07:::benchmark_data_smithi076_241159_object23614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.545105+0000", 2024-09-21T10:16:18.474 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.803886761, 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6345930740000001, 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146532 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.475 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545105+0000", 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545105+0000", 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545106+0000", 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5600000000000004e-07 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.476 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545110+0000", 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0230000000000003e-06 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545119+0000", 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3500000000000003e-06 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545126+0000", 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6630000000000004e-06 2024-09-21T10:16:18.477 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545202+0000", 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.6531000000000003e-05 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.545211+0000", 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6389999999999998e-06 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076250+0000", 2024-09-21T10:16:18.478 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.53103927100000003 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076257+0000", 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7689999999999999e-06 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169548+0000", 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.093291309 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.479 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.169667+0000", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011822199999999999 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.170145+0000", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000477854 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.173779+0000", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003634422 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.480 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179383+0000", 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056039610000000002 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179398+0000", 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5437000000000001e-05 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179511+0000", 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112378 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.481 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179565+0000", 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4737000000000001e-05 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179698+0000", 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000132552 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146549 8.15 8:ab5b0c7c:::benchmark_data_smithi076_241159_object23631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.482 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.574286+0000", 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.77470595300002, 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6095004479999999, 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146549 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.483 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574286+0000", 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574286+0000", 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574287+0000", 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.4900000000000005e-07 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.484 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574290+0000", 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4690000000000002e-06 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574299+0000", 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.816e-06 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574305+0000", 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.263e-06 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.485 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574370+0000", 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4814000000000007e-05 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.574383+0000", 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2775000000000001e-05 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076283+0000", 2024-09-21T10:16:18.486 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.501900558 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076292+0000", 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4579999999999993e-06 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.171703+0000", 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0954113809999999 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.487 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.171816+0000", 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000113424 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.172213+0000", 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00039656499999999998 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179404+0000", 2024-09-21T10:16:18.488 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0071912629999999998 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179437+0000", 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2344000000000001e-05 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179740+0000", 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00030363299999999998 2024-09-21T10:16:18.489 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.183689+0000", 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0039490039999999999 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.183723+0000", 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.3892e-05 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.183786+0000", 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.3040000000000006e-05 2024-09-21T10:16:18.490 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146584 8.15 8:aaf6a16e:::benchmark_data_smithi076_241159_object23666:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1234)", 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:13.689097+0000", 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 364.65989520400001, 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.494856561, 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146584 2024-09-21T10:16:18.491 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689097+0000", 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689097+0000", 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689098+0000", 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7379999999999999e-06 2024-09-21T10:16:18.492 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689104+0000", 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.259e-06 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689116+0000", 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2815e-05 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689125+0000", 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1330000000000005e-06 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.493 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689188+0000", 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2855999999999996e-05 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.689195+0000", 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.392e-06 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076315+0000", 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.38712031299999999 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.494 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:14.076322+0000", 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2799999999999998e-06 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.173854+0000", 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0975314389999999 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.173979+0000", 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012564499999999999 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.495 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 2,4,6", 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.174239+0000", 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000259275 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179480+0000", 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0052408519999999998 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.179807+0000", 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000327425 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.496 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.181367+0000", 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015600259999999999 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.183828+0000", 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002461179 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.183860+0000", 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1636e-05 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.497 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:15.183953+0000", 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3298e-05 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:219866 9.f 9:fc55a3f5:::benchmark_data_smithi076_269622_object105374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1285)", 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:39.298540+0000", 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 39.050452043, 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1077778679999999, 2024-09-21T10:16:18.498 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "reached pg", 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 219866 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298540+0000", 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.499 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298540+0000", 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298541+0000", 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.792e-06 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298548+0000", 2024-09-21T10:16:18.500 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4799999999999998e-06 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298565+0000", 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7527e-05 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298575+0000", 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0289999999999994e-06 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.501 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298639+0000", 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4213000000000003e-05 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298771+0000", 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000132612 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.298895+0000", 2024-09-21T10:16:18.502 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000123249 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.301725+0000", 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028303099999999999 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.307091+0000", 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.005366114 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.503 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.337621+0000", 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.030529984 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.337628+0000", 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.2690000000000002e-06 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.405247+0000", 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0676183990000001 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.504 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.406318+0000", 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010708899999999999 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:219876 9.f 9:f10ccb0f:::benchmark_data_smithi076_269622_object105384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1285)", 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:39.304609+0000", 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 39.044382783000003, 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1018986150000001, 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "reached pg", 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:16:18.505 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 219876 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.304609+0000", 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.304609+0000", 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.506 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.304610+0000", 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.92e-07 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.304613+0000", 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.7560000000000001e-06 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.507 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.304623+0000", 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.6830000000000003e-06 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.304630+0000", 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.5169999999999996e-06 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.307124+0000", 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0024944479999999998 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.508 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.307230+0000", 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000105732 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.307383+0000", 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000153432 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.312041+0000", 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0046581890000000001 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.509 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.312659+0000", 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00061771899999999997 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.337656+0000", 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.024996658000000001 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.337662+0000", 2024-09-21T10:16:18.510 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.711e-06 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.406429+0000", 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.068766965 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.406508+0000", 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9012999999999997e-05 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.511 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35664.0:219883 9.f 9:f6c0b366:::benchmark_data_smithi076_269622_object105391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1285)", 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:15:39.316334+0000", 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 39.032657714000003, 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1114335310000001, 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35664", 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/2967698104", 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 219883 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.512 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.316334+0000", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.316334+0000", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.316335+0000", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0699999999999996e-07 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:18.513 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.316339+0000", 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2169999999999996e-06 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.316350+0000", 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1103e-05 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.316355+0000", 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2100000000000001e-06 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.514 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.337731+0000", 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.021375879 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:39.337737+0000", 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2989999999999996e-06 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.406655+0000", 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0689181029999999 2024-09-21T10:16:18.515 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.406858+0000", 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000202776 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4", 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.407013+0000", 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000155835 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.413366+0000", 2024-09-21T10:16:18.516 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0063521890000000003 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.427634+0000", 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.014268820999999999 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.427676+0000", 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1693999999999998e-05 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:16:18.517 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:15:40.427768+0000", 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.1497999999999997e-05 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:18.518 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:16:18.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:18.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:18.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:18.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:18 smithi076 ceph-mon[106482]: pgmap v1947: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:18.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:18 smithi076 ceph-mon[110963]: pgmap v1947: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:18.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:18.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:18.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:18.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:18.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:18.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:18 smithi191 ceph-mon[102206]: pgmap v1947: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:18.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:18.922 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:18.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:18.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:19.023 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.045 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:19.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:19.147 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:19.172 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:19.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:19.299 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:19.375 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:16:19.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:19.439 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:19.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:19.564 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:19.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:19.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:19.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:19.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:19.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:19 smithi076 podman[295261]: 2024-09-21 10:16:19.73504441 +0000 UTC m=+7.090006951 container died 686a617b9fe6d90794984fec083eb001a90ee935afabb6c9c09a1294ffb910cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, maintainer=Guillaume Abrioux ) 2024-09-21T10:16:19.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:19.940 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:19.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:20.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:20.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:20.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:20.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:20.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:20.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:20.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:20.416 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:20.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:20.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:20.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:20.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:20.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:20 smithi076 ceph-mon[106482]: pgmap v1948: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:20.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:20 smithi076 ceph-mon[110963]: pgmap v1948: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:20.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:20 smithi076 podman[295928]: 2024-09-21 10:16:20.287652297 +0000 UTC m=+3.373167239 container remove 686a617b9fe6d90794984fec083eb001a90ee935afabb6c9c09a1294ffb910cd (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:16:20.665 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.688 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:20.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:20.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:20.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:20.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:20.841 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:16:20.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:20 smithi191 ceph-mon[102206]: pgmap v1948: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:20.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:20.935 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:20.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:21.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:21.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:21.109 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:21.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:21.182 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:21.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:21.306 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:21.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 podman[296400]: 2024-09-21 10:16:21.162548349 +0000 UTC m=+0.041215670 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:16:21.403 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:21.403 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_5' created 2024-09-21T10:16:21.406 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:21.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:21.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.442 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:21.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:21.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:21.564 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:21.663 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1501459729' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:16:21.663 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:21 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:16:21.664 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1501459729' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:16:21.664 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:21 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:16:21.664 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 podman[296400]: 2024-09-21 10:16:21.547008559 +0000 UTC m=+0.425675850 container create 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T10:16:21.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.688 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:21.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:21.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1501459729' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:16:21.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:21 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-21T10:16:21.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:21.933 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:21.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:21.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:16:21.997 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 podman[296400]: 2024-09-21 10:16:21.842560421 +0000 UTC m=+0.721227712 container init 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:16:21.998 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 podman[296400]: 2024-09-21 10:16:21.848673691 +0000 UTC m=+0.727340984 container start 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:16:21.998 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 bash[296400]: 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 2024-09-21T10:16:21.998 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 systemd[1]: Started Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:16:21.998 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:21 smithi076 ceph-osd[296613]: -- 172.21.15.76:0/405608846 <== mon.2 v2:172.21.15.76:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56344668fa40 con 0x56344745e000 2024-09-21T10:16:22.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:22.056 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:22.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:22.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:22.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:22.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:22.403 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:22.424 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:22.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:22.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:22 smithi076 ceph-mon[106482]: pgmap v1949: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:16:22.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:22 smithi076 ceph-mon[106482]: osdmap e1296: 8 total, 6 up, 4 in 2024-09-21T10:16:22.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:22 smithi076 ceph-mon[110963]: pgmap v1949: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:22.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:16:22.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:22 smithi076 ceph-mon[110963]: osdmap e1296: 8 total, 6 up, 4 in 2024-09-21T10:16:22.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:22.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:22.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:22.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:22.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:22 smithi191 ceph-mon[102206]: pgmap v1949: 73 pgs: 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-21T10:16:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:22 smithi191 ceph-mon[102206]: osdmap e1296: 8 total, 6 up, 4 in 2024-09-21T10:16:22.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:22.916 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:22.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:23.017 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:23.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:23.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:23.163 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:23.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:23.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:23.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:23.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-21T10:16:23.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:23.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:23.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:23.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:23.589 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:23 smithi076 ceph-mon[106482]: osdmap e1297: 8 total, 6 up, 4 in 2024-09-21T10:16:23.589 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:23 smithi076 ceph-mon[106482]: osdmap e1298: 8 total, 6 up, 4 in 2024-09-21T10:16:23.590 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:16:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:16:23] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-21T10:16:23.590 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:23 smithi076 ceph-mon[110963]: osdmap e1297: 8 total, 6 up, 4 in 2024-09-21T10:16:23.590 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:23 smithi076 ceph-mon[110963]: osdmap e1298: 8 total, 6 up, 4 in 2024-09-21T10:16:23.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:23.656 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:23.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:23.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:23 smithi191 ceph-mon[102206]: osdmap e1297: 8 total, 6 up, 4 in 2024-09-21T10:16:23.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:23 smithi191 ceph-mon[102206]: osdmap e1298: 8 total, 6 up, 4 in 2024-09-21T10:16:23.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[296572]: 2024-09-21T10:16:23.605+0000 7f57ee816640 -1 Falling back to public interface 2024-09-21T10:16:23.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:23.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:23.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:24.004 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:24.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:24.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:24.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:24.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:24.272 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:24.374 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:24.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:24.496 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:24.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:24.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:24.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:24.665 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:24 smithi076 ceph-mon[106482]: pgmap v1952: 89 pgs: 16 unknown, 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:24.665 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:24 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:16:24.665 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:24 smithi076 ceph-mon[110963]: pgmap v1952: 89 pgs: 16 unknown, 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:24.665 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:24 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:16:24.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:24.763 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:24 smithi191 ceph-mon[102206]: pgmap v1952: 89 pgs: 16 unknown, 6 active+clean+remapped, 44 active+undersized, 10 active+undersized+degraded, 13 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:24 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:16:24.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:24.887 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:24.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:24.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:24.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:25.134 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:25.236 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:25.257 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:25.358 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:25.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:25.417 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:16:25.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:25.504 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:25.602 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:25.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/583909408' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/583909408' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:25.627 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:25.728 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:25.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/583909408' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:16:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:25.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:25.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:25.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:25.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:26.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:26.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:26.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:26.240 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:26.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:26.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:26.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:26.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:26.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:26.485 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:26.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:26.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[110963]: pgmap v1954: 89 pgs: 9 unknown, 6 active+clean+remapped, 49 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[110963]: osdmap e1299: 8 total, 6 up, 4 in 2024-09-21T10:16:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:26.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[106482]: pgmap v1954: 89 pgs: 9 unknown, 6 active+clean+remapped, 49 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:26.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:26.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:26.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[106482]: osdmap e1299: 8 total, 6 up, 4 in 2024-09-21T10:16:26.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:26.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:26.712 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:26.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:26.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:26.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:26.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:26.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:26 smithi191 ceph-mon[102206]: pgmap v1954: 89 pgs: 9 unknown, 6 active+clean+remapped, 49 active+undersized, 10 active+undersized+degraded, 15 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:26 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:26 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:16:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:26 smithi191 ceph-mon[102206]: osdmap e1299: 8 total, 6 up, 4 in 2024-09-21T10:16:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:26.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:26.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:26.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:27.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:27.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:27.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:27.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.229 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:27.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:27.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:27.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:27.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:27.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:27.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:27.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:27.598 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:27.700 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:27.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:27.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:27.845 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:27.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:27.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:27.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:27.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:27.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:28.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:28.093 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:28.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:28.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:28.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:28.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:28.341 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:28.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:28.464 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:28.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:28.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:28.590 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:28.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:28.713 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:28.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:28.815 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:28.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:28.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:28 smithi191 ceph-mon[102206]: pgmap v1956: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:28.866 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:28 smithi076 ceph-mon[106482]: pgmap v1956: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:28.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:28 smithi076 ceph-mon[110963]: pgmap v1956: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:28.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:28.961 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:28.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:28.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:29.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:29.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:29.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:29.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:29.295 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:29.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:29.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore bluefs device info [] 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:config diff 2024-09-21T10:16:29.443 INFO:teuthology.orchestra.run.smithi076.stderr:admin_socket: invalid command 2024-09-21T10:16:29.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:29.542 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:29.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:29.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:29.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:29.768 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:29.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:29.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:29.917 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:29.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:30.018 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:30.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:30.142 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:30.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:30.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:30.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:30.289 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:30.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:30.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:30.516 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:30.531 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:30 smithi076 ceph-mon[106482]: pgmap v1957: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:30.531 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:30 smithi076 ceph-mon[110963]: pgmap v1957: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:30.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:30.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:30.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:30.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:30.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:30.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:30 smithi191 ceph-mon[102206]: pgmap v1957: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:30.887 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:30.908 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:30.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:30.942 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:16:30.943 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2024-09-21T10:16:31.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:31.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:31.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:31.231 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:16:31.232 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:16:31.232 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:16:31.232 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:16:31.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:31.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.382 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:31.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:31.505 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:31.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:31.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:31.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.751 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:31.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:31.836 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:16:31.847 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:16:31.847 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:31.865 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:16:31.865 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:31.865 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:16:31.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:31.882 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:16:31.882 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:31.895 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:31.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:31.898 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:16:31.898 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:31.915 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:16:31.915 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:31.931 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:16:31.931 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:31.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:32.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:32.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:16:32.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:32.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:32.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:32.263 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:32.364 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:32.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:32.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:32.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:32.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:32 smithi076 ceph-mon[106482]: pgmap v1958: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:32 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/339301083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:32 smithi076 ceph-mon[110963]: pgmap v1958: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:32.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:32 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/339301083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:32.631 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:32.706 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:16:32.706 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:16:32.733 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:32.754 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:32.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:32.856 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:32 smithi191 ceph-mon[102206]: pgmap v1958: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:32.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:32 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/339301083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:32.878 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:32.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:32.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:32.980 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:33.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:33.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:33.024 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:33.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:33.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:33.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:33.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:33.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.250 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:33.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:33.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:33.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:33.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:33.402 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:33.411 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:16:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:16:33] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.51.0" 2024-09-21T10:16:33.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:33.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:33.598 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:33.621 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:33.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:33.745 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:33.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:33.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:33.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:16:33.867 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:33.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:33.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:33.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:34.091 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:34.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:34.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:34.214 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:34.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:34.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:34.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:34.358 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:34.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:34.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:34.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:34.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:34.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:16:34.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:34.730 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:34.730 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:34 smithi076 ceph-mon[106482]: pgmap v1959: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:34.731 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:34 smithi076 ceph-mon[110963]: pgmap v1959: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:34.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:34.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:34 smithi191 ceph-mon[102206]: pgmap v1959: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:34.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:34.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:34.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:35.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:35.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:35.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:35.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:35.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:35.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:35.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:35.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:35.942 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:35.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:16:35.964 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:35.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:36.086 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:36.096 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:35 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[296572]: 2024-09-21T10:16:35.840+0000 7f57ee816640 -1 osd.3 1285 log_to_monitors true 2024-09-21T10:16:36.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:36.209 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:36.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:36.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:36.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:36.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:36.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:36.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:36.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:36.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:36.732 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:36 smithi076 ceph-mon[106482]: pgmap v1960: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:36.733 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:36 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:16:36.733 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:36 smithi076 ceph-mon[110963]: pgmap v1960: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:36.733 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:36 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:16:36.733 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:16:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[296572]: 2024-09-21T10:16:36.452+0000 7f57e25b8640 -1 osd.3 1285 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:16:36.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:36.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:36 smithi191 ceph-mon[102206]: pgmap v1960: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:36 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:16:36.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:36.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:36.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:36.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:37.072 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:37.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:37.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:37.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:37.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:37.429 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-21T10:16:37.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:37.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:37.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:37.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:37.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:37.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:37.814 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:37 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:16:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:37 smithi191 ceph-mon[102206]: osdmap e1300: 8 total, 6 up, 4 in 2024-09-21T10:16:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:37 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:16:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2282816086' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:16:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:37.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:16:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[110963]: osdmap e1300: 8 total, 6 up, 4 in 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2282816086' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[106482]: osdmap e1300: 8 total, 6 up, 4 in 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2282816086' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:37.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:37.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:37.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:37.952 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:38.038 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:38.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:38.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:38.184 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:38.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.308 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:38.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:38.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:38.432 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:38.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:38.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:38.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:38.657 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:38.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: pgmap v1962: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2282816086' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594] boot 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: osdmap e1301: 8 total, 7 up, 4 in 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:16:38.755 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[106482]: osdmap e1302: 8 total, 7 up, 4 in 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: pgmap v1962: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2282816086' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594] boot 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: osdmap e1301: 8 total, 7 up, 4 in 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:38.756 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:38 smithi076 ceph-mon[110963]: osdmap e1302: 8 total, 7 up, 4 in 2024-09-21T10:16:38.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:38.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: pgmap v1962: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2282816086' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: osd.3 [v2:172.21.15.76:6802/4223107594,v1:172.21.15.76:6803/4223107594] boot 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: osdmap e1301: 8 total, 7 up, 4 in 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:38 smithi191 ceph-mon[102206]: osdmap e1302: 8 total, 7 up, 4 in 2024-09-21T10:16:38.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:38.924 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:38.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:39.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:39.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:39.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:39.171 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:39.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:39.273 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:39.296 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:39.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:39.420 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:39.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:39.452 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:16:39.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:39.574 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: pgmap v1964: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/604860808' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:39.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[106482]: osdmap e1303: 8 total, 7 up, 4 in 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: pgmap v1964: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/604860808' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:39.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:39 smithi076 ceph-mon[110963]: osdmap e1303: 8 total, 7 up, 4 in 2024-09-21T10:16:39.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:39.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:39.799 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:39.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:39.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: pgmap v1964: 89 pgs: 6 active+clean+remapped, 56 active+undersized, 10 active+undersized+degraded, 17 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 18/78 objects degraded (23.077%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/604860808' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:16:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:39 smithi191 ceph-mon[102206]: osdmap e1303: 8 total, 7 up, 4 in 2024-09-21T10:16:39.923 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:39.945 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:39.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:40.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:40.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:40.172 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:40.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:40.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:40.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:40.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:40.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:40.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:40.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:40.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:40.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:40.671 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:40.694 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:40.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:40.813 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:40 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 16/78 objects degraded (20.513%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:40.813 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:40 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 16/78 objects degraded (20.513%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:40.818 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:40 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 16/78 objects degraded (20.513%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:16:40.920 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:40.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:40.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:41.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:41.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:41.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:41.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:41.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:41.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:41.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:41.446 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:41.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:41.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:41.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:41.619 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:41 smithi076 ceph-mon[106482]: pgmap v1967: 89 pgs: 6 active+clean+remapped, 41 active+undersized, 8 active+undersized+degraded, 34 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 16/78 objects degraded (20.513%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:41.620 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:41.620 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:41.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:41 smithi076 ceph-mon[110963]: pgmap v1967: 89 pgs: 6 active+clean+remapped, 41 active+undersized, 8 active+undersized+degraded, 34 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 16/78 objects degraded (20.513%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:41.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:41.620 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:41.643 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:16:41.643 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:16:41.644 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:16:41.644 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:16:41.644 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:16:41.644 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:16:41.644 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T10:16:41.650 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:41.650 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:41.653 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:16:41.654 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:16:41.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:41.681 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:16:41.681 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:16:41.699 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:16:41.699 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:16:41.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:41.715 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:16:41.716 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:16:41.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:41 smithi191 ceph-mon[102206]: pgmap v1967: 89 pgs: 6 active+clean+remapped, 41 active+undersized, 8 active+undersized+degraded, 34 active+clean; 582 KiB data, 2.8 GiB used, 355 GiB / 358 GiB avail; 16/78 objects degraded (20.513%); 2/78 objects misplaced (2.564%) 2024-09-21T10:16:41.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:41.764 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:41.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:41.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:41.929 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:41.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:41.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:42.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:42.074 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:42.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:42.128 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:16:42.128 DEBUG:teuthology.orchestra.run.smithi076:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-21T10:16:42.131 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:16:42.131 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:16:42.176 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:42.196 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:42.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:hints = 1 2024-09-21T10:16:42.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:16:42.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object prefix: benchmark_data_smithi076_299450 2024-09-21T10:16:42.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:16:42.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:16:42.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:42.329 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:42.431 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:42.466 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:42.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:42.568 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:42.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:42.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:42.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:42.813 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1389322030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:42.813 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1389322030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:42.838 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1389322030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:16:42.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:42.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:42.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:43.008 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:43.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:43.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:43.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 1 16 4144 4128 257.986 258 0.00400531 0.00386418 2024-09-21T10:16:43.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:43.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:43.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:43.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:43.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:43.516 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:16:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:16:43] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-21T10:16:43.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:43.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:43.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:43.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:43.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:43 smithi191 ceph-mon[102206]: pgmap v1968: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 355 GiB / 358 GiB avail 2024-09-21T10:16:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:43 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/78 objects degraded (20.513%), 8 pgs degraded) 2024-09-21T10:16:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:43 smithi076 ceph-mon[106482]: pgmap v1968: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 355 GiB / 358 GiB avail 2024-09-21T10:16:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:43 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/78 objects degraded (20.513%), 8 pgs degraded) 2024-09-21T10:16:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:43 smithi076 ceph-mon[110963]: pgmap v1968: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 355 GiB / 358 GiB avail 2024-09-21T10:16:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:43 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/78 objects degraded (20.513%), 8 pgs degraded) 2024-09-21T10:16:43.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:43.948 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:43.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:44.050 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:44.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:44.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:44.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:44.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 2 16 8039 8023 250.702 243.438 0.00310561 0.00398115 2024-09-21T10:16:44.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:44.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:44.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:44.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:44.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:44.631 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:44.733 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:44.766 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:44.824 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:16:44.825 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:16:44.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:44.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:44.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:45.003 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:45.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:45.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:45.175 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:45.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 3 16 11843 11827 246.378 237.75 0.00462966 0.00405233 2024-09-21T10:16:45.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:45.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:45.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:45.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:45.548 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:45.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:45 smithi076 ceph-mon[106482]: pgmap v1969: 89 pgs: 89 active+clean; 58 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 8.7 MiB/s wr, 139 op/s 2024-09-21T10:16:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:45 smithi076 ceph-mon[110963]: pgmap v1969: 89 pgs: 89 active+clean; 58 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 8.7 MiB/s wr, 139 op/s 2024-09-21T10:16:45.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:45.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:45.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:45.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:45 smithi191 ceph-mon[102206]: pgmap v1969: 89 pgs: 89 active+clean; 58 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 8.7 MiB/s wr, 139 op/s 2024-09-21T10:16:45.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:45.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:45.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:45.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:46.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:46.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:46.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:46.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 4 16 15245 15229 237.929 212.625 0.00473334 0.00419727 2024-09-21T10:16:46.239 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:16:46.239 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:16:46.239 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:16:46.239 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:16:46.239 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:16:46.257 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:46.358 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:46.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:46.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:46.526 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:46.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:46.663 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:46.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:46.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:46.798 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:46.825 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:16:46.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:46.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:46.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:46.960 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35901.0:17551 10.9 10:917b21d6:::benchmark_data_smithi076_299450_object17550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.955794+0000", 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.00035744100000000001, 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00038636200000000001, 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35901", 2024-09-21T10:16:46.961 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 17551 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955794+0000", 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955794+0000", 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.962 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955795+0000", 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3540000000000001e-06 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955817+0000", 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.1324000000000001e-05 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955829+0000", 2024-09-21T10:16:46.963 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1905e-05 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955837+0000", 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1799999999999996e-06 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.964 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955900+0000", 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2947000000000006e-05 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.956022+0000", 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000122483 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.956147+0000", 2024-09-21T10:16:46.965 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000124948 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35901.0:17535 10.8 10:13bbca21:::benchmark_data_smithi076_299450_object17534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.948003+0000", 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0081488059999999998, 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0082552489999999992, 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.966 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35901", 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 17535 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948003+0000", 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948003+0000", 2024-09-21T10:16:46.967 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948004+0000", 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3009999999999999e-06 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948029+0000", 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.4539e-05 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948039+0000", 2024-09-21T10:16:46.968 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0257e-05 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948047+0000", 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.9379999999999992e-06 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948366+0000", 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031910199999999998 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.969 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948494+0000", 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012771000000000001 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.948638+0000", 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014469400000000001 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955063+0000", 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0064243520000000004 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.970 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35901.0:17542 10.5 10:a02e2a5c:::benchmark_data_smithi076_299450_object17541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.950155+0000", 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0059966769999999997, 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.006168598, 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35901", 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 17542 2024-09-21T10:16:46.971 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950155+0000", 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950155+0000", 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.972 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950155+0000", 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6400000000000002e-07 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950176+0000", 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0591e-05 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950183+0000", 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7320000000000001e-06 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.973 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950188+0000", 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4060000000000004e-06 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950238+0000", 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9772999999999999e-05 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.974 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950416+0000", 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017799100000000001 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.950538+0000", 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012202 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.975 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954002+0000", 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034640640000000002 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955212+0000", 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001209985 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35901.0:17548 10.8 10:1e945fc0:::benchmark_data_smithi076_299450_object17547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.952711+0000", 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0034401560000000002, 2024-09-21T10:16:46.976 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003679061, 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35901", 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 17548 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952711+0000", 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.977 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952711+0000", 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952713+0000", 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2899999999999999e-06 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952732+0000", 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9385000000000002e-05 2024-09-21T10:16:46.978 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952749+0000", 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7095000000000001e-05 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952757+0000", 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2530000000000004e-06 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.979 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955121+0000", 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023636320000000001 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955277+0000", 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00015631900000000001 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955445+0000", 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016763100000000001 2024-09-21T10:16:46.980 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.35901.0:17549 10.5 10:a41e7c69:::benchmark_data_smithi076_299450_object17548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1303)", 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.952772+0000", 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0033793080000000001, 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0036781470000000001, 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.35901", 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:16:46.981 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 17549 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952772+0000", 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952772+0000", 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.982 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952773+0000", 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8999999999999996e-07 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952793+0000", 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0483999999999999e-05 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952804+0000", 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0642000000000001e-05 2024-09-21T10:16:46.983 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952810+0000", 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2790000000000004e-06 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.952918+0000", 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000107911 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.984 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.953097+0000", 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00017894100000000001 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.953241+0000", 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00014395200000000001 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.985 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955714+0000", 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002473004 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35901.0:17539 10.d e1303/1301 10:bbec8a3e:::benchmark_data_smithi076_299450_object17538:head v 1303'1097, mlcod=1303'1095)", 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.954062+0000", 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.002089142, 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0024739219999999999, 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.986 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954062+0000", 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954062+0000", 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.987 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954063+0000", 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.9900000000000001e-07 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954121+0000", 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8223000000000001e-05 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954211+0000", 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0057999999999997e-05 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.988 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954217+0000", 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7189999999999998e-06 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955240+0000", 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001023157 2024-09-21T10:16:46.989 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955256+0000", 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5869e-05 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35901.0:17541 10.7 e1303/1301 10:ead64580:::benchmark_data_smithi076_299450_object17540:head v 1303'1094, mlcod=1303'1092)", 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.954240+0000", 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.001911596, 2024-09-21T10:16:46.990 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023535349999999999, 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954240+0000", 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954240+0000", 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.991 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954240+0000", 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.58e-07 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954277+0000", 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6312000000000001e-05 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954284+0000", 2024-09-21T10:16:46.992 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.922e-06 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954288+0000", 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8380000000000001e-06 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954528+0000", 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000239702 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:46.993 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954544+0000", 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6167e-05 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35901.0:17543 10.d e1303/1301)", 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.954304+0000", 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0018475989999999999, 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023547059999999998, 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:16:46.994 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954304+0000", 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954304+0000", 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.995 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954304+0000", 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.4200000000000002e-07 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954340+0000", 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6176000000000001e-05 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954347+0000", 2024-09-21T10:16:46.996 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7750000000000004e-06 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954352+0000", 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0479999999999998e-06 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35901.0:17544 10.7 e1303/1301 10:e08df952:::benchmark_data_smithi076_299450_object17543:head v 1303'1095, mlcod=1303'1092)", 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.954360+0000", 2024-09-21T10:16:46.997 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0017916270000000001, 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023432940000000001, 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "started", 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954360+0000", 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954360+0000", 2024-09-21T10:16:46.998 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954360+0000", 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.8999999999999998e-07 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954396+0000", 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.6134000000000003e-05 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:46.999 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954403+0000", 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7959999999999998e-06 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954408+0000", 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.634e-06 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955893+0000", 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001485583 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:16:47.000 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.955916+0000", 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.2824999999999999e-05 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_repop(client.35901.0:17546 10.f e1303/1301)", 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:16:46.954415+0000", 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 0.0017363120000000001, 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0023349260000000002, 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "queued for pg", 2024-09-21T10:16:47.001 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954415+0000", 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954415+0000", 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954416+0000", 2024-09-21T10:16:47.002 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.39e-07 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954446+0000", 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.0839000000000001e-05 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954452+0000", 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8119999999999996e-06 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:16:46.954456+0000", 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1999999999999996e-06 2024-09-21T10:16:47.003 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:47.004 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:16:47.004 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:47.004 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:16:47.004 INFO:teuthology.orchestra.run.smithi076.stdout: ], 2024-09-21T10:16:47.004 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 10 2024-09-21T10:16:47.004 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:16:47.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:47.066 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:47.168 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:47.201 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:47.228 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 5 16 18365 18349 229.34 195 0.00241789 0.00435597 2024-09-21T10:16:47.303 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:47.338 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:47.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:47.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:47.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:47.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:47.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:47.609 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:47 smithi076 ceph-mon[106482]: pgmap v1970: 89 pgs: 89 active+clean; 58 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 7.2 MiB/s wr, 115 op/s 2024-09-21T10:16:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:47 smithi076 ceph-mon[110963]: pgmap v1970: 89 pgs: 89 active+clean; 58 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 7.2 MiB/s wr, 115 op/s 2024-09-21T10:16:47.711 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:47.745 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:47.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:47.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:47.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:47 smithi191 ceph-mon[102206]: pgmap v1970: 89 pgs: 89 active+clean; 58 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 7.2 MiB/s wr, 115 op/s 2024-09-21T10:16:47.882 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:47.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:47.984 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:47.986 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:47.995 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:16:48.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:48.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:48.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.165 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:48.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:48.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 6 16 21373 21357 222.435 188 0.00343113 0.00449033 2024-09-21T10:16:48.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:48.299 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:48.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:48.437 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:48.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:48.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:48.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:48.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:48.713 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:48.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:48.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:48.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:48 smithi191 ceph-mon[102206]: pgmap v1971: 89 pgs: 89 active+clean; 861 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:16:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:48 smithi076 ceph-mon[106482]: pgmap v1971: 89 pgs: 89 active+clean; 861 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:16:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:48 smithi076 ceph-mon[110963]: pgmap v1971: 89 pgs: 89 active+clean; 861 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:16:48.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:48.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:48.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:49.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:49.119 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:49.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:49.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 7 16 24334 24318 217.094 185.062 0.00350766 0.00459997 2024-09-21T10:16:49.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:49.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:49.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:49.399 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:49.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:49.534 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:49.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:49.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:49.670 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:49.772 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:49.806 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:49.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:49.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:49.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:49.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:50.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:50.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:50.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:50.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:50.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 8 16 27018 27002 210.925 167.75 0.0052135 0.00473596 2024-09-21T10:16:50.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:50.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:50.446 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:50.466 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:50.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:50.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:50.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:50.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:50.722 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:50.824 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:50.846 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:50.948 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:50.982 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:50.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:51.083 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:51.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:51.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:51.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 9 16 29114 29098 202.041 131 0.0018482 0.00493516 2024-09-21T10:16:51.239 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:51.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:51.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:51.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:51 smithi076 ceph-mon[106482]: pgmap v1972: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-21T10:16:51.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:51 smithi076 ceph-mon[110963]: pgmap v1972: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-21T10:16:51.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:51.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:51.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:51.512 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:51.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:51.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:51 smithi191 ceph-mon[102206]: pgmap v1972: 89 pgs: 89 active+clean; 1.0 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-21T10:16:51.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:51.649 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:51.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:51.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:51.873 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:51.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:51.910 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:51.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:52.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:52.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:52.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:52.148 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:52.169 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:52.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 10 16 31028 31012 193.799 119.625 0.00725192 0.00515521 2024-09-21T10:16:52.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:52.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:52.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:52.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:52.547 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:52.581 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:52.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:52.717 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:52.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:52.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:52.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:52.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:52.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:52.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:53.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.115 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:53.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:53.138 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:53.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:53.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 11 16 33089 33073 187.89 128.812 0.00558741 0.00531693 2024-09-21T10:16:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:53.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:53.349 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:16:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:16:53] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.51.0" 2024-09-21T10:16:53.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:53 smithi076 ceph-mon[106482]: pgmap v1973: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-21T10:16:53.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:53 smithi076 ceph-mon[110963]: pgmap v1973: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-21T10:16:53.389 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.491 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:53 smithi191 ceph-mon[102206]: pgmap v1973: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-09-21T10:16:53.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.688 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6] out_osds: [7, 5, 0, 2] dead_osds: [0] live_osds: [1, 2, 4, 5, 6, 7, 3] 2024-09-21T10:16:53.688 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:16:53.689 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-21T10:16:53.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-21T10:16:53.762 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:53.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:53.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:53.934 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:53.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:54.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:54.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:54.173 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:54.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:54.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 12 16 35065 35049 182.523 123.5 0.00705328 0.00547445 2024-09-21T10:16:54.310 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.332 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:54.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:54.433 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:54.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:54.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:54.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.605 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:54.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:54.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:54.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:54.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:54.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:54.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:54.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:54.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:55.000 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:55.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:55.139 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:55.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:55.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 13 16 37026 37010 177.91 122.562 0.00796258 0.00561547 2024-09-21T10:16:55.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:55.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:55.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:55.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:55 smithi076 ceph-mon[106482]: pgmap v1974: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-09-21T10:16:55.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:55 smithi076 ceph-mon[110963]: pgmap v1974: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-09-21T10:16:55.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:55.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:55.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:55.540 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:55.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:55 smithi191 ceph-mon[102206]: pgmap v1974: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 157 MiB/s wr, 2.51k op/s 2024-09-21T10:16:55.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:55.665 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:55.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.802 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:55.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:55.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:55.924 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:55.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:56.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:56.063 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:56.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:56.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:56.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:56.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:56.230 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 14 16 38696 38680 172.657 104.375 0.00519735 0.00578 2024-09-21T10:16:56.299 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:56.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:56.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:56.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:56.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:56.599 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:56.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:56.701 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:56.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:56.823 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:56.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:56.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:56 smithi191 ceph-mon[102206]: pgmap v1975: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-21T10:16:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:56 smithi076 ceph-mon[106482]: pgmap v1975: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-21T10:16:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:16:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:16:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:56 smithi076 ceph-mon[110963]: pgmap v1975: 89 pgs: 89 active+clean; 1.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-09-21T10:16:56.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:56.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:56.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:56.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:57.085 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:57.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:57.229 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 15 16 40252 40236 167.629 97.25 0.00493314 0.0059602 2024-09-21T10:16:57.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:57.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:57.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:57.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:57.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:57.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:57.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:16:57.732 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:57.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:57.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:57.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:57.921 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:57.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:58.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:58.057 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:58.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:58.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:16:58.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:58.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:16:58.230 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 16 16 41994 41978 163.957 108.875 0.00628113 0.00609449 2024-09-21T10:16:58.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:58.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:58.429 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:58.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.464 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:58.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:58.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:16:58.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:16:58.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:58.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:58.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:58.871 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:58.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:58.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:59.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:59.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:16:59.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:16:59 smithi076 ceph-mon[106482]: pgmap v1976: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 348 GiB / 358 GiB avail; 193 MiB/s wr, 3.08k op/s 2024-09-21T10:16:59.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:16:59 smithi076 ceph-mon[110963]: pgmap v1976: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 348 GiB / 358 GiB avail; 193 MiB/s wr, 3.08k op/s 2024-09-21T10:16:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.142 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:59.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:16:59.213 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:16:59.231 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 17 16 44132 44116 162.161 133.625 0.00954187 0.00616132 2024-09-21T10:16:59.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:59.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:59.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:16:59.415 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:16:59.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:59.551 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:59.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:59.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:16:59 smithi191 ceph-mon[102206]: pgmap v1976: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 348 GiB / 358 GiB avail; 193 MiB/s wr, 3.08k op/s 2024-09-21T10:16:59.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:16:59.687 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:16:59.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:16:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.821 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:16:59.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:16:59.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:16:59.955 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:16:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:16:59.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:00.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:00.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:00.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:00.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:00.232 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 18 16 46180 46164 160.263 128 0.00673029 0.00623499 2024-09-21T10:17:00.332 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:00.369 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:00.471 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:00.508 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:00.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:00.643 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:00.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:00.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:00.783 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:00.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:00.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:00.922 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:00.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:00.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:01.024 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:01.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:01.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:01.188 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:01.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:01.231 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 19 16 48054 48038 157.992 117.125 0.00448212 0.0063244 2024-09-21T10:17:01.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:01.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:01.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:01.426 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:01.450 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:01 smithi076 ceph-mon[106482]: pgmap v1977: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-21T10:17:01.450 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:01 smithi076 ceph-mon[110963]: pgmap v1977: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-21T10:17:01.463 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:01.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:01.591 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:01.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:01 smithi191 ceph-mon[102206]: pgmap v1977: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-21T10:17:01.693 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:01.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:01.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:01.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:01.867 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:01.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:01.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:02.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.016 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:02.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:02.117 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:02.132 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:17:02.150 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:02.231 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:17:02.230207+0000 min lat: 0.00112657 max lat: 0.219269 avg lat: 0.00642849 2024-09-21T10:17:02.231 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:17:02.231 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 20 16 49763 49747 155.433 106.812 0.00673327 0.00642849 2024-09-21T10:17:02.252 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:02.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:02.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:02.365 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/690485388' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:02.365 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:02 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:02.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/690485388' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:02.365 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:02 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:02.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:02.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:02.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:02.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:02.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/690485388' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:02 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:02.662 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:02.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:02.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:02.830 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:02.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:02.970 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:02.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:03.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:03.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:03.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:03.233 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 21 16 51689 51673 153.751 120.375 0.0146063 0.00649932 2024-09-21T10:17:03.239 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:03.341 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:03.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:03.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[106482]: pgmap v1978: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 158 MiB/s wr, 2.52k op/s 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[106482]: osdmap e1304: 8 total, 7 up, 4 in 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:17:03] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.51.0" 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[110963]: pgmap v1978: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 158 MiB/s wr, 2.52k op/s 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:03.487 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:03.488 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:03 smithi076 ceph-mon[110963]: osdmap e1304: 8 total, 7 up, 4 in 2024-09-21T10:17:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:03.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:03 smithi191 ceph-mon[102206]: pgmap v1978: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 158 MiB/s wr, 2.52k op/s 2024-09-21T10:17:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:03 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:03 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:03 smithi191 ceph-mon[102206]: osdmap e1304: 8 total, 7 up, 4 in 2024-09-21T10:17:03.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:03.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:03.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:03.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:03.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-21T10:17:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.798 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:03.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:03.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:03.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:03.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:03.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:04.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:04.077 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:04.179 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:04.211 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:04.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:04.234 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 22 16 53538 53522 152.009 115.562 0.00752847 0.00657347 2024-09-21T10:17:04.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:04.345 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:04.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:04.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:04.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:04.622 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:04.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:04.724 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:04.760 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:04.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:04.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:04.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:04.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:04.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:04.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:05.039 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:05.101 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:05.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:05.151 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:17:05.152 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:17:05.152 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:17:05.152 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:17:05.152 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.152 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145309 8.14 8:2cd4fe9c:::benchmark_data_smithi076_241159_object22391:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.153 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:17:05.153 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 413.03639320100001, 2024-09-21T10:17:05.153 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.067794157, 2024-09-21T10:17:05.153 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.153 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.153 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.154 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.154 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.154 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.154 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145309 2024-09-21T10:17:05.154 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.154 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.155 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.155 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.155 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:17:05.155 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.155 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.155 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.156 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107182+0000", 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.843e-06 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107187+0000", 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8749999999999999e-06 2024-09-21T10:17:05.157 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.158 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.158 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.158 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107196+0000", 2024-09-21T10:17:05.158 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3200000000000006e-06 2024-09-21T10:17:05.158 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.158 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.159 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.159 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107206+0000", 2024-09-21T10:17:05.159 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0093e-05 2024-09-21T10:17:05.159 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.159 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.159 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.160 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107256+0000", 2024-09-21T10:17:05.160 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0154999999999997e-05 2024-09-21T10:17:05.160 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.160 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.160 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.160 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107265+0000", 2024-09-21T10:17:05.161 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2089999999999994e-06 2024-09-21T10:17:05.161 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.161 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.161 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.161 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.170814+0000", 2024-09-21T10:17:05.161 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06354872 2024-09-21T10:17:05.162 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.162 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.162 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.162 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.170905+0000", 2024-09-21T10:17:05.162 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0631999999999996e-05 2024-09-21T10:17:05.162 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.163 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.163 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.163 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.171114+0000", 2024-09-21T10:17:05.163 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000209796 2024-09-21T10:17:05.163 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.163 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.164 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.164 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174576+0000", 2024-09-21T10:17:05.164 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034615789999999998 2024-09-21T10:17:05.164 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.164 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.164 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.165 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174585+0000", 2024-09-21T10:17:05.165 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0680000000000003e-06 2024-09-21T10:17:05.165 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.165 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.165 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.165 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174717+0000", 2024-09-21T10:17:05.166 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000131817 2024-09-21T10:17:05.166 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.166 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.166 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.166 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174784+0000", 2024-09-21T10:17:05.166 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7360999999999994e-05 2024-09-21T10:17:05.167 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.167 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.167 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.167 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174850+0000", 2024-09-21T10:17:05.167 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6104000000000003e-05 2024-09-21T10:17:05.167 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.168 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.168 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.168 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174974+0000", 2024-09-21T10:17:05.168 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000123585 2024-09-21T10:17:05.168 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.168 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.169 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.169 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.169 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.169 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145327 8.14 8:2bc8e99d:::benchmark_data_smithi076_241159_object22409:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.169 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:17:05.169 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 413.03634732900002, 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06996614, 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.170 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.171 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145327 2024-09-21T10:17:05.171 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.171 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.171 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.171 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.171 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.172 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0999999999999998e-07 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107229+0000", 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7199999999999998e-06 2024-09-21T10:17:05.173 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107238+0000", 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.755e-06 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107244+0000", 2024-09-21T10:17:05.174 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.392e-06 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107296+0000", 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2200999999999997e-05 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.175 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107303+0000", 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6379999999999998e-06 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172493+0000", 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.065190184 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172625+0000", 2024-09-21T10:17:05.176 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013188800000000001 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172812+0000", 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018733899999999999 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174726+0000", 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00191332 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176604+0000", 2024-09-21T10:17:05.177 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001878111 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176883+0000", 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027873299999999998 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177019+0000", 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013593700000000001 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.178 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177079+0000", 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0439000000000002e-05 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177192+0000", 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112873 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.179 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145341 8.14 8:2a80dff4:::benchmark_data_smithi076_241159_object22423:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 413.03632052400002, 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.070240276, 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145341 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.180 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107253+0000", 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4000000000000002e-07 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.181 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107259+0000", 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0270000000000001e-06 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107266+0000", 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.002e-06 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.182 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107273+0000", 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9380000000000003e-06 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108301+0000", 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010276829999999999 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108306+0000", 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2390000000000002e-06 2024-09-21T10:17:05.183 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173660+0000", 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0653539890000001 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173734+0000", 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3981999999999997e-05 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173865+0000", 2024-09-21T10:17:05.184 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013074199999999999 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176550+0000", 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002685532 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176826+0000", 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027604099999999998 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.185 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177260+0000", 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00043419999999999998 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177334+0000", 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3385000000000003e-05 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.186 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177396+0000", 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1779999999999995e-05 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177493+0000", 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7195999999999996e-05 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.187 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145379 8.14 8:2e4b539b:::benchmark_data_smithi076_241159_object22461:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 413.03625361600001, 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0704400940000001, 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145379 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.188 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107320+0000", 2024-09-21T10:17:05.189 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0800000000000004e-07 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107324+0000", 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.32e-06 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107332+0000", 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6440000000000006e-06 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.190 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107338+0000", 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8980000000000001e-06 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108338+0000", 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00099989499999999999 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.191 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108346+0000", 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0250000000000002e-06 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.192 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175028+0000", 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06668181 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175119+0000", 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0938000000000002e-05 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.193 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175301+0000", 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018216400000000001 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.194 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176843+0000", 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015422599999999999 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177544+0000", 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00070127500000000001 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.195 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177598+0000", 2024-09-21T10:17:05.196 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3390000000000002e-05 2024-09-21T10:17:05.196 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.196 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.196 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.196 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177645+0000", 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7596000000000003e-05 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177677+0000", 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1538999999999997e-05 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.197 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.198 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.198 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177759+0000", 2024-09-21T10:17:05.198 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2532000000000002e-05 2024-09-21T10:17:05.198 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.198 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.198 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145428 8.14 8:28ec7a73:::benchmark_data_smithi076_241159_object22510:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 413.03622650699998, 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0832395619999999, 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.199 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145428 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:17:05.200 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0999999999999999e-07 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.201 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107349+0000", 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0020000000000001e-06 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107355+0000", 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9989999999999997e-06 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107360+0000", 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.202 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108360+0000", 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010003779999999999 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108364+0000", 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.506e-06 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177840+0000", 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.069476358 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.203 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177971+0000", 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130772 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.178223+0000", 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025209899999999999 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183904+0000", 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056812440000000002 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.204 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183937+0000", 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2718000000000001e-05 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.186948+0000", 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030109640000000001 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190363+0000", 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003414753 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.205 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190424+0000", 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1289000000000002e-05 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190586+0000", 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016216000000000001 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145565 8.14 8:2ab87a67:::benchmark_data_smithi076_241159_object22647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.206 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.91929056200001, 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96663625200000003, 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145565 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.207 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224284+0000", 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.812e-06 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.208 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224290+0000", 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.874e-06 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224308+0000", 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8093000000000001e-05 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224317+0000", 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7960000000000001e-06 2024-09-21T10:17:05.209 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224386+0000", 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8879000000000002e-05 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224395+0000", 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6079999999999996e-06 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.210 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.211 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179328+0000", 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95493383300000001 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179451+0000", 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012236699999999999 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179658+0000", 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020740600000000001 2024-09-21T10:17:05.212 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183981+0000", 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043224200000000004 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.186993+0000", 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030123340000000002 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.213 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189973+0000", 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029805529999999999 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190741+0000", 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000767269 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190808+0000", 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7385999999999995e-05 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.214 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190919+0000", 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110622 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145636 8.14 8:2c7856f1:::benchmark_data_smithi076_241159_object22718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.86426160399998, 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91186562000000004, 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.215 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145636 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.216 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279313+0000", 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.612e-06 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279318+0000", 2024-09-21T10:17:05.217 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4700000000000001e-06 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279329+0000", 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0521e-05 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279336+0000", 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4960000000000001e-06 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.218 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279390+0000", 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4639000000000003e-05 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279399+0000", 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4519999999999997e-06 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.180800+0000", 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90140108500000005 2024-09-21T10:17:05.219 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.180917+0000", 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011759199999999999 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.181105+0000", 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000187345 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184031+0000", 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029260890000000002 2024-09-21T10:17:05.220 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187076+0000", 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030450529999999998 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189983+0000", 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029071829999999998 2024-09-21T10:17:05.221 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191024+0000", 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010408940000000001 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191081+0000", 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7377000000000003e-05 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191177+0000", 2024-09-21T10:17:05.222 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5811999999999997e-05 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145642 8.14 8:2c2c43d9:::benchmark_data_smithi076_241159_object22724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:17:05.223 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.85852544599999, 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90634175100000003, 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145642 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.224 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285049+0000", 2024-09-21T10:17:05.225 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5689999999999999e-06 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285054+0000", 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5680000000000001e-06 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285069+0000", 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5790999999999998e-05 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.226 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285077+0000", 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1380000000000001e-06 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285131+0000", 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4712000000000001e-05 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285140+0000", 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5930000000000006e-06 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.227 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182422+0000", 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89728208899999995 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182525+0000", 2024-09-21T10:17:05.228 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102975 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182730+0000", 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020512300000000001 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184080+0000", 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0013496820000000001 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187121+0000", 2024-09-21T10:17:05.229 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030413240000000002 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189992+0000", 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028713139999999998 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191253+0000", 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012601559999999999 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.230 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191297+0000", 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4267000000000002e-05 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191389+0000", 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2449999999999997e-05 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.231 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145647 8.14 8:2cdee3d5:::benchmark_data_smithi076_241159_object22729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.85393990099999, 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90206547199999998, 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145647 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.232 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289635+0000", 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.629e-06 2024-09-21T10:17:05.233 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289640+0000", 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5609999999999998e-06 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289653+0000", 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2500999999999999e-05 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.234 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289660+0000", 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7909999999999999e-06 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289716+0000", 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6434999999999998e-05 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289725+0000", 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.568e-06 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.235 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184119+0000", 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89439432600000002 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184251+0000", 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013181299999999999 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184465+0000", 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021390800000000001 2024-09-21T10:17:05.236 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187033+0000", 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025682499999999998 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190155+0000", 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031220950000000001 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190600+0000", 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00044475 2024-09-21T10:17:05.237 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191502+0000", 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00090238500000000001 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191592+0000", 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9889999999999995e-05 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191699+0000", 2024-09-21T10:17:05.238 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010657 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145668 8.14 8:2c4067c8:::benchmark_data_smithi076_241159_object22750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.83408785099999, 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88250502399999997, 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.239 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145668 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.240 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309487+0000", 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7969999999999999e-06 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309492+0000", 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0799999999999996e-06 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309506+0000", 2024-09-21T10:17:05.241 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4337e-05 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309537+0000", 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1062000000000002e-05 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.242 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309583+0000", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5954000000000001e-05 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309592+0000", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8950000000000004e-06 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185570+0000", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87597768499999995 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185659+0000", 2024-09-21T10:17:05.243 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8683000000000005e-05 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185852+0000", 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019369 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190093+0000", 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042406789999999998 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190606+0000", 2024-09-21T10:17:05.244 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00051294199999999996 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190699+0000", 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2977000000000001e-05 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191748+0000", 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00104901 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191835+0000", 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6559999999999998e-05 2024-09-21T10:17:05.245 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191990+0000", 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000155673 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145696 8.14 8:2d97a104:::benchmark_data_smithi076_241159_object22778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.80933451200002, 2024-09-21T10:17:05.246 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.863836041, 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145696 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.247 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334240+0000", 2024-09-21T10:17:05.248 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.654e-06 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334246+0000", 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.755e-06 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334262+0000", 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6543000000000001e-05 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.249 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334271+0000", 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1829999999999994e-06 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334339+0000", 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8495000000000001e-05 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334349+0000", 2024-09-21T10:17:05.250 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8679999999999994e-06 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187155+0000", 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85280550300000002 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187267+0000", 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011253500000000001 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187487+0000", 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021978000000000001 2024-09-21T10:17:05.251 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190612+0000", 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031255240000000002 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190657+0000", 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4929999999999998e-05 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.252 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191219+0000", 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056208899999999999 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197900+0000", 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066807790000000004 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197963+0000", 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2599000000000002e-05 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.253 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.198075+0000", 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000111804 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145737 8.14 8:2e5266a3:::benchmark_data_smithi076_241159_object22819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.77202020700003, 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.82838703400000002, 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.254 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.255 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.255 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.255 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:05.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145737 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.257 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371554+0000", 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5710000000000001e-06 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371559+0000", 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.101e-06 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.258 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371573+0000", 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3237999999999999e-05 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371580+0000", 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1770000000000001e-06 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371638+0000", 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8353999999999997e-05 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.259 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371644+0000", 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5829999999999997e-06 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188522+0000", 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.81687860199999995 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.260 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188653+0000", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013063699999999999 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188863+0000", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020946300000000001 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190934+0000", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020711869999999999 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.261 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190975+0000", 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1170000000000001e-05 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191443+0000", 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00046765700000000001 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199763+0000", 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0083201890000000004 2024-09-21T10:17:05.262 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199822+0000", 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8839999999999999e-05 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199940+0000", 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011826499999999999 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.263 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145790 8.14 8:29323519:::benchmark_data_smithi076_241159_object22872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.71203642299997, 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76945549899999999, 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145790 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.264 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431538+0000", 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.821e-06 2024-09-21T10:17:05.265 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431544+0000", 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3539999999999999e-06 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431558+0000", 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4664999999999999e-05 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.266 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431566+0000", 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1099999999999997e-06 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431630+0000", 2024-09-21T10:17:05.267 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4051000000000001e-05 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431644+0000", 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4489e-05 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192021+0000", 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76037687799999998 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.268 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192123+0000", 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102094 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192373+0000", 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024965499999999999 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197733+0000", 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053602290000000002 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.269 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197778+0000", 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4932999999999998e-05 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199979+0000", 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022014389999999999 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200021+0000", 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2040999999999999e-05 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.270 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200873+0000", 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00085203000000000004 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200992+0000", 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011870999999999999 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145960 8.14 8:2d57b90e:::benchmark_data_smithi076_241159_object23042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.271 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.51442197699998, 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.58195981299999999, 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145960 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.272 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.273 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629153+0000", 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8870000000000001e-06 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629158+0000", 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0730000000000004e-06 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629172+0000", 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4076e-05 2024-09-21T10:17:05.274 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629181+0000", 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0329999999999997e-06 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629231+0000", 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0161000000000003e-05 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629239+0000", 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0839999999999992e-06 2024-09-21T10:17:05.275 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193615+0000", 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.56437558099999996 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193705+0000", 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0105999999999994e-05 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193902+0000", 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019732800000000001 2024-09-21T10:17:05.276 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197795+0000", 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038931259999999998 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197850+0000", 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4941000000000001e-05 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201036+0000", 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031852819999999998 2024-09-21T10:17:05.277 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210835+0000", 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097991099999999998 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210934+0000", 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9510000000000001e-05 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211111+0000", 2024-09-21T10:17:05.278 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000176515 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145998 8.14 8:292bcf47:::benchmark_data_smithi076_241159_object23080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.424674676, 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.49255907199999999, 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.279 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145998 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.280 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718900+0000", 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9690000000000001e-06 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718906+0000", 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4670000000000003e-06 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718925+0000", 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9351000000000002e-05 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.281 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718931+0000", 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6620000000000002e-06 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718991+0000", 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0074000000000001e-05 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.719002+0000", 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0701e-05 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.282 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196200+0000", 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.477198858 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196319+0000", 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118368 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.283 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196556+0000", 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023754200000000001 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201345+0000", 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0047884770000000002 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.284 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210791+0000", 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0094463680000000001 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211125+0000", 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00033422200000000001 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211296+0000", 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000170445 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.285 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211355+0000", 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9215999999999997e-05 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211457+0000", 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102352 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.286 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146042 8.14 8:28c57b04:::benchmark_data_smithi076_241159_object23124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 412.29679718, 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.36551782100000002, 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146042 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.287 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846778+0000", 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7630000000000001e-06 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.288 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846783+0000", 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1340000000000003e-06 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846794+0000", 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1486e-05 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846800+0000", 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6969999999999998e-06 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.289 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846850+0000", 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0513999999999998e-05 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846856+0000", 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:17:05.290 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201389+0000", 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.35453316699999998 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201518+0000", 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012838199999999999 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201830+0000", 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031166100000000002 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.291 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211170+0000", 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.009340447 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211211+0000", 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1372000000000003e-05 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211254+0000", 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2997999999999997e-05 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.292 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211964+0000", 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00070925999999999997 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.212193+0000", 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000229161 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.212294+0000", 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010087899999999999 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.293 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.294 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:17:05.294 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:17:05.294 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:17:05.294 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 23 16 55320 55304 150.243 111.375 0.00445306 0.00665103 2024-09-21T10:17:05.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:05.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:05 smithi076 ceph-mon[106482]: pgmap v1980: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:17:05.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:05 smithi076 ceph-mon[110963]: pgmap v1980: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:17:05.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:05.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:05.495 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.531 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:05.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:05.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:05 smithi191 ceph-mon[102206]: pgmap v1980: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:17:05.632 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:05.670 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:05.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:05.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:05.806 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:05.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:05.847 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:05.848 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:05.863 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:17:05.863 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:05.894 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:17:05.894 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:05.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:05.930 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:17:05.930 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:05.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:05.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:06.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.093 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:06.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:06.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:06.225 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:06.234 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 24 16 57067 57051 148.532 109.188 0.00879251 0.00672779 2024-09-21T10:17:06.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:17:06.326 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:06.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:06.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:06.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:06.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4269783613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:06.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4269783613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:06.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4269783613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:06.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:06.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:06.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:06.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:06.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:06.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:06.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:06.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:17:07.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:07.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:07.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:07.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:07.234 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 25 16 58802 58786 146.928 108.438 0.00822173 0.00680123 2024-09-21T10:17:07.296 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:07 smithi076 ceph-mon[106482]: pgmap v1981: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:17:07.296 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:07 smithi076 ceph-mon[110963]: pgmap v1981: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:17:07.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:07.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:07.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:07 smithi191 ceph-mon[102206]: pgmap v1981: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-09-21T10:17:07.452 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:07.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:07.534 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:07.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:07.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:07.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:07.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:07.791 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:07.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:07.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:07.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:07.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:08.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:08.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:08.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:08.211 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:08.214 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:17:08.215 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:17:08.234 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 26 16 60414 60398 145.152 100.75 0.00940424 0.00688479 2024-09-21T10:17:08.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:08.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.350 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:08.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:08.451 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:08.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:08.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:08.613 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:08.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:08.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:08.852 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:08.874 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:08.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:08.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:08.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:09.020 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:09.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:09.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:09.235 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 27 16 61976 61960 143.391 97.625 0.0055697 0.00696944 2024-09-21T10:17:09.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:09.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:09.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:09 smithi191 ceph-mon[102206]: pgmap v1982: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-21T10:17:09.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:09 smithi076 ceph-mon[106482]: pgmap v1982: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-21T10:17:09.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:09 smithi076 ceph-mon[110963]: pgmap v1982: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-21T10:17:09.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:09.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:09.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:09.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:09.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:09.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:09.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:09.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:09.937 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:09.976 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:09.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:10.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:10.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:10.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:10.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 28 16 63588 63572 141.868 100.75 0.00837844 0.0070447 2024-09-21T10:17:10.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:10.258 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:10.259 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:10.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:10.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:10.376 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:10.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:10.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:10.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:10.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:10.696 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:10.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:10.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:10.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:10.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:10.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:10.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:17:11.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-21T10:17:11.074 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:11.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:11.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:11.235 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 29 16 65151 65135 140.343 97.6875 0.00995535 0.00712028 2024-09-21T10:17:11.264 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:11.277 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:11 smithi076 ceph-mon[106482]: pgmap v1983: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-21T10:17:11.277 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:11.278 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:11 smithi076 ceph-mon[110963]: pgmap v1983: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-21T10:17:11.278 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:11.347 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:11.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:11 smithi191 ceph-mon[102206]: pgmap v1983: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-21T10:17:11.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:11.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:11.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:11.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:11.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:11.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:11.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:11.657 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:11.688 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:11.710 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:11.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:11.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:11.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:11.988 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:11.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:11.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:12.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.123 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:12.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:12.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:12.235 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 30 16 66554 66538 138.587 87.6875 0.0104924 0.0072106 2024-09-21T10:17:12.254 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:12.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:12.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:12.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:12.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:12.543 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:12.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:12.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:12.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:12.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:12.787 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.824 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:12.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:12.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:12.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:12.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:13.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:13.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:13.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:13.181 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:17:13.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:13.235 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 31 16 67853 67837 136.736 81.1875 0.00745494 0.00730873 2024-09-21T10:17:13.247 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:13.348 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:13 smithi191 ceph-mon[102206]: pgmap v1984: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-21T10:17:13.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2184081545' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:13.378 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:13.480 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:13.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:13.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:13.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:13.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:13 smithi076 ceph-mon[106482]: pgmap v1984: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-21T10:17:13.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2184081545' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:17:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:17:13] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.51.0" 2024-09-21T10:17:13.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:13 smithi076 ceph-mon[110963]: pgmap v1984: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-21T10:17:13.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2184081545' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:13.651 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:13.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:13.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:13.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:13.925 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:13.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:14.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:14.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:14.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:14.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:14.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:14.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:14.235 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 32 16 69203 69187 135.099 84.375 0.00726324 0.00739732 2024-09-21T10:17:14.304 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:14 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2184081545' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:14.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:14 smithi076 ceph-mon[110963]: osdmap e1305: 8 total, 7 up, 4 in 2024-09-21T10:17:14.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:14 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:14.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2184081545' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:14.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:14 smithi076 ceph-mon[106482]: osdmap e1305: 8 total, 7 up, 4 in 2024-09-21T10:17:14.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:14.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.472 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:14.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:14.573 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:14.585 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:14 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:17:14.585 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2184081545' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:14.586 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:14 smithi191 ceph-mon[102206]: osdmap e1305: 8 total, 7 up, 4 in 2024-09-21T10:17:14.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:14.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:14.716 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:14.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:14.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:14.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:14.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:14.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.018 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:15.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:15.236 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 33 16 70642 70626 133.73 89.9375 0.0124022 0.00747292 2024-09-21T10:17:15.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:15.298 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:15.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:15 smithi076 ceph-mon[106482]: pgmap v1986: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:17:15.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:15 smithi076 ceph-mon[110963]: pgmap v1986: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:17:15.400 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:15.430 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:15.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:15.558 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:15.558 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:15.559 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:15 smithi191 ceph-mon[102206]: pgmap v1986: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:17:15.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:15.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:15.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:15.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:15.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:15.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:15.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:15.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:15.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:15.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:15.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:16.067 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:17:16.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:16.097 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:17:16.097 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:16.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:16.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:16.132 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:17:16.133 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:16.167 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:17:16.167 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:16.194 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:17:16.194 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:16.220 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:16.227 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:17:16.227 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:16.236 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 34 16 72183 72167 132.626 96.3125 0.0095962 0.00753496 2024-09-21T10:17:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.258 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:16.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:16.260 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:17:16.260 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:16.288 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:17:16.288 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:16.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:16.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2098912786' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:16.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2098912786' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:16.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.399 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:16.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:16.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:17:16.490 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2098912786' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:16.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:16.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:16.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:16.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:16.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:16.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:16.801 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:16.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:16.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:16.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:16.951 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:16.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:16.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:17.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:17.090 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:17.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:17.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:17.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:17.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:17.236 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 35 16 73625 73609 131.412 90.125 0.00980323 0.00760511 2024-09-21T10:17:17.333 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:17.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:17.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:17.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:17.505 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:17.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:17.531 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:17 smithi076 ceph-mon[106482]: pgmap v1987: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:17:17.532 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:17 smithi076 ceph-mon[110963]: pgmap v1987: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:17:17.574 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:17:17.575 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:17:17.606 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:17.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:17 smithi191 ceph-mon[102206]: pgmap v1987: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-21T10:17:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.643 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:17.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:17.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:17.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:17.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:17:17.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:17.940 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:17.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:17.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:18.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:18.085 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:18.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:18.187 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:18.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:18.236 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 36 16 75037 75021 130.213 88.25 0.00662287 0.00767475 2024-09-21T10:17:18.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:18.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:18.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:18.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:18.588 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:18.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:18.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:18.710 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:18.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:18.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:18.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:18.877 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:18.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:18.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:18.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:18.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-21T10:17:18.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:19.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:19.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.153 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:19.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 37 16 76241 76225 128.727 75.25 0.011931 0.00776304 2024-09-21T10:17:19.254 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:19.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:19.436 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:19.538 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:19.570 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:19 smithi076 ceph-mon[106482]: pgmap v1988: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-21T10:17:19.570 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:19 smithi076 ceph-mon[110963]: pgmap v1988: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-21T10:17:19.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.571 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:19.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:19.598 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:19 smithi191 ceph-mon[102206]: pgmap v1988: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-21T10:17:19.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.707 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:19.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:19.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:19.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:19.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:19.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:20.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.100 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:20.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:20.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 38 16 77564 77548 127.515 82.6875 0.0103639 0.00783771 2024-09-21T10:17:20.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:20.330 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:17:20.330 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:17:20.330 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45332.0:238255 8.4 e1257/1220 8:2731efb3:::benchmark_data_smithi076_241159_object115337:head v 1257'11498, mlcod=1257'11496)", 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:11:42.356221+0000", 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 337.96838568999999, 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036549491000000003, 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356221+0000", 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.331 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356221+0000", 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356220+0000", 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.332 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356227+0000", 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5080000000000002e-06 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356236+0000", 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8740000000000007e-06 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356242+0000", 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0480000000000004e-06 2024-09-21T10:17:20.333 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356285+0000", 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2250999999999999e-05 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.356302+0000", 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7482e-05 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.392709+0000", 2024-09-21T10:17:20.334 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.036407321999999999 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.392770+0000", 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0650999999999999e-05 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.45332.0:238253 8.c e1257/1208 8:328d83ec:::benchmark_data_smithi076_241159_object115335:head v 1257'14806, mlcod=1257'14804)", 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:11:42.358378+0000", 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 337.966228339, 2024-09-21T10:17:20.335 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034418753000000003, 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.358378+0000", 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.336 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.358378+0000", 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.358377+0000", 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.358385+0000", 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2970000000000004e-06 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.337 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.385774+0000", 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.027388597000000001 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.385787+0000", 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2961e-05 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.387236+0000", 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0014492750000000001 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.338 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.387258+0000", 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2325999999999999e-05 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.392775+0000", 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0055163879999999997 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:11:42.392797+0000", 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2036999999999999e-05 2024-09-21T10:17:20.339 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27517 10.d e1303/1301 10:b3f91c28:::benchmark_data_smithi076_299450_object27516:head v 1303'1716, mlcod=1303'1714)", 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414166+0000", 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.910440212000001, 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18228551500000001, 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.340 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414166+0000", 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414166+0000", 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414165+0000", 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.341 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414187+0000", 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1362999999999999e-05 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414197+0000", 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0569e-05 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.342 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414203+0000", 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8239999999999996e-06 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414243+0000", 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.002e-05 2024-09-21T10:17:20.343 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414263+0000", 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0213000000000001e-05 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.344 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596375+0000", 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.182111997 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596452+0000", 2024-09-21T10:17:20.345 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6447999999999997e-05 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27522 10.5 e1303/1301 10:a9158b13:::benchmark_data_smithi076_299450_object27521:head v 1303'1686, mlcod=1303'1684)", 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414175+0000", 2024-09-21T10:17:20.346 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.910431228, 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18425534199999999, 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.347 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414175+0000", 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414175+0000", 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.348 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.349 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.349 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414174+0000", 2024-09-21T10:17:20.349 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.349 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.349 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414238+0000", 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3662000000000001e-05 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414248+0000", 2024-09-21T10:17:20.350 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0324e-05 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414254+0000", 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1349999999999997e-06 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414395+0000", 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140345 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.351 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414419+0000", 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4470000000000001e-05 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598374+0000", 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.183954642 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598430+0000", 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6586999999999999e-05 2024-09-21T10:17:20.352 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27520 10.b e1303/1296 10:de5038e7:::benchmark_data_smithi076_299450_object27519:head v 1303'1709, mlcod=1303'1707)", 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.9103919, 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.184107515, 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:17:20.353 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.354 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414233+0000", 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8686e-05 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414239+0000", 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2450000000000003e-06 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414243+0000", 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3560000000000003e-06 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.355 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414334+0000", 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0291999999999995e-05 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414351+0000", 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6980999999999999e-05 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.597786+0000", 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18343555 2024-09-21T10:17:20.356 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598322+0000", 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00053571599999999999 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27519 10.0 e1303/1296 10:003e95b4:::benchmark_data_smithi076_299450_object27518:head v 1303'1711, mlcod=1303'1708)", 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.910352875000001, 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18245903899999999, 2024-09-21T10:17:20.357 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.358 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414271+0000", 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7893e-05 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414277+0000", 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.48e-06 2024-09-21T10:17:20.359 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414280+0000", 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8399999999999997e-06 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414310+0000", 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0008000000000001e-05 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414326+0000", 2024-09-21T10:17:20.360 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5532e-05 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596671+0000", 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18234499400000001 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596713+0000", 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1530999999999999e-05 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.361 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27523 10.c e1303/1301 10:3b9534aa:::benchmark_data_smithi076_299450_object27522:head v 1303'1837, mlcod=1303'1835)", 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.910345459999998, 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.182033742, 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.362 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.363 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.363 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.363 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:17:20.363 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.363 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414305+0000", 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4935000000000003e-05 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.365 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414315+0000", 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3649999999999993e-06 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414320+0000", 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6949999999999999e-06 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414374+0000", 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4620999999999998e-05 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.366 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414393+0000", 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9202e-05 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596251+0000", 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18185779899999999 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596295+0000", 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3504999999999998e-05 2024-09-21T10:17:20.367 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27518 10.6 e1303/1296 10:65ba96f9:::benchmark_data_smithi076_299450_object27517:head v 1303'1721, mlcod=1303'1718)", 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414731+0000", 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.909875668000002, 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.183657557, 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.368 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414731+0000", 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414731+0000", 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414730+0000", 2024-09-21T10:17:20.369 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414818+0000", 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8442999999999993e-05 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595101+0000", 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18028290699999999 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.370 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595114+0000", 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2323999999999999e-05 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595257+0000", 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014315899999999999 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595278+0000", 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1328e-05 2024-09-21T10:17:20.371 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598304+0000", 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003025537 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598388+0000", 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4610999999999993e-05 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.372 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27524 10.3 e1303/1301 10:cb011d2f:::benchmark_data_smithi076_299450_object27523:head v 1303'1719, mlcod=1303'1716)", 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.909585015000001, 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18332594199999999, 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.373 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.374 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415082+0000", 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1815000000000004e-05 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595106+0000", 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.180023974 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595118+0000", 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2024999999999999e-05 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.375 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595186+0000", 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7844999999999998e-05 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595212+0000", 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5483999999999999e-05 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598327+0000", 2024-09-21T10:17:20.376 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031148769999999998 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598347+0000", 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0596999999999999e-05 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27521 10.d e1303/1301 10:b20dfcd0:::benchmark_data_smithi076_299450_object27520:head v 1303'1717, mlcod=1303'1714)", 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.416052+0000", 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.908554456000001, 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18245150199999999, 2024-09-21T10:17:20.377 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416052+0000", 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.378 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416052+0000", 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416051+0000", 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.379 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416072+0000", 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0950000000000001e-05 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595128+0000", 2024-09-21T10:17:20.380 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17905605899999999 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595134+0000", 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4970000000000001e-06 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.381 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595327+0000", 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019367000000000001 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595345+0000", 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.785e-05 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.382 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598435+0000", 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003089699 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598503+0000", 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8544e-05 2024-09-21T10:17:20.383 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:28115 10.6 e1303/1296 10:6ba82360:::benchmark_data_smithi076_299450_object28114:head v 1303'1755, mlcod=1303'1753)", 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.807017+0000", 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.517588955000001, 2024-09-21T10:17:20.384 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.035477304000000001, 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807017+0000", 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.385 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807017+0000", 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807017+0000", 2024-09-21T10:17:20.386 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807157+0000", 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000139658 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.387 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807165+0000", 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9370000000000004e-06 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807169+0000", 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2200000000000003e-06 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.388 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807196+0000", 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7563999999999999e-05 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807214+0000", 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7509000000000001e-05 2024-09-21T10:17:20.389 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.808307+0000", 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001092911 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.842495+0000", 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.034187901999999999 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.390 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:28117 10.8 e1303/1301 10:103e7692:::benchmark_data_smithi076_299450_object28116:head v 1303'1685, mlcod=1303'1683)", 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.807297+0000", 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 29.517309118, 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.035216337, 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807297+0000", 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.391 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807297+0000", 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807297+0000", 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807375+0000", 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8374999999999994e-05 2024-09-21T10:17:20.392 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807384+0000", 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5909999999999996e-06 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807389+0000", 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3020000000000002e-06 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.393 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807436+0000", 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7302000000000002e-05 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.807449+0000", 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2272999999999999e-05 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.842484+0000", 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.035035076999999998 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.394 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.842514+0000", 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9975e-05 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76094 10.0 e1305/1296 10:0695fe9f:::benchmark_data_smithi076_299450_object76093:head v 1305'4744, mlcod=1305'4741)", 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.099315+0000", 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.2252912840000001, 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037246716999999999, 2024-09-21T10:17:20.395 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099315+0000", 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099315+0000", 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.396 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099314+0000", 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099343+0000", 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9056000000000001e-05 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099356+0000", 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3165e-05 2024-09-21T10:17:20.397 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099363+0000", 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0049999999999998e-06 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099435+0000", 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1920000000000003e-05 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.099454+0000", 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8128e-05 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.136470+0000", 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.037016694000000003 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.136562+0000", 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1561e-05 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.399 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76099 10.9 e1305/1301 10:9171c6b7:::benchmark_data_smithi076_299450_object76098:head v 1305'4799, mlcod=1305'4797)", 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.100879+0000", 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.223727368, 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038372087999999999, 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100879+0000", 2024-09-21T10:17:20.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100879+0000", 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100878+0000", 2024-09-21T10:17:20.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100918+0000", 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0411000000000003e-05 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100932+0000", 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4096e-05 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100940+0000", 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5610000000000001e-06 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100990+0000", 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0087999999999999e-05 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101010+0000", 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0125000000000001e-05 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139166+0000", 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038155441999999998 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139251+0000", 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5529000000000001e-05 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76095 10.b e1305/1296 10:dab84302:::benchmark_data_smithi076_299450_object76094:head v 1305'4754, mlcod=1305'4752)", 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.100936+0000", 2024-09-21T10:17:20.404 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.2236703150000001, 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038993450999999998, 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100936+0000", 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100936+0000", 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.405 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100935+0000", 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100971+0000", 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5941e-05 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100982+0000", 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0379e-05 2024-09-21T10:17:20.406 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100988+0000", 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0580000000000003e-06 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101091+0000", 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000103121 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101109+0000", 2024-09-21T10:17:20.407 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8243000000000001e-05 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139905+0000", 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.03879556 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139929+0000", 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4958999999999999e-05 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.408 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76097 10.4 e1305/1301 10:22958248:::benchmark_data_smithi076_299450_object76096:head v 1305'4737, mlcod=1305'4735)", 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.100957+0000", 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.223649312, 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038471367999999999, 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100957+0000", 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.409 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100957+0000", 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100957+0000", 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100988+0000", 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1470999999999997e-05 2024-09-21T10:17:20.410 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.100995+0000", 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1409999999999999e-06 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101001+0000", 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2619999999999998e-06 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101051+0000", 2024-09-21T10:17:20.411 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0362000000000003e-05 2024-09-21T10:17:20.412 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.412 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.412 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.412 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101069+0000", 2024-09-21T10:17:20.413 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8414e-05 2024-09-21T10:17:20.413 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.413 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.414 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.414 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139361+0000", 2024-09-21T10:17:20.414 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.038291948999999999 2024-09-21T10:17:20.414 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139428+0000", 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7120999999999997e-05 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76091 10.e e1305/1301 10:7c362795:::benchmark_data_smithi076_299450_object76090:head v 1305'4856, mlcod=1305'4854)", 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.101002+0000", 2024-09-21T10:17:20.415 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.223604836, 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042379459000000001, 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101002+0000", 2024-09-21T10:17:20.416 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101002+0000", 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101001+0000", 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.417 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101021+0000", 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9879999999999999e-05 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101027+0000", 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3999999999999997e-06 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101032+0000", 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5029999999999999e-06 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.418 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101072+0000", 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0265000000000002e-05 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101089+0000", 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6871000000000002e-05 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.419 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.143220+0000", 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.042130951999999999 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.143381+0000", 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000160915 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76096 10.f e1305/1301 10:f90348de:::benchmark_data_smithi076_299450_object76095:head v 1305'4726, mlcod=1305'4724)", 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.101415+0000", 2024-09-21T10:17:20.420 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.223191481, 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.043284807000000002, 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101415+0000", 2024-09-21T10:17:20.421 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101415+0000", 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101414+0000", 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.422 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101438+0000", 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3782e-05 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135069+0000", 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033631270999999997 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.423 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135079+0000", 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0156000000000001e-05 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135187+0000", 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107291 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135207+0000", 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0211999999999999e-05 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.424 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.144661+0000", 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094538309999999993 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.144700+0000", 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9082999999999998e-05 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.425 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76098 10.6 e1305/1296 10:6b097e1d:::benchmark_data_smithi076_299450_object76097:head v 1305'4710, mlcod=1305'4707)", 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.101879+0000", 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.222727589, 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.041335185000000003, 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101879+0000", 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.426 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101879+0000", 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101878+0000", 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.101920+0000", 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1656999999999998e-05 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.427 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135069+0000", 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.033149247999999999 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135080+0000", 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0662000000000001e-05 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135131+0000", 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1102e-05 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.428 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135153+0000", 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2138e-05 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.143155+0000", 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0080023799999999999 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.143214+0000", 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8743999999999999e-05 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.429 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:76103 10.9 e1305/1301 10:9918aa7a:::benchmark_data_smithi076_299450_object76102:head v 1305'4800, mlcod=1305'4797)", 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:19.106887+0000", 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 1.2177195300000001, 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.032444624999999998, 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.106887+0000", 2024-09-21T10:17:20.430 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.106887+0000", 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.106886+0000", 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.431 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.106919+0000", 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2839999999999997e-05 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135059+0000", 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.028140386 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135070+0000", 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1252e-05 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.432 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135450+0000", 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00038000000000000002 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.135469+0000", 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8870000000000001e-05 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139297+0000", 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0038277049999999998 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:17:20.433 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:19.139331+0000", 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4564999999999999e-05 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:17:20.434 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:17:20.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:20.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:20.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:20.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:20.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:20.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:20.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:20.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:20.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:20.976 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:20.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:20.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:21.099 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:21.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:21.221 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:21.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 39 16 78809 78793 126.241 77.8125 0.00945924 0.00791595 2024-09-21T10:17:21.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.355 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:21.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.457 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:21.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:21.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:21.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:21.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:21.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:21 smithi191 ceph-mon[102206]: pgmap v1989: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:17:21.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:21 smithi076 ceph-mon[106482]: pgmap v1989: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:17:21.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:21 smithi076 ceph-mon[110963]: pgmap v1989: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-21T10:17:21.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:21.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:21.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:21.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:21.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:21.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:21.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:21.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:17:21.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:22.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:22.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:22.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:22.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:17:22.236090+0000 min lat: 0.00112657 max lat: 0.219269 avg lat: 0.0079832 2024-09-21T10:17:22.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:17:22.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 40 16 80154 80138 125.187 84.0625 0.0207474 0.0079832 2024-09-21T10:17:22.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:22.284 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:22.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:22.405 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:22.437 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:22.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:22.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:22.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:22.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:22.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:22.800 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:22.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:22.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:22.937 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:22.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:22.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:23.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:23.096 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:23.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:23.198 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:23.223 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:23.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 41 16 81386 81370 124.011 77 0.0126004 0.0080594 2024-09-21T10:17:23.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:23.358 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:23 smithi076 ceph-mon[106482]: pgmap v1990: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-21T10:17:23.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:23 smithi076 ceph-mon[110963]: pgmap v1990: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-21T10:17:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:17:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:17:23] "GET /metrics HTTP/1.1" 200 36467 "" "Prometheus/2.51.0" 2024-09-21T10:17:23.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:23.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:23.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:23.577 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:23 smithi191 ceph-mon[102206]: pgmap v1990: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-21T10:17:23.596 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:23.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:23.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:23.768 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:23.870 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:23.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:23.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:23.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:24.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:24.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:24.171 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:24.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:24.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 42 15 82605 82590 122.874 76.25 0.0241811 0.00813375 2024-09-21T10:17:24.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:24.289 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:17:24.290 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:17:24.290 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:17:24.290 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:17:24.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:24.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:24.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.452 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:24.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:24.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:24.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:24.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:24.728 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:24.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:24.830 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:24.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.864 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:24.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:24.965 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:24.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:24.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:25.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:17:25.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:25.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.150 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:25.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:25.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 43 16 83824 83808 121.787 76.125 0.0104621 0.0082066 2024-09-21T10:17:25.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:25.302 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:25.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:25 smithi191 ceph-mon[102206]: pgmap v1991: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:17:25.403 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:25.404 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:25.430 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:25.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.567 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:25.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:25.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:25 smithi076 ceph-mon[106482]: pgmap v1991: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:17:25.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:25 smithi076 ceph-mon[110963]: pgmap v1991: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-21T10:17:25.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:25.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:25.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:25.823 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:25.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:25.925 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:25.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:25.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:26.063 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:26.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.202 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:26.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:26.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:26.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 44 16 84960 84944 120.633 71 0.00636712 0.00828435 2024-09-21T10:17:26.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:26.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:26.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:26.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:26.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:26.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:26.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:26.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:26.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:26.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.783 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:26.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:26 smithi191 ceph-mon[102206]: pgmap v1992: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:17:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:26 smithi076 ceph-mon[106482]: pgmap v1992: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:17:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:26 smithi076 ceph-mon[110963]: pgmap v1992: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-21T10:17:26.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:26.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:26.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:27.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:27.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:27.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:27.132 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:27.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:27.237 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 45 16 86250 86234 119.744 80.625 0.00633457 0.00834656 2024-09-21T10:17:27.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:27.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:27.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.430 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:27.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:27.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:27.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:27.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:27.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:27.692 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:27.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:27.805 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:27.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.831 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:27.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:27.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:27.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:27.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:27.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:28.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:28.081 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:28.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:28.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.208 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:28.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:28.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 46 16 87084 87068 118.274 52.125 0.0157628 0.00845053 2024-09-21T10:17:28.309 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:28.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:28.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:28.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:28.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:28.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:28.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:28.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:28.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:28.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.759 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:28.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:28.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:28.887 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:28.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:28.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:29.030 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:29.132 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:29.170 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:29.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:29.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 47 16 88360 88344 117.454 79.75 0.010432 0.00850993 2024-09-21T10:17:29.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:29.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:29.351 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:29 smithi191 ceph-mon[102206]: pgmap v1993: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:17:29.410 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:17:29.410 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:17:29.410 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:17:29.410 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:17:29.410 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:17:29.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:29.431 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:29 smithi076 ceph-mon[106482]: pgmap v1993: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:17:29.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:29 smithi076 ceph-mon[110963]: pgmap v1993: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:17:29.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:29.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:29.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:29.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:29.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:29.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:29.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:29.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:29.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:29.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:29.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:30.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:30.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:17:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:30.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:30.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 48 16 89585 89569 116.602 76.5625 0.00873492 0.00857191 2024-09-21T10:17:30.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:30.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:30.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:30.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:30.530 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:30.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:30.577 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:30.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:30.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:30.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:30.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:30.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:30.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:30.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:30.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:31.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:31.135 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:31.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:31.148 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:31.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:31.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 49 16 90896 90880 115.894 81.9375 0.0147976 0.00862433 2024-09-21T10:17:31.261 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:31 smithi076 ceph-mon[106482]: pgmap v1994: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-21T10:17:31.262 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:31 smithi076 ceph-mon[110963]: pgmap v1994: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-21T10:17:31.280 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:31.343 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:31 smithi191 ceph-mon[102206]: pgmap v1994: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-21T10:17:31.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:31.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:31.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:31.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:31.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:31.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:31.701 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:31.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:31.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:31.948 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:31.980 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:31.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:31.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:32.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:32.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:32.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:32.149 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:17:32.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:32.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 50 16 92359 92343 115.405 91.4375 0.00359656 0.00865981 2024-09-21T10:17:32.266 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:32.367 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:32 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3039794220' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:32.368 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:32 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:32.368 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:32 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3039794220' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:32.368 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:32 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:32.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:32.386 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:32 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3039794220' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:32.386 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:32 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:17:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.403 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:32.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:32.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:32.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:32.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:32.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.682 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:32.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:32.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:32.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:32.898 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:17:32.898 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:17:32.899 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:17:32.899 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [] 2024-09-21T10:17:32.899 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:17:32.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:32.954 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:32.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:33.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:33.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:33.190 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:33.232 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:33.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 51 16 93789 93773 114.895 89.375 0.00822067 0.00869911 2024-09-21T10:17:33.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:33.371 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:17:33] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-21T10:17:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[106482]: pgmap v1995: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:17:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:33.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[106482]: osdmap e1306: 8 total, 7 up, 4 in 2024-09-21T10:17:33.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[110963]: pgmap v1995: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:17:33.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:33.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:33.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:33 smithi076 ceph-mon[110963]: osdmap e1306: 8 total, 7 up, 4 in 2024-09-21T10:17:33.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:33.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:33.611 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:33.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:33 smithi191 ceph-mon[102206]: pgmap v1995: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-21T10:17:33.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:33 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:33.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:33 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:17:33.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:33 smithi191 ceph-mon[102206]: osdmap e1306: 8 total, 7 up, 4 in 2024-09-21T10:17:33.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:33.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:33.742 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:33.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:33.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:33.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:33.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:34.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:34.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:34.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:34.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:34.238 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 52 16 94974 94958 114.109 74.0625 0.0116264 0.00875911 2024-09-21T10:17:34.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.316 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:34.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:34.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:34.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:34.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:34.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:34.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:34.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:34.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:34.834 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.936 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:34.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:34.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:34.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:17:35.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:35.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:35.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:35.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:35.239 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 53 16 96219 96203 113.423 77.8125 0.00966739 0.00881229 2024-09-21T10:17:35.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:35.342 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:35.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:35.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:35 smithi191 ceph-mon[102206]: pgmap v1997: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:17:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:35.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:35.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:17:35.456 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:35 smithi076 ceph-mon[110963]: pgmap v1997: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:17:35.457 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:35 smithi076 ceph-mon[106482]: pgmap v1997: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:17:35.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:35.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:35.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:35.658 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:35.693 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:35.795 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:35.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:35.858 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:17:35.899 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:17:35.899 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:35.933 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:17:35.933 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:35.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:35.966 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:35.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:35.967 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:35.969 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:17:35.969 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:36.000 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:17:36.000 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:36.035 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:17:36.035 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:36.067 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:36.071 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:17:36.071 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:36.103 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:17:36.104 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:36.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:36.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:36.239 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 54 16 97597 97581 112.918 86.125 0.0240232 0.0088509 2024-09-21T10:17:36.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:36.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:36 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2157522904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:36 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2157522904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:36.381 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:36.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:36.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:36.520 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:36.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:36.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:36 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2157522904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:36.622 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:36.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:36.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:36.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:36.785 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:36.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:36.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:36.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:36.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:37.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:37.066 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:37.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:37.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:37.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:37.239 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 55 16 98830 98814 112.266 77.0625 0.00914531 0.00890293 2024-09-21T10:17:37.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:37.345 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:37.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:37.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:37 smithi191 ceph-mon[102206]: pgmap v1998: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:17:37.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:37.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:37.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:37.618 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:37.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:37 smithi076 ceph-mon[106482]: pgmap v1998: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:17:37.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:37 smithi076 ceph-mon[110963]: pgmap v1998: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-21T10:17:37.719 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:37.720 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:37.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:37.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:37.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:37.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:37.900 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:37.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:37.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:38.002 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:38.034 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:38.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:38.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:38.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:38.239 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 56 16 100293 100277 111.894 91.4375 0.00983368 0.00893264 2024-09-21T10:17:38.279 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.315 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:38.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:38.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:38.504 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:17:38.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:17:38.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:38.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:38.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:38.611 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:17:38.611 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:17:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:17:38.647 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:17:38.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:38.726 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:38.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:38.828 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:38.861 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:38.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:38.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:38.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:39.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:39.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:39.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:39.240 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 57 16 101592 101576 111.354 81.1875 0.0210147 0.00897571 2024-09-21T10:17:39.246 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:39.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:39.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:39 smithi191 ceph-mon[102206]: pgmap v1999: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:17:39.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:39.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:39.509 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:39 smithi076 ceph-mon[106482]: pgmap v1999: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:17:39.509 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:39 smithi076 ceph-mon[110963]: pgmap v1999: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-21T10:17:39.539 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:39.580 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:39.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:39.585 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:39.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:39.724 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:39.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:39.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:39.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:39.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:39.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:40.001 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:40.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:40.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:40.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:40.173 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6] out_osds: [7, 5, 0, 2] dead_osds: [0] live_osds: [1, 2, 4, 5, 6, 7, 3] 2024-09-21T10:17:40.173 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:17:40.173 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-09-21T10:17:40.173 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 5 2024-09-21T10:17:40.240 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 58 16 102959 102943 110.907 85.4375 0.00854177 0.00901235 2024-09-21T10:17:40.257 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:40.298 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:40.400 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:40.432 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:40.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:40.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:40.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:17:40.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:40.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:40.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:40.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:40.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:41.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:41.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:41.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:41.158 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:41.197 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:17:41.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:41 smithi191 ceph-mon[102206]: pgmap v2000: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:17:41.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:41 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/459246163' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:41.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:41 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:41.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:41.216 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:41.227 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:41.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:41.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 59 16 104251 104235 110.394 80.75 0.0145206 0.00905393 2024-09-21T10:17:41.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:41.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:41.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:41.502 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:41.559 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[110963]: pgmap v2000: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:17:41.559 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/459246163' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:41.559 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:41.559 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:41.559 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:41.560 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[106482]: pgmap v2000: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:17:41.560 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/459246163' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:41.560 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:17:41.560 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:41.560 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:41.603 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:41.640 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:41.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:41.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:41.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:41.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:41.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:41.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:41.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:42.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:42.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:42.151 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:42.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:42.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:17:42.240018+0000 min lat: 0.00112657 max lat: 0.219269 avg lat: 0.00908449 2024-09-21T10:17:42.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:17:42.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 60 16 105661 105645 110.023 88.125 0.00359511 0.00908449 2024-09-21T10:17:42.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:42.300 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:42 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:42.300 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:42 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:42.300 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:42 smithi076 ceph-mon[106482]: osdmap e1307: 8 total, 7 up, 4 in 2024-09-21T10:17:42.300 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:42 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:42.300 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:42 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:42.300 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:42 smithi076 ceph-mon[110963]: osdmap e1307: 8 total, 7 up, 4 in 2024-09-21T10:17:42.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:42 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:42 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:17:42.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:42 smithi191 ceph-mon[102206]: osdmap e1307: 8 total, 7 up, 4 in 2024-09-21T10:17:42.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.454 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:42.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:42.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:42.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:42.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:42.715 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:42.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:42.799 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:42.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:42.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:42.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:42.991 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:42.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:43.093 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:43.119 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:43.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:43.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:43.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 61 16 106738 106722 109.322 67.3125 0.00832278 0.00914272 2024-09-21T10:17:43.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:43.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:43.361 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:43.395 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:43.497 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:43.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:43 smithi191 ceph-mon[102206]: pgmap v2002: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-21T10:17:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:17:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:17:43] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.51.0" 2024-09-21T10:17:43.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:43 smithi076 ceph-mon[106482]: pgmap v2002: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-21T10:17:43.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:43 smithi076 ceph-mon[110963]: pgmap v2002: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-21T10:17:43.627 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:43.658 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:43.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:43.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.798 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:43.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:43.899 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:43.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:43.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:44.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:44.041 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:17:44.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:44.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:44.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:44.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:44.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:44.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 62 16 107861 107845 108.691 70.1875 0.0099988 0.00919627 2024-09-21T10:17:44.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:44.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:44.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:44.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.478 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:44.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:44.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:44.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:44.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:44.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:44.737 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:44.737 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.759 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:44.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:44.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:44.894 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:44.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:44.995 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:45.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:45.123 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:45.145 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:45.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:45.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 63 16 108992 108976 108.088 70.6875 0.00953089 0.00924725 2024-09-21T10:17:45.247 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:45.278 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:45.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:45.380 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:45.410 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:45.424 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:45 smithi191 ceph-mon[102206]: pgmap v2003: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:17:45.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:17:45.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:45.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.563 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:45.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:45 smithi076 ceph-mon[106482]: pgmap v2003: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:17:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:45 smithi076 ceph-mon[110963]: pgmap v2003: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:17:45.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:45.712 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:45.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:45.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:45.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:45.950 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:45.978 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:45.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:45.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:46.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:46.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:46.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:46.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 64 16 110322 110306 107.698 83.125 0.0081508 0.00928133 2024-09-21T10:17:46.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:46.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:46.388 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:46.490 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:46.518 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:46.538 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:46.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:46.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:46.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:46.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:46.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:46.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:46.896 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:46.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:46.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:46.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:47.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:47.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:47.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:47.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:47.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:47.241 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 65 16 111596 111580 107.266 79.625 0.0148031 0.00931828 2024-09-21T10:17:47.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:47.339 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:47.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:47 smithi191 ceph-mon[102206]: pgmap v2004: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:17:47.441 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:47.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:47.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:47.575 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:47.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:47.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:47 smithi076 ceph-mon[106482]: pgmap v2004: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:17:47.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:47 smithi076 ceph-mon[110963]: pgmap v2004: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-21T10:17:47.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:47.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:47.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:47.885 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:47.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:47.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:48.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:48.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.159 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:48.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:48.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 66 16 113004 112988 106.97 88 0.00592242 0.00934409 2024-09-21T10:17:48.260 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:48.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:48.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:48.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:48.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:48.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:48.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:48.712 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:48.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:48.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:48.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:48.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:48.993 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:48.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:49.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:49.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:49.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:49.231 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:49.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 67 16 114300 114284 106.583 81 0.0081137 0.00937824 2024-09-21T10:17:49.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:49 smithi191 ceph-mon[102206]: pgmap v2005: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:17:49.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:49.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:49.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:49.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:49.531 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:49 smithi076 ceph-mon[106482]: pgmap v2005: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:17:49.532 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:49 smithi076 ceph-mon[110963]: pgmap v2005: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-21T10:17:49.552 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:49.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:49.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.696 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:49.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:49.714 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:17:49.714 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:17:49.788 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:49 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:17:49.618+0000 7f240edc2640 -1 mon.a@0(leader).osd e1307 definitely_dead 0 2024-09-21T10:17:49.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:49.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:49.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:49.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:49.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:49.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:49.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:50.058 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:17:50.058 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:50.058 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:50.058 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:17:50.058 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:17:50.058 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:50.059 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:50.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:50.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:50.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:50.233 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.5. 2024-09-21T10:17:50.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 68 16 115558 115542 106.172 78.625 0.0076297 0.00941303 2024-09-21T10:17:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.262 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:50.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2035575471' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-09-21T10:17:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2889831275' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:50.363 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2035575471' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-09-21T10:17:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2889831275' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2035575471' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-09-21T10:17:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2889831275' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:17:50.384 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:50.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:50.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:50.507 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:50.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:50.629 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:50.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:50.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:50.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:50.872 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:17:50.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:50.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:50.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:50.995 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:50.998 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-09-21T10:17:50.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:51.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:51.087 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:51.101 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.124 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:51.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:51.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:51.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 69 16 115578 115562 104.646 1.25 0.00330054 0.0094148 2024-09-21T10:17:51.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:51.359 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:51.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:51.482 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:51.504 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:51.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:51.538 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:51 smithi191 ceph-mon[102206]: pgmap v2006: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:17:51.538 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2035575471' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-09-21T10:17:51.538 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:51 smithi191 ceph-mon[102206]: osdmap e1308: 8 total, 7 up, 5 in 2024-09-21T10:17:51.538 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:51 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:51.538 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:51 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:51.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[106482]: pgmap v2006: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2035575471' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[106482]: osdmap e1308: 8 total, 7 up, 5 in 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[110963]: pgmap v2006: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2035575471' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-09-21T10:17:51.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[110963]: osdmap e1308: 8 total, 7 up, 5 in 2024-09-21T10:17:51.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:51.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:51 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:51.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:51.728 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:51.750 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:51.852 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:17:51.864 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:17:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:51.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:51.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:51.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:51.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:52.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:52.120 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:52.222 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:52.243 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:52.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:52.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 70 16 115578 115562 103.151 0 - 0.0094148 2024-09-21T10:17:52.345 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:52.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:52.468 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:52.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:52.580 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:52 smithi191 ceph-mon[102206]: osdmap e1309: 8 total, 7 up, 5 in 2024-09-21T10:17:52.591 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:52.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.613 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:52.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:52.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:52 smithi076 ceph-mon[106482]: osdmap e1309: 8 total, 7 up, 5 in 2024-09-21T10:17:52.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:52 smithi076 ceph-mon[110963]: osdmap e1309: 8 total, 7 up, 5 in 2024-09-21T10:17:52.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:52.736 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:52.790 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:17:52.790 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:17:52.790 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:17:52.790 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:17:52.839 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:52.860 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:52.962 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:52.984 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:52.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:53.086 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:53.122 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:53.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:53.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 71 16 115596 115580 101.715 0.5625 2.98892 0.0096977 2024-09-21T10:17:53.248 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:53.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:53.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:53.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.384 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:53.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:53.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:53.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: pgmap v2009: 89 pgs: 5 remapped+peering, 8 activating, 6 activating+degraded, 70 active+clean; 7.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 7/343785 objects degraded (0.002%); 512 B/s, 0 objects/s recovering 2024-09-21T10:17:53.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 9 pgs inactive (PG_AVAILABILITY) 2024-09-21T10:17:53.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 7/343785 objects degraded (0.002%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:53.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7313 2024-09-21T10:17:53.497 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7140 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.a starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7271 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7140 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7372 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.3 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7336 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: osdmap e1310: 8 total, 7 up, 5 in 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7157 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.1 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7199 2024-09-21T10:17:53.498 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:53 smithi191 ceph-mon[102206]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7091 2024-09-21T10:17:53.514 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:53.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:17:53.616 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:17:53] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.51.0" 2024-09-21T10:17:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: pgmap v2009: 89 pgs: 5 remapped+peering, 8 activating, 6 activating+degraded, 70 active+clean; 7.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 7/343785 objects degraded (0.002%); 512 B/s, 0 objects/s recovering 2024-09-21T10:17:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 9 pgs inactive (PG_AVAILABILITY) 2024-09-21T10:17:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 7/343785 objects degraded (0.002%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:53.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7313 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7140 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.a starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7271 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7140 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7372 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.3 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7336 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: osdmap e1310: 8 total, 7 up, 5 in 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7157 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.1 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7199 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[106482]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7091 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: pgmap v2009: 89 pgs: 5 remapped+peering, 8 activating, 6 activating+degraded, 70 active+clean; 7.0 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 74 MiB/s wr, 1.19k op/s; 7/343785 objects degraded (0.002%); 512 B/s, 0 objects/s recovering 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 9 pgs inactive (PG_AVAILABILITY) 2024-09-21T10:17:53.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 7/343785 objects degraded (0.002%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7313 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7140 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.a starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7271 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7140 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7372 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.3 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7336 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: osdmap e1310: 8 total, 7 up, 5 in 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7157 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.1 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7199 2024-09-21T10:17:53.625 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:53 smithi076 ceph-mon[110963]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 1307'7091 2024-09-21T10:17:53.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:53.764 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:53.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:53.865 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:17:53.897 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:53.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:53.984 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:53.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:54.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:54.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:54.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:54.248 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 72 16 116677 116661 101.24 67.5625 0.010246 0.00987267 2024-09-21T10:17:54.260 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:54.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:54.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:54.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.409 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:54.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:54.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:54.553 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:54.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:54.693 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:54.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:54.720 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:54.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:54.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:54.938 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:54.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:54.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:54.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:55.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:55.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:55.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:55.217 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:55.248 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 73 16 117813 117797 100.826 71 0.00971019 0.00991313 2024-09-21T10:17:55.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:55.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:55.267 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:17:55.353 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:55.384 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:55.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:55.524 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:55.545 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:55 smithi191 ceph-mon[102206]: pgmap v2011: 89 pgs: 4 active+remapped+backfill_wait, 5 remapped+peering, 8 activating, 6 activating+degraded, 66 active+clean; 7.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 59 MiB/s wr, 946 op/s; 7/347160 objects degraded (0.002%); 28884/347160 objects misplaced (8.320%); 43 KiB/s, 1 objects/s recovering 2024-09-21T10:17:55.546 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2830725392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:55.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:55 smithi076 ceph-mon[106482]: pgmap v2011: 89 pgs: 4 active+remapped+backfill_wait, 5 remapped+peering, 8 activating, 6 activating+degraded, 66 active+clean; 7.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 59 MiB/s wr, 946 op/s; 7/347160 objects degraded (0.002%); 28884/347160 objects misplaced (8.320%); 43 KiB/s, 1 objects/s recovering 2024-09-21T10:17:55.583 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2830725392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:55.583 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:55 smithi076 ceph-mon[110963]: pgmap v2011: 89 pgs: 4 active+remapped+backfill_wait, 5 remapped+peering, 8 activating, 6 activating+degraded, 66 active+clean; 7.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 59 MiB/s wr, 946 op/s; 7/347160 objects degraded (0.002%); 28884/347160 objects misplaced (8.320%); 43 KiB/s, 1 objects/s recovering 2024-09-21T10:17:55.583 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2830725392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:17:55.627 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:55.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:55.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:55.794 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:55.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:55.895 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:55.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:55.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:55.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:55.998 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6, 5] out_osds: [7, 0, 2] dead_osds: [0] live_osds: [1, 2, 4, 5, 6, 7, 3] 2024-09-21T10:17:55.998 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:17:55.999 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-21T10:17:55.999 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.913268 2024-09-21T10:17:55.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 3 0.9132680268826356 2024-09-21T10:17:56.043 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:56.046 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:17:56.046 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:17:56.046 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:17:56.046 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:17:56.046 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:17:56.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:56.182 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:56.219 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:56.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 74 16 118914 118898 100.391 68.8125 0.0103877 0.009957 2024-09-21T10:17:56.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:56.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:56.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:56.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:56.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:56.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:56 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2830725392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:56 smithi191 ceph-mon[102206]: osdmap e1311: 8 total, 7 up, 5 in 2024-09-21T10:17:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2830725392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[106482]: osdmap e1311: 8 total, 7 up, 5 in 2024-09-21T10:17:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:17:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2830725392' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:17:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[110963]: osdmap e1311: 8 total, 7 up, 5 in 2024-09-21T10:17:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:17:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.637 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:56.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:56.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:56.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:17:56.792 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:56.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:56.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:56.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:56.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:56.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:57.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:57.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:57.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:57.170 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:17:57.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:57.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:57.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 75 16 119977 119961 99.9385 66.4375 0.0252724 0.0100018 2024-09-21T10:17:57.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:17:57.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:17:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:57 smithi076 ceph-mon[106482]: pgmap v2013: 89 pgs: 4 active+remapped+backfill_wait, 5 remapped+peering, 8 activating, 6 activating+degraded, 66 active+clean; 7.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 48 MiB/s wr, 771 op/s; 7/347160 objects degraded (0.002%); 28884/347160 objects misplaced (8.320%); 44 KiB/s, 2 objects/s recovering 2024-09-21T10:17:57.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:57 smithi076 ceph-mon[110963]: pgmap v2013: 89 pgs: 4 active+remapped+backfill_wait, 5 remapped+peering, 8 activating, 6 activating+degraded, 66 active+clean; 7.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 48 MiB/s wr, 771 op/s; 7/347160 objects degraded (0.002%); 28884/347160 objects misplaced (8.320%); 44 KiB/s, 2 objects/s recovering 2024-09-21T10:17:57.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:57.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:57.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:57.516 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:57 smithi191 ceph-mon[102206]: pgmap v2013: 89 pgs: 4 active+remapped+backfill_wait, 5 remapped+peering, 8 activating, 6 activating+degraded, 66 active+clean; 7.1 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 48 MiB/s wr, 771 op/s; 7/347160 objects degraded (0.002%); 28884/347160 objects misplaced (8.320%); 44 KiB/s, 2 objects/s recovering 2024-09-21T10:17:57.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:57.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:57.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:57.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:57.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:57.921 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:57.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:58.023 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:58.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:58.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:58.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:58.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:58.249 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 76 16 121083 121067 99.5331 69.125 0.010369 0.0100422 2024-09-21T10:17:58.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.340 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:58.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:58.440 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.476 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:58.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:58.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:17:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.627 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:58.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:17:58.702 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:17:58.728 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:17:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.763 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:58.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:17:58.864 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:58.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:58.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:59.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.077 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:59.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:59.146 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:17:59.170 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:17:59.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:59.210 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:17:59.210 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:17:59.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:59.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:59.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:17:59.246 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:17:59.253 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 77 16 122152 122136 99.1066 66.8125 0.0244567 0.0100857 2024-09-21T10:17:59.282 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.282 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.282 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[106482]: pgmap v2014: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.2 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s; 65803/354615 objects misplaced (18.556%); 4.1 MiB/s, 67 objects/s recovering 2024-09-21T10:17:59.282 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:17:59.282 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:17:59.282 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.283 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.283 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.283 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[110963]: pgmap v2014: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.2 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s; 65803/354615 objects misplaced (18.556%); 4.1 MiB/s, 67 objects/s recovering 2024-09-21T10:17:59.283 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:17:59.283 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:17:59.283 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:17:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.283 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:17:59.283 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:17:59.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:59 smithi191 ceph-mon[102206]: pgmap v2014: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.2 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 64 MiB/s wr, 1.02k op/s; 65803/354615 objects misplaced (18.556%); 4.1 MiB/s, 67 objects/s recovering 2024-09-21T10:17:59.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:17:59.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:17:59.288 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:17:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:17:59.317 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:17:59.318 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:17:59.341 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:17:59.353 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:17:59.353 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:17:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:59.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:17:59.387 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:17:59.388 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:17:59.422 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:17:59.422 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:17:59.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:17:59.524 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:17:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:17:59.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:17:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:59.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:17:59.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:59.804 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:17:59.906 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:17:59.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:17:59.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:17:59.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:00.011 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:00.011 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:00.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:00.081 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:00.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:00.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:00.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:00.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 78 16 123177 123161 98.6573 64.0625 0.00991461 0.0101309 2024-09-21T10:18:00.325 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:00.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:00 smithi191 ceph-mon[102206]: pgmap v2015: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 50 MiB/s wr, 804 op/s; 67175/359745 objects misplaced (18.673%); 4.2 MiB/s, 69 objects/s recovering 2024-09-21T10:18:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:00 smithi191 ceph-mon[102206]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T10:18:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:00 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive) 2024-09-21T10:18:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:00 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/347160 objects degraded (0.002%), 6 pgs degraded) 2024-09-21T10:18:00.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4274546798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[106482]: pgmap v2015: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 50 MiB/s wr, 804 op/s; 67175/359745 objects misplaced (18.673%); 4.2 MiB/s, 69 objects/s recovering 2024-09-21T10:18:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[106482]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T10:18:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive) 2024-09-21T10:18:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/347160 objects degraded (0.002%), 6 pgs degraded) 2024-09-21T10:18:00.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4274546798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[110963]: pgmap v2015: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 50 MiB/s wr, 804 op/s; 67175/359745 objects misplaced (18.673%); 4.2 MiB/s, 69 objects/s recovering 2024-09-21T10:18:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[110963]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T10:18:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive) 2024-09-21T10:18:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/347160 objects degraded (0.002%), 6 pgs degraded) 2024-09-21T10:18:00.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4274546798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:00.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:00.496 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:00.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:00.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:00.633 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:00.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:00.769 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:18:00.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:00.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:00.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:00.918 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:00.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:00.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:01.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:01.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:01.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:01.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:01.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 79 16 124222 124206 98.2305 65.3125 0.0221994 0.0101751 2024-09-21T10:18:01.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:01.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:01.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:01.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:01.486 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:01.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:01.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:01.621 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:01.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:01.723 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:01.734 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:01.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:01.818 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:01.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:01.910 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:01.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:02.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:02.043 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:02.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:02.144 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:02.186 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:02 smithi076 ceph-mon[106482]: pgmap v2016: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 39 MiB/s wr, 630 op/s; 67175/359745 objects misplaced (18.673%); 3.3 MiB/s, 54 objects/s recovering 2024-09-21T10:18:02.187 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:02 smithi076 ceph-mon[110963]: pgmap v2016: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 39 MiB/s wr, 630 op/s; 67175/359745 objects misplaced (18.673%); 3.3 MiB/s, 54 objects/s recovering 2024-09-21T10:18:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.188 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:02.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:02.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:18:02.254019+0000 min lat: 0.00112657 max lat: 3.00327 avg lat: 0.0102148 2024-09-21T10:18:02.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:18:02.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 80 16 125310 125294 97.8526 68 0.0201637 0.0102148 2024-09-21T10:18:02.288 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:02.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:02.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:02.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:02 smithi191 ceph-mon[102206]: pgmap v2016: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 39 MiB/s wr, 630 op/s; 67175/359745 objects misplaced (18.673%); 3.3 MiB/s, 54 objects/s recovering 2024-09-21T10:18:02.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:02.467 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:02.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:02.613 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:02.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:02.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:02.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:02.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:02.857 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:02.858 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:02.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:02.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:02.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:03.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:03.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:03.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:03.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 81 16 126377 126361 97.4679 66.6875 0.0128469 0.0102547 2024-09-21T10:18:03.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:03.301 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:03.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:03.432 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:03.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:03.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.565 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:03.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:18:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:18:03] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-21T10:18:03.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:03.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:03.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:03.856 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:03.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:03.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:03.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:04.095 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:04.133 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:04.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:04.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 82 16 127501 127485 97.136 70.25 0.0145311 0.0102901 2024-09-21T10:18:04.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:04.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:04 smithi191 ceph-mon[102206]: pgmap v2017: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.5 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 57 MiB/s wr, 913 op/s; 68120/370761 objects misplaced (18.373%); 8.1 MiB/s, 130 objects/s recovering 2024-09-21T10:18:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:04 smithi076 ceph-mon[106482]: pgmap v2017: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.5 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 57 MiB/s wr, 913 op/s; 68120/370761 objects misplaced (18.373%); 8.1 MiB/s, 130 objects/s recovering 2024-09-21T10:18:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:04 smithi076 ceph-mon[110963]: pgmap v2017: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.5 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 57 MiB/s wr, 913 op/s; 68120/370761 objects misplaced (18.373%); 8.1 MiB/s, 130 objects/s recovering 2024-09-21T10:18:04.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:04.424 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:04.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:04.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:04.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:04.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:04.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:04.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.845 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:04.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:04.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:18:04.946 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:04.985 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:04.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:04.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:05.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:05.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:05.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:05.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:05.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 83 16 128551 128535 96.7564 65.625 0.00632792 0.0103299 2024-09-21T10:18:05.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:05.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:05.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:05.412 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:05.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:05.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:05.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:05.662 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:05.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:05.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:05.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:05.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:05.988 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:05.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:06.060 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.3 primary-affinity to 0.913268 (8598512) 2024-09-21T10:18:06.090 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:06.113 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:06 smithi076 ceph-mon[106482]: pgmap v2018: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.6 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 69463/375813 objects misplaced (18.483%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:18:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3496597799' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356} : dispatch 2024-09-21T10:18:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:06 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356} : dispatch 2024-09-21T10:18:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:06 smithi076 ceph-mon[110963]: pgmap v2018: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.6 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 69463/375813 objects misplaced (18.483%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:18:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3496597799' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356} : dispatch 2024-09-21T10:18:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:06 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356} : dispatch 2024-09-21T10:18:06.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:06.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:06.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 84 16 129462 129446 96.2824 56.9375 0.0182438 0.0103815 2024-09-21T10:18:06.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:06 smithi191 ceph-mon[102206]: pgmap v2018: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.6 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 69463/375813 objects misplaced (18.483%); 7.4 MiB/s, 119 objects/s recovering 2024-09-21T10:18:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3496597799' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356} : dispatch 2024-09-21T10:18:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:06 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356} : dispatch 2024-09-21T10:18:06.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:06.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:06.480 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:06.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:06.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:06.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:06.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:06.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:06.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:06.763 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:06.865 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:06.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:06.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:06.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:07.015 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:07.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:07.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:07.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 85 16 130297 130281 95.7637 52.1875 0.0171679 0.0104376 2024-09-21T10:18:07.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:07.290 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:07.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:07 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356}]': finished 2024-09-21T10:18:07.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:07 smithi191 ceph-mon[102206]: osdmap e1312: 8 total, 7 up, 5 in 2024-09-21T10:18:07.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:07 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356}]': finished 2024-09-21T10:18:07.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:07 smithi076 ceph-mon[106482]: osdmap e1312: 8 total, 7 up, 5 in 2024-09-21T10:18:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:07 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9132680268826356}]': finished 2024-09-21T10:18:07.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:07 smithi076 ceph-mon[110963]: osdmap e1312: 8 total, 7 up, 5 in 2024-09-21T10:18:07.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:07.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:07.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:07.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:07.665 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:07.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:07.802 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:07.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:07.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:07.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:07.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:07.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:07.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:08.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:08.125 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:08.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:08.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:08.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 86 16 131280 131264 95.3646 61.4375 0.0365735 0.0104816 2024-09-21T10:18:08.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:08 smithi191 ceph-mon[102206]: pgmap v2020: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 7.9 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 70410/386475 objects misplaced (18.219%); 9.4 MiB/s, 149 objects/s recovering 2024-09-21T10:18:08.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:08 smithi191 ceph-mon[102206]: osdmap e1313: 8 total, 7 up, 5 in 2024-09-21T10:18:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:08 smithi076 ceph-mon[110963]: pgmap v2020: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 7.9 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 70410/386475 objects misplaced (18.219%); 9.4 MiB/s, 149 objects/s recovering 2024-09-21T10:18:08.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:08 smithi076 ceph-mon[110963]: osdmap e1313: 8 total, 7 up, 5 in 2024-09-21T10:18:08.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:08 smithi076 ceph-mon[106482]: pgmap v2020: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 7.9 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 70410/386475 objects misplaced (18.219%); 9.4 MiB/s, 149 objects/s recovering 2024-09-21T10:18:08.375 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:08 smithi076 ceph-mon[106482]: osdmap e1313: 8 total, 7 up, 5 in 2024-09-21T10:18:08.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:08.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:08.518 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:08.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:08.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:08.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:08.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:08.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:08.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:08.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:08.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:08.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:09.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:09.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:09.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:09.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:09.209 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:09.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 87 16 132334 132318 95.0256 65.875 0.0153342 0.010519 2024-09-21T10:18:09.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:09.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:09.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:09.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:09.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:09.551 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:09.551 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:09.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.620 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:09.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:09.721 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.768 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:09.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:09.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:09.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:09.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:10.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:10.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:10.092 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:18:10.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:10.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:10.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:10.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 88 16 133347 133331 94.6653 63.3125 0.00784587 0.0105584 2024-09-21T10:18:10.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:10.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:10.332 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:10 smithi076 ceph-mon[110963]: pgmap v2022: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 8.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 71607/391008 objects misplaced (18.313%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:18:10.332 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/54806110' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:10.332 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:10.333 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:10 smithi076 ceph-mon[106482]: pgmap v2022: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 8.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 71607/391008 objects misplaced (18.313%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:18:10.333 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/54806110' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:10.333 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:10 smithi191 ceph-mon[102206]: pgmap v2022: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 8.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 71607/391008 objects misplaced (18.313%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:18:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/54806110' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:10.364 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:18:10.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:10.446 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:10.548 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:10.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:10.639 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:10.695 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:10.735 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:10.837 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:10.867 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:10.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:10.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:11.005 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:11.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:11.106 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:11.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:11.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:11.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:11.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 89 16 134322 134306 94.2863 60.9375 0.0238038 0.0106015 2024-09-21T10:18:11.289 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:11.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:11.347 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:11.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:18:11.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:11.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[106482]: osdmap e1314: 8 total, 7 up, 5 in 2024-09-21T10:18:11.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:18:11.348 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:11.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:11.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:18:11.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:11.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[110963]: osdmap e1314: 8 total, 7 up, 5 in 2024-09-21T10:18:11.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:18:11.349 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:11 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:18:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:11 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:18:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:11 smithi191 ceph-mon[102206]: osdmap e1314: 8 total, 7 up, 5 in 2024-09-21T10:18:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:18:11.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:11.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:11.420 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:11.521 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:11.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.559 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:11.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:11.660 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:11.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:11.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:11.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:11.949 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:11.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:11.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:11.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:12.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:12.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:12.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:12.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:12.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total time run: 90.0055 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total writes made: 135176 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Write size: 65536 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object size: 65536 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Bandwidth (MB/sec): 93.8665 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Bandwidth: 45.0281 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max bandwidth (MB/sec): 258 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:18:12.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average IOPS: 1501 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev IOPS: 720.45 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max IOPS: 4128 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min IOPS: 0 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average Latency(s): 0.0106497 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Latency(s): 0.033654 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max latency(s): 3.00327 2024-09-21T10:18:12.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min latency(s): 0.00112657 2024-09-21T10:18:12.263 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:18:12.347 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:12 smithi076 ceph-mon[106482]: pgmap v2024: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 8.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s; 71607/391008 objects misplaced (18.313%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:18:12.347 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:12 smithi076 ceph-mon[110963]: pgmap v2024: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 8.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s; 71607/391008 objects misplaced (18.313%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:18:12.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:12 smithi191 ceph-mon[102206]: pgmap v2024: 89 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 78 active+clean; 8.0 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 53 MiB/s wr, 843 op/s; 71607/391008 objects misplaced (18.313%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:18:12.379 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:12.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.520 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:12.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:12.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:12.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:12.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:12.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:12.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:12.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:12.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:13.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:13.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:13.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:13.198 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:13.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:13.293 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:18:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:18:13] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-21T10:18:13.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:13.300 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:18:13.301 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:13.330 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:18:13.330 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:13.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:13.352 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:18:13.368 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:18:13.368 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:13.399 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:18:13.399 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:13.435 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:18:13.435 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:13.436 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:13.475 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:18:13.475 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:13.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:13.502 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:18:13.502 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:13.534 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:18:13.534 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:13.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:13.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:13.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:13.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:13.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:13.864 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:13.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:13.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:13.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:13.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:14.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:14.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:14.180 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6, 5] out_osds: [7, 0, 2] dead_osds: [0] live_osds: [1, 2, 4, 5, 6, 7, 3] 2024-09-21T10:18:14.180 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2024-09-21T10:18:14.180 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2024-09-21T10:18:14.180 DEBUG:teuthology.orchestra.run.smithi076:osd.0> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T10:18:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.181 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:14.184 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl start ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:18:14.212 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:14 smithi076 ceph-mon[106482]: pgmap v2025: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.2 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 72370/400911 objects misplaced (18.051%); 12 MiB/s, 194 objects/s recovering 2024-09-21T10:18:14.212 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3024835732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:14.214 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:14 smithi076 ceph-mon[110963]: pgmap v2025: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.2 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 72370/400911 objects misplaced (18.051%); 12 MiB/s, 194 objects/s recovering 2024-09-21T10:18:14.214 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3024835732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:14.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.323 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:14 smithi191 ceph-mon[102206]: pgmap v2025: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.2 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 72370/400911 objects misplaced (18.051%); 12 MiB/s, 194 objects/s recovering 2024-09-21T10:18:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3024835732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:14.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:14.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:14.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:14.607 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:14.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:14 smithi076 systemd[1]: Starting Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:18:14.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:14.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:14.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:14.882 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:14.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:14.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:14.991 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:15.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:15.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:15.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:15.169 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:15.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:15.277 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:15.313 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:15.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:15.451 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:15.553 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:15.581 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:15.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:15.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:15.726 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:15.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:15.828 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:15.861 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:15.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:15.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:15.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:15.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:16.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:16.125 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:16.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:16.226 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:16.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:16.317 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:16 smithi076 ceph-mon[110963]: pgmap v2026: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.2 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 47 MiB/s wr, 794 op/s; 72968/403137 objects misplaced (18.100%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:18:16.317 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:16 smithi076 ceph-mon[106482]: pgmap v2026: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.2 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 47 MiB/s wr, 794 op/s; 72968/403137 objects misplaced (18.100%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:18:16.317 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:16 smithi076 podman[309730]: 2024-09-21 10:18:16.050377131 +0000 UTC m=+0.268181189 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:18:16.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:16 smithi191 ceph-mon[102206]: pgmap v2026: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.2 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 47 MiB/s wr, 794 op/s; 72968/403137 objects misplaced (18.100%); 5.3 MiB/s, 84 objects/s recovering 2024-09-21T10:18:16.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:16.416 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:16.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:16.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:16.548 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:16.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:16.650 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:16.680 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:16.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:16.813 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:16.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:16.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:18:16.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:16.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:16.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:16.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:17.044 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:18:17.044 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:18:17.044 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:18:17.044 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:18:17.044 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:18:17.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.091 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:17.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:17.139 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:17 smithi076 podman[309730]: 2024-09-21 10:18:17.136934953 +0000 UTC m=+1.354739006 container create d9f876fe746f41a582203ced9280bc50467af3da5ee774e846d784ba9640473b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T10:18:17.192 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:17.235 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:17.337 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:17.376 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:17.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:17.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:17.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:17.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:17.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:17.749 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:17.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:17.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:17.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:17.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:18.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:18.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:18.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:18.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.180 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:18.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:18.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:18.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:18 smithi191 ceph-mon[102206]: pgmap v2027: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.0 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 106 B/s rd, 41 MiB/s wr, 1.04k op/s; 71172/394782 objects misplaced (18.028%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T10:18:18.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:18 smithi076 ceph-mon[106482]: pgmap v2027: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.0 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 106 B/s rd, 41 MiB/s wr, 1.04k op/s; 71172/394782 objects misplaced (18.028%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T10:18:18.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:18 smithi076 ceph-mon[110963]: pgmap v2027: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 8.0 GiB data, 28 GiB used, 419 GiB / 447 GiB avail; 106 B/s rd, 41 MiB/s wr, 1.04k op/s; 71172/394782 objects misplaced (18.028%); 8.5 MiB/s, 136 objects/s recovering 2024-09-21T10:18:18.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:18.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:18.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:18.559 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:18.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:18.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:18.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:18.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:18.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:18.985 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:18.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:19.086 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:19.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:19.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:19.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:19.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:19.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:19.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:19.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:19.528 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:19.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:19.630 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:19.659 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:19.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:19.791 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:19.892 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:19.927 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:19.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:20.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:20.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:20.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:20.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:20.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:20.337 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:20 smithi191 ceph-mon[102206]: pgmap v2028: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.9 GiB data, 28 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.01k op/s; 69897/390006 objects misplaced (17.922%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:18:20.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:20 smithi076 ceph-mon[106482]: pgmap v2028: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.9 GiB data, 28 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.01k op/s; 69897/390006 objects misplaced (17.922%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:18:20.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:20 smithi076 ceph-mon[110963]: pgmap v2028: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.9 GiB data, 28 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.01k op/s; 69897/390006 objects misplaced (17.922%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:18:20.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:20.471 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:20.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:20.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:20.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:20.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:20.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:20.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:20.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:20.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:20.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:21.016 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:21.117 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:21.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:21.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:21.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:21.285 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:21.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:18:21.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:21.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:21.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:21.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:21.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:21 smithi076 podman[309730]: 2024-09-21 10:18:21.443831626 +0000 UTC m=+5.661635683 container init d9f876fe746f41a582203ced9280bc50467af3da5ee774e846d784ba9640473b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T10:18:21.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:21 smithi076 podman[309730]: 2024-09-21 10:18:21.454216221 +0000 UTC m=+5.672020283 container start d9f876fe746f41a582203ced9280bc50467af3da5ee774e846d784ba9640473b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:18:21.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.705 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:21.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:21.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:21.837 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:21.939 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:21.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:21.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.036 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:21 smithi076 podman[309730]: 2024-09-21 10:18:21.676860357 +0000 UTC m=+5.894664416 container attach d9f876fe746f41a582203ced9280bc50467af3da5ee774e846d784ba9640473b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T10:18:22.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:22.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:22.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.254 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:22.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:22.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:22.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:22 smithi191 ceph-mon[102206]: pgmap v2029: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.9 GiB data, 28 GiB used, 420 GiB / 447 GiB avail; 96 B/s rd, 29 MiB/s wr, 948 op/s; 69897/390006 objects misplaced (17.922%); 7.8 MiB/s, 124 objects/s recovering 2024-09-21T10:18:22.366 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:22 smithi076 ceph-mon[106482]: pgmap v2029: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.9 GiB data, 28 GiB used, 420 GiB / 447 GiB avail; 96 B/s rd, 29 MiB/s wr, 948 op/s; 69897/390006 objects misplaced (17.922%); 7.8 MiB/s, 124 objects/s recovering 2024-09-21T10:18:22.366 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:22 smithi076 ceph-mon[110963]: pgmap v2029: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.9 GiB data, 28 GiB used, 420 GiB / 447 GiB avail; 96 B/s rd, 29 MiB/s wr, 948 op/s; 69897/390006 objects misplaced (17.922%); 7.8 MiB/s, 124 objects/s recovering 2024-09-21T10:18:22.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:22.422 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:22.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.535 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.636 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:22.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:22.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:22.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:22.909 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:22.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:22.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:22.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:23.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:23.079 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:23.110 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:18:23.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.322 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:23 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/910553271' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:23 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:23.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:23.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:23 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/910553271' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:23 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:18:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:18:23] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-21T10:18:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:23 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/910553271' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:23 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:18:23.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.501 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:23.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:23.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:23.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:23.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:23.737 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:23.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:23.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:23.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:23.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:23.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:24.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:24.036 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:24.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:24.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.180 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:24.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:24.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:24.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:24 smithi191 ceph-mon[102206]: pgmap v2030: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.7 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 85 B/s rd, 25 MiB/s wr, 1.15k op/s; 67690/378894 objects misplaced (17.865%); 10 MiB/s, 166 objects/s recovering 2024-09-21T10:18:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:24 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:24 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:18:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:24 smithi191 ceph-mon[102206]: osdmap e1315: 8 total, 7 up, 5 in 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[106482]: pgmap v2030: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.7 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 85 B/s rd, 25 MiB/s wr, 1.15k op/s; 67690/378894 objects misplaced (17.865%); 10 MiB/s, 166 objects/s recovering 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[106482]: osdmap e1315: 8 total, 7 up, 5 in 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[110963]: pgmap v2030: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.7 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 85 B/s rd, 25 MiB/s wr, 1.15k op/s; 67690/378894 objects misplaced (17.865%); 10 MiB/s, 166 objects/s recovering 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:18:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:24 smithi076 ceph-mon[110963]: osdmap e1315: 8 total, 7 up, 5 in 2024-09-21T10:18:24.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:24.450 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:24.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:24.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:24.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:24.721 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:24.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:24.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:24.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:24.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:24.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:24.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:25.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:25.127 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:25.229 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:25.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:25.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:25.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:25.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:25.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:25.541 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:25.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:25.677 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:25.779 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:25.810 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:25.912 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:25.945 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:25.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:25.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:26.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:26.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.203 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:26.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:26 smithi191 ceph-mon[102206]: pgmap v2032: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.6 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 2.6 MiB/s wr, 1.05k op/s; 66438/374238 objects misplaced (17.753%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:18:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:18:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-mon[106482]: pgmap v2032: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.6 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 2.6 MiB/s wr, 1.05k op/s; 66438/374238 objects misplaced (17.753%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:18:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:18:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-mon[110963]: pgmap v2032: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.6 GiB data, 27 GiB used, 420 GiB / 447 GiB avail; 102 B/s rd, 2.6 MiB/s wr, 1.05k op/s; 66438/374238 objects misplaced (17.753%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:18:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:18:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:26.436 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:26.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:26.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:26.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:26.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:26.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:26.738 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:26.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:26.880 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:26.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:26.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:26.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:27.020 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:27.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:27.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:27.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:18:27.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:26 smithi076 bash[309730]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:18:27.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:26 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:18:27.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:26 smithi076 bash[309730]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:18:27.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:27.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:27.281 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:27.383 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:27.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.419 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:27.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:27.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:27.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:27.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:27.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:27.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:27.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:27.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:27.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:27.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:28.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:28.084 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:28.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:28.112 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:27 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:18:28.112 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:27 smithi076 bash[309730]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:18:28.112 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:27 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:18:28.112 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:27 smithi076 bash[309730]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T10:18:28.186 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:28.207 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:28.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:28.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-mon[106482]: pgmap v2033: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.4 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 64440/363864 objects misplaced (17.710%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:28.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-mon[110963]: pgmap v2033: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.4 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 64440/363864 objects misplaced (17.710%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 bash[309730]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 bash[309730]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 bash[309730]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:18:28.374 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 bash[309730]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:18:28.375 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate[310006]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:18:28.375 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 bash[309730]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T10:18:28.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:28.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:28.581 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:28.614 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:28 smithi191 ceph-mon[102206]: pgmap v2033: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.4 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 64440/363864 objects misplaced (17.710%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:28.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:28.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:28.718 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:28.747 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:28.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:28.873 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:28 smithi076 podman[309730]: 2024-09-21 10:18:28.570432879 +0000 UTC m=+12.788236937 container died d9f876fe746f41a582203ced9280bc50467af3da5ee774e846d784ba9640473b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T10:18:28.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:28.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:28.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:28.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:29.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:29.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:29.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:29.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:29.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.303 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:29.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:29.404 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:29.423 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:29 smithi076 podman[311035]: 2024-09-21 10:18:29.269627721 +0000 UTC m=+0.919919609 container remove d9f876fe746f41a582203ced9280bc50467af3da5ee774e846d784ba9640473b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b) 2024-09-21T10:18:29.439 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:29.445 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:29.542 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:29.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:29.688 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:29.722 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:29.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:29.861 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:29.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:29.963 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:30.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:30.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:30.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:30.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:30.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:30.260 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:30 smithi076 ceph-mon[106482]: pgmap v2034: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 63190/359169 objects misplaced (17.593%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:30.261 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:30 smithi076 ceph-mon[110963]: pgmap v2034: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 63190/359169 objects misplaced (17.593%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:30.281 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:30.383 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.422 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:30.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:30.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:30.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.561 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:30.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:30.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:30 smithi191 ceph-mon[102206]: pgmap v2034: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 63190/359169 objects misplaced (17.593%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:30.662 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:30.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.693 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:30.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:30.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:30.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.831 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:30.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:30.932 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:30.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:30.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:31.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:31.105 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:31.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:31.187 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:31 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2725122009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:31.187 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:31 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2725122009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:31.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:31.245 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:18:31.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:18:31.246 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:18:31.246 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-21T10:18:31.246 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:31.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:31.247 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:31.247 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:18:31.247 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:18:31.248 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:18:31.248 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:18:31.248 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:31.248 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:31.248 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:31.249 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.250 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:31.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:31.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.386 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:31.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:31.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:18:31.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:31.509 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:31 smithi076 podman[311296]: 2024-09-21 10:18:31.134306792 +0000 UTC m=+0.569349332 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:18:31.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:31.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:31 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2725122009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:31.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:31.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:31.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:31.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:31.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:31.923 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:31.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:32.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:32.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:32.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:32.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:32.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:32.305 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:32 smithi076 ceph-mon[106482]: pgmap v2035: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 63190/359169 objects misplaced (17.593%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:32.305 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:32 smithi076 ceph-mon[110963]: pgmap v2035: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 63190/359169 objects misplaced (17.593%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:32.306 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:32 smithi076 podman[311296]: 2024-09-21 10:18:32.091799196 +0000 UTC m=+1.526841740 container create 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:18:32.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:32.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:32.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:32.480 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:32.520 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:32 smithi191 ceph-mon[102206]: pgmap v2035: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.3 GiB data, 26 GiB used, 421 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 63190/359169 objects misplaced (17.593%); 8.1 MiB/s, 128 objects/s recovering 2024-09-21T10:18:32.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:32.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:32 smithi076 podman[311296]: 2024-09-21 10:18:32.440255226 +0000 UTC m=+1.875297766 container init 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T10:18:32.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:32 smithi076 podman[311296]: 2024-09-21 10:18:32.448472877 +0000 UTC m=+1.883515411 container start 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:18:32.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:32.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:32.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:32.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:32.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:18:32.900 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:18:32.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:32.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:33.030 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:33.035 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:18:33.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:33.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:33 smithi076 bash[311296]: 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 2024-09-21T10:18:33.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:33 smithi076 systemd[1]: Started Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:18:33.162 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:33.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:33.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:33.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:33.346 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:33.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:33.448 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:33.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:33.497 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:18:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:18:33] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.51.0" 2024-09-21T10:18:33.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:33.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:33.668 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:33.668 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:33.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:33.763 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:33.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:33.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:33.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:33.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:34.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:34.031 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:34.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:34.174 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:34.261 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:34 smithi076 ceph-mon[106482]: pgmap v2036: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.1 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 61234/348579 objects misplaced (17.567%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:18:34.262 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:34 smithi076 ceph-mon[110963]: pgmap v2036: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.1 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 61234/348579 objects misplaced (17.567%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:18:34.262 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[311601]: 2024-09-21T10:18:33.891+0000 7f00b2873640 -1 Falling back to public interface 2024-09-21T10:18:34.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.316 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:34.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:34.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:34.454 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:34.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:34.588 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:34.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:34.591 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:18:34.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:34 smithi191 ceph-mon[102206]: pgmap v2036: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.1 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 61234/348579 objects misplaced (17.567%); 7.5 MiB/s, 120 objects/s recovering 2024-09-21T10:18:34.619 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:34.619 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:34.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.735 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:34.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:34.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:34.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:34.876 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:34.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:34.901 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:18:34.977 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:35.007 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:35.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:35.109 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:35.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:35.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:35.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:35.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:35.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:35.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:35.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:35.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:35.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:35.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:35.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:35.807 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:35.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:35.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:35.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:35.977 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:35.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:36.078 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:36.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:36.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:36.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:36.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:36.354 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:36.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:36.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:36.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:36.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:36.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:36 smithi191 ceph-mon[102206]: pgmap v2037: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.0 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 59844/343407 objects misplaced (17.427%); 6.5 MiB/s, 103 objects/s recovering 2024-09-21T10:18:36.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:36 smithi076 ceph-mon[106482]: pgmap v2037: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.0 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 59844/343407 objects misplaced (17.427%); 6.5 MiB/s, 103 objects/s recovering 2024-09-21T10:18:36.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:36 smithi076 ceph-mon[110963]: pgmap v2037: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 7.0 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 59844/343407 objects misplaced (17.427%); 6.5 MiB/s, 103 objects/s recovering 2024-09-21T10:18:36.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:36.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:36.756 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:36.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:36.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:36.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:36.907 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:36.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:36.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:36.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:37.045 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:37.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:37.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:37.202 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:37.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:37.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:37.314 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:37.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:37.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:37.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:37.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:37.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:37.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:37.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:37.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:37.871 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:37.903 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:37.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:38.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:38.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:38.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:38.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:38.176 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:38.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:38.278 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:38.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:38.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:38.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:38.450 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:38.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:38.552 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:38.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:38.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:38 smithi191 ceph-mon[102206]: pgmap v2038: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s; 57800/332346 objects misplaced (17.392%); 9.3 MiB/s, 149 objects/s recovering 2024-09-21T10:18:38.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:38 smithi076 ceph-mon[106482]: pgmap v2038: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s; 57800/332346 objects misplaced (17.392%); 9.3 MiB/s, 149 objects/s recovering 2024-09-21T10:18:38.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:38 smithi076 ceph-mon[110963]: pgmap v2038: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s; 57800/332346 objects misplaced (17.392%); 9.3 MiB/s, 149 objects/s recovering 2024-09-21T10:18:38.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.722 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:38.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:38.823 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:38.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:38.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:38.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:38.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:38.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:38.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:39.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:39.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:39.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:39.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:39.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:39.367 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:39.399 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:39.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:39.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:39.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:39.631 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:39.665 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:39.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:39.766 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:39.808 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:39.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:39.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:39.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:40.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:40.082 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:40.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:39 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[311601]: 2024-09-21T10:18:39.735+0000 7f00b2873640 -1 osd.0 1292 log_to_monitors true 2024-09-21T10:18:40.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:40.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:40.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.445 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:40.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:40.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:40.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:40 smithi191 ceph-mon[102206]: pgmap v2039: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.7 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 56491/327378 objects misplaced (17.256%); 6.1 MiB/s, 98 objects/s recovering 2024-09-21T10:18:40.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:40 smithi191 ceph-mon[102206]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:18:40.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:40.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:40.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:40 smithi076 ceph-mon[106482]: pgmap v2039: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.7 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 56491/327378 objects misplaced (17.256%); 6.1 MiB/s, 98 objects/s recovering 2024-09-21T10:18:40.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:40 smithi076 ceph-mon[106482]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:18:40.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:40 smithi076 ceph-mon[110963]: pgmap v2039: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.7 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 56491/327378 objects misplaced (17.256%); 6.1 MiB/s, 98 objects/s recovering 2024-09-21T10:18:40.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:40 smithi076 ceph-mon[110963]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:18:40.624 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:18:40 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[311601]: 2024-09-21T10:18:40.160+0000 7f00a6615640 -1 osd.0 1292 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:18:40.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:40.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:40.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:40.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:40.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:40.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:40.992 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:41.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:41.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:41.122 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:41.153 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:41.255 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:41.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.298 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:41.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[106482]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:18:41.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[106482]: osdmap e1316: 8 total, 7 up, 5 in 2024-09-21T10:18:41.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[106482]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:18:41.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:41.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[110963]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:18:41.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[110963]: osdmap e1316: 8 total, 7 up, 5 in 2024-09-21T10:18:41.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[110963]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:18:41.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:41.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:41.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:41 smithi191 ceph-mon[102206]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:18:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:41 smithi191 ceph-mon[102206]: osdmap e1316: 8 total, 7 up, 5 in 2024-09-21T10:18:41.515 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:41 smithi191 ceph-mon[102206]: from='osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:18:41.515 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:41.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.569 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:41.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:41.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:41.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:41.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:41.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:41.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:41.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:41.973 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:41.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:42.074 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:42.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:42.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:42.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:42.293 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:42 smithi191 ceph-mon[102206]: pgmap v2041: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.7 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.06k op/s; 56491/327378 objects misplaced (17.256%); 7.4 MiB/s, 117 objects/s recovering 2024-09-21T10:18:42.293 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:42 smithi191 ceph-mon[102206]: osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421] boot 2024-09-21T10:18:42.293 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:42 smithi191 ceph-mon[102206]: osdmap e1317: 8 total, 8 up, 5 in 2024-09-21T10:18:42.293 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:18:42.293 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/771357568' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:42.293 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:42 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:42.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:42.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:42.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:42.429 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[106482]: pgmap v2041: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.7 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.06k op/s; 56491/327378 objects misplaced (17.256%); 7.4 MiB/s, 117 objects/s recovering 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[106482]: osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421] boot 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[106482]: osdmap e1317: 8 total, 8 up, 5 in 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/771357568' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[110963]: pgmap v2041: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.7 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 0 B/s wr, 1.06k op/s; 56491/327378 objects misplaced (17.256%); 7.4 MiB/s, 117 objects/s recovering 2024-09-21T10:18:42.430 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[110963]: osd.0 [v2:172.21.15.76:6826/399575421,v1:172.21.15.76:6827/399575421] boot 2024-09-21T10:18:42.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[110963]: osdmap e1317: 8 total, 8 up, 5 in 2024-09-21T10:18:42.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:18:42.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/771357568' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:42.431 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:42 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:18:42.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:42.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:42.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:42.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:42.659 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:42.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:42.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:42.889 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:42.918 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:42.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:43.020 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:43.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:43.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:43.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:43.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:43.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:43.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.334 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:43.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:43.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:43.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:43.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:43.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:43 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:43.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:43 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:43.481 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:43 smithi191 ceph-mon[102206]: osdmap e1318: 8 total, 8 up, 5 in 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:18:43] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-mon[106482]: osdmap e1318: 8 total, 8 up, 5 in 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:18:43.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:43 smithi076 ceph-mon[110963]: osdmap e1318: 8 total, 8 up, 5 in 2024-09-21T10:18:43.565 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:43.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:43.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:43.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:43.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:43.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:43.876 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:43.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:43.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:43.978 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.023 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:44.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:44.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:44.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:44.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:44.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:44.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:44.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:44.426 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:44.528 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:44.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:44.585 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:44 smithi076 ceph-mon[106482]: pgmap v2044: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 870 op/s; 54584/316680 objects misplaced (17.236%); 5.6 MiB/s, 88 objects/s recovering 2024-09-21T10:18:44.585 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:44 smithi076 ceph-mon[110963]: pgmap v2044: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 870 op/s; 54584/316680 objects misplaced (17.236%); 5.6 MiB/s, 88 objects/s recovering 2024-09-21T10:18:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:44 smithi191 ceph-mon[102206]: pgmap v2044: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 870 op/s; 54584/316680 objects misplaced (17.236%); 5.6 MiB/s, 88 objects/s recovering 2024-09-21T10:18:44.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:44.698 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:44.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:44.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.845 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:44.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:44.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:44.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:44.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:45.092 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:45.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:45.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:45.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:45.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:45.368 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:45.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:45.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:45.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:45.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:45.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:45.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:45.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:45.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:45.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:45.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:46.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:46.095 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:46.197 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:46.218 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:18:46.218 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:18:46.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:46.342 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:46.374 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:46.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:46.512 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:46.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:46 smithi191 ceph-mon[102206]: pgmap v2045: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 847 op/s; 53389/312120 objects misplaced (17.105%); 5.6 MiB/s, 88 objects/s recovering 2024-09-21T10:18:46.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:46.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:46 smithi076 ceph-mon[106482]: pgmap v2045: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 847 op/s; 53389/312120 objects misplaced (17.105%); 5.6 MiB/s, 88 objects/s recovering 2024-09-21T10:18:46.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:46 smithi076 ceph-mon[110963]: pgmap v2045: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 847 op/s; 53389/312120 objects misplaced (17.105%); 5.6 MiB/s, 88 objects/s recovering 2024-09-21T10:18:46.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:46.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:46.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:46.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:46.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:46.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:46.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:46.940 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:46.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:47.041 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:47.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:47.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:47.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:47.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:47.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:47.364 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:47.466 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:47.494 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:47.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:47.625 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:47.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:47.727 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.761 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:47.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:47.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:47.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:47.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:47.996 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:18:47.997 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T10:18:48.012 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:48.027 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:18:48.027 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:18:48.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:48.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:48.056 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:18:48.056 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:18:48.083 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:18:48.084 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:18:48.107 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:18:48.108 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:18:48.145 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:18:48.145 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:18:48.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:48.181 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:18:48.181 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:18:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.185 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:48.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:48.221 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:18:48.221 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:18:48.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:48 smithi076 ceph-mon[106482]: pgmap v2046: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.1 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.29k op/s; 51486/301929 objects misplaced (17.052%); 11 MiB/s, 172 objects/s recovering 2024-09-21T10:18:48.249 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1450059701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:48.249 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:48 smithi076 ceph-mon[110963]: pgmap v2046: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.1 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.29k op/s; 51486/301929 objects misplaced (17.052%); 11 MiB/s, 172 objects/s recovering 2024-09-21T10:18:48.249 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1450059701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:48.282 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:18:48.282 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:18:48.282 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:18:48.282 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:18:48.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:48.325 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:48.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:48.427 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:48.457 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:48.559 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:48.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.589 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:48.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:48 smithi191 ceph-mon[102206]: pgmap v2046: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.1 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.29k op/s; 51486/301929 objects misplaced (17.052%); 11 MiB/s, 172 objects/s recovering 2024-09-21T10:18:48.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1450059701' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:18:48.690 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:48.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:48.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:48.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:48.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:48.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:48.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:49.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:49.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:49.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:49.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:49.250 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:49.280 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:49.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:49.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:49.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:49.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:49.658 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:49.685 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:49.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:49.759 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:18:49.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:49.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:49.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:49.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:49.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:50.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:18:50.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:50.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:50.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:50.231 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:50.273 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:50.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:50.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:50 smithi076 ceph-mon[106482]: pgmap v2047: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s; 50211/297150 objects misplaced (16.898%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T10:18:50.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:50 smithi076 ceph-mon[110963]: pgmap v2047: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s; 50211/297150 objects misplaced (16.898%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T10:18:50.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:50.415 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:50.517 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:50.545 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:50.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:50 smithi191 ceph-mon[102206]: pgmap v2047: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.26k op/s; 50211/297150 objects misplaced (16.898%); 8.8 MiB/s, 141 objects/s recovering 2024-09-21T10:18:50.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:50.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:18:50.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:50.692 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:50.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:50.831 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:50.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:50.973 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:50.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:51.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:51.120 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:51.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:51.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:51.353 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:51.381 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:51.483 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:51.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:18:51.522 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:51.576 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:51.577 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:51.624 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:51.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:51.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:51.797 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:51.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:51.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:51.934 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:51.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:52.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:52.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:52.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:52.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:52.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:52.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:52.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:52.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:52.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:52.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:52.487 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:52 smithi076 ceph-mon[106482]: pgmap v2048: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s; 50211/297150 objects misplaced (16.898%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:18:52.488 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:52 smithi076 ceph-mon[110963]: pgmap v2048: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s; 50211/297150 objects misplaced (16.898%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:18:52.571 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:52.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:52.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:52.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:52 smithi191 ceph-mon[102206]: pgmap v2048: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s; 50211/297150 objects misplaced (16.898%); 7.4 MiB/s, 118 objects/s recovering 2024-09-21T10:18:52.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:52.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:52.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:52.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:52.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:52.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:52.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:52.990 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:53.020 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:53.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:53.158 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:53.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:53.260 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:53.289 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:53.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:53.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:53.416 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:53.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:53.529 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:18:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:18:53] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.51.0" 2024-09-21T10:18:53.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:53.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:53.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:53.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:53.769 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:53.792 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:53.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:53.916 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:53.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:53.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:54.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:54.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:54.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:54.165 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:54.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:54.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:54.306 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:54.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:54.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:54.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:54.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:54.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:54.547 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:54 smithi076 ceph-mon[106482]: pgmap v2049: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 963 op/s; 48237/286230 objects misplaced (16.853%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T10:18:54.547 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:54 smithi076 ceph-mon[106482]: osdmap e1319: 8 total, 8 up, 5 in 2024-09-21T10:18:54.548 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:54 smithi076 ceph-mon[110963]: pgmap v2049: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 963 op/s; 48237/286230 objects misplaced (16.853%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T10:18:54.548 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:54 smithi076 ceph-mon[110963]: osdmap e1319: 8 total, 8 up, 5 in 2024-09-21T10:18:54.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:54 smithi191 ceph-mon[102206]: pgmap v2049: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 963 op/s; 48237/286230 objects misplaced (16.853%); 7.1 MiB/s, 113 objects/s recovering 2024-09-21T10:18:54.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:54 smithi191 ceph-mon[102206]: osdmap e1319: 8 total, 8 up, 5 in 2024-09-21T10:18:54.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:54.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.711 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:54.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:54.812 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:54.842 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:54.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:54.973 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:54.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:55.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:55.108 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:55.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.247 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:55.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:55.348 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:55.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:55.478 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:55.488 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:55 smithi076 ceph-mon[106482]: osdmap e1320: 8 total, 8 up, 5 in 2024-09-21T10:18:55.488 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:55 smithi076 ceph-mon[110963]: osdmap e1320: 8 total, 8 up, 5 in 2024-09-21T10:18:55.515 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:55.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:55 smithi191 ceph-mon[102206]: osdmap e1320: 8 total, 8 up, 5 in 2024-09-21T10:18:55.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:55.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:55.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:55.846 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:18:55.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:55.948 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:55.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:56.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:56.090 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:56.193 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:56.226 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:56.243 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:18:56.246 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:18:56.327 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:56.368 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:56.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:56.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:18:56.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:18:56.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:56.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:56 smithi191 ceph-mon[102206]: pgmap v2052: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 853 op/s; 46965/281445 objects misplaced (16.687%); 4.7 MiB/s, 74 objects/s recovering 2024-09-21T10:18:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3619490558' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:56.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1100380422' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[106482]: pgmap v2052: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 853 op/s; 46965/281445 objects misplaced (16.687%); 4.7 MiB/s, 74 objects/s recovering 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3619490558' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1100380422' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[110963]: pgmap v2052: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 853 op/s; 46965/281445 objects misplaced (16.687%); 4.7 MiB/s, 74 objects/s recovering 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:18:56.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3619490558' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:18:56.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1100380422' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:18:56.649 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:56.751 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:56.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:56.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:56.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:56.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:56.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:57.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:57.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:57.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:18:57.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:18:57.314 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:57.353 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:57.374 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:18:57.375 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:18:57.375 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:18:57.375 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:18:57.375 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:18:57.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:57.484 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:57.586 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:57 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:57 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3619490558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:57 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1100380422' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:18:57.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:57 smithi191 ceph-mon[102206]: osdmap e1321: 8 total, 8 up, 5 in 2024-09-21T10:18:57.615 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:57.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3619490558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1100380422' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[110963]: osdmap e1321: 8 total, 8 up, 5 in 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3619490558' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1100380422' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:18:57.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:57 smithi076 ceph-mon[106482]: osdmap e1321: 8 total, 8 up, 5 in 2024-09-21T10:18:57.719 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:57.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:18:57.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:57.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:57.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:58.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:58.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:58.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:58.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:58.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:58.300 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:58.332 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:58.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:58 smithi076 ceph-mon[106482]: pgmap v2054: 89 pgs: 8 active+remapped+backfill_wait, 81 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s; 46536/278376 objects misplaced (16.717%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:18:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:58 smithi076 ceph-mon[110963]: pgmap v2054: 89 pgs: 8 active+remapped+backfill_wait, 81 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s; 46536/278376 objects misplaced (16.717%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:18:58.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:58.466 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:58.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:58.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:58.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:58 smithi191 ceph-mon[102206]: pgmap v2054: 89 pgs: 8 active+remapped+backfill_wait, 81 active+clean; 5.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 1.04k op/s; 46536/278376 objects misplaced (16.717%); 7.9 MiB/s, 126 objects/s recovering 2024-09-21T10:18:58.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:58.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:58.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:58.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:18:58.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:58.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:18:58.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:18:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.018 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:59.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:18:59.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:59.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:18:59.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:59.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:59.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:59.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:18:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:18:59.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:18:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:18:59.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:59.420 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:59.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:59.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:18:59.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:18:59.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:18:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:18:59.674 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:18:59.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:18:59.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:18:59.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:18:59.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:18:59.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:18:59.982 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:18:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:18:59.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:00.084 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:00.112 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:00.214 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:00.245 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:00.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:00.384 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:00.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:00.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.525 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:00.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:00.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:00 smithi191 ceph-mon[102206]: pgmap v2055: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 43359/267816 objects misplaced (16.190%); 14 MiB/s, 218 objects/s recovering 2024-09-21T10:19:00.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:00 smithi076 ceph-mon[106482]: pgmap v2055: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 43359/267816 objects misplaced (16.190%); 14 MiB/s, 218 objects/s recovering 2024-09-21T10:19:00.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:00 smithi076 ceph-mon[110963]: pgmap v2055: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 43359/267816 objects misplaced (16.190%); 14 MiB/s, 218 objects/s recovering 2024-09-21T10:19:00.626 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:00.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:00.757 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:00.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:00.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:00.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:00.961 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:00.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:01.062 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:01.115 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:01.216 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.244 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:01.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:01.344 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.377 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:01.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:01.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:01.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:01.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.632 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:01.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:01.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.768 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:01.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:01.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:01.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:01.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:02.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:02.036 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:02.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:02.138 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:02.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:02.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:02.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:02.300 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:02.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:02.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:02.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:02.535 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:02.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:02.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:02.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:02 smithi191 ceph-mon[102206]: pgmap v2056: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 819 op/s; 43359/267816 objects misplaced (16.190%); 11 MiB/s, 174 objects/s recovering 2024-09-21T10:19:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:02 smithi076 ceph-mon[106482]: pgmap v2056: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 819 op/s; 43359/267816 objects misplaced (16.190%); 11 MiB/s, 174 objects/s recovering 2024-09-21T10:19:02.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:02 smithi076 ceph-mon[110963]: pgmap v2056: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 819 op/s; 43359/267816 objects misplaced (16.190%); 11 MiB/s, 174 objects/s recovering 2024-09-21T10:19:02.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:02.708 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:02.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:02.849 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:02.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:02.951 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:02.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:02.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:03.091 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:03.117 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:03.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:03.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:03.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:03.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:03.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:03.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:03.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:03.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:19:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:19:03] "GET /metrics HTTP/1.1" 200 36637 "" "Prometheus/2.51.0" 2024-09-21T10:19:03.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:03.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:03.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:03.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:03.873 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:03.899 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:03.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:03.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:04.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.042 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:04.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:04.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:04.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:04.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:04.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:19:04.316 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:04.418 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:04.477 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:04.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:04.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:04.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:04 smithi191 ceph-mon[102206]: pgmap v2057: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 42151/257811 objects misplaced (16.350%); 9.6 MiB/s, 154 objects/s recovering 2024-09-21T10:19:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:04 smithi076 ceph-mon[106482]: pgmap v2057: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 42151/257811 objects misplaced (16.350%); 9.6 MiB/s, 154 objects/s recovering 2024-09-21T10:19:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:04 smithi076 ceph-mon[110963]: pgmap v2057: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 927 op/s; 42151/257811 objects misplaced (16.350%); 9.6 MiB/s, 154 objects/s recovering 2024-09-21T10:19:04.624 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:04.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:04.726 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:04.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:04.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:04.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:04.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:05.031 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:05.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:05.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:05.190 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:05.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:05.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:05.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.455 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:05.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:05.556 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:05.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:05.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:05.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:05.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:05.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:05.943 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:05.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:05.964 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:05.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:06.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:06.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.103 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:06.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:06.204 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:06.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:06.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:06.365 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:06.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:06.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:06.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:06 smithi191 ceph-mon[102206]: pgmap v2058: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 929 op/s; 40481/253560 objects misplaced (15.965%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:19:06.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:06 smithi076 ceph-mon[106482]: pgmap v2058: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 929 op/s; 40481/253560 objects misplaced (15.965%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:19:06.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:06 smithi076 ceph-mon[110963]: pgmap v2058: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 929 op/s; 40481/253560 objects misplaced (15.965%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:19:06.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:06.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:06.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:06.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:06.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:06.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:06.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:07.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:07.132 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:07.176 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:07.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:07.278 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:07.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:07.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:07.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:07.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:07.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:07.591 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:07.693 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:07.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:07.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:07.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:07.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:07.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:07.966 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:07.999 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:08.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:08.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:08.145 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:08.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:08.203 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:08.246 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:08.276 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:08.319 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:08 smithi076 ceph-mon[106482]: pgmap v2059: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 40229/250221 objects misplaced (16.077%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:19:08.319 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:08 smithi076 ceph-mon[110963]: pgmap v2059: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 40229/250221 objects misplaced (16.077%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:19:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:08 smithi191 ceph-mon[102206]: pgmap v2059: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 5.1 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 40229/250221 objects misplaced (16.077%); 11 MiB/s, 181 objects/s recovering 2024-09-21T10:19:08.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:08.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:08.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:08.521 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:08.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:08.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:08.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:08.696 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:08.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:08.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:08.933 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:08.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:08.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:09.007 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:09.042 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:19:09.042 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:09.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:19:09.073 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:19:09.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:09.097 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:19:09.097 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:09.121 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:19:09.121 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:09.121 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:09.159 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:19:09.159 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:09.200 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:19:09.201 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:09.223 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:09.232 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:19:09.232 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:09.259 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:19:09.260 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:09.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:09.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:09.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:09.391 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:09.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:09.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:09.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:19:09.532 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:09.633 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:09.667 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:09.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[106482]: pgmap v2060: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 37672/239904 objects misplaced (15.703%); 12 MiB/s, 198 objects/s recovering 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3448718355' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[110963]: pgmap v2060: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 37672/239904 objects misplaced (15.703%); 12 MiB/s, 198 objects/s recovering 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:09.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:09 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3448718355' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:09.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:09.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:09.911 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:09.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:09.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:09.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:10.048 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:10.082 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:09 smithi191 ceph-mon[102206]: pgmap v2060: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 37672/239904 objects misplaced (15.703%); 12 MiB/s, 198 objects/s recovering 2024-09-21T10:19:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:09 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:09 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3448718355' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:10.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.215 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:10.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:10.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:10.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:10.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:10.462 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:10.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:10.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:10.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:10.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:10.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:10.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:10.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:10.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:10.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:10.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:11.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:11.018 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:11.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:11.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:11.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:11.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:11.331 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:11.432 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:11.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:11.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:11.570 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:11.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:11.712 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:11.743 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:11.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:11.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:11.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:11.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:11.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:12.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:12.103 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:11 smithi191 ceph-mon[102206]: pgmap v2061: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 775 op/s; 37672/239904 objects misplaced (15.703%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:19:12.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:11 smithi076 ceph-mon[106482]: pgmap v2061: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 775 op/s; 37672/239904 objects misplaced (15.703%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:19:12.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:11 smithi076 ceph-mon[110963]: pgmap v2061: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 775 op/s; 37672/239904 objects misplaced (15.703%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:19:12.125 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.226 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:12.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:12.356 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:12.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:12.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:12.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:12.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:12.657 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:12.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:12.788 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:12.817 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:12.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:12.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:12.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:13.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.096 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:13.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:13.197 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:13.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:13.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:13.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:13.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:13.385 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:19:13.416 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:19:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:19:13] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.51.0" 2024-09-21T10:19:13.473 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:13.518 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:13.620 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:13.656 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:13.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:13.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:13.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:13.924 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:13.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:13.998 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:13 smithi076 ceph-mon[106482]: pgmap v2062: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 36450/229389 objects misplaced (15.890%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:19:13.998 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:13 smithi076 ceph-mon[110963]: pgmap v2062: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 36450/229389 objects misplaced (15.890%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:19:14.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:14.071 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:14.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:14.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:13 smithi191 ceph-mon[102206]: pgmap v2062: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 36450/229389 objects misplaced (15.890%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:19:14.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:14.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:14.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:14.318 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:14.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:14.365 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:14.365 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:14.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:14.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:14.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:14.642 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:14.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:14.783 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:14.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:14.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:14.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:14.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:14.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:15.017 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:15.056 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:15.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:15.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:15.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:15.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.330 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:15.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:15.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:19:15.431 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.472 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:15.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:15.557 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1176826704' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:19:15.557 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:15 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:19:15.557 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:19:15.558 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1176826704' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:19:15.558 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:15 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:19:15.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:15.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1176826704' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:19:15.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:15 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:19:15.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:15.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.739 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:15.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:15.840 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:15.870 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:15.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:15.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:15.931 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:15.972 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:16.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:16.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:16.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:16.258 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:16.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:16.335 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:16 smithi076 ceph-mon[106482]: pgmap v2063: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 912 op/s; 34738/224940 objects misplaced (15.443%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:19:16.335 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:16 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:19:16.335 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:16 smithi076 ceph-mon[106482]: osdmap e1322: 8 total, 8 up, 5 in 2024-09-21T10:19:16.335 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:16 smithi076 ceph-mon[110963]: pgmap v2063: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 912 op/s; 34738/224940 objects misplaced (15.443%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:19:16.335 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:16 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:19:16.336 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:16 smithi076 ceph-mon[110963]: osdmap e1322: 8 total, 8 up, 5 in 2024-09-21T10:19:16.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:19:16.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:16.429 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.479 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:16.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:16.580 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:16.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:16 smithi191 ceph-mon[102206]: pgmap v2063: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 912 op/s; 34738/224940 objects misplaced (15.443%); 9.9 MiB/s, 158 objects/s recovering 2024-09-21T10:19:16.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:16 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:19:16.615 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:16 smithi191 ceph-mon[102206]: osdmap e1322: 8 total, 8 up, 5 in 2024-09-21T10:19:16.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:16.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:16.720 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:16.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:16.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:16.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:16.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:16.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:16.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:16.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:17.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:17.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.170 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:17.271 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:17.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:17.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:17.458 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:17.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:17.594 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:17.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:17.716 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:17.818 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:17.855 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:17.957 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:17.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:17.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:18.087 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:18.117 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:18.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:18.218 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:18.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:18.353 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:18.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:18.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:18.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:18.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:18.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:18.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:18.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:18 smithi191 ceph-mon[102206]: pgmap v2065: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.5 GiB data, 18 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 971 op/s; 34412/221070 objects misplaced (15.566%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:19:18.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:18 smithi076 ceph-mon[106482]: pgmap v2065: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.5 GiB data, 18 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 971 op/s; 34412/221070 objects misplaced (15.566%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:19:18.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:18 smithi076 ceph-mon[110963]: pgmap v2065: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.5 GiB data, 18 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 971 op/s; 34412/221070 objects misplaced (15.566%); 8.2 MiB/s, 131 objects/s recovering 2024-09-21T10:19:18.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:18.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:18.792 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:18.894 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:18.937 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:18.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:18.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:19.039 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:19.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:19.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:19.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:19.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:19.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:19.343 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:19.444 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:19.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:19.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:19.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:19.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:19.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:19.842 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:19.872 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:19.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:19.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:19.974 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:20.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:20.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:20.138 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:20.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:20.276 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:20.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:20.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.415 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:20.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:20.516 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:20.537 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:20.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:20 smithi191 ceph-mon[102206]: pgmap v2066: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 993 op/s; 31857/210075 objects misplaced (15.165%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:20.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:20 smithi076 ceph-mon[106482]: pgmap v2066: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 993 op/s; 31857/210075 objects misplaced (15.165%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:20.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:20 smithi076 ceph-mon[110963]: pgmap v2066: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 993 op/s; 31857/210075 objects misplaced (15.165%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:20.638 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:20.665 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:20.767 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.803 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:20.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:20.904 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:20.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:20.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:20.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:21.045 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:21.096 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:21.198 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:21.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:21.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:21.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:21.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:21.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:21.505 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:21.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.647 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:21.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:21.748 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:21.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:21.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:21.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:21.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:22.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:22.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:22.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:22.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:22.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:22.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:22.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:22.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:22.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:22 smithi076 ceph-mon[106482]: pgmap v2067: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 31857/210075 objects misplaced (15.165%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:22 smithi076 ceph-mon[110963]: pgmap v2067: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 31857/210075 objects misplaced (15.165%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:22.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.467 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:22.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:22.605 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:22.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:22.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:22 smithi191 ceph-mon[102206]: pgmap v2067: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 31857/210075 objects misplaced (15.165%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:22.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.748 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:22.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:22.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:22.886 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:22.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:22.988 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:23.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:23.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:23.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:23.151 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:23.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:23.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:23.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:23.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:23.435 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:23.453 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:19:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:19:23] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.51.0" 2024-09-21T10:19:23.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:23.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:23.675 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:23.714 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:23.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:23.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.859 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:23.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:23.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:23.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:23.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:24.076 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:19:24.076 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:19:24.076 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:19:24.076 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:19:24.077 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:24.078 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:24.078 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:24.099 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:24.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:24.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:24.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:24.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:24.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:24.320 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:24.320 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:24 smithi076 ceph-mon[106482]: pgmap v2068: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 978 op/s; 30606/200034 objects misplaced (15.300%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1195127091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:24 smithi076 ceph-mon[110963]: pgmap v2068: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 978 op/s; 30606/200034 objects misplaced (15.300%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1195127091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:24.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:24.423 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:24.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:24.565 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:24.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:24 smithi191 ceph-mon[102206]: pgmap v2068: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 978 op/s; 30606/200034 objects misplaced (15.300%); 7.8 MiB/s, 125 objects/s recovering 2024-09-21T10:19:24.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1195127091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:24.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:24.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.703 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:24.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:24.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.838 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:24.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:24.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:24.986 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:24.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:25.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.230 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:25.253 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:19:25.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:25.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:25.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:25.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:25.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.514 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:25.544 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:25.551 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4057157394' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:25.552 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4057157394' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:25.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4057157394' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:25.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:25.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:25.783 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:25.817 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:25.919 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:25.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:25.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:26.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.080 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:26.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:26.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:26.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.357 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:26.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:26.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:26.490 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:26.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: pgmap v2069: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 28916/195099 objects misplaced (14.821%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4057157394' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: osdmap e1323: 8 total, 8 up, 5 in 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: pgmap v2069: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 28916/195099 objects misplaced (14.821%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4057157394' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: osdmap e1323: 8 total, 8 up, 5 in 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:26.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: pgmap v2069: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 994 op/s; 28916/195099 objects misplaced (14.821%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:19:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:19:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:19:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4057157394' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:19:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: osdmap e1323: 8 total, 8 up, 5 in 2024-09-21T10:19:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:26.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:26.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:26.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:26.770 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:26.872 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:26.902 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:26.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.004 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:27.034 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:27.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:27.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:27.297 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:27.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:27.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:27.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:27.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:27.570 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:27.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:19:27.672 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:27.726 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:27.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:27.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:27.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:27.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:27.998 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:27.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:28.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.141 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:28.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:28.242 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:28.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:28 smithi076 ceph-mon[106482]: pgmap v2071: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 897 op/s; 28916/194124 objects misplaced (14.896%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:19:28.436 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:28 smithi076 ceph-mon[110963]: pgmap v2071: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 897 op/s; 28916/194124 objects misplaced (14.896%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:19:28.436 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:28.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.469 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:28.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:28.569 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:28.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:28.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:28 smithi191 ceph-mon[102206]: pgmap v2071: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 897 op/s; 28916/194124 objects misplaced (14.896%); 7.3 MiB/s, 117 objects/s recovering 2024-09-21T10:19:28.706 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:28.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:28.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:28.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:28.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:28.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:28.982 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:29.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:29.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.125 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:29.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:29.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:29.291 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.393 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:29.427 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:29.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:29.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:29.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:29.707 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:29.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:29.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:29.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:29.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:29.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:29.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:30.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:30.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:30.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.469 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:30 smithi076 ceph-mon[106482]: pgmap v2072: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26062/179574 objects misplaced (14.513%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:30.469 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:30 smithi076 ceph-mon[110963]: pgmap v2072: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26062/179574 objects misplaced (14.513%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:30.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:30.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:30 smithi191 ceph-mon[102206]: pgmap v2072: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26062/179574 objects misplaced (14.513%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:30.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:30.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:30.777 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:30.806 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:30.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:30.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:30.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:30.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:31.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:31.096 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:31.198 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:31.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:31.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:31.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:31.363 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:31.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:31.464 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:31.497 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:31.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:31.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:31.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:31.721 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:31.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:31.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:31.871 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:31.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:31.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:31.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:31.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:32.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:32.117 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:32.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:32.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:32.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:32.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:32.285 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:19:32.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:32.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:32.474 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:32.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:32.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:32.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:32 smithi191 ceph-mon[102206]: pgmap v2073: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26062/179574 objects misplaced (14.513%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:32 smithi191 ceph-mon[102206]: osdmap e1324: 8 total, 8 up, 5 in 2024-09-21T10:19:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:32 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3313347115' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:19:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:32 smithi076 ceph-mon[106482]: pgmap v2073: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26062/179574 objects misplaced (14.513%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:32 smithi076 ceph-mon[106482]: osdmap e1324: 8 total, 8 up, 5 in 2024-09-21T10:19:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:32 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3313347115' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:19:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:32 smithi076 ceph-mon[110963]: pgmap v2073: 89 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 81 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26062/179574 objects misplaced (14.513%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:32 smithi076 ceph-mon[110963]: osdmap e1324: 8 total, 8 up, 5 in 2024-09-21T10:19:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:32 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3313347115' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:19:32.632 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:32.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:32.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:32.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:32.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:32.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:32.886 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:32.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:32.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:32.957 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:19:32.987 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:33.033 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:33.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:33.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:33.201 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:19:33.201 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:19:33.201 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:19:33.201 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:19:33.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:33.313 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:33.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:33.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:33.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:33.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:33.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:33.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:33 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3313347115' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:19:33.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:33 smithi191 ceph-mon[102206]: osdmap e1325: 8 total, 8 up, 5 in 2024-09-21T10:19:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:19:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:19:33] "GET /metrics HTTP/1.1" 200 36640 "" "Prometheus/2.51.0" 2024-09-21T10:19:33.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:33 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3313347115' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:19:33.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:33 smithi076 ceph-mon[106482]: osdmap e1325: 8 total, 8 up, 5 in 2024-09-21T10:19:33.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:33 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3313347115' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:19:33.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:33 smithi076 ceph-mon[110963]: osdmap e1325: 8 total, 8 up, 5 in 2024-09-21T10:19:33.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:33.739 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:33.841 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:33.868 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:33.969 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:33.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:33.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:34.098 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:34.139 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:34.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:34.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:34.249 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:34.261 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:34.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:34.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:19:34.363 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:34.404 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:34.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:34.515 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:34 smithi076 ceph-mon[110963]: pgmap v2076: 89 pgs: 1 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.19k op/s; 24308/168537 objects misplaced (14.423%); 9.4 MiB/s, 151 objects/s recovering 2024-09-21T10:19:34.516 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:34 smithi076 ceph-mon[106482]: pgmap v2076: 89 pgs: 1 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.19k op/s; 24308/168537 objects misplaced (14.423%); 9.4 MiB/s, 151 objects/s recovering 2024-09-21T10:19:34.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:34.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:34 smithi191 ceph-mon[102206]: pgmap v2076: 89 pgs: 1 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.19k op/s; 24308/168537 objects misplaced (14.423%); 9.4 MiB/s, 151 objects/s recovering 2024-09-21T10:19:34.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:34.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:34.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:34.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:34.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:34.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:34.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:34.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.001 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:35.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:35.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:35.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:35.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:35.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:35.425 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:35.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:35.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:35.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:35.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:35.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:35.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:35.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:35.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:35.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:35.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:36.075 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.114 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:36.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:36.215 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:36.258 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:36.360 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:36.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:36.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:36.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:36.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:36.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:36 smithi191 ceph-mon[102206]: pgmap v2077: 89 pgs: 1 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.19k op/s; 23542/165471 objects misplaced (14.227%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:19:36.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:36 smithi076 ceph-mon[106482]: pgmap v2077: 89 pgs: 1 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.19k op/s; 23542/165471 objects misplaced (14.227%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:19:36.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:36 smithi076 ceph-mon[110963]: pgmap v2077: 89 pgs: 1 peering, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 81 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.19k op/s; 23542/165471 objects misplaced (14.227%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:19:36.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:36.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:36.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:36.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:19:36.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:36.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:36.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:36.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:36.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:37.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:37.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:37.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:37.189 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:37.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:37.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:37.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:37.375 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:37.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:37.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.523 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:37.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:37.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:37.652 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:37.754 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:37.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:37.882 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:37.918 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:37.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:38.019 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:19:38.020 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:19:38.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:19:38.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:19:38.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:19:38.021 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:19:38.021 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:19:38.021 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: pgmap v2078: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 23542/163815 objects misplaced (14.371%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3544643017' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: pgmap v2078: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 23542/163815 objects misplaced (14.371%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.022 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.023 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3544643017' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:38.023 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:19:38.023 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:19:38.023 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:37 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.023 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:38.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:38.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: pgmap v2078: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 23542/163815 objects misplaced (14.371%); 3.8 MiB/s, 60 objects/s recovering 2024-09-21T10:19:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3544643017' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:19:38.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:19:38.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:19:38.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:37 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:38.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.192 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:38.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:38.293 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:38.322 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:38.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:38.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:38.465 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:38.567 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:38.602 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:38.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:38.704 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:38.734 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:38.836 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:38.871 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:38.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:38.971 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:39.011 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:39.113 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:39.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:39.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:39.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.271 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:39.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:39.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:39.406 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:39.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:39.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:39.624 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:39.631 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:39.646 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:39.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:39.796 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:39.838 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:39.940 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:39.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:39.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:39.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:40.070 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:40.097 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:40.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:39 smithi191 ceph-mon[102206]: pgmap v2079: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s; 20908/153198 objects misplaced (13.648%); 12 MiB/s, 185 objects/s recovering 2024-09-21T10:19:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:39 smithi076 ceph-mon[106482]: pgmap v2079: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s; 20908/153198 objects misplaced (13.648%); 12 MiB/s, 185 objects/s recovering 2024-09-21T10:19:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:39 smithi076 ceph-mon[110963]: pgmap v2079: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s; 20908/153198 objects misplaced (13.648%); 12 MiB/s, 185 objects/s recovering 2024-09-21T10:19:40.199 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:40.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:40.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:40.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:40.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:40.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:40.484 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:40.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:40.626 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:40.725 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:40.728 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:40.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:40.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:40.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:40.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:40.983 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:40.984 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:40.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:41.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:41.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:41.144 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.186 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:41.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:41.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:41.330 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:41.431 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:41.459 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:41.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:41.590 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:41.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.727 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:41.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:41.827 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:41.858 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:41.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:41.960 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:41.987 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:41.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:42.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:41 smithi191 ceph-mon[102206]: pgmap v2080: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 931 op/s; 20908/153198 objects misplaced (13.648%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:19:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:41 smithi076 ceph-mon[106482]: pgmap v2080: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 931 op/s; 20908/153198 objects misplaced (13.648%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:19:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:41 smithi076 ceph-mon[110963]: pgmap v2080: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 931 op/s; 20908/153198 objects misplaced (13.648%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:19:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:42.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:42.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:42.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:42.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:42.384 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:42.486 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:42.523 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:42.624 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:42.660 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:42.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:42.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:19:42.762 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.833 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:42.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:42.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:42.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:42.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:43.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:43.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:43.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:43.242 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:43.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:43.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:43.369 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:43.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:43.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:43.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:43.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:43.542 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:19:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:19:43] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-21T10:19:43.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:43.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:43.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:43.741 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:43.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:43.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:43 smithi076 ceph-mon[106482]: pgmap v2081: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 979 op/s; 18157/137883 objects misplaced (13.168%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:19:43.794 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:43 smithi076 ceph-mon[110963]: pgmap v2081: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 979 op/s; 18157/137883 objects misplaced (13.168%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:19:43.891 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:43.928 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:43.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:43.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:44.030 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:44.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:44.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:43 smithi191 ceph-mon[102206]: pgmap v2081: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 979 op/s; 18157/137883 objects misplaced (13.168%); 9.9 MiB/s, 157 objects/s recovering 2024-09-21T10:19:44.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:44.197 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:44.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:44.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:44.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:44.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:44.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:44.475 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:44.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:44.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:44.707 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:44.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:44.762 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:19:44.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4085276107' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:19:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4085276107' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:19:44.876 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:44.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:44.977 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:45.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:45.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:45.108 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4085276107' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:19:45.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:45.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:45.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:45.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:45.416 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:45.518 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:45.556 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:45.658 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:45.688 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:45.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:45.833 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:45.895 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:19:45.895 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:19:45.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:45.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:45.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:46.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:46.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:45 smithi191 ceph-mon[102206]: pgmap v2082: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 851 op/s; 18157/137883 objects misplaced (13.168%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:19:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4085276107' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:19:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:45 smithi191 ceph-mon[102206]: osdmap e1326: 8 total, 8 up, 5 in 2024-09-21T10:19:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:45 smithi076 ceph-mon[106482]: pgmap v2082: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 851 op/s; 18157/137883 objects misplaced (13.168%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:19:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4085276107' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:19:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:45 smithi076 ceph-mon[106482]: osdmap e1326: 8 total, 8 up, 5 in 2024-09-21T10:19:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:45 smithi076 ceph-mon[110963]: pgmap v2082: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 851 op/s; 18157/137883 objects misplaced (13.168%); 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:19:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4085276107' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:19:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:45 smithi076 ceph-mon[110963]: osdmap e1326: 8 total, 8 up, 5 in 2024-09-21T10:19:46.203 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:46.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:46.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:46.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:46.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:46.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:46.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:46.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:46.658 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:19:46.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:46.799 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:46.900 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:46.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:46.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:47.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:47.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:47.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:47.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:47.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:47.311 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:47.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.365 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:47.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:47.465 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:47.508 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:47.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:47.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:47.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:47.779 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:19:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.783 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:47.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:47.884 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:47.918 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:19:47.939 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:47.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:48.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:48.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:47 smithi191 ceph-mon[102206]: pgmap v2084: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 930 op/s; 18157/135894 objects misplaced (13.361%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:19:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:47 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/92439773' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:47 smithi076 ceph-mon[106482]: pgmap v2084: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 930 op/s; 18157/135894 objects misplaced (13.361%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:19:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/92439773' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:48.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:47 smithi076 ceph-mon[110963]: pgmap v2084: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 930 op/s; 18157/135894 objects misplaced (13.361%); 10 MiB/s, 164 objects/s recovering 2024-09-21T10:19:48.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/92439773' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:19:48.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:48.208 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:48.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:48.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:48.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:48.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:48.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:48.572 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:48.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:48.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:48.711 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:48.743 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:48.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:48.872 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:48.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:48.974 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:49.001 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:49.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:49.044 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:48 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:49.044 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/92439773' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:49.044 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:48 smithi076 ceph-mon[106482]: osdmap e1327: 8 total, 8 up, 5 in 2024-09-21T10:19:49.044 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:48 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:49.044 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/92439773' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:49.044 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:48 smithi076 ceph-mon[110963]: osdmap e1327: 8 total, 8 up, 5 in 2024-09-21T10:19:49.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:48 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:19:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/92439773' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:19:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:48 smithi191 ceph-mon[102206]: osdmap e1327: 8 total, 8 up, 5 in 2024-09-21T10:19:49.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:49.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.279 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:49.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:49.380 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:49.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:49.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:49.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:49.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:49.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:49.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:49.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:49.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:49.805 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:49.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:49.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:49 smithi076 ceph-mon[110963]: pgmap v2086: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.29k op/s; 15482/122154 objects misplaced (12.674%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:19:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:49 smithi076 ceph-mon[106482]: pgmap v2086: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.29k op/s; 15482/122154 objects misplaced (12.674%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:19:49.918 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:49.944 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:49.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:50.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.084 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:50.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:49 smithi191 ceph-mon[102206]: pgmap v2086: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.29k op/s; 15482/122154 objects misplaced (12.674%); 10 MiB/s, 162 objects/s recovering 2024-09-21T10:19:50.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:50.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:50.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:50.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:50.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:50.400 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:50.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:50.484 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:50.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:50.623 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:50.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:50.724 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:50.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:50.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:50.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:50.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:50.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:50.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:51.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:51.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:51.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:19:51.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:51.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:51.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:51.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:51.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:51.379 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:51.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:51.503 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:51.525 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:51.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:51.626 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:51.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:51.750 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:51.772 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:51.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:51.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:51.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:52.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:52.055 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:51 smithi191 ceph-mon[102206]: pgmap v2087: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 655 op/s; 15482/122154 objects misplaced (12.674%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:19:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:51 smithi191 ceph-mon[102206]: osdmap e1328: 8 total, 8 up, 5 in 2024-09-21T10:19:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:51 smithi076 ceph-mon[106482]: pgmap v2087: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 655 op/s; 15482/122154 objects misplaced (12.674%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:19:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:51 smithi076 ceph-mon[106482]: osdmap e1328: 8 total, 8 up, 5 in 2024-09-21T10:19:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:51 smithi076 ceph-mon[110963]: pgmap v2087: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 655 op/s; 15482/122154 objects misplaced (12.674%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:19:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:51 smithi076 ceph-mon[110963]: osdmap e1328: 8 total, 8 up, 5 in 2024-09-21T10:19:52.157 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:52.184 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:52.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:52.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:52.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:52.448 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:52.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:52.591 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:52.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:52.732 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:52.833 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:52.860 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:52.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:52.986 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:52.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:52.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:53.035 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:52 smithi076 ceph-mon[106482]: osdmap e1329: 8 total, 8 up, 5 in 2024-09-21T10:19:53.038 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:52 smithi076 ceph-mon[110963]: osdmap e1329: 8 total, 8 up, 5 in 2024-09-21T10:19:53.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:53.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:52 smithi191 ceph-mon[102206]: osdmap e1329: 8 total, 8 up, 5 in 2024-09-21T10:19:53.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:53.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.262 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:53.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:53.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:19:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:19:53] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-21T10:19:53.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:53.491 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:53.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.536 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:53.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:53.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:53.665 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:53.766 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:53.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:53.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:53.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:53.931 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:53.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:53.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:54.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:54.066 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:53 smithi191 ceph-mon[102206]: pgmap v2090: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.52k op/s; 12838/108492 objects misplaced (11.833%); 17 MiB/s, 274 objects/s recovering 2024-09-21T10:19:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:53 smithi076 ceph-mon[106482]: pgmap v2090: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.52k op/s; 12838/108492 objects misplaced (11.833%); 17 MiB/s, 274 objects/s recovering 2024-09-21T10:19:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:53 smithi076 ceph-mon[110963]: pgmap v2090: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.52k op/s; 12838/108492 objects misplaced (11.833%); 17 MiB/s, 274 objects/s recovering 2024-09-21T10:19:54.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:54.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:54.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.342 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:54.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:54.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:54.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:54.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:54.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:54.611 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:54.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:54.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.753 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:54.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:54.854 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:54.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:54.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:54.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:54.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:55.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:55.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:55.109 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:55.136 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:55.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:55.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:55.376 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:55.403 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:55.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:55.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:55.540 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:55.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:55.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:55.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:55.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:55.820 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:55.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:55.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:55.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:56.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:56.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:56.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:55 smithi191 ceph-mon[102206]: pgmap v2091: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 12838/108492 objects misplaced (11.833%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:19:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:55 smithi076 ceph-mon[106482]: pgmap v2091: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 12838/108492 objects misplaced (11.833%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:19:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:55 smithi076 ceph-mon[110963]: pgmap v2091: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 12838/108492 objects misplaced (11.833%); 8.9 MiB/s, 142 objects/s recovering 2024-09-21T10:19:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:19:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:19:56.191 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:56.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:56.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:56.359 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:56.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:56.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.498 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:56.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:56.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:56.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:56.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:56.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:56.771 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:19:56.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:56.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:56.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:57.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:57.045 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:57.146 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:57.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:57.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:57.273 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:57.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:57.409 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:57.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:57.537 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:57.563 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:57.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:57.665 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:57.691 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:57.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:57.831 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:57.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:57.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:57.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:57.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:58.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:58.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.108 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:58.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:58.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:57 smithi191 ceph-mon[102206]: pgmap v2092: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 639 op/s; 12838/106791 objects misplaced (12.022%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:57 smithi076 ceph-mon[106482]: pgmap v2092: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 639 op/s; 12838/106791 objects misplaced (12.022%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:57 smithi076 ceph-mon[110963]: pgmap v2092: 89 pgs: 1 active+remapped+backfilling, 1 peering, 5 active+remapped+backfill_wait, 82 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 639 op/s; 12838/106791 objects misplaced (12.022%); 7.7 MiB/s, 123 objects/s recovering 2024-09-21T10:19:58.209 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:58.248 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:58.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:58.350 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:19:58.387 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:19:58.489 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:19:58.516 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:58.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:19:58.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:19:58.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:58.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:19:58.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.804 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:58.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:58.905 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:58.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:58.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:59.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:19:59.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:59.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:19:59.164 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:59.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:59.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:59.330 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:59.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:19:59.467 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:19:59.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:19:59.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:59.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:19:59.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:19:59.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:19:59.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:19:59.826 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:19:59.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:19:59.895 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:19:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:19:59.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:19:59.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:59 smithi076 ceph-mon[106482]: pgmap v2093: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.17k op/s; 10725/94023 objects misplaced (11.407%); 11 MiB/s, 174 objects/s recovering 2024-09-21T10:19:59.978 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:19:59 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/850905525' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:59.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:59 smithi076 ceph-mon[110963]: pgmap v2093: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.17k op/s; 10725/94023 objects misplaced (11.407%); 11 MiB/s, 174 objects/s recovering 2024-09-21T10:19:59.979 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:19:59 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/850905525' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:19:59.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:00.036 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:59 smithi191 ceph-mon[102206]: pgmap v2093: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.17k op/s; 10725/94023 objects misplaced (11.407%); 11 MiB/s, 174 objects/s recovering 2024-09-21T10:20:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:19:59 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/850905525' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:00.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:00.172 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:00.274 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:00.318 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:00.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.458 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:00.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:00.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:00.590 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:00.631 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:20:00.631 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:20:00.631 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:20:00.631 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:20:00.631 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:20:00.632 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:00.633 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:00.633 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:00.633 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:00.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:00.727 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:00.829 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:00.870 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:00.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:00.961 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/850905525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[106482]: osdmap e1330: 8 total, 8 up, 5 in 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[106482]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[106482]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4241842922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/850905525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[110963]: osdmap e1330: 8 total, 8 up, 5 in 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[110963]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[110963]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:00.962 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4241842922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:00.972 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.021 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:01.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:00 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/850905525' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:00 smithi191 ceph-mon[102206]: osdmap e1330: 8 total, 8 up, 5 in 2024-09-21T10:20:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:00 smithi191 ceph-mon[102206]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:00 smithi191 ceph-mon[102206]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-09-21T10:20:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4241842922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:01.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:01.162 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:01.263 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:01.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:01.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:01.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:01.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:01.537 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:20:01.537 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:20:01.537 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [ 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145309 8.14 8:2cd4fe9c:::benchmark_data_smithi076_241159_object22391:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.422259973, 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.067794157, 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.538 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145309 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107180+0000", 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.539 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107182+0000", 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.843e-06 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107187+0000", 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.8749999999999999e-06 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107196+0000", 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.3200000000000006e-06 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.540 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107206+0000", 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0093e-05 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107256+0000", 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0154999999999997e-05 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107265+0000", 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2089999999999994e-06 2024-09-21T10:20:01.541 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.170814+0000", 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06354872 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.170905+0000", 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0631999999999996e-05 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.542 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.171114+0000", 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000209796 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174576+0000", 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0034615789999999998 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174585+0000", 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0680000000000003e-06 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.543 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174717+0000", 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000131817 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174784+0000", 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7360999999999994e-05 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174850+0000", 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6104000000000003e-05 2024-09-21T10:20:01.544 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174974+0000", 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000123585 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145327 8.14 8:2bc8e99d:::benchmark_data_smithi076_241159_object22409:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.42221410100001, 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06996614, 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.545 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145327 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.546 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107226+0000", 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0999999999999998e-07 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107229+0000", 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.7199999999999998e-06 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.547 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107238+0000", 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.755e-06 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107244+0000", 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.392e-06 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107296+0000", 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2200999999999997e-05 2024-09-21T10:20:01.548 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107303+0000", 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.6379999999999998e-06 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172493+0000", 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.065190184 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172625+0000", 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013188800000000001 2024-09-21T10:20:01.549 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.172812+0000", 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018733899999999999 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.174726+0000", 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00191332 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176604+0000", 2024-09-21T10:20:01.550 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.001878111 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176883+0000", 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027873299999999998 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.551 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177019+0000", 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013593700000000001 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177079+0000", 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0439000000000002e-05 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.553 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177192+0000", 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000112873 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145341 8.14 8:2a80dff4:::benchmark_data_smithi076_241159_object22423:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.42218729599995, 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.070240276, 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.554 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145341 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.555 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107252+0000", 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107253+0000", 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4000000000000002e-07 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107259+0000", 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0270000000000001e-06 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.556 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107266+0000", 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.002e-06 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107273+0000", 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.9380000000000003e-06 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108301+0000", 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010276829999999999 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.557 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108306+0000", 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.2390000000000002e-06 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173660+0000", 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0653539890000001 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173734+0000", 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3981999999999997e-05 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.558 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.173865+0000", 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013074199999999999 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176550+0000", 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.002685532 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176826+0000", 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00027604099999999998 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.559 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177260+0000", 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00043419999999999998 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177334+0000", 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.3385000000000003e-05 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177396+0000", 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1779999999999995e-05 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.560 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177493+0000", 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.7195999999999996e-05 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145379 8.14 8:2e4b539b:::benchmark_data_smithi076_241159_object22461:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.42212038800005, 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0704400940000001, 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.561 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145379 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.562 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107319+0000", 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107320+0000", 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0800000000000004e-07 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107324+0000", 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.32e-06 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.563 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107332+0000", 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6440000000000006e-06 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107338+0000", 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8980000000000001e-06 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108338+0000", 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00099989499999999999 2024-09-21T10:20:01.564 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108346+0000", 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0250000000000002e-06 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175028+0000", 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.06668181 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175119+0000", 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0938000000000002e-05 2024-09-21T10:20:01.565 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.175301+0000", 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00018216400000000001 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.176843+0000", 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0015422599999999999 2024-09-21T10:20:01.566 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177544+0000", 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00070127500000000001 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177598+0000", 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3390000000000002e-05 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177645+0000", 2024-09-21T10:20:01.567 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.7596000000000003e-05 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177677+0000", 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1538999999999997e-05 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177759+0000", 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.2532000000000002e-05 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.568 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145428 8.14 8:28ec7a73:::benchmark_data_smithi076_241159_object22510:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.42209327900002, 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0832395619999999, 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.569 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145428 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.570 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107347+0000", 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.0999999999999999e-07 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107349+0000", 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 2.0020000000000001e-06 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107355+0000", 2024-09-21T10:20:01.571 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9989999999999997e-06 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.107360+0000", 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108360+0000", 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010003779999999999 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.572 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.108364+0000", 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.506e-06 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177840+0000", 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.069476358 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.177971+0000", 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000130772 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.573 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.178223+0000", 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00025209899999999999 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183904+0000", 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0056812440000000002 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183937+0000", 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.2718000000000001e-05 2024-09-21T10:20:01.574 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.186948+0000", 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030109640000000001 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190363+0000", 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.003414753 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190424+0000", 2024-09-21T10:20:01.575 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.1289000000000002e-05 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190586+0000", 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00016216000000000001 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145565 8.14 8:2ab87a67:::benchmark_data_smithi076_241159_object22647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.305157334, 2024-09-21T10:20:01.576 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.96663625200000003, 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145565 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.577 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224282+0000", 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224284+0000", 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.812e-06 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224290+0000", 2024-09-21T10:20:01.578 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.874e-06 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224308+0000", 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8093000000000001e-05 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224317+0000", 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.7960000000000001e-06 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.579 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224386+0000", 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8879000000000002e-05 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.224395+0000", 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6079999999999996e-06 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179328+0000", 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.95493383300000001 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.580 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179451+0000", 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012236699999999999 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.179658+0000", 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020740600000000001 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.183981+0000", 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0043224200000000004 2024-09-21T10:20:01.581 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.186993+0000", 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030123340000000002 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189973+0000", 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029805529999999999 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.582 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190741+0000", 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000767269 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190808+0000", 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7385999999999995e-05 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190919+0000", 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000110622 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.583 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145636 8.14 8:2c7856f1:::benchmark_data_smithi076_241159_object22718:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.25012837600002, 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.91186562000000004, 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145636 2024-09-21T10:20:01.584 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279311+0000", 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.585 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279313+0000", 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.612e-06 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279318+0000", 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4700000000000001e-06 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279329+0000", 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0521e-05 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.586 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279336+0000", 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4960000000000001e-06 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279390+0000", 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4639000000000003e-05 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.279399+0000", 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.4519999999999997e-06 2024-09-21T10:20:01.587 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.180800+0000", 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90140108500000005 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.180917+0000", 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011759199999999999 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.181105+0000", 2024-09-21T10:20:01.588 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000187345 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184031+0000", 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029260890000000002 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187076+0000", 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030450529999999998 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.589 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189983+0000", 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0029071829999999998 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191024+0000", 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0010408940000000001 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191081+0000", 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.7377000000000003e-05 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.590 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191177+0000", 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.5811999999999997e-05 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145642 8.14 8:2c2c43d9:::benchmark_data_smithi076_241159_object22724:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.24439221800003, 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90634175100000003, 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.591 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145642 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.592 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285048+0000", 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285049+0000", 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5689999999999999e-06 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285054+0000", 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5680000000000001e-06 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.593 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285069+0000", 2024-09-21T10:20:01.594 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5790999999999998e-05 2024-09-21T10:20:01.594 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.594 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.594 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285077+0000", 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1380000000000001e-06 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285131+0000", 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4712000000000001e-05 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.596 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.285140+0000", 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.5930000000000006e-06 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182422+0000", 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89728208899999995 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182525+0000", 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102975 2024-09-21T10:20:01.597 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.182730+0000", 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020512300000000001 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184080+0000", 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0013496820000000001 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187121+0000", 2024-09-21T10:20:01.598 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0030413240000000002 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.189992+0000", 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0028713139999999998 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191253+0000", 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0012601559999999999 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.599 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191297+0000", 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4267000000000002e-05 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191389+0000", 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2449999999999997e-05 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145647 8.14 8:2cdee3d5:::benchmark_data_smithi076_241159_object22729:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.600 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.23980667299998, 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.90206547199999998, 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145647 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.601 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289633+0000", 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289635+0000", 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.629e-06 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289640+0000", 2024-09-21T10:20:01.602 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5609999999999998e-06 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289653+0000", 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.2500999999999999e-05 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289660+0000", 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.7909999999999999e-06 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.603 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289716+0000", 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6434999999999998e-05 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.289725+0000", 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.568e-06 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184119+0000", 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.89439432600000002 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.604 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184251+0000", 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013181299999999999 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.184465+0000", 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021390800000000001 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187033+0000", 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0025682499999999998 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.605 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190155+0000", 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031220950000000001 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190600+0000", 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00044475 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191502+0000", 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00090238500000000001 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.606 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191592+0000", 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.9889999999999995e-05 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191699+0000", 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010657 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145668 8.14 8:2c4067c8:::benchmark_data_smithi076_241159_object22750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.607 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.21995462300004, 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.88250502399999997, 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145668 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.608 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309485+0000", 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309487+0000", 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7969999999999999e-06 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.609 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309492+0000", 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0799999999999996e-06 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309506+0000", 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4337e-05 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309537+0000", 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 3.1062000000000002e-05 2024-09-21T10:20:01.610 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309583+0000", 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.5954000000000001e-05 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.309592+0000", 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8950000000000004e-06 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185570+0000", 2024-09-21T10:20:01.611 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.87597768499999995 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185659+0000", 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.8683000000000005e-05 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.185852+0000", 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019369 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.612 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190093+0000", 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0042406789999999998 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190606+0000", 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00051294199999999996 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190699+0000", 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.2977000000000001e-05 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.613 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191748+0000", 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00104901 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191835+0000", 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.6559999999999998e-05 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191990+0000", 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000155673 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.614 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145696 8.14 8:2d97a104:::benchmark_data_smithi076_241159_object22778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.19520128399995, 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.863836041, 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.615 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145696 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334238+0000", 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.616 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334240+0000", 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.654e-06 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334246+0000", 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.755e-06 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334262+0000", 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.6543000000000001e-05 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.617 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334271+0000", 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.1829999999999994e-06 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334339+0000", 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.8495000000000001e-05 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.334349+0000", 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.8679999999999994e-06 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.618 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187155+0000", 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.85280550300000002 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187267+0000", 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011253500000000001 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.187487+0000", 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00021978000000000001 2024-09-21T10:20:01.619 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190612+0000", 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031255240000000002 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190657+0000", 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4929999999999998e-05 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191219+0000", 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00056208899999999999 2024-09-21T10:20:01.620 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197900+0000", 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0066807790000000004 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197963+0000", 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.2599000000000002e-05 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.198075+0000", 2024-09-21T10:20:01.621 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000111804 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145737 8.14 8:2e5266a3:::benchmark_data_smithi076_241159_object22819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.15788697899995, 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.82838703400000002, 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.622 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145737 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371553+0000", 2024-09-21T10:20:01.623 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371554+0000", 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.5710000000000001e-06 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371559+0000", 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.101e-06 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.624 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371573+0000", 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.3237999999999999e-05 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371580+0000", 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1770000000000001e-06 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371638+0000", 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8353999999999997e-05 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.625 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.371644+0000", 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.5829999999999997e-06 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188522+0000", 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.81687860199999995 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188653+0000", 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00013063699999999999 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.626 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.188863+0000", 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00020946300000000001 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190934+0000", 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0020711869999999999 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.190975+0000", 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1170000000000001e-05 2024-09-21T10:20:01.627 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.191443+0000", 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00046765700000000001 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199763+0000", 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0083201890000000004 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199822+0000", 2024-09-21T10:20:01.628 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.8839999999999999e-05 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199940+0000", 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011826499999999999 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145790 8.14 8:29323519:::benchmark_data_smithi076_241159_object22872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 589.09790319499996, 2024-09-21T10:20:01.629 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76945549899999999, 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145790 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:20:01.630 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431537+0000", 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431538+0000", 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.821e-06 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431544+0000", 2024-09-21T10:20:01.631 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.3539999999999999e-06 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431558+0000", 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4664999999999999e-05 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431566+0000", 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 7.1099999999999997e-06 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.632 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431630+0000", 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.4051000000000001e-05 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.431644+0000", 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4489e-05 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192021+0000", 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.76037687799999998 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.633 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192123+0000", 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102094 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.192373+0000", 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00024965499999999999 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197733+0000", 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0053602290000000002 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.634 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197778+0000", 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.4932999999999998e-05 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.199979+0000", 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0022014389999999999 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200021+0000", 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2040999999999999e-05 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.635 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200873+0000", 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00085203000000000004 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.200992+0000", 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00011870999999999999 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.636 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145960 8.14 8:2d57b90e:::benchmark_data_smithi076_241159_object23042:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 588.90028874899997, 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.58195981299999999, 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145960 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.637 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629151+0000", 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629153+0000", 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.8870000000000001e-06 2024-09-21T10:20:01.638 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629158+0000", 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0730000000000004e-06 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629172+0000", 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.4076e-05 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629181+0000", 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0329999999999997e-06 2024-09-21T10:20:01.639 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629231+0000", 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0161000000000003e-05 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.629239+0000", 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 8.0839999999999992e-06 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193615+0000", 2024-09-21T10:20:01.640 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.56437558099999996 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193705+0000", 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.0105999999999994e-05 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.193902+0000", 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00019732800000000001 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197795+0000", 2024-09-21T10:20:01.641 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0038931259999999998 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.197850+0000", 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4941000000000001e-05 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201036+0000", 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0031852819999999998 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.642 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210835+0000", 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0097991099999999998 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210934+0000", 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 9.9510000000000001e-05 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211111+0000", 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000176515 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.643 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:145998 8.14 8:292bcf47:::benchmark_data_smithi076_241159_object23080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 588.81054144799998, 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.49255907199999999, 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 145998 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.644 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718898+0000", 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718900+0000", 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9690000000000001e-06 2024-09-21T10:20:01.645 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718906+0000", 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.4670000000000003e-06 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718925+0000", 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.9351000000000002e-05 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718931+0000", 2024-09-21T10:20:01.646 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6620000000000002e-06 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.718991+0000", 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 6.0074000000000001e-05 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.719002+0000", 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.0701e-05 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196200+0000", 2024-09-21T10:20:01.647 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.477198858 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196319+0000", 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000118368 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.196556+0000", 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00023754200000000001 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201345+0000", 2024-09-21T10:20:01.648 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0047884770000000002 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.210791+0000", 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.0094463680000000001 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211125+0000", 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00033422200000000001 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211296+0000", 2024-09-21T10:20:01.649 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000170445 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211355+0000", 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9215999999999997e-05 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211457+0000", 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000102352 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.650 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "description": "osd_op(client.45332.0:146042 8.14 8:28c57b04:::benchmark_data_smithi076_241159_object23124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1233)", 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "initiated_at": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "age": 588.68266395199998, 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.36551782100000002, 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "continuous": false, 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "type_data": { 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "client_info": { 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "client": "client.45332", 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "client_addr": "172.21.15.76:0/3056129108", 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "tid": 146042 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.651 INFO:teuthology.orchestra.run.smithi076.stdout: "events": [ 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "initiated", 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "header_read", 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846776+0000", 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "throttled", 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846778+0000", 2024-09-21T10:20:01.652 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.7630000000000001e-06 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "all_read", 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846783+0000", 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.1340000000000003e-06 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "dispatched", 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846794+0000", 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 1.1486e-05 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "queued_for_pg", 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846800+0000", 2024-09-21T10:20:01.653 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.6969999999999998e-06 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846850+0000", 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.0513999999999998e-05 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for peered", 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:12.846856+0000", 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "reached_pg", 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201389+0000", 2024-09-21T10:20:01.654 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.35453316699999998 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "started", 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201518+0000", 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00012838199999999999 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "waiting for subops from 1,4,6", 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.201830+0000", 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00031166100000000002 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "op_commit", 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211170+0000", 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.009340447 2024-09-21T10:20:01.655 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211211+0000", 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.1372000000000003e-05 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211254+0000", 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 4.2997999999999997e-05 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.211964+0000", 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00070925999999999997 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.656 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "commit_sent", 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.212193+0000", 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.000229161 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: }, 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: { 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: "event": "done", 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: "time": "2024-09-21T10:10:13.212294+0000", 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 0.00010087899999999999 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: } 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout: ] 2024-09-21T10:20:01.657 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:20:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:01.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:01.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:01.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:01.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:01.860 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:20:01.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:01.946 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:01.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:01.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:02.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:02.095 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:01 smithi191 ceph-mon[102206]: pgmap v2095: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 885 op/s; 10725/94023 objects misplaced (11.407%); 6.6 MiB/s, 104 objects/s recovering 2024-09-21T10:20:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3828103710' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:20:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:01 smithi076 ceph-mon[106482]: pgmap v2095: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 885 op/s; 10725/94023 objects misplaced (11.407%); 6.6 MiB/s, 104 objects/s recovering 2024-09-21T10:20:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3828103710' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:20:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:01 smithi076 ceph-mon[110963]: pgmap v2095: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 885 op/s; 10725/94023 objects misplaced (11.407%); 6.6 MiB/s, 104 objects/s recovering 2024-09-21T10:20:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3828103710' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:20:02.197 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:02.229 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:02.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:02.331 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:02.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:02.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:02.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:02.500 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:02.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:02.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:02.729 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:02.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:02.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:02.918 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:02.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:03.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:03.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3828103710' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:20:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:02 smithi191 ceph-mon[102206]: osdmap e1331: 8 total, 8 up, 5 in 2024-09-21T10:20:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3828103710' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:20:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:02 smithi076 ceph-mon[106482]: osdmap e1331: 8 total, 8 up, 5 in 2024-09-21T10:20:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3828103710' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:20:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:02 smithi076 ceph-mon[110963]: osdmap e1331: 8 total, 8 up, 5 in 2024-09-21T10:20:03.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:03.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:03.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:03.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:03.438 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:03.476 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:03.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:03.609 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:20:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:20:03] "GET /metrics HTTP/1.1" 200 36638 "" "Prometheus/2.51.0" 2024-09-21T10:20:03.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:20:03.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:03.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:03.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:03.919 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:03.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:04.020 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:04.044 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:04.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:04.088 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:20:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:03 smithi191 ceph-mon[102206]: pgmap v2097: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s; 8603/78879 objects misplaced (10.907%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:20:04.113 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:03 smithi076 ceph-mon[106482]: pgmap v2097: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s; 8603/78879 objects misplaced (10.907%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:20:04.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:03 smithi076 ceph-mon[110963]: pgmap v2097: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.23k op/s; 8603/78879 objects misplaced (10.907%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:20:04.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:04.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:04.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:04.269 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:20:04.269 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:20:04.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:04.346 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:04.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:04.371 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:04.447 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:04.489 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:04.591 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:04.615 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:04.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:04.761 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:04.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:04.906 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:04.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:04.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:04.936 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:05.008 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.037 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:05.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:05.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:05.182 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:05.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:05.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:05.421 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:05.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:05.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:05.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:05.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:05.678 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:05.721 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:05.822 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:05.871 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:05.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:05.878 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:05.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:05.999 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:05.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:06.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:05 smithi191 ceph-mon[102206]: pgmap v2098: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s; 8603/78879 objects misplaced (10.907%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:20:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:05 smithi076 ceph-mon[106482]: pgmap v2098: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s; 8603/78879 objects misplaced (10.907%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:20:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:05 smithi076 ceph-mon[110963]: pgmap v2098: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.16k op/s; 8603/78879 objects misplaced (10.907%); 6.4 MiB/s, 102 objects/s recovering 2024-09-21T10:20:06.129 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:06.231 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:06.266 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:06.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:06.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:06.407 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:06.509 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:06.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:06.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:06.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:06.691 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:06.792 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:06.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:06.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:06.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:06.954 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:06.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:06.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:07.056 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:07.083 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:07.185 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:07.223 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:07.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:07.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:07.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:07.466 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:07.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:07.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:07.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:07.629 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:07.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:07.760 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:07.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:07.887 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:07.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:07.988 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:08.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.020 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:08.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:08.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:07 smithi076 ceph-mon[106482]: pgmap v2099: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 704 op/s; 8603/77112 objects misplaced (11.156%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:20:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:07 smithi076 ceph-mon[110963]: pgmap v2099: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 704 op/s; 8603/77112 objects misplaced (11.156%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:20:08.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:08.184 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:08.184 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249980 10.1 e1331/1309 10:84be2631:::benchmark_data_smithi076_299450_object114801:head v 1331'15595, mlcod=1331'15591)", 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.168195+0000", 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.012932806, 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012959345000000001, 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.168195+0000", 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.168195+0000", 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.168194+0000", 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.168202+0000", 2024-09-21T10:20:08.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3130000000000001e-06 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.168222+0000", 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0177999999999999e-05 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.168229+0000", 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0910000000000004e-06 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.172137+0000", 2024-09-21T10:20:08.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0039079370000000002 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.172158+0000", 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1203e-05 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249985 10.4 e1331/1301 10:2daca485:::benchmark_data_smithi076_299450_object114806:head v 1331'15624, mlcod=1331'15622)", 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.169018+0000", 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.012110500999999999, 2024-09-21T10:20:08.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012199916999999999, 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169018+0000", 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169018+0000", 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169017+0000", 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169022+0000", 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0930000000000002e-06 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169028+0000", 2024-09-21T10:20:08.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5040000000000002e-06 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169032+0000", 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1350000000000002e-06 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169069+0000", 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7044e-05 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.169086+0000", 2024-09-21T10:20:08.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7309e-05 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249981 10.1 e1331/1309 10:85f1dcab:::benchmark_data_smithi076_299450_object114802:head v 1331'15596, mlcod=1331'15591)", 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.171117+0000", 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.010011348, 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010153987, 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171117+0000", 2024-09-21T10:20:08.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171117+0000", 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171116+0000", 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171123+0000", 2024-09-21T10:20:08.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2200000000000003e-06 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171137+0000", 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4289e-05 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171144+0000", 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8390000000000001e-06 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178437+0000", 2024-09-21T10:20:08.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0072925079999999996 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178460+0000", 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3023999999999998e-05 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249989 10.4 e1331/1301 10:2acb887d:::benchmark_data_smithi076_299450_object114810:head v 1331'15625, mlcod=1331'15622)", 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.171995+0000", 2024-09-21T10:20:08.195 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0091331490000000001, 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0093278370000000003, 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171995+0000", 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171995+0000", 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171994+0000", 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.171998+0000", 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0029999999999996e-06 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.172005+0000", 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0149999999999998e-06 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.172010+0000", 2024-09-21T10:20:08.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7319999999999998e-06 2024-09-21T10:20:08.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179208+0000", 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0071980100000000003 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179229+0000", 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1061e-05 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.199 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249982 10.f e1331/1301 10:fc3d72b7:::benchmark_data_smithi076_299450_object114803:head v 1331'15598, mlcod=1331'15595)", 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.173533+0000", 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0075951050000000004, 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078425549999999993, 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173533+0000", 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173533+0000", 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173532+0000", 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173538+0000", 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4380000000000003e-06 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173551+0000", 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2894999999999999e-05 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173557+0000", 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2450000000000003e-06 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173594+0000", 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6672999999999999e-05 2024-09-21T10:20:08.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173611+0000", 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7765999999999999e-05 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249988 10.e e1331/1309 10:70004c48:::benchmark_data_smithi076_299450_object114809:head v 1331'15920, mlcod=1331'15917)", 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.173567+0000", 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0075609639999999999, 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0078677999999999994, 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.203 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173567+0000", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173567+0000", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173567+0000", 2024-09-21T10:20:08.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967296 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173570+0000", 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4240000000000002e-06 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173575+0000", 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1240000000000004e-06 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173579+0000", 2024-09-21T10:20:08.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0210000000000001e-06 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173719+0000", 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000139677 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.173741+0000", 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2178999999999999e-05 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249983 10.7 e1331/1309 10:ee311400:::benchmark_data_smithi076_299450_object114804:head v 1331'15484, mlcod=1331'15482)", 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.175849+0000", 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0052786839999999996, 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056376289999999999, 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175849+0000", 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175849+0000", 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175848+0000", 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175856+0000", 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6930000000000007e-06 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175867+0000", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0793e-05 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175873+0000", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1700000000000002e-06 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175934+0000", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0815e-05 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.175955+0000", 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1586e-05 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249986 10.9 e1331/1319)", 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.178953+0000", 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0021752070000000002, 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025822079999999999, 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2024-09-21T10:20:08.210 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178953+0000", 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178953+0000", 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178952+0000", 2024-09-21T10:20:08.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178958+0000", 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3559999999999998e-06 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178971+0000", 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2377e-05 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.178978+0000", 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2679999999999997e-06 2024-09-21T10:20:08.212 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249987 10.f e1331/1301 10:f429061d:::benchmark_data_smithi076_299450_object114808:head v 1331'15599, mlcod=1331'15595)", 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.179199+0000", 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0019288879999999999, 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023808000000000002, 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179199+0000", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179199+0000", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179198+0000", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179205+0000", 2024-09-21T10:20:08.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9970000000000001e-06 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179216+0000", 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0977e-05 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179223+0000", 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2090000000000002e-06 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.180028+0000", 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000805155 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.180046+0000", 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8749e-05 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:249992 10.0 e1331/1296 10:0397b9df:::benchmark_data_smithi076_299450_object114813:head v 1331'15583, mlcod=1331'15581)", 2024-09-21T10:20:08.216 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.179234+0000", 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0018942799999999999, 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00239777, 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179234+0000", 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:08.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179234+0000", 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179234+0000", 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967296 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179237+0000", 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6289999999999999e-06 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179243+0000", 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6910000000000002e-06 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179247+0000", 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0799999999999999e-06 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179305+0000", 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8245000000000002e-05 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:08.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.179323+0000", 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7941e-05 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 10 2024-09-21T10:20:08.220 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:08.220 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:07 smithi191 ceph-mon[102206]: pgmap v2099: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.6 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 704 op/s; 8603/77112 objects misplaced (11.156%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:20:08.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:08.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:08.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:08.435 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:08.537 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:08.564 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:08.665 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:08.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:08.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:08.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:08.904 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:20:08.923 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:08.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:08.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:09.063 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:09.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:20:09.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:09.174 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:08 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1314937193' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:09.174 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:08 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:09.175 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:08 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1314937193' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:09.175 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:08 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:09.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:09.212 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:09.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:09.314 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:09.321 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:09.337 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:09.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:08 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1314937193' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:09.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:08 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:09.439 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:09.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.461 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:09.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:09.562 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:09.583 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:09.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:09.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:09.707 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:09.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:09.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:09.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:09.934 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:09.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:09.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:09.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:10.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:10.079 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:09 smithi191 ceph-mon[102206]: pgmap v2100: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s; 6726/64680 objects misplaced (10.399%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:20:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:09 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:09 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:09 smithi191 ceph-mon[102206]: osdmap e1332: 8 total, 8 up, 5 in 2024-09-21T10:20:10.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:10.203 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:10.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:10.304 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.333 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:10.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:10.358 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[106482]: pgmap v2100: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s; 6726/64680 objects misplaced (10.399%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:20:10.358 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:10.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:10.359 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[106482]: osdmap e1332: 8 total, 8 up, 5 in 2024-09-21T10:20:10.359 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[110963]: pgmap v2100: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s; 6726/64680 objects misplaced (10.399%); 5.3 MiB/s, 85 objects/s recovering 2024-09-21T10:20:10.359 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:10.359 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:10.359 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:09 smithi076 ceph-mon[110963]: osdmap e1332: 8 total, 8 up, 5 in 2024-09-21T10:20:10.434 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:10.467 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:10.568 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:10.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:10.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:10.636 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:10.703 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:10.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:10.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:10.883 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:10.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:10.895 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:20:10.898 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:20:10.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.013 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:10 smithi191 ceph-mon[102206]: osdmap e1333: 8 total, 8 up, 5 in 2024-09-21T10:20:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:11.114 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:10 smithi076 ceph-mon[106482]: osdmap e1333: 8 total, 8 up, 5 in 2024-09-21T10:20:11.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:10 smithi076 ceph-mon[110963]: osdmap e1333: 8 total, 8 up, 5 in 2024-09-21T10:20:11.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:11.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:11.244 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:11.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:11.307 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:20:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:20:11.307 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:20:11.308 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:11.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:11.417 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:11.519 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:11.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:11.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:11.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:11.687 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2103,"stamp":"2024-09-21T10:20:10.729597+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1411782653,"num_objects":21560,"num_object_clones":0,"num_object_copies":64680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6726,"num_objects_unfound":0,"num_objects_dirty":21560,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":249099,"num_write_kb":8653714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17315,"num_bytes_recovered":1132552029,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89428,"ondisk_log_size":89428,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8630224,"kb_used_data":4605032,"kb_used_omap":94,"kb_used_meta":4024929,"kb_avail":460095536,"statfs":{"total":479975178240,"available":471137828864,"internally_reserved":0,"allocated":4715552768,"data_stored":4699507527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96735,"internal_metadata":4121527841},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,15,5,9],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-310181888,"num_objects":-4733,"num_object_clones":0,"num_object_copies":-14199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1877,"num_objects_unfound":0,"num_objects_dirty":-4733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4733,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":349,"num_bytes_recovered":22872064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002103"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":479,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872517+0000","last_change":"2024-09-21T10:17:51.249951+0000","last_active":"2024-09-21T10:20:01.872517+0000","last_peered":"2024-09-21T10:20:01.872517+0000","last_clean":"2024-09-21T10:20:01.872517+0000","last_became_active":"2024-09-21T10:17:51.249158+0000","last_became_peered":"2024-09-21T10:17:51.249158+0000","last_unstale":"2024-09-21T10:20:01.872517+0000","last_undegraded":"2024-09-21T10:20:01.872517+0000","last_fullsized":"2024-09-21T10:20:01.872517+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:53.769122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":208,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839965+0000","last_change":"2024-09-21T10:15:19.141623+0000","last_active":"2024-09-21T10:19:59.839965+0000","last_peered":"2024-09-21T10:19:59.839965+0000","last_clean":"2024-09-21T10:19:59.839965+0000","last_became_active":"2024-09-21T10:15:19.111927+0000","last_became_peered":"2024-09-21T10:15:19.111927+0000","last_unstale":"2024-09-21T10:19:59.839965+0000","last_undegraded":"2024-09-21T10:19:59.839965+0000","last_fullsized":"2024-09-21T10:19:59.839965+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:10.426159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":33,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840941+0000","last_change":"2024-09-21T10:17:51.258024+0000","last_active":"2024-09-21T10:19:59.840941+0000","last_peered":"2024-09-21T10:19:59.840941+0000","last_clean":"2024-09-21T10:19:59.840941+0000","last_became_active":"2024-09-21T10:17:51.257639+0000","last_became_peered":"2024-09-21T10:17:51.257639+0000","last_unstale":"2024-09-21T10:19:59.840941+0000","last_undegraded":"2024-09-21T10:19:59.840941+0000","last_fullsized":"2024-09-21T10:19:59.840941+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:09.112335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1796,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872256+0000","last_change":"2024-09-21T10:17:51.250405+0000","last_active":"2024-09-21T10:20:01.872256+0000","last_peered":"2024-09-21T10:20:01.872256+0000","last_clean":"2024-09-21T10:20:01.872256+0000","last_became_active":"2024-09-21T10:17:51.248030+0000","last_became_peered":"2024-09-21T10:17:51.248030+0000","last_unstale":"2024-09-21T10:20:01.872256+0000","last_undegraded":"2024-09-21T10:20:01.872256+0000","last_fullsized":"2024-09-21T10:20:01.872256+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:45.889214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1795,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872766+0000","last_change":"2024-09-21T10:17:51.249710+0000","last_active":"2024-09-21T10:20:01.872766+0000","last_peered":"2024-09-21T10:20:01.872766+0000","last_clean":"2024-09-21T10:20:01.872766+0000","last_became_active":"2024-09-21T10:17:51.249284+0000","last_became_peered":"2024-09-21T10:17:51.249284+0000","last_unstale":"2024-09-21T10:20:01.872766+0000","last_undegraded":"2024-09-21T10:20:01.872766+0000","last_fullsized":"2024-09-21T10:20:01.872766+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:09.927717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839820+0000","last_change":"2024-09-21T10:16:38.451401+0000","last_active":"2024-09-21T10:19:59.839820+0000","last_peered":"2024-09-21T10:19:59.839820+0000","last_clean":"2024-09-21T10:19:59.839820+0000","last_became_active":"2024-09-21T10:16:38.450094+0000","last_became_peered":"2024-09-21T10:16:38.450094+0000","last_unstale":"2024-09-21T10:19:59.839820+0000","last_undegraded":"2024-09-21T10:19:59.839820+0000","last_fullsized":"2024-09-21T10:19:59.839820+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:22:50.798278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":472,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872760+0000","last_change":"2024-09-21T10:16:38.451461+0000","last_active":"2024-09-21T10:20:01.872760+0000","last_peered":"2024-09-21T10:20:01.872760+0000","last_clean":"2024-09-21T10:20:01.872760+0000","last_became_active":"2024-09-21T10:16:38.451137+0000","last_became_peered":"2024-09-21T10:16:38.451137+0000","last_unstale":"2024-09-21T10:20:01.872760+0000","last_undegraded":"2024-09-21T10:20:01.872760+0000","last_fullsized":"2024-09-21T10:20:01.872760+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:40.847463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":269,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840106+0000","last_change":"2024-09-21T10:17:51.269584+0000","last_active":"2024-09-21T10:19:59.840106+0000","last_peered":"2024-09-21T10:19:59.840106+0000","last_clean":"2024-09-21T10:19:59.840106+0000","last_became_active":"2024-09-21T10:17:51.260673+0000","last_became_peered":"2024-09-21T10:17:51.260673+0000","last_unstale":"2024-09-21T10:19:59.840106+0000","last_undegraded":"2024-09-21T10:19:59.840106+0000","last_fullsized":"2024-09-21T10:19:59.840106+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:16.649818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"1331'15413","reported_seq":15503,"reported_epoch":1331,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:06.811589+0000","last_change":"2024-09-21T10:17:53.268636+0000","last_active":"2024-09-21T10:20:06.811589+0000","last_peered":"2024-09-21T10:20:06.811589+0000","last_clean":"2024-09-21T10:17:50.209821+0000","last_became_active":"2024-09-21T10:17:53.218553+0000","last_became_peered":"2024-09-21T10:17:53.218553+0000","last_unstale":"2024-09-21T10:20:06.811589+0000","last_undegraded":"2024-09-21T10:20:06.811589+0000","last_fullsized":"2024-09-21T10:20:06.811589+0000","mapping_epoch":1309,"log_start":"1315'9800","ondisk_log_start":"1315'9800","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87621632,"num_objects":1337,"num_object_clones":0,"num_object_copies":4011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1337,"num_objects_unfound":0,"num_objects_dirty":1337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15413,"num_write_kb":536000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1337}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":382,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868143+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:20:01.868143+0000","last_peered":"2024-09-21T10:20:01.868143+0000","last_clean":"2024-09-21T10:20:01.868143+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:20:01.868143+0000","last_undegraded":"2024-09-21T10:20:01.868143+0000","last_fullsized":"2024-09-21T10:20:01.868143+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":34,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.364947+0000","last_change":"2024-09-21T10:17:51.256335+0000","last_active":"2024-09-21T10:20:01.364947+0000","last_peered":"2024-09-21T10:20:01.364947+0000","last_clean":"2024-09-21T10:20:01.364947+0000","last_became_active":"2024-09-21T10:17:51.251630+0000","last_became_peered":"2024-09-21T10:17:51.251630+0000","last_unstale":"2024-09-21T10:20:01.364947+0000","last_undegraded":"2024-09-21T10:20:01.364947+0000","last_fullsized":"2024-09-21T10:20:01.364947+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:31.141409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"1331'15414","reported_seq":16686,"reported_epoch":1331,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:06.861559+0000","last_change":"2024-09-21T10:18:54.252039+0000","last_active":"2024-09-21T10:20:06.861559+0000","last_peered":"2024-09-21T10:20:06.861559+0000","last_clean":"2024-09-21T10:17:50.234745+0000","last_became_active":"2024-09-21T10:17:53.221056+0000","last_became_peered":"2024-09-21T10:17:53.221056+0000","last_unstale":"2024-09-21T10:20:06.861559+0000","last_undegraded":"2024-09-21T10:20:06.861559+0000","last_fullsized":"2024-09-21T10:20:06.861559+0000","mapping_epoch":1309,"log_start":"1315'9800","ondisk_log_start":"1315'9800","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5614,"log_dups_size":0,"ondisk_log_size":5614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:41.368294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89128960,"num_objects":1360,"num_object_clones":0,"num_object_copies":4080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1228,"num_objects_unfound":0,"num_objects_dirty":1360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15414,"num_write_kb":536768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":604,"num_bytes_recovered":39583744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,4],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":1360}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":82,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840115+0000","last_change":"2024-09-21T10:17:51.281550+0000","last_active":"2024-09-21T10:19:59.840115+0000","last_peered":"2024-09-21T10:19:59.840115+0000","last_clean":"2024-09-21T10:19:59.840115+0000","last_became_active":"2024-09-21T10:17:51.262457+0000","last_became_peered":"2024-09-21T10:17:51.262457+0000","last_unstale":"2024-09-21T10:19:59.840115+0000","last_undegraded":"2024-09-21T10:19:59.840115+0000","last_fullsized":"2024-09-21T10:19:59.840115+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:49.212195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1456,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871515+0000","last_change":"2024-09-21T10:16:38.461867+0000","last_active":"2024-09-21T10:20:01.871515+0000","last_peered":"2024-09-21T10:20:01.871515+0000","last_clean":"2024-09-21T10:20:01.871515+0000","last_became_active":"2024-09-21T10:16:38.461416+0000","last_became_peered":"2024-09-21T10:16:38.461416+0000","last_unstale":"2024-09-21T10:20:01.871515+0000","last_undegraded":"2024-09-21T10:20:01.871515+0000","last_fullsized":"2024-09-21T10:20:01.871515+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:15.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"1331'15543","reported_seq":15614,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.652162+0000","last_change":"2024-09-21T10:16:38.451467+0000","last_active":"2024-09-21T10:20:06.652162+0000","last_peered":"2024-09-21T10:20:06.652162+0000","last_clean":"2024-09-21T10:20:06.652162+0000","last_became_active":"2024-09-21T10:16:38.446328+0000","last_became_peered":"2024-09-21T10:16:38.446328+0000","last_unstale":"2024-09-21T10:20:06.652162+0000","last_undegraded":"2024-09-21T10:20:06.652162+0000","last_fullsized":"2024-09-21T10:20:06.652162+0000","mapping_epoch":1301,"log_start":"1318'10400","ondisk_log_start":"1318'10400","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5143,"log_dups_size":0,"ondisk_log_size":5143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:27.117612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86179840,"num_objects":1315,"num_object_clones":0,"num_object_copies":3945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15543,"num_write_kb":539456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":38,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.874504+0000","last_change":"2024-09-21T10:17:51.283569+0000","last_active":"2024-09-21T10:20:01.874504+0000","last_peered":"2024-09-21T10:20:01.874504+0000","last_clean":"2024-09-21T10:20:01.874504+0000","last_became_active":"2024-09-21T10:17:51.259187+0000","last_became_peered":"2024-09-21T10:17:51.259187+0000","last_unstale":"2024-09-21T10:20:01.874504+0000","last_undegraded":"2024-09-21T10:20:01.874504+0000","last_fullsized":"2024-09-21T10:20:01.874504+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:55.002645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":221,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871961+0000","last_change":"2024-09-21T10:17:51.248805+0000","last_active":"2024-09-21T10:20:01.871961+0000","last_peered":"2024-09-21T10:20:01.871961+0000","last_clean":"2024-09-21T10:20:01.871961+0000","last_became_active":"2024-09-21T10:17:51.248157+0000","last_became_peered":"2024-09-21T10:17:51.248157+0000","last_unstale":"2024-09-21T10:20:01.871961+0000","last_undegraded":"2024-09-21T10:20:01.871961+0000","last_fullsized":"2024-09-21T10:20:01.871961+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:36.573842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1331'15373","reported_seq":21344,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.657178+0000","last_change":"2024-09-21T10:19:51.830838+0000","last_active":"2024-09-21T10:20:06.657178+0000","last_peered":"2024-09-21T10:20:06.657178+0000","last_clean":"2024-09-21T10:20:06.657178+0000","last_became_active":"2024-09-21T10:19:51.804737+0000","last_became_peered":"2024-09-21T10:19:51.804737+0000","last_unstale":"2024-09-21T10:20:06.657178+0000","last_undegraded":"2024-09-21T10:20:06.657178+0000","last_fullsized":"2024-09-21T10:20:06.657178+0000","mapping_epoch":1328,"log_start":"1316'10200","ondisk_log_start":"1316'10200","created":1296,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5173,"log_dups_size":0,"ondisk_log_size":5173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:46.112314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90243072,"num_objects":1377,"num_object_clones":0,"num_object_copies":4131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15373,"num_write_kb":536000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2940,"num_bytes_recovered":192675840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":34,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.874423+0000","last_change":"2024-09-21T10:17:51.259331+0000","last_active":"2024-09-21T10:20:01.874423+0000","last_peered":"2024-09-21T10:20:01.874423+0000","last_clean":"2024-09-21T10:20:01.874423+0000","last_became_active":"2024-09-21T10:17:51.258943+0000","last_became_peered":"2024-09-21T10:17:51.258943+0000","last_unstale":"2024-09-21T10:20:01.874423+0000","last_undegraded":"2024-09-21T10:20:01.874423+0000","last_fullsized":"2024-09-21T10:20:01.874423+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:59.304035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1842,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868421+0000","last_change":"2024-09-21T10:16:38.461644+0000","last_active":"2024-09-21T10:20:01.868421+0000","last_peered":"2024-09-21T10:20:01.868421+0000","last_clean":"2024-09-21T10:20:01.868421+0000","last_became_active":"2024-09-21T10:16:38.461179+0000","last_became_peered":"2024-09-21T10:16:38.461179+0000","last_unstale":"2024-09-21T10:20:01.868421+0000","last_undegraded":"2024-09-21T10:20:01.868421+0000","last_fullsized":"2024-09-21T10:20:01.868421+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:43.638430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"1331'15443","reported_seq":15528,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.835530+0000","last_change":"2024-09-21T10:16:38.459692+0000","last_active":"2024-09-21T10:20:06.835530+0000","last_peered":"2024-09-21T10:20:06.835530+0000","last_clean":"2024-09-21T10:20:06.835530+0000","last_became_active":"2024-09-21T10:16:38.459132+0000","last_became_peered":"2024-09-21T10:16:38.459132+0000","last_unstale":"2024-09-21T10:20:06.835530+0000","last_undegraded":"2024-09-21T10:20:06.835530+0000","last_fullsized":"2024-09-21T10:20:06.835530+0000","mapping_epoch":1301,"log_start":"1315'9800","ondisk_log_start":"1315'9800","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:36:53.847003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063424,"num_objects":1359,"num_object_clones":0,"num_object_copies":4077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15443,"num_write_kb":537664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":425,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872260+0000","last_change":"2024-09-21T10:16:38.459890+0000","last_active":"2024-09-21T10:20:01.872260+0000","last_peered":"2024-09-21T10:20:01.872260+0000","last_clean":"2024-09-21T10:20:01.872260+0000","last_became_active":"2024-09-21T10:16:38.454382+0000","last_became_peered":"2024-09-21T10:16:38.454382+0000","last_unstale":"2024-09-21T10:20:01.872260+0000","last_undegraded":"2024-09-21T10:20:01.872260+0000","last_fullsized":"2024-09-21T10:20:01.872260+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:02.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1469,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872831+0000","last_change":"2024-09-21T10:16:38.454504+0000","last_active":"2024-09-21T10:20:01.872831+0000","last_peered":"2024-09-21T10:20:01.872831+0000","last_clean":"2024-09-21T10:20:01.872831+0000","last_became_active":"2024-09-21T10:16:38.448833+0000","last_became_peered":"2024-09-21T10:16:38.448833+0000","last_unstale":"2024-09-21T10:20:01.872831+0000","last_undegraded":"2024-09-21T10:20:01.872831+0000","last_fullsized":"2024-09-21T10:20:01.872831+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:11.905645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"1331'15756","reported_seq":15838,"reported_epoch":1331,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:06.685660+0000","last_change":"2024-09-21T10:17:53.216962+0000","last_active":"2024-09-21T10:20:06.685660+0000","last_peered":"2024-09-21T10:20:06.685660+0000","last_clean":"2024-09-21T10:17:50.226719+0000","last_became_active":"2024-09-21T10:17:53.190608+0000","last_became_peered":"2024-09-21T10:17:53.190608+0000","last_unstale":"2024-09-21T10:20:06.685660+0000","last_undegraded":"2024-09-21T10:20:06.685660+0000","last_fullsized":"2024-09-21T10:20:06.685660+0000","mapping_epoch":1309,"log_start":"1315'10200","ondisk_log_start":"1315'10200","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5556,"log_dups_size":0,"ondisk_log_size":5556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88080384,"num_objects":1344,"num_object_clones":0,"num_object_copies":4032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1344,"num_objects_unfound":0,"num_objects_dirty":1344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15756,"num_write_kb":547200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":1344}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":188,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.870758+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:20:01.870758+0000","last_peered":"2024-09-21T10:20:01.870758+0000","last_clean":"2024-09-21T10:20:01.870758+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:20:01.870758+0000","last_undegraded":"2024-09-21T10:20:01.870758+0000","last_fullsized":"2024-09-21T10:20:01.870758+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":69,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.873287+0000","last_change":"2024-09-21T10:16:39.439192+0000","last_active":"2024-09-21T10:20:01.873287+0000","last_peered":"2024-09-21T10:20:01.873287+0000","last_clean":"2024-09-21T10:20:01.873287+0000","last_became_active":"2024-09-21T10:16:39.438778+0000","last_became_peered":"2024-09-21T10:16:39.438778+0000","last_unstale":"2024-09-21T10:20:01.873287+0000","last_undegraded":"2024-09-21T10:20:01.873287+0000","last_fullsized":"2024-09-21T10:20:01.873287+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:49.643063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.0","version":"1331'15519","reported_seq":15564,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.893465+0000","last_change":"2024-09-21T10:16:23.239046+0000","last_active":"2024-09-21T10:20:06.893465+0000","last_peered":"2024-09-21T10:20:06.893465+0000","last_clean":"2024-09-21T10:20:06.893465+0000","last_became_active":"2024-09-21T10:16:23.238681+0000","last_became_peered":"2024-09-21T10:16:23.238681+0000","last_unstale":"2024-09-21T10:20:06.893465+0000","last_undegraded":"2024-09-21T10:20:06.893465+0000","last_fullsized":"2024-09-21T10:20:06.893465+0000","mapping_epoch":1296,"log_start":"1315'9900","ondisk_log_start":"1315'9900","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:15:42.666111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88801280,"num_objects":1355,"num_object_clones":0,"num_object_copies":4065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15519,"num_write_kb":539968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1501,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871353+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:20:01.871353+0000","last_peered":"2024-09-21T10:20:01.871353+0000","last_clean":"2024-09-21T10:20:01.871353+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:20:01.871353+0000","last_undegraded":"2024-09-21T10:20:01.871353+0000","last_fullsized":"2024-09-21T10:20:01.871353+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.841012+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:19:59.841012+0000","last_peered":"2024-09-21T10:19:59.841012+0000","last_clean":"2024-09-21T10:19:59.841012+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:19:59.841012+0000","last_undegraded":"2024-09-21T10:19:59.841012+0000","last_fullsized":"2024-09-21T10:19:59.841012+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"1331'15512","reported_seq":15603,"reported_epoch":1331,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:06.671402+0000","last_change":"2024-09-21T10:17:53.251533+0000","last_active":"2024-09-21T10:20:06.671402+0000","last_peered":"2024-09-21T10:20:06.671402+0000","last_clean":"2024-09-21T10:17:50.241218+0000","last_became_active":"2024-09-21T10:17:53.213630+0000","last_became_peered":"2024-09-21T10:17:53.213630+0000","last_unstale":"2024-09-21T10:20:06.671402+0000","last_undegraded":"2024-09-21T10:20:06.671402+0000","last_fullsized":"2024-09-21T10:20:06.671402+0000","mapping_epoch":1309,"log_start":"1315'9900","ondisk_log_start":"1315'9900","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89522176,"num_objects":1366,"num_object_clones":0,"num_object_copies":4098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1366,"num_objects_unfound":0,"num_objects_dirty":1366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15512,"num_write_kb":540096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":1366}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":450,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872339+0000","last_change":"2024-09-21T10:16:38.454278+0000","last_active":"2024-09-21T10:20:01.872339+0000","last_peered":"2024-09-21T10:20:01.872339+0000","last_clean":"2024-09-21T10:20:01.872339+0000","last_became_active":"2024-09-21T10:16:38.449058+0000","last_became_peered":"2024-09-21T10:16:38.449058+0000","last_unstale":"2024-09-21T10:20:01.872339+0000","last_undegraded":"2024-09-21T10:20:01.872339+0000","last_fullsized":"2024-09-21T10:20:01.872339+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:02.332428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1846,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868355+0000","last_change":"2024-09-21T10:16:38.460495+0000","last_active":"2024-09-21T10:20:01.868355+0000","last_peered":"2024-09-21T10:20:01.868355+0000","last_clean":"2024-09-21T10:20:01.868355+0000","last_became_active":"2024-09-21T10:16:38.458759+0000","last_became_peered":"2024-09-21T10:16:38.458759+0000","last_unstale":"2024-09-21T10:20:01.868355+0000","last_undegraded":"2024-09-21T10:20:01.868355+0000","last_fullsized":"2024-09-21T10:20:01.868355+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:37.031677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1331'15694","reported_seq":15739,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.909252+0000","last_change":"2024-09-21T10:16:23.237256+0000","last_active":"2024-09-21T10:20:06.909252+0000","last_peered":"2024-09-21T10:20:06.909252+0000","last_clean":"2024-09-21T10:20:06.909252+0000","last_became_active":"2024-09-21T10:16:23.236945+0000","last_became_peered":"2024-09-21T10:16:23.236945+0000","last_unstale":"2024-09-21T10:20:06.909252+0000","last_undegraded":"2024-09-21T10:20:06.909252+0000","last_fullsized":"2024-09-21T10:20:06.909252+0000","mapping_epoch":1296,"log_start":"1315'10100","ondisk_log_start":"1315'10100","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5594,"log_dups_size":0,"ondisk_log_size":5594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:50.314871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86376448,"num_objects":1318,"num_object_clones":0,"num_object_copies":3954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1318,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15694,"num_write_kb":544384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":884,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839906+0000","last_change":"2024-09-21T10:15:22.139134+0000","last_active":"2024-09-21T10:19:59.839906+0000","last_peered":"2024-09-21T10:19:59.839906+0000","last_clean":"2024-09-21T10:19:59.839906+0000","last_became_active":"2024-09-21T10:15:22.138788+0000","last_became_peered":"2024-09-21T10:15:22.138788+0000","last_unstale":"2024-09-21T10:19:59.839906+0000","last_undegraded":"2024-09-21T10:19:59.839906+0000","last_fullsized":"2024-09-21T10:19:59.839906+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:35.831693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":38,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.838570+0000","last_change":"2024-09-21T10:17:51.287937+0000","last_active":"2024-09-21T10:19:59.838570+0000","last_peered":"2024-09-21T10:19:59.838570+0000","last_clean":"2024-09-21T10:19:59.838570+0000","last_became_active":"2024-09-21T10:17:51.256610+0000","last_became_peered":"2024-09-21T10:17:51.256610+0000","last_unstale":"2024-09-21T10:19:59.838570+0000","last_undegraded":"2024-09-21T10:19:59.838570+0000","last_fullsized":"2024-09-21T10:19:59.838570+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:15.137072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":481,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840310+0000","last_change":"2024-09-21T10:18:07.086426+0000","last_active":"2024-09-21T10:19:59.840310+0000","last_peered":"2024-09-21T10:19:59.840310+0000","last_clean":"2024-09-21T10:19:59.840310+0000","last_became_active":"2024-09-21T10:18:07.086157+0000","last_became_peered":"2024-09-21T10:18:07.086157+0000","last_unstale":"2024-09-21T10:19:59.840310+0000","last_undegraded":"2024-09-21T10:19:59.840310+0000","last_fullsized":"2024-09-21T10:19:59.840310+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:24:34.289605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":212,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839630+0000","last_change":"2024-09-21T10:16:38.450231+0000","last_active":"2024-09-21T10:19:59.839630+0000","last_peered":"2024-09-21T10:19:59.839630+0000","last_clean":"2024-09-21T10:19:59.839630+0000","last_became_active":"2024-09-21T10:16:38.449941+0000","last_became_peered":"2024-09-21T10:16:38.449941+0000","last_unstale":"2024-09-21T10:19:59.839630+0000","last_undegraded":"2024-09-21T10:19:59.839630+0000","last_fullsized":"2024-09-21T10:19:59.839630+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:51.559013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"1331'15304","reported_seq":24531,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.316793+0000","last_change":"2024-09-21T10:19:32.344481+0000","last_active":"2024-09-21T10:20:06.316793+0000","last_peered":"2024-09-21T10:20:06.316793+0000","last_clean":"2024-09-21T10:20:06.316793+0000","last_became_active":"2024-09-21T10:19:32.294085+0000","last_became_peered":"2024-09-21T10:19:32.294085+0000","last_unstale":"2024-09-21T10:20:06.316793+0000","last_undegraded":"2024-09-21T10:20:06.316793+0000","last_fullsized":"2024-09-21T10:20:06.316793+0000","mapping_epoch":1324,"log_start":"1315'9300","ondisk_log_start":"1315'9300","created":1296,"last_epoch_clean":1325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6004,"log_dups_size":0,"ondisk_log_size":6004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:58.316719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90963968,"num_objects":1388,"num_object_clones":0,"num_object_copies":4164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15304,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":299827200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1138,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.867817+0000","last_change":"2024-09-21T10:17:51.264078+0000","last_active":"2024-09-21T10:20:01.867817+0000","last_peered":"2024-09-21T10:20:01.867817+0000","last_clean":"2024-09-21T10:20:01.867817+0000","last_became_active":"2024-09-21T10:17:51.263721+0000","last_became_peered":"2024-09-21T10:17:51.263721+0000","last_unstale":"2024-09-21T10:20:01.867817+0000","last_undegraded":"2024-09-21T10:20:01.867817+0000","last_fullsized":"2024-09-21T10:20:01.867817+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:46:02.104778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1627,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871942+0000","last_change":"2024-09-21T10:17:51.251859+0000","last_active":"2024-09-21T10:20:01.871942+0000","last_peered":"2024-09-21T10:20:01.871942+0000","last_clean":"2024-09-21T10:20:01.871942+0000","last_became_active":"2024-09-21T10:17:51.251525+0000","last_became_peered":"2024-09-21T10:17:51.251525+0000","last_unstale":"2024-09-21T10:20:01.871942+0000","last_undegraded":"2024-09-21T10:20:01.871942+0000","last_fullsized":"2024-09-21T10:20:01.871942+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:24.106509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1131,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871427+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:20:01.871427+0000","last_peered":"2024-09-21T10:20:01.871427+0000","last_clean":"2024-09-21T10:20:01.871427+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:20:01.871427+0000","last_undegraded":"2024-09-21T10:20:01.871427+0000","last_fullsized":"2024-09-21T10:20:01.871427+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"1331'15740","reported_seq":30627,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.317672+0000","last_change":"2024-09-21T10:18:54.245834+0000","last_active":"2024-09-21T10:20:06.317672+0000","last_peered":"2024-09-21T10:20:06.317672+0000","last_clean":"2024-09-21T10:20:06.317672+0000","last_became_active":"2024-09-21T10:18:54.227580+0000","last_became_peered":"2024-09-21T10:18:54.227580+0000","last_unstale":"2024-09-21T10:20:06.317672+0000","last_undegraded":"2024-09-21T10:20:06.317672+0000","last_fullsized":"2024-09-21T10:20:06.317672+0000","mapping_epoch":1319,"log_start":"1315'9700","ondisk_log_start":"1315'9700","created":1296,"last_epoch_clean":1320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6040,"log_dups_size":0,"ondisk_log_size":6040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:07.095924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15740,"num_write_kb":548480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7408,"num_bytes_recovered":485490688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1717,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872196+0000","last_change":"2024-09-21T10:16:38.456464+0000","last_active":"2024-09-21T10:20:01.872196+0000","last_peered":"2024-09-21T10:20:01.872196+0000","last_clean":"2024-09-21T10:20:01.872196+0000","last_became_active":"2024-09-21T10:16:38.455345+0000","last_became_peered":"2024-09-21T10:16:38.455345+0000","last_unstale":"2024-09-21T10:20:01.872196+0000","last_undegraded":"2024-09-21T10:20:01.872196+0000","last_fullsized":"2024-09-21T10:20:01.872196+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:39.393023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1862,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868081+0000","last_change":"2024-09-21T10:17:51.254069+0000","last_active":"2024-09-21T10:20:01.868081+0000","last_peered":"2024-09-21T10:20:01.868081+0000","last_clean":"2024-09-21T10:20:01.868081+0000","last_became_active":"2024-09-21T10:17:51.253723+0000","last_became_peered":"2024-09-21T10:17:51.253723+0000","last_unstale":"2024-09-21T10:20:01.868081+0000","last_undegraded":"2024-09-21T10:20:01.868081+0000","last_fullsized":"2024-09-21T10:20:01.868081+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:22.160096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1862,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868233+0000","last_change":"2024-09-21T10:17:51.249147+0000","last_active":"2024-09-21T10:20:01.868233+0000","last_peered":"2024-09-21T10:20:01.868233+0000","last_clean":"2024-09-21T10:20:01.868233+0000","last_became_active":"2024-09-21T10:17:51.248713+0000","last_became_peered":"2024-09-21T10:17:51.248713+0000","last_unstale":"2024-09-21T10:20:01.868233+0000","last_undegraded":"2024-09-21T10:20:01.868233+0000","last_fullsized":"2024-09-21T10:20:01.868233+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:33.757896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1331'15626","reported_seq":15688,"reported_epoch":1331,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:06.913590+0000","last_change":"2024-09-21T10:17:53.257522+0000","last_active":"2024-09-21T10:20:06.913590+0000","last_peered":"2024-09-21T10:20:06.913590+0000","last_clean":"2024-09-21T10:17:50.228509+0000","last_became_active":"2024-09-21T10:17:53.220833+0000","last_became_peered":"2024-09-21T10:17:53.220833+0000","last_unstale":"2024-09-21T10:20:06.913590+0000","last_undegraded":"2024-09-21T10:20:06.913590+0000","last_fullsized":"2024-09-21T10:20:06.913590+0000","mapping_epoch":1309,"log_start":"1315'10000","ondisk_log_start":"1315'10000","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5626,"log_dups_size":0,"ondisk_log_size":5626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:07.029873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84803584,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1294,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15626,"num_write_kb":541440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":1294}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":38,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840486+0000","last_change":"2024-09-21T10:17:51.286472+0000","last_active":"2024-09-21T10:19:59.840486+0000","last_peered":"2024-09-21T10:19:59.840486+0000","last_clean":"2024-09-21T10:19:59.840486+0000","last_became_active":"2024-09-21T10:17:51.249949+0000","last_became_peered":"2024-09-21T10:17:51.249949+0000","last_unstale":"2024-09-21T10:19:59.840486+0000","last_undegraded":"2024-09-21T10:19:59.840486+0000","last_fullsized":"2024-09-21T10:19:59.840486+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:31.760578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1298,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839644+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:19:59.839644+0000","last_peered":"2024-09-21T10:19:59.839644+0000","last_clean":"2024-09-21T10:19:59.839644+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:19:59.839644+0000","last_undegraded":"2024-09-21T10:19:59.839644+0000","last_fullsized":"2024-09-21T10:19:59.839644+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1803,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872599+0000","last_change":"2024-09-21T10:16:39.440143+0000","last_active":"2024-09-21T10:20:01.872599+0000","last_peered":"2024-09-21T10:20:01.872599+0000","last_clean":"2024-09-21T10:20:01.872599+0000","last_became_active":"2024-09-21T10:16:39.439370+0000","last_became_peered":"2024-09-21T10:16:39.439370+0000","last_unstale":"2024-09-21T10:20:01.872599+0000","last_undegraded":"2024-09-21T10:20:01.872599+0000","last_fullsized":"2024-09-21T10:20:01.872599+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:36.417381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"1331'15520","reported_seq":15591,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.665473+0000","last_change":"2024-09-21T10:16:38.458835+0000","last_active":"2024-09-21T10:20:06.665473+0000","last_peered":"2024-09-21T10:20:06.665473+0000","last_clean":"2024-09-21T10:20:06.665473+0000","last_became_active":"2024-09-21T10:16:38.451145+0000","last_became_peered":"2024-09-21T10:16:38.451145+0000","last_unstale":"2024-09-21T10:20:06.665473+0000","last_undegraded":"2024-09-21T10:20:06.665473+0000","last_fullsized":"2024-09-21T10:20:06.665473+0000","mapping_epoch":1301,"log_start":"1316'10400","ondisk_log_start":"1316'10400","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:14.844302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85065728,"num_objects":1298,"num_object_clones":0,"num_object_copies":3894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15520,"num_write_kb":538176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":34,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.841179+0000","last_change":"2024-09-21T10:17:51.259438+0000","last_active":"2024-09-21T10:19:59.841179+0000","last_peered":"2024-09-21T10:19:59.841179+0000","last_clean":"2024-09-21T10:19:59.841179+0000","last_became_active":"2024-09-21T10:17:51.258704+0000","last_became_peered":"2024-09-21T10:17:51.258704+0000","last_unstale":"2024-09-21T10:19:59.841179+0000","last_undegraded":"2024-09-21T10:19:59.841179+0000","last_fullsized":"2024-09-21T10:19:59.841179+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:16.479749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1876,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.867834+0000","last_change":"2024-09-21T10:16:38.460671+0000","last_active":"2024-09-21T10:20:01.867834+0000","last_peered":"2024-09-21T10:20:01.867834+0000","last_clean":"2024-09-21T10:20:01.867834+0000","last_became_active":"2024-09-21T10:16:38.459939+0000","last_became_peered":"2024-09-21T10:16:38.459939+0000","last_unstale":"2024-09-21T10:20:01.867834+0000","last_undegraded":"2024-09-21T10:20:01.867834+0000","last_fullsized":"2024-09-21T10:20:01.867834+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:51.143325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":402,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868112+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:20:01.868112+0000","last_peered":"2024-09-21T10:20:01.868112+0000","last_clean":"2024-09-21T10:20:01.868112+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:20:01.868112+0000","last_undegraded":"2024-09-21T10:20:01.868112+0000","last_fullsized":"2024-09-21T10:20:01.868112+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"1331'15576","reported_seq":15628,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.886618+0000","last_change":"2024-09-21T10:16:38.446821+0000","last_active":"2024-09-21T10:20:06.886618+0000","last_peered":"2024-09-21T10:20:06.886618+0000","last_clean":"2024-09-21T10:20:06.886618+0000","last_became_active":"2024-09-21T10:16:38.446505+0000","last_became_peered":"2024-09-21T10:16:38.446505+0000","last_unstale":"2024-09-21T10:20:06.886618+0000","last_undegraded":"2024-09-21T10:20:06.886618+0000","last_fullsized":"2024-09-21T10:20:06.886618+0000","mapping_epoch":1301,"log_start":"1315'10000","ondisk_log_start":"1315'10000","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:38.183784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89063448,"num_objects":1360,"num_object_clones":0,"num_object_copies":4080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1360,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15576,"num_write_kb":541889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":76,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872329+0000","last_change":"2024-09-21T10:17:51.250313+0000","last_active":"2024-09-21T10:20:01.872329+0000","last_peered":"2024-09-21T10:20:01.872329+0000","last_clean":"2024-09-21T10:20:01.872329+0000","last_became_active":"2024-09-21T10:17:51.249931+0000","last_became_peered":"2024-09-21T10:17:51.249931+0000","last_unstale":"2024-09-21T10:20:01.872329+0000","last_undegraded":"2024-09-21T10:20:01.872329+0000","last_fullsized":"2024-09-21T10:20:01.872329+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:38:15.879964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1015,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.867961+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:20:01.867961+0000","last_peered":"2024-09-21T10:20:01.867961+0000","last_clean":"2024-09-21T10:20:01.867961+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:20:01.867961+0000","last_undegraded":"2024-09-21T10:20:01.867961+0000","last_fullsized":"2024-09-21T10:20:01.867961+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":34,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.838640+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:19:59.838640+0000","last_peered":"2024-09-21T10:19:59.838640+0000","last_clean":"2024-09-21T10:19:59.838640+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:19:59.838640+0000","last_undegraded":"2024-09-21T10:19:59.838640+0000","last_fullsized":"2024-09-21T10:19:59.838640+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"1331'15844","reported_seq":19296,"reported_epoch":1331,"state":"active+remapped+backfilling","last_fresh":"2024-09-21T10:20:06.922820+0000","last_change":"2024-09-21T10:19:50.200340+0000","last_active":"2024-09-21T10:20:06.922820+0000","last_peered":"2024-09-21T10:20:06.922820+0000","last_clean":"2024-09-21T10:17:50.228737+0000","last_became_active":"2024-09-21T10:17:53.226658+0000","last_became_peered":"2024-09-21T10:17:53.226658+0000","last_unstale":"2024-09-21T10:20:06.922820+0000","last_undegraded":"2024-09-21T10:20:06.922820+0000","last_fullsized":"2024-09-21T10:20:06.922820+0000","mapping_epoch":1309,"log_start":"1315'10200","ondisk_log_start":"1315'10200","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:56.871462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87818240,"num_objects":1340,"num_object_clones":0,"num_object_copies":4020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":157,"num_objects_unfound":0,"num_objects_dirty":1340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15844,"num_write_kb":549888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1696,"num_bytes_recovered":111149056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":1340}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":429,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840045+0000","last_change":"2024-09-21T10:18:07.090592+0000","last_active":"2024-09-21T10:19:59.840045+0000","last_peered":"2024-09-21T10:19:59.840045+0000","last_clean":"2024-09-21T10:19:59.840045+0000","last_became_active":"2024-09-21T10:18:07.090188+0000","last_became_peered":"2024-09-21T10:18:07.090188+0000","last_unstale":"2024-09-21T10:19:59.840045+0000","last_undegraded":"2024-09-21T10:19:59.840045+0000","last_fullsized":"2024-09-21T10:19:59.840045+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.763114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":69,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872630+0000","last_change":"2024-09-21T10:16:39.439344+0000","last_active":"2024-09-21T10:20:01.872630+0000","last_peered":"2024-09-21T10:20:01.872630+0000","last_clean":"2024-09-21T10:20:01.872630+0000","last_became_active":"2024-09-21T10:16:39.438987+0000","last_became_peered":"2024-09-21T10:16:39.438987+0000","last_unstale":"2024-09-21T10:20:01.872630+0000","last_undegraded":"2024-09-21T10:20:01.872630+0000","last_fullsized":"2024-09-21T10:20:01.872630+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:17.163136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":431,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872201+0000","last_change":"2024-09-21T10:17:51.250156+0000","last_active":"2024-09-21T10:20:01.872201+0000","last_peered":"2024-09-21T10:20:01.872201+0000","last_clean":"2024-09-21T10:20:01.872201+0000","last_became_active":"2024-09-21T10:17:51.248282+0000","last_became_peered":"2024-09-21T10:17:51.248282+0000","last_unstale":"2024-09-21T10:20:01.872201+0000","last_undegraded":"2024-09-21T10:20:01.872201+0000","last_fullsized":"2024-09-21T10:20:01.872201+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:58.954423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"1331'15543","reported_seq":15593,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:06.907750+0000","last_change":"2024-09-21T10:16:38.454005+0000","last_active":"2024-09-21T10:20:06.907750+0000","last_peered":"2024-09-21T10:20:06.907750+0000","last_clean":"2024-09-21T10:20:06.907750+0000","last_became_active":"2024-09-21T10:16:38.453756+0000","last_became_peered":"2024-09-21T10:16:38.453756+0000","last_unstale":"2024-09-21T10:20:06.907750+0000","last_undegraded":"2024-09-21T10:20:06.907750+0000","last_fullsized":"2024-09-21T10:20:06.907750+0000","mapping_epoch":1301,"log_start":"1315'9900","ondisk_log_start":"1315'9900","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:30.976913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86704128,"num_objects":1323,"num_object_clones":0,"num_object_copies":3969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15543,"num_write_kb":539712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":387,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871511+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:20:01.871511+0000","last_peered":"2024-09-21T10:20:01.871511+0000","last_clean":"2024-09-21T10:20:01.871511+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:20:01.871511+0000","last_undegraded":"2024-09-21T10:20:01.871511+0000","last_fullsized":"2024-09-21T10:20:01.871511+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":212,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871843+0000","last_change":"2024-09-21T10:16:38.455089+0000","last_active":"2024-09-21T10:20:01.871843+0000","last_peered":"2024-09-21T10:20:01.871843+0000","last_clean":"2024-09-21T10:20:01.871843+0000","last_became_active":"2024-09-21T10:16:38.454804+0000","last_became_peered":"2024-09-21T10:16:38.454804+0000","last_unstale":"2024-09-21T10:20:01.871843+0000","last_undegraded":"2024-09-21T10:20:01.871843+0000","last_fullsized":"2024-09-21T10:20:01.871843+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:16.448016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.844703+0000","last_change":"2024-09-21T10:17:51.250324+0000","last_active":"2024-09-21T10:19:59.844703+0000","last_peered":"2024-09-21T10:19:59.844703+0000","last_clean":"2024-09-21T10:19:59.844703+0000","last_became_active":"2024-09-21T10:17:51.249933+0000","last_became_peered":"2024-09-21T10:17:51.249933+0000","last_unstale":"2024-09-21T10:19:59.844703+0000","last_undegraded":"2024-09-21T10:19:59.844703+0000","last_fullsized":"2024-09-21T10:19:59.844703+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:23.880577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":187,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839872+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:19:59.839872+0000","last_peered":"2024-09-21T10:19:59.839872+0000","last_clean":"2024-09-21T10:19:59.839872+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:19:59.839872+0000","last_undegraded":"2024-09-21T10:19:59.839872+0000","last_fullsized":"2024-09-21T10:19:59.839872+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":250,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.839682+0000","last_change":"2024-09-21T10:17:51.277615+0000","last_active":"2024-09-21T10:19:59.839682+0000","last_peered":"2024-09-21T10:19:59.839682+0000","last_clean":"2024-09-21T10:19:59.839682+0000","last_became_active":"2024-09-21T10:17:51.261756+0000","last_became_peered":"2024-09-21T10:17:51.261756+0000","last_unstale":"2024-09-21T10:19:59.839682+0000","last_undegraded":"2024-09-21T10:19:59.839682+0000","last_fullsized":"2024-09-21T10:19:59.839682+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:05:35.608372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1849,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868445+0000","last_change":"2024-09-21T10:17:51.251440+0000","last_active":"2024-09-21T10:20:01.868445+0000","last_peered":"2024-09-21T10:20:01.868445+0000","last_clean":"2024-09-21T10:20:01.868445+0000","last_became_active":"2024-09-21T10:17:51.249109+0000","last_became_peered":"2024-09-21T10:17:51.249109+0000","last_unstale":"2024-09-21T10:20:01.868445+0000","last_undegraded":"2024-09-21T10:20:01.868445+0000","last_fullsized":"2024-09-21T10:20:01.868445+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:59.577984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1359,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.873303+0000","last_change":"2024-09-21T10:16:38.453215+0000","last_active":"2024-09-21T10:20:01.873303+0000","last_peered":"2024-09-21T10:20:01.873303+0000","last_clean":"2024-09-21T10:20:01.873303+0000","last_became_active":"2024-09-21T10:16:38.452464+0000","last_became_peered":"2024-09-21T10:16:38.452464+0000","last_unstale":"2024-09-21T10:20:01.873303+0000","last_undegraded":"2024-09-21T10:20:01.873303+0000","last_fullsized":"2024-09-21T10:20:01.873303+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:16.818394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":33,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.843601+0000","last_change":"2024-09-21T10:17:51.248299+0000","last_active":"2024-09-21T10:19:59.843601+0000","last_peered":"2024-09-21T10:19:59.843601+0000","last_clean":"2024-09-21T10:19:59.843601+0000","last_became_active":"2024-09-21T10:17:51.247851+0000","last_became_peered":"2024-09-21T10:17:51.247851+0000","last_unstale":"2024-09-21T10:19:59.843601+0000","last_undegraded":"2024-09-21T10:19:59.843601+0000","last_fullsized":"2024-09-21T10:19:59.843601+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:04.319143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":507,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871991+0000","last_change":"2024-09-21T10:17:51.248187+0000","last_active":"2024-09-21T10:20:01.871991+0000","last_peered":"2024-09-21T10:20:01.871991+0000","last_clean":"2024-09-21T10:20:01.871991+0000","last_became_active":"2024-09-21T10:17:51.247436+0000","last_became_peered":"2024-09-21T10:17:51.247436+0000","last_unstale":"2024-09-21T10:20:01.871991+0000","last_undegraded":"2024-09-21T10:20:01.871991+0000","last_fullsized":"2024-09-21T10:20:01.871991+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:33.505149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":33,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840311+0000","last_change":"2024-09-21T10:17:51.258577+0000","last_active":"2024-09-21T10:19:59.840311+0000","last_peered":"2024-09-21T10:19:59.840311+0000","last_clean":"2024-09-21T10:19:59.840311+0000","last_became_active":"2024-09-21T10:17:51.257820+0000","last_became_peered":"2024-09-21T10:17:51.257820+0000","last_unstale":"2024-09-21T10:19:59.840311+0000","last_undegraded":"2024-09-21T10:19:59.840311+0000","last_fullsized":"2024-09-21T10:19:59.840311+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:00.922957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":491,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871972+0000","last_change":"2024-09-21T10:16:38.456905+0000","last_active":"2024-09-21T10:20:01.871972+0000","last_peered":"2024-09-21T10:20:01.871972+0000","last_clean":"2024-09-21T10:20:01.871972+0000","last_became_active":"2024-09-21T10:16:38.453276+0000","last_became_peered":"2024-09-21T10:16:38.453276+0000","last_unstale":"2024-09-21T10:20:01.871972+0000","last_undegraded":"2024-09-21T10:20:01.871972+0000","last_fullsized":"2024-09-21T10:20:01.871972+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:38.174569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":230,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840165+0000","last_change":"2024-09-21T10:17:51.262911+0000","last_active":"2024-09-21T10:19:59.840165+0000","last_peered":"2024-09-21T10:19:59.840165+0000","last_clean":"2024-09-21T10:19:59.840165+0000","last_became_active":"2024-09-21T10:17:51.262314+0000","last_became_peered":"2024-09-21T10:17:51.262314+0000","last_unstale":"2024-09-21T10:19:59.840165+0000","last_undegraded":"2024-09-21T10:19:59.840165+0000","last_fullsized":"2024-09-21T10:19:59.840165+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:06.241585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":479,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.874395+0000","last_change":"2024-09-21T10:17:51.254289+0000","last_active":"2024-09-21T10:20:01.874395+0000","last_peered":"2024-09-21T10:20:01.874395+0000","last_clean":"2024-09-21T10:20:01.874395+0000","last_became_active":"2024-09-21T10:17:51.253922+0000","last_became_peered":"2024-09-21T10:17:51.253922+0000","last_unstale":"2024-09-21T10:20:01.874395+0000","last_undegraded":"2024-09-21T10:20:01.874395+0000","last_fullsized":"2024-09-21T10:20:01.874395+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:57.111361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1742,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872848+0000","last_change":"2024-09-21T10:17:51.249960+0000","last_active":"2024-09-21T10:20:01.872848+0000","last_peered":"2024-09-21T10:20:01.872848+0000","last_clean":"2024-09-21T10:20:01.872848+0000","last_became_active":"2024-09-21T10:17:51.248883+0000","last_became_peered":"2024-09-21T10:17:51.248883+0000","last_unstale":"2024-09-21T10:20:01.872848+0000","last_undegraded":"2024-09-21T10:20:01.872848+0000","last_fullsized":"2024-09-21T10:20:01.872848+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:20.076539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":134,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.871437+0000","last_change":"2024-09-21T10:16:38.461877+0000","last_active":"2024-09-21T10:20:01.871437+0000","last_peered":"2024-09-21T10:20:01.871437+0000","last_clean":"2024-09-21T10:20:01.871437+0000","last_became_active":"2024-09-21T10:16:38.461554+0000","last_became_peered":"2024-09-21T10:16:38.461554+0000","last_unstale":"2024-09-21T10:20:01.871437+0000","last_undegraded":"2024-09-21T10:20:01.871437+0000","last_fullsized":"2024-09-21T10:20:01.871437+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:46.372637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":419,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.873405+0000","last_change":"2024-09-21T10:16:38.447369+0000","last_active":"2024-09-21T10:20:01.873405+0000","last_peered":"2024-09-21T10:20:01.873405+0000","last_clean":"2024-09-21T10:20:01.873405+0000","last_became_active":"2024-09-21T10:16:38.447085+0000","last_became_peered":"2024-09-21T10:16:38.447085+0000","last_unstale":"2024-09-21T10:20:01.873405+0000","last_undegraded":"2024-09-21T10:20:01.873405+0000","last_fullsized":"2024-09-21T10:20:01.873405+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:59.110578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":406,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868593+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:20:01.868593+0000","last_peered":"2024-09-21T10:20:01.868593+0000","last_clean":"2024-09-21T10:20:01.868593+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:20:01.868593+0000","last_undegraded":"2024-09-21T10:20:01.868593+0000","last_fullsized":"2024-09-21T10:20:01.868593+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1735,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.867941+0000","last_change":"2024-09-21T10:16:38.460957+0000","last_active":"2024-09-21T10:20:01.867941+0000","last_peered":"2024-09-21T10:20:01.867941+0000","last_clean":"2024-09-21T10:20:01.867941+0000","last_became_active":"2024-09-21T10:16:38.460325+0000","last_became_peered":"2024-09-21T10:16:38.460325+0000","last_unstale":"2024-09-21T10:20:01.867941+0000","last_undegraded":"2024-09-21T10:20:01.867941+0000","last_fullsized":"2024-09-21T10:20:01.867941+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:34.529895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1809,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.868509+0000","last_change":"2024-09-21T10:16:38.461576+0000","last_active":"2024-09-21T10:20:01.868509+0000","last_peered":"2024-09-21T10:20:01.868509+0000","last_clean":"2024-09-21T10:20:01.868509+0000","last_became_active":"2024-09-21T10:16:38.461205+0000","last_became_peered":"2024-09-21T10:16:38.461205+0000","last_unstale":"2024-09-21T10:20:01.868509+0000","last_undegraded":"2024-09-21T10:20:01.868509+0000","last_fullsized":"2024-09-21T10:20:01.868509+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:17.166658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1401,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872158+0000","last_change":"2024-09-21T10:16:38.459378+0000","last_active":"2024-09-21T10:20:01.872158+0000","last_peered":"2024-09-21T10:20:01.872158+0000","last_clean":"2024-09-21T10:20:01.872158+0000","last_became_active":"2024-09-21T10:16:38.451582+0000","last_became_peered":"2024-09-21T10:16:38.451582+0000","last_unstale":"2024-09-21T10:20:01.872158+0000","last_undegraded":"2024-09-21T10:20:01.872158+0000","last_fullsized":"2024-09-21T10:20:01.872158+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:34.820762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":37,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.838887+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:19:59.838887+0000","last_peered":"2024-09-21T10:19:59.838887+0000","last_clean":"2024-09-21T10:19:59.838887+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:19:59.838887+0000","last_undegraded":"2024-09-21T10:19:59.838887+0000","last_fullsized":"2024-09-21T10:19:59.838887+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1418,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872069+0000","last_change":"2024-09-21T10:16:38.453928+0000","last_active":"2024-09-21T10:20:01.872069+0000","last_peered":"2024-09-21T10:20:01.872069+0000","last_clean":"2024-09-21T10:20:01.872069+0000","last_became_active":"2024-09-21T10:16:38.450890+0000","last_became_peered":"2024-09-21T10:16:38.450890+0000","last_unstale":"2024-09-21T10:20:01.872069+0000","last_undegraded":"2024-09-21T10:20:01.872069+0000","last_fullsized":"2024-09-21T10:20:01.872069+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:13.500644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":37,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.841024+0000","last_change":"2024-09-21T10:17:51.281734+0000","last_active":"2024-09-21T10:19:59.841024+0000","last_peered":"2024-09-21T10:19:59.841024+0000","last_clean":"2024-09-21T10:19:59.841024+0000","last_became_active":"2024-09-21T10:17:51.258206+0000","last_became_peered":"2024-09-21T10:17:51.258206+0000","last_unstale":"2024-09-21T10:19:59.841024+0000","last_undegraded":"2024-09-21T10:19:59.841024+0000","last_fullsized":"2024-09-21T10:19:59.841024+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:16.184203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":479,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872089+0000","last_change":"2024-09-21T10:17:51.251289+0000","last_active":"2024-09-21T10:20:01.872089+0000","last_peered":"2024-09-21T10:20:01.872089+0000","last_clean":"2024-09-21T10:20:01.872089+0000","last_became_active":"2024-09-21T10:17:51.247868+0000","last_became_peered":"2024-09-21T10:17:51.247868+0000","last_unstale":"2024-09-21T10:20:01.872089+0000","last_undegraded":"2024-09-21T10:20:01.872089+0000","last_fullsized":"2024-09-21T10:20:01.872089+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:39:49.784298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":481,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-09-21T10:20:01.872876+0000","last_change":"2024-09-21T10:16:38.451567+0000","last_active":"2024-09-21T10:20:01.872876+0000","last_peered":"2024-09-21T10:20:01.872876+0000","last_clean":"2024-09-21T10:20:01.872876+0000","last_became_active":"2024-09-21T10:16:38.451245+0000","last_became_peered":"2024-09-21T10:16:38.451245+0000","last_unstale":"2024-09-21T10:20:01.872876+0000","last_undegraded":"2024-09-21T10:20:01.872876+0000","last_fullsized":"2024-09-21T10:20:01.872876+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:27.022083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":196,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840178+0000","last_change":"2024-09-21T10:15:19.115911+0000","last_active":"2024-09-21T10:19:59.840178+0000","last_peered":"2024-09-21T10:19:59.840178+0000","last_clean":"2024-09-21T10:19:59.840178+0000","last_became_active":"2024-09-21T10:15:19.115615+0000","last_became_peered":"2024-09-21T10:15:19.115615+0000","last_unstale":"2024-09-21T10:19:59.840178+0000","last_undegraded":"2024-09-21T10:19:59.840178+0000","last_fullsized":"2024-09-21T10:19:59.840178+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":40,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-09-21T10:19:59.840233+0000","last_change":"2024-09-21T10:17:51.280012+0000","last_active":"2024-09-21T10:19:59.840233+0000","last_peered":"2024-09-21T10:19:59.840233+0000","last_clean":"2024-09-21T10:19:59.840233+0000","last_became_active":"2024-09-21T10:17:51.257359+0000","last_became_peered":"2024-09-21T10:17:51.257359+0000","last_unstale":"2024-09-21T10:19:59.840233+0000","last_undegraded":"2024-09-21T10:19:59.840233+0000","last_fullsized":"2024-09-21T10:19:59.840233+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:44.522556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1411186712,"num_objects":21534,"num_object_clones":0,"num_object_copies":64602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6726,"num_objects_unfound":0,"num_objects_dirty":21534,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":248820,"num_write_kb":8651265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17227,"num_bytes_recovered":1128988672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4565512192,"data_stored":4565499976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89220,"ondisk_log_size":89220,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":21092,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17031,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":6}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234531,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496167,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2292932,"kb_used_data":1128316,"kb_used_omap":1,"kb_used_meta":1164542,"kb_avail":91452220,"statfs":{"total":95995035648,"available":93647073280,"internally_reserved":0,"allocated":1155395584,"data_stored":1152175039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1192491463},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,9,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757806,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":758968,"kb_used_data":362864,"kb_used_omap":1,"kb_used_meta":396094,"kb_avail":92986184,"statfs":{"total":95995035648,"available":95217852416,"internally_reserved":0,"allocated":371572736,"data_stored":368363809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":405600710},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019442,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2362176,"kb_used_data":1352376,"kb_used_omap":1,"kb_used_meta":1009790,"kb_avail":91382976,"statfs":{"total":95995035648,"available":93576167424,"internally_reserved":0,"allocated":1384833024,"data_stored":1381616508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1034025414},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":1301,"seq":5587752452139,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1426024,"kb_used_data":1065248,"kb_used_omap":48,"kb_used_meta":360655,"kb_avail":92319128,"statfs":{"total":95995035648,"available":94534787072,"internally_reserved":0,"allocated":1090813952,"data_stored":1087616156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49883,"internal_metadata":369311013},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793967978,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047216,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1790124,"kb_used_data":696228,"kb_used_omap":41,"kb_used_meta":1093846,"kb_avail":91955028,"statfs":{"total":95995035648,"available":94161948672,"internally_reserved":0,"allocated":712937472,"data_stored":709736015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1120099241},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928851,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7559,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":701825024,"data_stored":701825024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1058869248,"data_stored":1058865176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1330188288,"data_stored":1330184216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":359333888,"data_stored":359333888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1115295744,"data_stored":1115291672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:20:11.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:20:11.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:11.815 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:11.916 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:11.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:11.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:12.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.082 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:12.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:12.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:12.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:12.308 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:12.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:11 smithi076 ceph-mon[106482]: pgmap v2103: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 591 op/s; 6726/64680 objects misplaced (10.399%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:20:12.327 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:11 smithi076 ceph-mon[106482]: from='client.36105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:12.327 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:11 smithi076 ceph-mon[110963]: pgmap v2103: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 591 op/s; 6726/64680 objects misplaced (10.399%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:20:12.327 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:11 smithi076 ceph-mon[110963]: from='client.36105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:12.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:12.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:11 smithi191 ceph-mon[102206]: pgmap v2103: 89 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 83 active+clean; 1.3 GiB data, 8.2 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 591 op/s; 6726/64680 objects misplaced (10.399%); 2.7 MiB/s, 43 objects/s recovering 2024-09-21T10:20:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:11 smithi191 ceph-mon[102206]: from='client.36105 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:12.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:20:12.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:12.501 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:12.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:12.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:12.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:12.726 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:12.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:12.856 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:12.864 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:12.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:12.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:12.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:13.014 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:13.101 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:13.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:13.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:13.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:13.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:13.294 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:13.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:20:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:20:13] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.51.0" 2024-09-21T10:20:13.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:13.428 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:13.530 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:13.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:13.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:13.712 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:13.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:13.813 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:13.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:13.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:13.970 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:13.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:14.072 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:14.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:14.115 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:20:14.116 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:20:14.117 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:20:14.117 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:20:14.117 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:14.117 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:14.117 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:14.117 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:14.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.241 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:14.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:14.343 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:13 smithi191 ceph-mon[102206]: pgmap v2104: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s; 4556/53724 objects misplaced (8.480%); 11 MiB/s, 183 objects/s recovering 2024-09-21T10:20:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/701993980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:13 smithi076 ceph-mon[110963]: pgmap v2104: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s; 4556/53724 objects misplaced (8.480%); 11 MiB/s, 183 objects/s recovering 2024-09-21T10:20:14.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/701993980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:13 smithi076 ceph-mon[106482]: pgmap v2104: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 1.05k op/s; 4556/53724 objects misplaced (8.480%); 11 MiB/s, 183 objects/s recovering 2024-09-21T10:20:14.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/701993980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:14.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:14.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:14.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:14.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:14.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:14.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:14.737 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:14.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:14.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:14.859 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:14.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:14.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:14.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:14.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.012 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:15.154 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:15.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:15.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:20:15.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.283 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:15.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:15.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:15.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:15.613 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:15.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:15.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:15.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:15.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:15.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:15.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:15.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:16.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:16.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:16.013 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:15 smithi191 ceph-mon[102206]: pgmap v2105: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 974 op/s; 4556/53724 objects misplaced (8.480%); 11 MiB/s, 183 objects/s recovering 2024-09-21T10:20:16.112 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:15 smithi076 ceph-mon[106482]: pgmap v2105: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 974 op/s; 4556/53724 objects misplaced (8.480%); 11 MiB/s, 183 objects/s recovering 2024-09-21T10:20:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:15 smithi076 ceph-mon[110963]: pgmap v2105: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 974 op/s; 4556/53724 objects misplaced (8.480%); 11 MiB/s, 183 objects/s recovering 2024-09-21T10:20:16.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:16.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:16.283 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:16.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:16.385 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:16.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:16.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:16.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:16.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:16.706 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:16.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:16.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:16.839 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:16.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:16.967 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:16.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:17.069 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:17.093 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:17.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:17.195 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:17.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:17.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:17.353 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:17.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:17.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:17.481 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:17.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:17.607 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:17.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:17.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:17.742 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:17.742 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:17.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.757 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:17.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:17.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:17.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:17.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:17.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:17.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:17 smithi076 ceph-mon[106482]: pgmap v2106: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 521 op/s; 4556/52161 objects misplaced (8.734%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:20:17.997 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:17 smithi076 ceph-mon[110963]: pgmap v2106: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 521 op/s; 4556/52161 objects misplaced (8.734%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:20:18.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:18.032 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:18.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:18.172 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:18.273 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:18.296 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:18.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:17 smithi191 ceph-mon[102206]: pgmap v2106: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 1.1 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 521 op/s; 4556/52161 objects misplaced (8.734%); 8.8 MiB/s, 140 objects/s recovering 2024-09-21T10:20:18.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:18.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.444 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:18.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:18.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:18.587 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:18.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:18.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:18.817 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:18.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:18.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:18.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:18.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:18.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:18.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:19.067 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:20:19.067 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:20:19.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:19.124 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:19.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:20:19.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:19.274 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:19.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:19.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:19.513 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:19.551 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:19.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:19.687 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:19.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:19.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:19.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:19.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:19.848 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:20:19.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:19.929 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:20:19.960 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:19.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:20.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.103 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:20.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:19 smithi191 ceph-mon[102206]: pgmap v2107: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 841 MiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 821 op/s; 3026/40437 objects misplaced (7.483%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:20:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:19 smithi191 ceph-mon[102206]: from='client.46010 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:19 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3462831775' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:19 smithi076 ceph-mon[106482]: pgmap v2107: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 841 MiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 821 op/s; 3026/40437 objects misplaced (7.483%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:20:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:19 smithi076 ceph-mon[106482]: from='client.46010 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:19 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3462831775' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:19 smithi076 ceph-mon[110963]: pgmap v2107: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 841 MiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 821 op/s; 3026/40437 objects misplaced (7.483%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:20:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:19 smithi076 ceph-mon[110963]: from='client.46010 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:19 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3462831775' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:20.204 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:20.255 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:20.280 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2107,"stamp":"2024-09-21T10:20:18.731971+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":882186237,"num_objects":13479,"num_object_clones":0,"num_object_copies":40437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3026,"num_objects_unfound":0,"num_objects_dirty":13479,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":257180,"num_write_kb":8653714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18890,"num_bytes_recovered":1235705693,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89809,"ondisk_log_size":89809,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":273,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":7080048,"kb_used_data":3050440,"kb_used_omap":94,"kb_used_meta":4029089,"kb_avail":461645712,"statfs":{"total":479975178240,"available":472725209088,"internally_reserved":0,"allocated":3123650560,"data_stored":3107526784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96735,"internal_metadata":4125787681},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,4,5,2,2,0,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-529596416,"num_objects":-8081,"num_object_clones":0,"num_object_copies":-24243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-3700,"num_objects_unfound":0,"num_objects_dirty":-8081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8081,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1575,"num_bytes_recovered":103153664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.838078"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":483,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906372+0000","last_change":"2024-09-21T10:17:51.249951+0000","last_active":"2024-09-21T10:20:09.906372+0000","last_peered":"2024-09-21T10:20:09.906372+0000","last_clean":"2024-09-21T10:20:09.906372+0000","last_became_active":"2024-09-21T10:17:51.249158+0000","last_became_peered":"2024-09-21T10:17:51.249158+0000","last_unstale":"2024-09-21T10:20:09.906372+0000","last_undegraded":"2024-09-21T10:20:09.906372+0000","last_fullsized":"2024-09-21T10:20:09.906372+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:53.769122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":210,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.913991+0000","last_change":"2024-09-21T10:15:19.141623+0000","last_active":"2024-09-21T10:20:08.913991+0000","last_peered":"2024-09-21T10:20:08.913991+0000","last_clean":"2024-09-21T10:20:08.913991+0000","last_became_active":"2024-09-21T10:15:19.111927+0000","last_became_peered":"2024-09-21T10:15:19.111927+0000","last_unstale":"2024-09-21T10:20:08.913991+0000","last_undegraded":"2024-09-21T10:20:08.913991+0000","last_fullsized":"2024-09-21T10:20:08.913991+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:10.426159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":36,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.900293+0000","last_change":"2024-09-21T10:17:51.258024+0000","last_active":"2024-09-21T10:20:09.900293+0000","last_peered":"2024-09-21T10:20:09.900293+0000","last_clean":"2024-09-21T10:20:09.900293+0000","last_became_active":"2024-09-21T10:17:51.257639+0000","last_became_peered":"2024-09-21T10:17:51.257639+0000","last_unstale":"2024-09-21T10:20:09.900293+0000","last_undegraded":"2024-09-21T10:20:09.900293+0000","last_fullsized":"2024-09-21T10:20:09.900293+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:09.112335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1800,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.905978+0000","last_change":"2024-09-21T10:17:51.250405+0000","last_active":"2024-09-21T10:20:09.905978+0000","last_peered":"2024-09-21T10:20:09.905978+0000","last_clean":"2024-09-21T10:20:09.905978+0000","last_became_active":"2024-09-21T10:17:51.248030+0000","last_became_peered":"2024-09-21T10:17:51.248030+0000","last_unstale":"2024-09-21T10:20:09.905978+0000","last_undegraded":"2024-09-21T10:20:09.905978+0000","last_fullsized":"2024-09-21T10:20:09.905978+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:45.889214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1799,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.912331+0000","last_change":"2024-09-21T10:17:51.249710+0000","last_active":"2024-09-21T10:20:09.912331+0000","last_peered":"2024-09-21T10:20:09.912331+0000","last_clean":"2024-09-21T10:20:09.912331+0000","last_became_active":"2024-09-21T10:17:51.249284+0000","last_became_peered":"2024-09-21T10:17:51.249284+0000","last_unstale":"2024-09-21T10:20:09.912331+0000","last_undegraded":"2024-09-21T10:20:09.912331+0000","last_fullsized":"2024-09-21T10:20:09.912331+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:09.927717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":213,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.909242+0000","last_change":"2024-09-21T10:16:38.451401+0000","last_active":"2024-09-21T10:20:08.909242+0000","last_peered":"2024-09-21T10:20:08.909242+0000","last_clean":"2024-09-21T10:20:08.909242+0000","last_became_active":"2024-09-21T10:16:38.450094+0000","last_became_peered":"2024-09-21T10:16:38.450094+0000","last_unstale":"2024-09-21T10:20:08.909242+0000","last_undegraded":"2024-09-21T10:20:08.909242+0000","last_fullsized":"2024-09-21T10:20:08.909242+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:22:50.798278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":476,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906754+0000","last_change":"2024-09-21T10:16:38.451461+0000","last_active":"2024-09-21T10:20:09.906754+0000","last_peered":"2024-09-21T10:20:09.906754+0000","last_clean":"2024-09-21T10:20:09.906754+0000","last_became_active":"2024-09-21T10:16:38.451137+0000","last_became_peered":"2024-09-21T10:16:38.451137+0000","last_unstale":"2024-09-21T10:20:09.906754+0000","last_undegraded":"2024-09-21T10:20:09.906754+0000","last_fullsized":"2024-09-21T10:20:09.906754+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:40.847463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":271,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.909191+0000","last_change":"2024-09-21T10:17:51.269584+0000","last_active":"2024-09-21T10:20:08.909191+0000","last_peered":"2024-09-21T10:20:08.909191+0000","last_clean":"2024-09-21T10:20:08.909191+0000","last_became_active":"2024-09-21T10:17:51.260673+0000","last_became_peered":"2024-09-21T10:17:51.260673+0000","last_unstale":"2024-09-21T10:20:08.909191+0000","last_undegraded":"2024-09-21T10:20:08.909191+0000","last_fullsized":"2024-09-21T10:20:08.909191+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:16.649818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"1333'15960","reported_seq":16055,"reported_epoch":1333,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:16.801802+0000","last_change":"2024-09-21T10:17:53.268636+0000","last_active":"2024-09-21T10:20:16.801802+0000","last_peered":"2024-09-21T10:20:16.801802+0000","last_clean":"2024-09-21T10:17:50.209821+0000","last_became_active":"2024-09-21T10:17:53.218553+0000","last_became_peered":"2024-09-21T10:17:53.218553+0000","last_unstale":"2024-09-21T10:20:16.801802+0000","last_undegraded":"2024-09-21T10:20:16.801802+0000","last_fullsized":"2024-09-21T10:20:16.801802+0000","mapping_epoch":1309,"log_start":"1318'10400","ondisk_log_start":"1318'10400","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5560,"log_dups_size":0,"ondisk_log_size":5560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51773440,"num_objects":790,"num_object_clones":0,"num_object_copies":2370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":790,"num_objects_unfound":0,"num_objects_dirty":790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15960,"num_write_kb":536000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":790}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":386,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903011+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:20:09.903011+0000","last_peered":"2024-09-21T10:20:09.903011+0000","last_clean":"2024-09-21T10:20:09.903011+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:20:09.903011+0000","last_undegraded":"2024-09-21T10:20:09.903011+0000","last_fullsized":"2024-09-21T10:20:09.903011+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":36,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.909189+0000","last_change":"2024-09-21T10:17:51.256335+0000","last_active":"2024-09-21T10:20:08.909189+0000","last_peered":"2024-09-21T10:20:08.909189+0000","last_clean":"2024-09-21T10:20:08.909189+0000","last_became_active":"2024-09-21T10:17:51.251630+0000","last_became_peered":"2024-09-21T10:17:51.251630+0000","last_unstale":"2024-09-21T10:20:08.909189+0000","last_undegraded":"2024-09-21T10:20:08.909189+0000","last_fullsized":"2024-09-21T10:20:08.909189+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:31.141409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"1333'15921","reported_seq":17196,"reported_epoch":1333,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:16.915391+0000","last_change":"2024-09-21T10:18:54.252039+0000","last_active":"2024-09-21T10:20:16.915391+0000","last_peered":"2024-09-21T10:20:16.915391+0000","last_clean":"2024-09-21T10:17:50.234745+0000","last_became_active":"2024-09-21T10:17:53.221056+0000","last_became_peered":"2024-09-21T10:17:53.221056+0000","last_unstale":"2024-09-21T10:20:16.915391+0000","last_undegraded":"2024-09-21T10:20:16.915391+0000","last_fullsized":"2024-09-21T10:20:16.915391+0000","mapping_epoch":1309,"log_start":"1317'10300","ondisk_log_start":"1317'10300","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5621,"log_dups_size":0,"ondisk_log_size":5621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:41.368294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":768,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15921,"num_write_kb":536768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":604,"num_bytes_recovered":39583744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,4],"avail_no_missing":["6","3","4"],"object_location_counts":[{"shards":"3,4,6","objects":853}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":85,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902231+0000","last_change":"2024-09-21T10:17:51.281550+0000","last_active":"2024-09-21T10:20:09.902231+0000","last_peered":"2024-09-21T10:20:09.902231+0000","last_clean":"2024-09-21T10:20:09.902231+0000","last_became_active":"2024-09-21T10:17:51.262457+0000","last_became_peered":"2024-09-21T10:17:51.262457+0000","last_unstale":"2024-09-21T10:20:09.902231+0000","last_undegraded":"2024-09-21T10:20:09.902231+0000","last_fullsized":"2024-09-21T10:20:09.902231+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:49.212195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1460,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902762+0000","last_change":"2024-09-21T10:16:38.461867+0000","last_active":"2024-09-21T10:20:09.902762+0000","last_peered":"2024-09-21T10:20:09.902762+0000","last_clean":"2024-09-21T10:20:09.902762+0000","last_became_active":"2024-09-21T10:16:38.461416+0000","last_became_peered":"2024-09-21T10:16:38.461416+0000","last_unstale":"2024-09-21T10:20:09.902762+0000","last_undegraded":"2024-09-21T10:20:09.902762+0000","last_fullsized":"2024-09-21T10:20:09.902762+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:15.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"1333'16053","reported_seq":16128,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.752183+0000","last_change":"2024-09-21T10:16:38.451467+0000","last_active":"2024-09-21T10:20:16.752183+0000","last_peered":"2024-09-21T10:20:16.752183+0000","last_clean":"2024-09-21T10:20:16.752183+0000","last_became_active":"2024-09-21T10:16:38.446328+0000","last_became_peered":"2024-09-21T10:16:38.446328+0000","last_unstale":"2024-09-21T10:20:16.752183+0000","last_undegraded":"2024-09-21T10:20:16.752183+0000","last_fullsized":"2024-09-21T10:20:16.752183+0000","mapping_epoch":1301,"log_start":"1318'10800","ondisk_log_start":"1318'10800","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5253,"log_dups_size":0,"ondisk_log_size":5253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:27.117612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52756480,"num_objects":805,"num_object_clones":0,"num_object_copies":2415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16053,"num_write_kb":539456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":40,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.906109+0000","last_change":"2024-09-21T10:17:51.283569+0000","last_active":"2024-09-21T10:20:08.906109+0000","last_peered":"2024-09-21T10:20:08.906109+0000","last_clean":"2024-09-21T10:20:08.906109+0000","last_became_active":"2024-09-21T10:17:51.259187+0000","last_became_peered":"2024-09-21T10:17:51.259187+0000","last_unstale":"2024-09-21T10:20:08.906109+0000","last_undegraded":"2024-09-21T10:20:08.906109+0000","last_fullsized":"2024-09-21T10:20:08.906109+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:55.002645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":223,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.907779+0000","last_change":"2024-09-21T10:17:51.248805+0000","last_active":"2024-09-21T10:20:08.907779+0000","last_peered":"2024-09-21T10:20:08.907779+0000","last_clean":"2024-09-21T10:20:08.907779+0000","last_became_active":"2024-09-21T10:17:51.248157+0000","last_became_peered":"2024-09-21T10:17:51.248157+0000","last_unstale":"2024-09-21T10:20:08.907779+0000","last_undegraded":"2024-09-21T10:20:08.907779+0000","last_fullsized":"2024-09-21T10:20:08.907779+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:36.573842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1333'15914","reported_seq":21889,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.735120+0000","last_change":"2024-09-21T10:19:51.830838+0000","last_active":"2024-09-21T10:20:16.735120+0000","last_peered":"2024-09-21T10:20:16.735120+0000","last_clean":"2024-09-21T10:20:16.735120+0000","last_became_active":"2024-09-21T10:19:51.804737+0000","last_became_peered":"2024-09-21T10:19:51.804737+0000","last_unstale":"2024-09-21T10:20:16.735120+0000","last_undegraded":"2024-09-21T10:20:16.735120+0000","last_fullsized":"2024-09-21T10:20:16.735120+0000","mapping_epoch":1328,"log_start":"1318'10700","ondisk_log_start":"1318'10700","created":1296,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5214,"log_dups_size":0,"ondisk_log_size":5214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:46.112314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15914,"num_write_kb":536000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2940,"num_bytes_recovered":192675840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":36,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.901839+0000","last_change":"2024-09-21T10:17:51.259331+0000","last_active":"2024-09-21T10:20:08.901839+0000","last_peered":"2024-09-21T10:20:08.901839+0000","last_clean":"2024-09-21T10:20:08.901839+0000","last_became_active":"2024-09-21T10:17:51.258943+0000","last_became_peered":"2024-09-21T10:17:51.258943+0000","last_unstale":"2024-09-21T10:20:08.901839+0000","last_undegraded":"2024-09-21T10:20:08.901839+0000","last_fullsized":"2024-09-21T10:20:08.901839+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:59.304035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1846,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903771+0000","last_change":"2024-09-21T10:16:38.461644+0000","last_active":"2024-09-21T10:20:09.903771+0000","last_peered":"2024-09-21T10:20:09.903771+0000","last_clean":"2024-09-21T10:20:09.903771+0000","last_became_active":"2024-09-21T10:16:38.461179+0000","last_became_peered":"2024-09-21T10:16:38.461179+0000","last_unstale":"2024-09-21T10:20:09.903771+0000","last_undegraded":"2024-09-21T10:20:09.903771+0000","last_fullsized":"2024-09-21T10:20:09.903771+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:43.638430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"1333'15940","reported_seq":16029,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.815622+0000","last_change":"2024-09-21T10:16:38.459692+0000","last_active":"2024-09-21T10:20:16.815622+0000","last_peered":"2024-09-21T10:20:16.815622+0000","last_clean":"2024-09-21T10:20:16.815622+0000","last_became_active":"2024-09-21T10:16:38.459132+0000","last_became_peered":"2024-09-21T10:16:38.459132+0000","last_unstale":"2024-09-21T10:20:16.815622+0000","last_undegraded":"2024-09-21T10:20:16.815622+0000","last_fullsized":"2024-09-21T10:20:16.815622+0000","mapping_epoch":1301,"log_start":"1317'10300","ondisk_log_start":"1317'10300","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:36:53.847003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15940,"num_write_kb":537664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":429,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906059+0000","last_change":"2024-09-21T10:16:38.459890+0000","last_active":"2024-09-21T10:20:09.906059+0000","last_peered":"2024-09-21T10:20:09.906059+0000","last_clean":"2024-09-21T10:20:09.906059+0000","last_became_active":"2024-09-21T10:16:38.454382+0000","last_became_peered":"2024-09-21T10:16:38.454382+0000","last_unstale":"2024-09-21T10:20:09.906059+0000","last_undegraded":"2024-09-21T10:20:09.906059+0000","last_fullsized":"2024-09-21T10:20:09.906059+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:02.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1473,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906649+0000","last_change":"2024-09-21T10:16:38.454504+0000","last_active":"2024-09-21T10:20:09.906649+0000","last_peered":"2024-09-21T10:20:09.906649+0000","last_clean":"2024-09-21T10:20:09.906649+0000","last_became_active":"2024-09-21T10:16:38.448833+0000","last_became_peered":"2024-09-21T10:16:38.448833+0000","last_unstale":"2024-09-21T10:20:09.906649+0000","last_undegraded":"2024-09-21T10:20:09.906649+0000","last_fullsized":"2024-09-21T10:20:09.906649+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:11.905645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"1333'16250","reported_seq":18035,"reported_epoch":1333,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:16.839426+0000","last_change":"2024-09-21T10:20:10.183861+0000","last_active":"2024-09-21T10:20:16.839426+0000","last_peered":"2024-09-21T10:20:16.839426+0000","last_clean":"2024-09-21T10:17:50.226719+0000","last_became_active":"2024-09-21T10:17:53.190608+0000","last_became_peered":"2024-09-21T10:17:53.190608+0000","last_unstale":"2024-09-21T10:20:16.839426+0000","last_undegraded":"2024-09-21T10:20:16.839426+0000","last_fullsized":"2024-09-21T10:20:16.839426+0000","mapping_epoch":1309,"log_start":"1318'10600","ondisk_log_start":"1318'10600","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5650,"log_dups_size":0,"ondisk_log_size":5650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":266,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16250,"num_write_kb":547200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":851,"num_bytes_recovered":55771136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":850}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":190,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.907645+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:20:08.907645+0000","last_peered":"2024-09-21T10:20:08.907645+0000","last_clean":"2024-09-21T10:20:08.907645+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:20:08.907645+0000","last_undegraded":"2024-09-21T10:20:08.907645+0000","last_fullsized":"2024-09-21T10:20:08.907645+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":73,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906415+0000","last_change":"2024-09-21T10:16:39.439192+0000","last_active":"2024-09-21T10:20:09.906415+0000","last_peered":"2024-09-21T10:20:09.906415+0000","last_clean":"2024-09-21T10:20:09.906415+0000","last_became_active":"2024-09-21T10:16:39.438778+0000","last_became_peered":"2024-09-21T10:16:39.438778+0000","last_unstale":"2024-09-21T10:20:09.906415+0000","last_undegraded":"2024-09-21T10:20:09.906415+0000","last_fullsized":"2024-09-21T10:20:09.906415+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:49.643063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.0","version":"1333'16041","reported_seq":16088,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.936125+0000","last_change":"2024-09-21T10:16:23.239046+0000","last_active":"2024-09-21T10:20:16.936125+0000","last_peered":"2024-09-21T10:20:16.936125+0000","last_clean":"2024-09-21T10:20:16.936125+0000","last_became_active":"2024-09-21T10:16:23.238681+0000","last_became_peered":"2024-09-21T10:16:23.238681+0000","last_unstale":"2024-09-21T10:20:16.936125+0000","last_undegraded":"2024-09-21T10:20:16.936125+0000","last_fullsized":"2024-09-21T10:20:16.936125+0000","mapping_epoch":1296,"log_start":"1318'10400","ondisk_log_start":"1318'10400","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5641,"log_dups_size":0,"ondisk_log_size":5641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:15:42.666111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54591488,"num_objects":833,"num_object_clones":0,"num_object_copies":2499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16041,"num_write_kb":539968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1505,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902522+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:20:09.902522+0000","last_peered":"2024-09-21T10:20:09.902522+0000","last_clean":"2024-09-21T10:20:09.902522+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:20:09.902522+0000","last_undegraded":"2024-09-21T10:20:09.902522+0000","last_fullsized":"2024-09-21T10:20:09.902522+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.917036+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:20:08.917036+0000","last_peered":"2024-09-21T10:20:08.917036+0000","last_clean":"2024-09-21T10:20:08.917036+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:20:08.917036+0000","last_undegraded":"2024-09-21T10:20:08.917036+0000","last_fullsized":"2024-09-21T10:20:08.917036+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"1333'16031","reported_seq":17271,"reported_epoch":1333,"state":"active+remapped+backfilling","last_fresh":"2024-09-21T10:20:16.838284+0000","last_change":"2024-09-21T10:20:10.246223+0000","last_active":"2024-09-21T10:20:16.838284+0000","last_peered":"2024-09-21T10:20:16.838284+0000","last_clean":"2024-09-21T10:17:50.241218+0000","last_became_active":"2024-09-21T10:17:53.213630+0000","last_became_peered":"2024-09-21T10:17:53.213630+0000","last_unstale":"2024-09-21T10:20:16.838284+0000","last_undegraded":"2024-09-21T10:20:16.838284+0000","last_fullsized":"2024-09-21T10:20:16.838284+0000","mapping_epoch":1309,"log_start":"1318'10400","ondisk_log_start":"1318'10400","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":386,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16031,"num_write_kb":540096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":573,"num_bytes_recovered":37552128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,6],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4,6","objects":847}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":454,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.905807+0000","last_change":"2024-09-21T10:16:38.454278+0000","last_active":"2024-09-21T10:20:09.905807+0000","last_peered":"2024-09-21T10:20:09.905807+0000","last_clean":"2024-09-21T10:20:09.905807+0000","last_became_active":"2024-09-21T10:16:38.449058+0000","last_became_peered":"2024-09-21T10:16:38.449058+0000","last_unstale":"2024-09-21T10:20:09.905807+0000","last_undegraded":"2024-09-21T10:20:09.905807+0000","last_fullsized":"2024-09-21T10:20:09.905807+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:02.332428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1850,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903132+0000","last_change":"2024-09-21T10:16:38.460495+0000","last_active":"2024-09-21T10:20:09.903132+0000","last_peered":"2024-09-21T10:20:09.903132+0000","last_clean":"2024-09-21T10:20:09.903132+0000","last_became_active":"2024-09-21T10:16:38.458759+0000","last_became_peered":"2024-09-21T10:16:38.458759+0000","last_unstale":"2024-09-21T10:20:09.903132+0000","last_undegraded":"2024-09-21T10:20:09.903132+0000","last_fullsized":"2024-09-21T10:20:09.903132+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:37.031677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1333'16177","reported_seq":16224,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.936285+0000","last_change":"2024-09-21T10:16:23.237256+0000","last_active":"2024-09-21T10:20:16.936285+0000","last_peered":"2024-09-21T10:20:16.936285+0000","last_clean":"2024-09-21T10:20:16.936285+0000","last_became_active":"2024-09-21T10:16:23.236945+0000","last_became_peered":"2024-09-21T10:16:23.236945+0000","last_unstale":"2024-09-21T10:20:16.936285+0000","last_undegraded":"2024-09-21T10:20:16.936285+0000","last_fullsized":"2024-09-21T10:20:16.936285+0000","mapping_epoch":1296,"log_start":"1318'10600","ondisk_log_start":"1318'10600","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5577,"log_dups_size":0,"ondisk_log_size":5577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:50.314871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54722560,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16177,"num_write_kb":544384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":886,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.909057+0000","last_change":"2024-09-21T10:15:22.139134+0000","last_active":"2024-09-21T10:20:08.909057+0000","last_peered":"2024-09-21T10:20:08.909057+0000","last_clean":"2024-09-21T10:20:08.909057+0000","last_became_active":"2024-09-21T10:15:22.138788+0000","last_became_peered":"2024-09-21T10:15:22.138788+0000","last_unstale":"2024-09-21T10:20:08.909057+0000","last_undegraded":"2024-09-21T10:20:08.909057+0000","last_fullsized":"2024-09-21T10:20:08.909057+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:35.831693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":40,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.904107+0000","last_change":"2024-09-21T10:17:51.287937+0000","last_active":"2024-09-21T10:20:08.904107+0000","last_peered":"2024-09-21T10:20:08.904107+0000","last_clean":"2024-09-21T10:20:08.904107+0000","last_became_active":"2024-09-21T10:17:51.256610+0000","last_became_peered":"2024-09-21T10:17:51.256610+0000","last_unstale":"2024-09-21T10:20:08.904107+0000","last_undegraded":"2024-09-21T10:20:08.904107+0000","last_fullsized":"2024-09-21T10:20:08.904107+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:15.137072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":484,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.900092+0000","last_change":"2024-09-21T10:18:07.086426+0000","last_active":"2024-09-21T10:20:09.900092+0000","last_peered":"2024-09-21T10:20:09.900092+0000","last_clean":"2024-09-21T10:20:09.900092+0000","last_became_active":"2024-09-21T10:18:07.086157+0000","last_became_peered":"2024-09-21T10:18:07.086157+0000","last_unstale":"2024-09-21T10:20:09.900092+0000","last_undegraded":"2024-09-21T10:20:09.900092+0000","last_fullsized":"2024-09-21T10:20:09.900092+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:24:34.289605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":214,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.911182+0000","last_change":"2024-09-21T10:16:38.450231+0000","last_active":"2024-09-21T10:20:08.911182+0000","last_peered":"2024-09-21T10:20:08.911182+0000","last_clean":"2024-09-21T10:20:08.911182+0000","last_became_active":"2024-09-21T10:16:38.449941+0000","last_became_peered":"2024-09-21T10:16:38.449941+0000","last_unstale":"2024-09-21T10:20:08.911182+0000","last_undegraded":"2024-09-21T10:20:08.911182+0000","last_fullsized":"2024-09-21T10:20:08.911182+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:51.559013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"1333'15767","reported_seq":24996,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.311223+0000","last_change":"2024-09-21T10:19:32.344481+0000","last_active":"2024-09-21T10:20:16.311223+0000","last_peered":"2024-09-21T10:20:16.311223+0000","last_clean":"2024-09-21T10:20:16.311223+0000","last_became_active":"2024-09-21T10:19:32.294085+0000","last_became_peered":"2024-09-21T10:19:32.294085+0000","last_unstale":"2024-09-21T10:20:16.311223+0000","last_undegraded":"2024-09-21T10:20:16.311223+0000","last_fullsized":"2024-09-21T10:20:16.311223+0000","mapping_epoch":1324,"log_start":"1315'9700","ondisk_log_start":"1315'9700","created":1296,"last_epoch_clean":1325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6067,"log_dups_size":0,"ondisk_log_size":6067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:58.316719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60620800,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15767,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":299827200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1142,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902617+0000","last_change":"2024-09-21T10:17:51.264078+0000","last_active":"2024-09-21T10:20:09.902617+0000","last_peered":"2024-09-21T10:20:09.902617+0000","last_clean":"2024-09-21T10:20:09.902617+0000","last_became_active":"2024-09-21T10:17:51.263721+0000","last_became_peered":"2024-09-21T10:17:51.263721+0000","last_unstale":"2024-09-21T10:20:09.902617+0000","last_undegraded":"2024-09-21T10:20:09.902617+0000","last_fullsized":"2024-09-21T10:20:09.902617+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:46:02.104778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1631,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906499+0000","last_change":"2024-09-21T10:17:51.251859+0000","last_active":"2024-09-21T10:20:09.906499+0000","last_peered":"2024-09-21T10:20:09.906499+0000","last_clean":"2024-09-21T10:20:09.906499+0000","last_became_active":"2024-09-21T10:17:51.251525+0000","last_became_peered":"2024-09-21T10:17:51.251525+0000","last_unstale":"2024-09-21T10:20:09.906499+0000","last_undegraded":"2024-09-21T10:20:09.906499+0000","last_fullsized":"2024-09-21T10:20:09.906499+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:24.106509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1135,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902988+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:20:09.902988+0000","last_peered":"2024-09-21T10:20:09.902988+0000","last_clean":"2024-09-21T10:20:09.902988+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:20:09.902988+0000","last_undegraded":"2024-09-21T10:20:09.902988+0000","last_fullsized":"2024-09-21T10:20:09.902988+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"1333'16250","reported_seq":31139,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.351795+0000","last_change":"2024-09-21T10:18:54.245834+0000","last_active":"2024-09-21T10:20:16.351795+0000","last_peered":"2024-09-21T10:20:16.351795+0000","last_clean":"2024-09-21T10:20:16.351795+0000","last_became_active":"2024-09-21T10:18:54.227580+0000","last_became_peered":"2024-09-21T10:18:54.227580+0000","last_unstale":"2024-09-21T10:20:16.351795+0000","last_undegraded":"2024-09-21T10:20:16.351795+0000","last_fullsized":"2024-09-21T10:20:16.351795+0000","mapping_epoch":1319,"log_start":"1315'10200","ondisk_log_start":"1315'10200","created":1296,"last_epoch_clean":1320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:07.095924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58327040,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16250,"num_write_kb":548480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7408,"num_bytes_recovered":485490688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1721,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906119+0000","last_change":"2024-09-21T10:16:38.456464+0000","last_active":"2024-09-21T10:20:09.906119+0000","last_peered":"2024-09-21T10:20:09.906119+0000","last_clean":"2024-09-21T10:20:09.906119+0000","last_became_active":"2024-09-21T10:16:38.455345+0000","last_became_peered":"2024-09-21T10:16:38.455345+0000","last_unstale":"2024-09-21T10:20:09.906119+0000","last_undegraded":"2024-09-21T10:20:09.906119+0000","last_fullsized":"2024-09-21T10:20:09.906119+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:39.393023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1866,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902728+0000","last_change":"2024-09-21T10:17:51.254069+0000","last_active":"2024-09-21T10:20:09.902728+0000","last_peered":"2024-09-21T10:20:09.902728+0000","last_clean":"2024-09-21T10:20:09.902728+0000","last_became_active":"2024-09-21T10:17:51.253723+0000","last_became_peered":"2024-09-21T10:17:51.253723+0000","last_unstale":"2024-09-21T10:20:09.902728+0000","last_undegraded":"2024-09-21T10:20:09.902728+0000","last_fullsized":"2024-09-21T10:20:09.902728+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:22.160096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1866,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903019+0000","last_change":"2024-09-21T10:17:51.249147+0000","last_active":"2024-09-21T10:20:09.903019+0000","last_peered":"2024-09-21T10:20:09.903019+0000","last_clean":"2024-09-21T10:20:09.903019+0000","last_became_active":"2024-09-21T10:17:51.248713+0000","last_became_peered":"2024-09-21T10:17:51.248713+0000","last_unstale":"2024-09-21T10:20:09.903019+0000","last_undegraded":"2024-09-21T10:20:09.903019+0000","last_fullsized":"2024-09-21T10:20:09.903019+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:33.757896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1333'16104","reported_seq":16170,"reported_epoch":1333,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-21T10:20:16.946165+0000","last_change":"2024-09-21T10:17:53.257522+0000","last_active":"2024-09-21T10:20:16.946165+0000","last_peered":"2024-09-21T10:20:16.946165+0000","last_clean":"2024-09-21T10:17:50.228509+0000","last_became_active":"2024-09-21T10:17:53.220833+0000","last_became_peered":"2024-09-21T10:17:53.220833+0000","last_unstale":"2024-09-21T10:20:16.946165+0000","last_undegraded":"2024-09-21T10:20:16.946165+0000","last_fullsized":"2024-09-21T10:20:16.946165+0000","mapping_epoch":1309,"log_start":"1318'10500","ondisk_log_start":"1318'10500","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5604,"log_dups_size":0,"ondisk_log_size":5604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:07.029873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":816,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16104,"num_write_kb":541440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":816}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":40,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.902360+0000","last_change":"2024-09-21T10:17:51.286472+0000","last_active":"2024-09-21T10:20:08.902360+0000","last_peered":"2024-09-21T10:20:08.902360+0000","last_clean":"2024-09-21T10:20:08.902360+0000","last_became_active":"2024-09-21T10:17:51.249949+0000","last_became_peered":"2024-09-21T10:17:51.249949+0000","last_unstale":"2024-09-21T10:20:08.902360+0000","last_undegraded":"2024-09-21T10:20:08.902360+0000","last_fullsized":"2024-09-21T10:20:08.902360+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:31.760578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1300,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.913692+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:20:08.913692+0000","last_peered":"2024-09-21T10:20:08.913692+0000","last_clean":"2024-09-21T10:20:08.913692+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:20:08.913692+0000","last_undegraded":"2024-09-21T10:20:08.913692+0000","last_fullsized":"2024-09-21T10:20:08.913692+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1807,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906444+0000","last_change":"2024-09-21T10:16:39.440143+0000","last_active":"2024-09-21T10:20:09.906444+0000","last_peered":"2024-09-21T10:20:09.906444+0000","last_clean":"2024-09-21T10:20:09.906444+0000","last_became_active":"2024-09-21T10:16:39.439370+0000","last_became_peered":"2024-09-21T10:16:39.439370+0000","last_unstale":"2024-09-21T10:20:09.906444+0000","last_undegraded":"2024-09-21T10:20:09.906444+0000","last_fullsized":"2024-09-21T10:20:09.906444+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:36.417381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"1333'15999","reported_seq":16074,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.752304+0000","last_change":"2024-09-21T10:16:38.458835+0000","last_active":"2024-09-21T10:20:16.752304+0000","last_peered":"2024-09-21T10:20:16.752304+0000","last_clean":"2024-09-21T10:20:16.752304+0000","last_became_active":"2024-09-21T10:16:38.451145+0000","last_became_peered":"2024-09-21T10:16:38.451145+0000","last_unstale":"2024-09-21T10:20:16.752304+0000","last_undegraded":"2024-09-21T10:20:16.752304+0000","last_fullsized":"2024-09-21T10:20:16.752304+0000","mapping_epoch":1301,"log_start":"1318'10800","ondisk_log_start":"1318'10800","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5199,"log_dups_size":0,"ondisk_log_size":5199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:14.844302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53673984,"num_objects":819,"num_object_clones":0,"num_object_copies":2457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15999,"num_write_kb":538176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":36,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.906178+0000","last_change":"2024-09-21T10:17:51.259438+0000","last_active":"2024-09-21T10:20:08.906178+0000","last_peered":"2024-09-21T10:20:08.906178+0000","last_clean":"2024-09-21T10:20:08.906178+0000","last_became_active":"2024-09-21T10:17:51.258704+0000","last_became_peered":"2024-09-21T10:17:51.258704+0000","last_unstale":"2024-09-21T10:20:08.906178+0000","last_undegraded":"2024-09-21T10:20:08.906178+0000","last_fullsized":"2024-09-21T10:20:08.906178+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:16.479749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1880,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902618+0000","last_change":"2024-09-21T10:16:38.460671+0000","last_active":"2024-09-21T10:20:09.902618+0000","last_peered":"2024-09-21T10:20:09.902618+0000","last_clean":"2024-09-21T10:20:09.902618+0000","last_became_active":"2024-09-21T10:16:38.459939+0000","last_became_peered":"2024-09-21T10:16:38.459939+0000","last_unstale":"2024-09-21T10:20:09.902618+0000","last_undegraded":"2024-09-21T10:20:09.902618+0000","last_fullsized":"2024-09-21T10:20:09.902618+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:51.143325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":406,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902465+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:20:09.902465+0000","last_peered":"2024-09-21T10:20:09.902465+0000","last_clean":"2024-09-21T10:20:09.902465+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:20:09.902465+0000","last_undegraded":"2024-09-21T10:20:09.902465+0000","last_fullsized":"2024-09-21T10:20:09.902465+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"1333'16103","reported_seq":16156,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.955741+0000","last_change":"2024-09-21T10:16:38.446821+0000","last_active":"2024-09-21T10:20:16.955741+0000","last_peered":"2024-09-21T10:20:16.955741+0000","last_clean":"2024-09-21T10:20:16.955741+0000","last_became_active":"2024-09-21T10:16:38.446505+0000","last_became_peered":"2024-09-21T10:16:38.446505+0000","last_unstale":"2024-09-21T10:20:16.955741+0000","last_undegraded":"2024-09-21T10:20:16.955741+0000","last_fullsized":"2024-09-21T10:20:16.955741+0000","mapping_epoch":1301,"log_start":"1318'10500","ondisk_log_start":"1318'10500","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5603,"log_dups_size":0,"ondisk_log_size":5603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:38.183784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525976,"num_objects":833,"num_object_clones":0,"num_object_copies":2499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":833,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16103,"num_write_kb":541889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":80,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906200+0000","last_change":"2024-09-21T10:17:51.250313+0000","last_active":"2024-09-21T10:20:09.906200+0000","last_peered":"2024-09-21T10:20:09.906200+0000","last_clean":"2024-09-21T10:20:09.906200+0000","last_became_active":"2024-09-21T10:17:51.249931+0000","last_became_peered":"2024-09-21T10:17:51.249931+0000","last_unstale":"2024-09-21T10:20:09.906200+0000","last_undegraded":"2024-09-21T10:20:09.906200+0000","last_fullsized":"2024-09-21T10:20:09.906200+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:38:15.879964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1020,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903132+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:20:09.903132+0000","last_peered":"2024-09-21T10:20:09.903132+0000","last_clean":"2024-09-21T10:20:09.903132+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:20:09.903132+0000","last_undegraded":"2024-09-21T10:20:09.903132+0000","last_fullsized":"2024-09-21T10:20:09.903132+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":36,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.904180+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:20:08.904180+0000","last_peered":"2024-09-21T10:20:08.904180+0000","last_clean":"2024-09-21T10:20:08.904180+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:20:08.904180+0000","last_undegraded":"2024-09-21T10:20:08.904180+0000","last_fullsized":"2024-09-21T10:20:08.904180+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"1333'16340","reported_seq":20118,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.947204+0000","last_change":"2024-09-21T10:20:10.240181+0000","last_active":"2024-09-21T10:20:16.947204+0000","last_peered":"2024-09-21T10:20:16.947204+0000","last_clean":"2024-09-21T10:20:16.947204+0000","last_became_active":"2024-09-21T10:20:10.136698+0000","last_became_peered":"2024-09-21T10:20:10.136698+0000","last_unstale":"2024-09-21T10:20:16.947204+0000","last_undegraded":"2024-09-21T10:20:16.947204+0000","last_fullsized":"2024-09-21T10:20:16.947204+0000","mapping_epoch":1332,"log_start":"1318'10700","ondisk_log_start":"1318'10700","created":1296,"last_epoch_clean":1333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:44.765170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55312384,"num_objects":844,"num_object_clones":0,"num_object_copies":2532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16340,"num_write_kb":549888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1850,"num_bytes_recovered":121176064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":432,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.900492+0000","last_change":"2024-09-21T10:18:07.090592+0000","last_active":"2024-09-21T10:20:09.900492+0000","last_peered":"2024-09-21T10:20:09.900492+0000","last_clean":"2024-09-21T10:20:09.900492+0000","last_became_active":"2024-09-21T10:18:07.090188+0000","last_became_peered":"2024-09-21T10:18:07.090188+0000","last_unstale":"2024-09-21T10:20:09.900492+0000","last_undegraded":"2024-09-21T10:20:09.900492+0000","last_fullsized":"2024-09-21T10:20:09.900492+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.763114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":73,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.912254+0000","last_change":"2024-09-21T10:16:39.439344+0000","last_active":"2024-09-21T10:20:09.912254+0000","last_peered":"2024-09-21T10:20:09.912254+0000","last_clean":"2024-09-21T10:20:09.912254+0000","last_became_active":"2024-09-21T10:16:39.438987+0000","last_became_peered":"2024-09-21T10:16:39.438987+0000","last_unstale":"2024-09-21T10:20:09.912254+0000","last_undegraded":"2024-09-21T10:20:09.912254+0000","last_fullsized":"2024-09-21T10:20:09.912254+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:17.163136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":435,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.905743+0000","last_change":"2024-09-21T10:17:51.250156+0000","last_active":"2024-09-21T10:20:09.905743+0000","last_peered":"2024-09-21T10:20:09.905743+0000","last_clean":"2024-09-21T10:20:09.905743+0000","last_became_active":"2024-09-21T10:17:51.248282+0000","last_became_peered":"2024-09-21T10:17:51.248282+0000","last_unstale":"2024-09-21T10:20:09.905743+0000","last_undegraded":"2024-09-21T10:20:09.905743+0000","last_fullsized":"2024-09-21T10:20:09.905743+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:58.954423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"1333'16051","reported_seq":16104,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:16.940213+0000","last_change":"2024-09-21T10:16:38.454005+0000","last_active":"2024-09-21T10:20:16.940213+0000","last_peered":"2024-09-21T10:20:16.940213+0000","last_clean":"2024-09-21T10:20:16.940213+0000","last_became_active":"2024-09-21T10:16:38.453756+0000","last_became_peered":"2024-09-21T10:16:38.453756+0000","last_unstale":"2024-09-21T10:20:16.940213+0000","last_undegraded":"2024-09-21T10:20:16.940213+0000","last_fullsized":"2024-09-21T10:20:16.940213+0000","mapping_epoch":1301,"log_start":"1317'10400","ondisk_log_start":"1317'10400","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5651,"log_dups_size":0,"ondisk_log_size":5651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:30.976913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53411840,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16051,"num_write_kb":539712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":391,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903049+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:20:09.903049+0000","last_peered":"2024-09-21T10:20:09.903049+0000","last_clean":"2024-09-21T10:20:09.903049+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:20:09.903049+0000","last_undegraded":"2024-09-21T10:20:09.903049+0000","last_fullsized":"2024-09-21T10:20:09.903049+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":214,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.913844+0000","last_change":"2024-09-21T10:16:38.455089+0000","last_active":"2024-09-21T10:20:08.913844+0000","last_peered":"2024-09-21T10:20:08.913844+0000","last_clean":"2024-09-21T10:20:08.913844+0000","last_became_active":"2024-09-21T10:16:38.454804+0000","last_became_peered":"2024-09-21T10:16:38.454804+0000","last_unstale":"2024-09-21T10:20:08.913844+0000","last_undegraded":"2024-09-21T10:20:08.913844+0000","last_fullsized":"2024-09-21T10:20:08.913844+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:16.448016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":36,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.907726+0000","last_change":"2024-09-21T10:17:51.250324+0000","last_active":"2024-09-21T10:20:08.907726+0000","last_peered":"2024-09-21T10:20:08.907726+0000","last_clean":"2024-09-21T10:20:08.907726+0000","last_became_active":"2024-09-21T10:17:51.249933+0000","last_became_peered":"2024-09-21T10:17:51.249933+0000","last_unstale":"2024-09-21T10:20:08.907726+0000","last_undegraded":"2024-09-21T10:20:08.907726+0000","last_fullsized":"2024-09-21T10:20:08.907726+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:23.880577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":189,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.913907+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:20:08.913907+0000","last_peered":"2024-09-21T10:20:08.913907+0000","last_clean":"2024-09-21T10:20:08.913907+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:20:08.913907+0000","last_undegraded":"2024-09-21T10:20:08.913907+0000","last_fullsized":"2024-09-21T10:20:08.913907+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":252,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.907621+0000","last_change":"2024-09-21T10:17:51.277615+0000","last_active":"2024-09-21T10:20:08.907621+0000","last_peered":"2024-09-21T10:20:08.907621+0000","last_clean":"2024-09-21T10:20:08.907621+0000","last_became_active":"2024-09-21T10:17:51.261756+0000","last_became_peered":"2024-09-21T10:17:51.261756+0000","last_unstale":"2024-09-21T10:20:08.907621+0000","last_undegraded":"2024-09-21T10:20:08.907621+0000","last_fullsized":"2024-09-21T10:20:08.907621+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:05:35.608372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1853,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902778+0000","last_change":"2024-09-21T10:17:51.251440+0000","last_active":"2024-09-21T10:20:09.902778+0000","last_peered":"2024-09-21T10:20:09.902778+0000","last_clean":"2024-09-21T10:20:09.902778+0000","last_became_active":"2024-09-21T10:17:51.249109+0000","last_became_peered":"2024-09-21T10:17:51.249109+0000","last_unstale":"2024-09-21T10:20:09.902778+0000","last_undegraded":"2024-09-21T10:20:09.902778+0000","last_fullsized":"2024-09-21T10:20:09.902778+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:59.577984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1363,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906449+0000","last_change":"2024-09-21T10:16:38.453215+0000","last_active":"2024-09-21T10:20:09.906449+0000","last_peered":"2024-09-21T10:20:09.906449+0000","last_clean":"2024-09-21T10:20:09.906449+0000","last_became_active":"2024-09-21T10:16:38.452464+0000","last_became_peered":"2024-09-21T10:16:38.452464+0000","last_unstale":"2024-09-21T10:20:09.906449+0000","last_undegraded":"2024-09-21T10:20:09.906449+0000","last_fullsized":"2024-09-21T10:20:09.906449+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:16.818394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":36,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.900570+0000","last_change":"2024-09-21T10:17:51.248299+0000","last_active":"2024-09-21T10:20:09.900570+0000","last_peered":"2024-09-21T10:20:09.900570+0000","last_clean":"2024-09-21T10:20:09.900570+0000","last_became_active":"2024-09-21T10:17:51.247851+0000","last_became_peered":"2024-09-21T10:17:51.247851+0000","last_unstale":"2024-09-21T10:20:09.900570+0000","last_undegraded":"2024-09-21T10:20:09.900570+0000","last_fullsized":"2024-09-21T10:20:09.900570+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:04.319143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":511,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.905829+0000","last_change":"2024-09-21T10:17:51.248187+0000","last_active":"2024-09-21T10:20:09.905829+0000","last_peered":"2024-09-21T10:20:09.905829+0000","last_clean":"2024-09-21T10:20:09.905829+0000","last_became_active":"2024-09-21T10:17:51.247436+0000","last_became_peered":"2024-09-21T10:17:51.247436+0000","last_unstale":"2024-09-21T10:20:09.905829+0000","last_undegraded":"2024-09-21T10:20:09.905829+0000","last_fullsized":"2024-09-21T10:20:09.905829+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:33.505149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.900615+0000","last_change":"2024-09-21T10:17:51.258577+0000","last_active":"2024-09-21T10:20:09.900615+0000","last_peered":"2024-09-21T10:20:09.900615+0000","last_clean":"2024-09-21T10:20:09.900615+0000","last_became_active":"2024-09-21T10:17:51.257820+0000","last_became_peered":"2024-09-21T10:17:51.257820+0000","last_unstale":"2024-09-21T10:20:09.900615+0000","last_undegraded":"2024-09-21T10:20:09.900615+0000","last_fullsized":"2024-09-21T10:20:09.900615+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:00.922957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":495,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.905712+0000","last_change":"2024-09-21T10:16:38.456905+0000","last_active":"2024-09-21T10:20:09.905712+0000","last_peered":"2024-09-21T10:20:09.905712+0000","last_clean":"2024-09-21T10:20:09.905712+0000","last_became_active":"2024-09-21T10:16:38.453276+0000","last_became_peered":"2024-09-21T10:16:38.453276+0000","last_unstale":"2024-09-21T10:20:09.905712+0000","last_undegraded":"2024-09-21T10:20:09.905712+0000","last_fullsized":"2024-09-21T10:20:09.905712+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:38.174569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":232,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.913705+0000","last_change":"2024-09-21T10:17:51.262911+0000","last_active":"2024-09-21T10:20:08.913705+0000","last_peered":"2024-09-21T10:20:08.913705+0000","last_clean":"2024-09-21T10:20:08.913705+0000","last_became_active":"2024-09-21T10:17:51.262314+0000","last_became_peered":"2024-09-21T10:17:51.262314+0000","last_unstale":"2024-09-21T10:20:08.913705+0000","last_undegraded":"2024-09-21T10:20:08.913705+0000","last_fullsized":"2024-09-21T10:20:08.913705+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:06.241585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":483,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906094+0000","last_change":"2024-09-21T10:17:51.254289+0000","last_active":"2024-09-21T10:20:09.906094+0000","last_peered":"2024-09-21T10:20:09.906094+0000","last_clean":"2024-09-21T10:20:09.906094+0000","last_became_active":"2024-09-21T10:17:51.253922+0000","last_became_peered":"2024-09-21T10:17:51.253922+0000","last_unstale":"2024-09-21T10:20:09.906094+0000","last_undegraded":"2024-09-21T10:20:09.906094+0000","last_fullsized":"2024-09-21T10:20:09.906094+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:57.111361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1746,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.912330+0000","last_change":"2024-09-21T10:17:51.249960+0000","last_active":"2024-09-21T10:20:09.912330+0000","last_peered":"2024-09-21T10:20:09.912330+0000","last_clean":"2024-09-21T10:20:09.912330+0000","last_became_active":"2024-09-21T10:17:51.248883+0000","last_became_peered":"2024-09-21T10:17:51.248883+0000","last_unstale":"2024-09-21T10:20:09.912330+0000","last_undegraded":"2024-09-21T10:20:09.912330+0000","last_fullsized":"2024-09-21T10:20:09.912330+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:20.076539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":139,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902675+0000","last_change":"2024-09-21T10:16:38.461877+0000","last_active":"2024-09-21T10:20:09.902675+0000","last_peered":"2024-09-21T10:20:09.902675+0000","last_clean":"2024-09-21T10:20:09.902675+0000","last_became_active":"2024-09-21T10:16:38.461554+0000","last_became_peered":"2024-09-21T10:16:38.461554+0000","last_unstale":"2024-09-21T10:20:09.902675+0000","last_undegraded":"2024-09-21T10:20:09.902675+0000","last_fullsized":"2024-09-21T10:20:09.902675+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:46.372637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":423,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906561+0000","last_change":"2024-09-21T10:16:38.447369+0000","last_active":"2024-09-21T10:20:09.906561+0000","last_peered":"2024-09-21T10:20:09.906561+0000","last_clean":"2024-09-21T10:20:09.906561+0000","last_became_active":"2024-09-21T10:16:38.447085+0000","last_became_peered":"2024-09-21T10:16:38.447085+0000","last_unstale":"2024-09-21T10:20:09.906561+0000","last_undegraded":"2024-09-21T10:20:09.906561+0000","last_fullsized":"2024-09-21T10:20:09.906561+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:59.110578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":410,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902889+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:20:09.902889+0000","last_peered":"2024-09-21T10:20:09.902889+0000","last_clean":"2024-09-21T10:20:09.902889+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:20:09.902889+0000","last_undegraded":"2024-09-21T10:20:09.902889+0000","last_fullsized":"2024-09-21T10:20:09.902889+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1739,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.902776+0000","last_change":"2024-09-21T10:16:38.460957+0000","last_active":"2024-09-21T10:20:09.902776+0000","last_peered":"2024-09-21T10:20:09.902776+0000","last_clean":"2024-09-21T10:20:09.902776+0000","last_became_active":"2024-09-21T10:16:38.460325+0000","last_became_peered":"2024-09-21T10:16:38.460325+0000","last_unstale":"2024-09-21T10:20:09.902776+0000","last_undegraded":"2024-09-21T10:20:09.902776+0000","last_fullsized":"2024-09-21T10:20:09.902776+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:34.529895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1813,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.903799+0000","last_change":"2024-09-21T10:16:38.461576+0000","last_active":"2024-09-21T10:20:09.903799+0000","last_peered":"2024-09-21T10:20:09.903799+0000","last_clean":"2024-09-21T10:20:09.903799+0000","last_became_active":"2024-09-21T10:16:38.461205+0000","last_became_peered":"2024-09-21T10:16:38.461205+0000","last_unstale":"2024-09-21T10:20:09.903799+0000","last_undegraded":"2024-09-21T10:20:09.903799+0000","last_fullsized":"2024-09-21T10:20:09.903799+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:17.166658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1405,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906037+0000","last_change":"2024-09-21T10:16:38.459378+0000","last_active":"2024-09-21T10:20:09.906037+0000","last_peered":"2024-09-21T10:20:09.906037+0000","last_clean":"2024-09-21T10:20:09.906037+0000","last_became_active":"2024-09-21T10:16:38.451582+0000","last_became_peered":"2024-09-21T10:16:38.451582+0000","last_unstale":"2024-09-21T10:20:09.906037+0000","last_undegraded":"2024-09-21T10:20:09.906037+0000","last_fullsized":"2024-09-21T10:20:09.906037+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:34.820762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":39,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.901769+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:20:08.901769+0000","last_peered":"2024-09-21T10:20:08.901769+0000","last_clean":"2024-09-21T10:20:08.901769+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:20:08.901769+0000","last_undegraded":"2024-09-21T10:20:08.901769+0000","last_fullsized":"2024-09-21T10:20:08.901769+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1422,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.905969+0000","last_change":"2024-09-21T10:16:38.453928+0000","last_active":"2024-09-21T10:20:09.905969+0000","last_peered":"2024-09-21T10:20:09.905969+0000","last_clean":"2024-09-21T10:20:09.905969+0000","last_became_active":"2024-09-21T10:16:38.450890+0000","last_became_peered":"2024-09-21T10:16:38.450890+0000","last_unstale":"2024-09-21T10:20:09.905969+0000","last_undegraded":"2024-09-21T10:20:09.905969+0000","last_fullsized":"2024-09-21T10:20:09.905969+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:13.500644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":39,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.906026+0000","last_change":"2024-09-21T10:17:51.281734+0000","last_active":"2024-09-21T10:20:08.906026+0000","last_peered":"2024-09-21T10:20:08.906026+0000","last_clean":"2024-09-21T10:20:08.906026+0000","last_became_active":"2024-09-21T10:17:51.258206+0000","last_became_peered":"2024-09-21T10:17:51.258206+0000","last_unstale":"2024-09-21T10:20:08.906026+0000","last_undegraded":"2024-09-21T10:20:08.906026+0000","last_fullsized":"2024-09-21T10:20:08.906026+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:16.184203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":483,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.906019+0000","last_change":"2024-09-21T10:17:51.251289+0000","last_active":"2024-09-21T10:20:09.906019+0000","last_peered":"2024-09-21T10:20:09.906019+0000","last_clean":"2024-09-21T10:20:09.906019+0000","last_became_active":"2024-09-21T10:17:51.247868+0000","last_became_peered":"2024-09-21T10:17:51.247868+0000","last_unstale":"2024-09-21T10:20:09.906019+0000","last_undegraded":"2024-09-21T10:20:09.906019+0000","last_fullsized":"2024-09-21T10:20:09.906019+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:39:49.784298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":485,"reported_epoch":1333,"state":"active+clean","last_fresh":"2024-09-21T10:20:09.912397+0000","last_change":"2024-09-21T10:16:38.451567+0000","last_active":"2024-09-21T10:20:09.912397+0000","last_peered":"2024-09-21T10:20:09.912397+0000","last_clean":"2024-09-21T10:20:09.912397+0000","last_became_active":"2024-09-21T10:16:38.451245+0000","last_became_peered":"2024-09-21T10:16:38.451245+0000","last_unstale":"2024-09-21T10:20:09.912397+0000","last_undegraded":"2024-09-21T10:20:09.912397+0000","last_fullsized":"2024-09-21T10:20:09.912397+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:27.022083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":198,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.909259+0000","last_change":"2024-09-21T10:15:19.115911+0000","last_active":"2024-09-21T10:20:08.909259+0000","last_peered":"2024-09-21T10:20:08.909259+0000","last_clean":"2024-09-21T10:20:08.909259+0000","last_became_active":"2024-09-21T10:15:19.115615+0000","last_became_peered":"2024-09-21T10:15:19.115615+0000","last_unstale":"2024-09-21T10:20:08.909259+0000","last_undegraded":"2024-09-21T10:20:08.909259+0000","last_fullsized":"2024-09-21T10:20:08.909259+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":42,"reported_epoch":1332,"state":"active+clean","last_fresh":"2024-09-21T10:20:08.912764+0000","last_change":"2024-09-21T10:17:51.280012+0000","last_active":"2024-09-21T10:20:08.912764+0000","last_peered":"2024-09-21T10:20:08.912764+0000","last_clean":"2024-09-21T10:20:08.912764+0000","last_became_active":"2024-09-21T10:17:51.257359+0000","last_became_peered":"2024-09-21T10:17:51.257359+0000","last_unstale":"2024-09-21T10:20:08.912764+0000","last_undegraded":"2024-09-21T10:20:08.912764+0000","last_fullsized":"2024-09-21T10:20:08.912764+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:44.522556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":881590296,"num_objects":13453,"num_object_clones":0,"num_object_copies":40359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3026,"num_objects_unfound":0,"num_objects_dirty":13453,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":256901,"num_write_kb":8651265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18802,"num_bytes_recovered":1232142336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2954047488,"data_stored":2954035272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89601,"ondisk_log_size":89601,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":21092,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17031,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":6}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234533,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496169,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1944420,"kb_used_data":779868,"kb_used_omap":1,"kb_used_meta":1164542,"kb_avail":91800732,"statfs":{"total":95995035648,"available":94003949568,"internally_reserved":0,"allocated":798584832,"data_stored":795348580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1192491463},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,4,5,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757808,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":703000,"kb_used_data":315088,"kb_used_omap":1,"kb_used_meta":387902,"kb_avail":93042152,"statfs":{"total":95995035648,"available":95275163648,"internally_reserved":0,"allocated":322650112,"data_stored":319425478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":397212102},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019444,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1893216,"kb_used_data":874904,"kb_used_omap":1,"kb_used_meta":1017982,"kb_avail":91851936,"statfs":{"total":95995035648,"available":94056382464,"internally_reserved":0,"allocated":895901696,"data_stored":892669473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1042414022},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":1301,"seq":5587752452141,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000968,"kb_used_data":636096,"kb_used_omap":48,"kb_used_meta":364815,"kb_avail":92744184,"statfs":{"total":95995035648,"available":94970044416,"internally_reserved":0,"allocated":651362304,"data_stored":648148801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49883,"internal_metadata":373570853},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793967979,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047218,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1538444,"kb_used_data":444484,"kb_used_omap":41,"kb_used_meta":1093846,"kb_avail":92206708,"statfs":{"total":95995035648,"available":94419668992,"internally_reserved":0,"allocated":455151616,"data_stored":451934452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1120099241},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928852,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7559,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":435814400,"data_stored":435814400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":610734080,"data_stored":610730008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":838799360,"data_stored":838795288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":306380800,"data_stored":306380800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":762318848,"data_stored":762314776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:20:20.284 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T10:20:20.287 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:20.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:20.394 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:20.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.536 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:20.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:20.637 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:20.665 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:20.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:20.795 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:20.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:20.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:20.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:20.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:21.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:21.061 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:21.109 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:21.119 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:21.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:21.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3462831775' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[106482]: osdmap e1334: 8 total, 8 up, 5 in 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3462831775' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:21.239 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:20 smithi076 ceph-mon[110963]: osdmap e1334: 8 total, 8 up, 5 in 2024-09-21T10:20:21.288 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:21.310 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:21.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:20 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:20:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:20 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:20:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:20 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3462831775' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:20 smithi191 ceph-mon[102206]: osdmap e1334: 8 total, 8 up, 5 in 2024-09-21T10:20:21.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:21.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:21.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:21.560 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:21.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:21.661 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:21.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:21.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:21.806 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:21.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:21.930 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:21.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:21.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:22.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:22.055 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:21 smithi076 ceph-mon[106482]: pgmap v2109: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 841 MiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 807 op/s; 3026/40437 objects misplaced (7.483%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:20:22.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:21 smithi076 ceph-mon[106482]: osdmap e1335: 8 total, 8 up, 5 in 2024-09-21T10:20:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:21 smithi076 ceph-mon[110963]: pgmap v2109: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 841 MiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 807 op/s; 3026/40437 objects misplaced (7.483%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:20:22.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:21 smithi076 ceph-mon[110963]: osdmap e1335: 8 total, 8 up, 5 in 2024-09-21T10:20:22.157 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:22.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:22.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:22.279 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:21 smithi191 ceph-mon[102206]: pgmap v2109: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 841 MiB data, 6.8 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 807 op/s; 3026/40437 objects misplaced (7.483%); 9.8 MiB/s, 157 objects/s recovering 2024-09-21T10:20:22.279 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:21 smithi191 ceph-mon[102206]: osdmap e1335: 8 total, 8 up, 5 in 2024-09-21T10:20:22.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:22.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:22.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:22.333 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:22.333 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:22.427 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:22.449 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:22.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:22.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:22.572 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:22.673 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.695 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:22.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:22.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:22.818 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:22.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:22.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:22.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:23.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:23.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:23.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:22 smithi076 ceph-mon[110963]: osdmap e1336: 8 total, 8 up, 5 in 2024-09-21T10:20:23.079 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:22 smithi076 ceph-mon[106482]: osdmap e1336: 8 total, 8 up, 5 in 2024-09-21T10:20:23.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:20:23.120 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:20:23.172 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:23.205 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:23.307 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:23.349 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:22 smithi191 ceph-mon[102206]: osdmap e1336: 8 total, 8 up, 5 in 2024-09-21T10:20:23.363 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:20:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:20:23] "GET /metrics HTTP/1.1" 200 36628 "" "Prometheus/2.51.0" 2024-09-21T10:20:23.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:23.484 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.511 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:23.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:23.612 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.635 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:23.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:23.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:23.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:23.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:23.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:23.911 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:23.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:24.012 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:24.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:24.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:24.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:24.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:24.262 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:23 smithi076 ceph-mon[106482]: pgmap v2112: 89 pgs: 2 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 84 active+clean; 590 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s; 1316/28359 objects misplaced (4.641%); 14 MiB/s, 225 objects/s recovering 2024-09-21T10:20:24.263 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:23 smithi076 ceph-mon[106482]: osdmap e1337: 8 total, 8 up, 5 in 2024-09-21T10:20:24.263 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:23 smithi076 ceph-mon[110963]: pgmap v2112: 89 pgs: 2 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 84 active+clean; 590 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s; 1316/28359 objects misplaced (4.641%); 14 MiB/s, 225 objects/s recovering 2024-09-21T10:20:24.263 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:23 smithi076 ceph-mon[110963]: osdmap e1337: 8 total, 8 up, 5 in 2024-09-21T10:20:24.267 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:24.288 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:24.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:23 smithi191 ceph-mon[102206]: pgmap v2112: 89 pgs: 2 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 84 active+clean; 590 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s; 1316/28359 objects misplaced (4.641%); 14 MiB/s, 225 objects/s recovering 2024-09-21T10:20:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:23 smithi191 ceph-mon[102206]: osdmap e1337: 8 total, 8 up, 5 in 2024-09-21T10:20:24.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:24.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:24.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:24.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:24.639 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:24.660 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:24.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:24.761 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:24.784 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:24.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:24.801 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:20:24.801 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:24.821 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:20:24.821 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:24.837 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:20:24.838 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:24.853 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:20:24.853 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:24.870 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:20:24.870 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:24.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:24.887 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:20:24.887 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:24.904 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:20:24.905 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:24.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:24.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:24.922 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:20:24.922 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:25.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:25.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:24 smithi191 ceph-mon[102206]: osdmap e1338: 8 total, 8 up, 5 in 2024-09-21T10:20:25.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1502966932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:25.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:24 smithi076 ceph-mon[106482]: osdmap e1338: 8 total, 8 up, 5 in 2024-09-21T10:20:25.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1502966932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:25.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:24 smithi076 ceph-mon[110963]: osdmap e1338: 8 total, 8 up, 5 in 2024-09-21T10:20:25.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1502966932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:25.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:25.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:25.262 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:25.283 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:25.385 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:25.387 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:20:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:20:25.387 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:20:25.387 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:25.406 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:25.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:25.416 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:20:25.416 INFO:teuthology.orchestra.run.smithi076.stdout:{"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2128,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1339,"num_osds":8,"num_up_osds":8,"osd_up_since":1726913921,"num_in_osds":5,"osd_in_since":1726913870,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":84},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+remapped+backfilling","count":2},{"state_name":"peering","count":1}],"num_pgs":89,"num_pools":5,"num_objects":9453,"data_bytes":618338301,"bytes_used":6382247936,"bytes_avail":473592930304,"bytes_total":479975178240,"inactive_pgs_ratio":0.01123595517128706,"misplaced_objects":1316,"misplaced_total":28359,"misplaced_ratio":0.046405021333615432,"recovering_objects_per_sec":225,"recovering_bytes_per_sec":14774478,"recovering_keys_per_sec":0,"num_objects_recovered":902,"num_bytes_recovered":59113472,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":1006},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi076.jymebx","status":"up:active","gid":14631}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.76:8443/","prometheus":"http://172.21.15.76:9283/"}},"servicemap":{"epoch":31,"modified":"2024-09-21T10:19:40.722377+0000","services":{"osd":{"daemons":{"summary":"","1":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-09-21T10:20:25.508 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:25.529 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:25.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:25.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:25.754 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:25.776 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:25.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:25.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:25.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:25.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:26.000 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:26.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:26.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:26.078 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[106482]: pgmap v2115: 89 pgs: 2 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 84 active+clean; 590 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 1316/28359 objects misplaced (4.641%); 14 MiB/s, 225 objects/s recovering 2024-09-21T10:20:26.078 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[106482]: osdmap e1339: 8 total, 8 up, 5 in 2024-09-21T10:20:26.078 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/264977746' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:20:26.078 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:26.078 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:26.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[110963]: pgmap v2115: 89 pgs: 2 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 84 active+clean; 590 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 1316/28359 objects misplaced (4.641%); 14 MiB/s, 225 objects/s recovering 2024-09-21T10:20:26.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[110963]: osdmap e1339: 8 total, 8 up, 5 in 2024-09-21T10:20:26.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/264977746' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:20:26.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:26.079 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:26.105 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:25 smithi191 ceph-mon[102206]: pgmap v2115: 89 pgs: 2 active+remapped+backfilling, 1 peering, 2 active+remapped+backfill_wait, 84 active+clean; 590 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 1316/28359 objects misplaced (4.641%); 14 MiB/s, 225 objects/s recovering 2024-09-21T10:20:26.105 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:25 smithi191 ceph-mon[102206]: osdmap e1339: 8 total, 8 up, 5 in 2024-09-21T10:20:26.105 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/264977746' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-21T10:20:26.105 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:26.105 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:26.109 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-21T10:20:26.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:20:26.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:26.140 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:20:26.162 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:26.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:26.263 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:26.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:26.394 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:26.459 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:26.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:26.534 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:26.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:26.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:26.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:26.688 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:26.705 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:26.722 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:26.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:26.823 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:26.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:26.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:26.961 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:26.986 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:26.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:27.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:27.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.264 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:27.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.295 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:26 smithi076 ceph-mon[106482]: osdmap e1340: 8 total, 8 up, 5 in 2024-09-21T10:20:27.295 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:26 smithi076 ceph-mon[110963]: osdmap e1340: 8 total, 8 up, 5 in 2024-09-21T10:20:27.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:26 smithi191 ceph-mon[102206]: osdmap e1340: 8 total, 8 up, 5 in 2024-09-21T10:20:27.366 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:27.535 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:27.637 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.676 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:27.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:27.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:27.817 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:27.919 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:27.944 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:27.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:27.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:28.001 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:28.047 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:28.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:28.107 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:27 smithi191 ceph-mon[102206]: pgmap v2118: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 86 active+clean; 560 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 117 op/s; 1100/26952 objects misplaced (4.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:20:28.187 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:28.213 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:28.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:28.318 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:27 smithi076 ceph-mon[106482]: pgmap v2118: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 86 active+clean; 560 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 117 op/s; 1100/26952 objects misplaced (4.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:20:28.318 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:27 smithi076 ceph-mon[110963]: pgmap v2118: 89 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+remapped+backfill_wait, 86 active+clean; 560 MiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 117 op/s; 1100/26952 objects misplaced (4.081%); 3.0 MiB/s, 47 objects/s recovering 2024-09-21T10:20:28.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:28.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:28.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:28.578 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:28.586 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:28.586 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:20:28.586 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:20:28.586 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:20:28.586 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115541 10.9 10.eef0e4d9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.260116+0000", 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 158.32045247100001, 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97780290700000005, 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115541 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.587 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260116+0000", 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260116+0000", 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260114+0000", 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.588 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260152+0000", 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7388000000000002e-05 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260165+0000", 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3342e-05 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260173+0000", 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0390000000000005e-06 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.589 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260226+0000", 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2772999999999998e-05 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260235+0000", 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8079999999999994e-06 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.237906+0000", 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.977671701 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.590 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.237919+0000", 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2441e-05 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115555 10.6 10.c9dda6f6 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.260191+0000", 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 158.320377622, 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97924076599999998, 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.591 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115555 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260191+0000", 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260191+0000", 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.592 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.260190+0000", 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.261177+0000", 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00098687300000000009 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.261188+0000", 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1148999999999999e-05 2024-09-21T10:20:28.593 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.261195+0000", 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2710000000000003e-06 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.261239+0000", 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3523999999999999e-05 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.261247+0000", 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.695e-06 2024-09-21T10:20:28.594 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.239423+0000", 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97817628000000001 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.239431+0000", 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5539999999999998e-06 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.595 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115571 10.8 10.3a41c458 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.263296+0000", 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 158.31727260599999, 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97570074299999998, 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115571 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.596 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263296+0000", 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263296+0000", 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263294+0000", 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.597 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263348+0000", 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3798999999999998e-05 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263361+0000", 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.324e-05 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263369+0000", 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.5429999999999996e-06 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.598 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263464+0000", 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4806e-05 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.263471+0000", 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6590000000000007e-06 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.238986+0000", 2024-09-21T10:20:28.599 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97551455399999998 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.238996+0000", 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0709e-05 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177657 10.9 10:9662158c:::benchmark_data_smithi076_299450_object42478:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223326+0000", 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.357242697999993, 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0223126890000001, 2024-09-21T10:20:28.600 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177657 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223326+0000", 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.601 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223326+0000", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223324+0000", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223328+0000", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6950000000000002e-06 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.602 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223336+0000", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2190000000000004e-06 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223342+0000", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0689999999999998e-06 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223487+0000", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014458400000000001 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223495+0000", 2024-09-21T10:20:28.603 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8650000000000001e-06 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245576+0000", 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.022081561 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245638+0000", 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2071999999999998e-05 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.604 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177665 10.9 10:9b6f0096:::benchmark_data_smithi076_299450_object42486:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.357217934999994, 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.022590052, 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177665 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:20:28.605 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223352+0000", 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8759999999999999e-06 2024-09-21T10:20:28.606 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223357+0000", 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.135e-06 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223362+0000", 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5129999999999998e-06 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223513+0000", 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015170400000000001 2024-09-21T10:20:28.607 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223519+0000", 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3199999999999999e-06 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245899+0000", 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.022380165 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245940+0000", 2024-09-21T10:20:28.608 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1669999999999999e-05 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177668 10.9 10:9f5283a7:::benchmark_data_smithi076_299450_object42489:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.357203089999999, 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.02263262, 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.609 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177668 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.610 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223367+0000", 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8330000000000001e-06 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223372+0000", 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.611 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223376+0000", 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1840000000000001e-06 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223541+0000", 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016516 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223548+0000", 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3440000000000003e-06 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.612 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245963+0000", 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0224149629999999 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245998+0000", 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4524999999999999e-05 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.613 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177671 10.9 10:91d97354:::benchmark_data_smithi076_299450_object42492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.357189438999995, 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0226838979999999, 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177671 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.614 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.615 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223380+0000", 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.751e-06 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223385+0000", 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6759999999999998e-06 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223389+0000", 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0249999999999996e-06 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.616 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223568+0000", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178627 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223573+0000", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5690000000000004e-06 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246018+0000", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0224451210000001 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.617 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246063+0000", 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4397999999999997e-05 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177675 10.9 10:94de0b4c:::benchmark_data_smithi076_299450_object42496:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.357164999000005, 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0379785429999999, 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.618 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177675 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.619 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223405+0000", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.26e-06 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223419+0000", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3491e-05 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223424+0000", 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0769999999999998e-06 2024-09-21T10:20:28.620 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223593+0000", 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016949200000000001 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223600+0000", 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6189999999999996e-06 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246083+0000", 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0224835459999999 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.621 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246159+0000", 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6076999999999997e-05 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246280+0000", 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120429 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256305+0000", 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010025552 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.622 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256340+0000", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4144999999999999e-05 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261293+0000", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049532669999999999 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261340+0000", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6830999999999999e-05 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.623 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261382+0000", 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2138999999999998e-05 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177677 10.9 10:95d800f0:::benchmark_data_smithi076_299450_object42498:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.357139896999996, 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0381621590000001, 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177677 2024-09-21T10:20:28.624 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.625 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223430+0000", 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3779999999999999e-06 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223436+0000", 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5709999999999997e-06 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.626 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223440+0000", 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.177e-06 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223620+0000", 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179741 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223625+0000", 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3550000000000004e-06 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.627 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.248686+0000", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.025060804 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.248795+0000", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108602 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.248903+0000", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108515 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256371+0000", 2024-09-21T10:20:28.628 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074677620000000002 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256433+0000", 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1599000000000004e-05 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261503+0000", 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050703149999999997 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261555+0000", 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1925000000000002e-05 2024-09-21T10:20:28.629 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261591+0000", 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5796000000000001e-05 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177687 10.9 10:9283661d:::benchmark_data_smithi076_299450_object42508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.232288+0000", 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.348280747999993, 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0293759579999999, 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.630 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177687 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232288+0000", 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.631 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232288+0000", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232286+0000", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232292+0000", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.241e-06 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232309+0000", 2024-09-21T10:20:28.632 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6951000000000001e-05 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232316+0000", 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8619999999999997e-06 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232359+0000", 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2562999999999999e-05 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232366+0000", 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7519999999999996e-06 2024-09-21T10:20:28.633 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.251387+0000", 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.019020714 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.251505+0000", 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011822799999999999 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.251606+0000", 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010082 2024-09-21T10:20:28.634 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256399+0000", 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047931110000000001 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256465+0000", 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5335999999999999e-05 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261599+0000", 2024-09-21T10:20:28.635 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051342990000000002 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261634+0000", 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5091e-05 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261664+0000", 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9558000000000001e-05 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.636 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177694 10.9 10:9c545240:::benchmark_data_smithi076_299450_object42515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.239830+0000", 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.340738462999994, 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021904913, 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177694 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.637 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239830+0000", 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239830+0000", 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.638 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239828+0000", 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239846+0000", 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7907999999999999e-05 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239858+0000", 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1528999999999999e-05 2024-09-21T10:20:28.639 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239865+0000", 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7819999999999996e-06 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239908+0000", 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3652000000000002e-05 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239916+0000", 2024-09-21T10:20:28.640 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2840000000000006e-06 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.253937+0000", 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0140201680000001 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.254041+0000", 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104066 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.254152+0000", 2024-09-21T10:20:28.641 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111322 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261447+0000", 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0072946529999999999 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261495+0000", 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8523999999999998e-05 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.642 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261673+0000", 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017774699999999999 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261706+0000", 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2935999999999998e-05 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261735+0000", 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.887e-05 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.643 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177696 10.9 10:9bab1062:::benchmark_data_smithi076_299450_object42517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.240479+0000", 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.340089594999995, 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0300966309999999, 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177696 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.644 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240479+0000", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240479+0000", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240478+0000", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240482+0000", 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6600000000000001e-06 2024-09-21T10:20:28.645 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240489+0000", 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2840000000000001e-06 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240494+0000", 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6759999999999998e-06 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240529+0000", 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5552000000000001e-05 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.646 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240537+0000", 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9049999999999997e-06 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256486+0000", 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0159486790000001 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256597+0000", 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111092 2024-09-21T10:20:28.647 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256702+0000", 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105289 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261745+0000", 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050424700000000003 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261803+0000", 2024-09-21T10:20:28.648 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8353999999999997e-05 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270493+0000", 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086903450000000004 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270539+0000", 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5556000000000002e-05 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270575+0000", 2024-09-21T10:20:28.649 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6482000000000001e-05 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177703 10.9 10:9803cc08:::benchmark_data_smithi076_299450_object42524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.247427+0000", 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.333141787000002, 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034124346, 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.650 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177703 2024-09-21T10:20:28.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:28.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:28.652 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.652 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.652 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.652 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.652 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247427+0000", 2024-09-21T10:20:28.652 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247427+0000", 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247425+0000", 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247439+0000", 2024-09-21T10:20:28.653 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3509000000000001e-05 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247450+0000", 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1494999999999999e-05 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247457+0000", 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0670000000000003e-06 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247549+0000", 2024-09-21T10:20:28.654 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1928999999999993e-05 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247557+0000", 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8029999999999997e-06 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.258917+0000", 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0113603289999999 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.259032+0000", 2024-09-21T10:20:28.655 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114831 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.259114+0000", 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1827000000000001e-05 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261861+0000", 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027469 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270614+0000", 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087532370000000005 2024-09-21T10:20:28.656 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.281470+0000", 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010855492 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.281512+0000", 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1931999999999998e-05 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.281551+0000", 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9348999999999999e-05 2024-09-21T10:20:28.657 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177707 10.9 10:91aa2076:::benchmark_data_smithi076_299450_object42528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.249447+0000", 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.331121608000004, 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0387998350000001, 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.658 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177707 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249447+0000", 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249447+0000", 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.659 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249445+0000", 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249452+0000", 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.443e-06 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249468+0000", 2024-09-21T10:20:28.660 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6738000000000001e-05 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249476+0000", 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9359999999999999e-06 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249886+0000", 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00041004700000000003 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249894+0000", 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4359999999999996e-06 2024-09-21T10:20:28.661 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261889+0000", 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0119947090000001 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261992+0000", 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010364100000000001 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.262098+0000", 2024-09-21T10:20:28.662 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010531600000000001 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270646+0000", 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085488430000000004 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270678+0000", 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1634000000000002e-05 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288170+0000", 2024-09-21T10:20:28.663 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017492167999999999 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288211+0000", 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0664000000000002e-05 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288247+0000", 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5704999999999998e-05 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.664 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177709 10.9 10:9ef77a4b:::benchmark_data_smithi076_299450_object42530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.253684+0000", 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.326884663000001, 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034637657, 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177709 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.665 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253684+0000", 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253684+0000", 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253682+0000", 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.666 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253689+0000", 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4110000000000001e-06 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253701+0000", 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2897e-05 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253708+0000", 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9909999999999996e-06 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.667 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253755+0000", 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6978999999999999e-05 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253762+0000", 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8600000000000004e-06 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.267340+0000", 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0135781589999999 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.668 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.267450+0000", 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109311 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.267709+0000", 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025922800000000002 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.276075+0000", 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0083659670000000002 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.669 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.276104+0000", 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8580000000000001e-05 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288254+0000", 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012150564000000001 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288291+0000", 2024-09-21T10:20:28.670 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6832000000000002e-05 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288321+0000", 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0388000000000001e-05 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177726 10.9 10:9d8bfc47:::benchmark_data_smithi076_299450_object42547:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.275006+0000", 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 95.305562679999994, 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0134493389999999, 2024-09-21T10:20:28.671 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177726 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275006+0000", 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.672 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275006+0000", 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275004+0000", 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275009+0000", 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.259e-06 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.673 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275021+0000", 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1382999999999999e-05 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275027+0000", 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275069+0000", 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1944999999999999e-05 2024-09-21T10:20:28.674 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275077+0000", 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270703+0000", 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99562528800000005 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270841+0000", 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013829900000000001 2024-09-21T10:20:28.675 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270967+0000", 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012629699999999999 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.276132+0000", 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051644009999999999 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288328+0000", 2024-09-21T10:20:28.676 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012195869 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288385+0000", 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7873e-05 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288425+0000", 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9493000000000003e-05 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288455+0000", 2024-09-21T10:20:28.677 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0040999999999999e-05 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:214746 10.8 10:1a04690b:::benchmark_data_smithi076_299450_object79567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1324)", 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:19:31.325536+0000", 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 57.255032100999998, 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0216948720000001, 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.678 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 214746 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325536+0000", 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325536+0000", 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.679 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325534+0000", 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325541+0000", 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4420000000000003e-06 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325553+0000", 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2490999999999999e-05 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.680 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325560+0000", 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8190000000000003e-06 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325609+0000", 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8715999999999999e-05 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325617+0000", 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8560000000000007e-06 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.681 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.341627+0000", 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.016009951 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.341745+0000", 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0001188 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.341863+0000", 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117589 2024-09-21T10:20:28.682 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347084+0000", 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052205990000000002 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347129+0000", 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5185000000000002e-05 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347164+0000", 2024-09-21T10:20:28.683 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5385000000000001e-05 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347193+0000", 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8297e-05 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347231+0000", 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8546e-05 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.684 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:214773 10.8 10:1cc0885d:::benchmark_data_smithi076_299450_object79594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1324)", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:19:31.353782+0000", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 57.226786093999998, 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99976494100000002, 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 214773 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353782+0000", 2024-09-21T10:20:28.685 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353782+0000", 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353782+0000", 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.686 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353785+0000", 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.869e-06 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353791+0000", 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0079999999999999e-06 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353795+0000", 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6949999999999999e-06 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.687 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353868+0000", 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2113999999999994e-05 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.353877+0000", 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7429999999999997e-06 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.344571+0000", 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99069381199999995 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.688 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.344705+0000", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013391400000000001 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.344779+0000", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3894999999999996e-05 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347159+0000", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0023801410000000001 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.689 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.353450+0000", 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062909050000000003 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.353464+0000", 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.398e-05 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.353513+0000", 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9413999999999998e-05 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.690 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.353547+0000", 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3852e-05 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262305 10.3 10:c1fd82fe:::benchmark_data_smithi076_299450_object127126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.038678+0000", 2024-09-21T10:20:28.691 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.5418906789999998, 2024-09-21T10:20:28.691 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:28.692 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0040169240000001, 2024-09-21T10:20:28.692 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.692 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.692 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262305 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038678+0000", 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.693 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038678+0000", 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038676+0000", 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038683+0000", 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5610000000000004e-06 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.694 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038695+0000", 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2646999999999999e-05 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038703+0000", 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.306e-06 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038749+0000", 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6431999999999999e-05 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.695 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038758+0000", 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6060000000000003e-06 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.036325+0000", 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99756721800000003 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.036437+0000", 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111712 2024-09-21T10:20:28.696 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.036526+0000", 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9777000000000005e-05 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042505+0000", 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059781929999999997 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042513+0000", 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1859999999999992e-06 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.697 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042626+0000", 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112977 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042662+0000", 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6607999999999999e-05 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042695+0000", 2024-09-21T10:20:28.698 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2162000000000001e-05 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262335 10.3 10:c86c8333:::benchmark_data_smithi076_299450_object127156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.065235+0000", 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 3.5153333170000001, 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98971618900000002, 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:28.699 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262335 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065235+0000", 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065235+0000", 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.700 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065234+0000", 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065239+0000", 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8570000000000003e-06 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065249+0000", 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0647e-05 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.701 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065256+0000", 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5409999999999997e-06 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065308+0000", 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2441999999999997e-05 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.065317+0000", 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.2890000000000003e-06 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.702 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.039738+0000", 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97442098799999999 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.039843+0000", 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104091 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.039948+0000", 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105167 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.703 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.048482+0000", 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085344359999999994 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.048540+0000", 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7519999999999998e-05 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.054830+0000", 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0062904550000000004 2024-09-21T10:20:28.704 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.054916+0000", 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5844e-05 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.054951+0000", 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5250999999999998e-05 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:28.705 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:28.752 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:28.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:28.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:28.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:28.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:28.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:29.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:29.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:29.150 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:29.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:29.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:29.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:29.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:29.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:29.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:29.551 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:29.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:29.681 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:29.696 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:20:29.708 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:29.810 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:29.837 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:29.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:29.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:29.939 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:30.013 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:20:30.015 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:30.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:30.064 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:30.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:30 smithi076 ceph-mon[106482]: pgmap v2119: 89 pgs: 89 active+clean; 447 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 13 MiB/s, 208 objects/s recovering 2024-09-21T10:20:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:30 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2293998600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:30.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:30 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:30 smithi076 ceph-mon[110963]: pgmap v2119: 89 pgs: 89 active+clean; 447 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 13 MiB/s, 208 objects/s recovering 2024-09-21T10:20:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:30 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2293998600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:30.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:30 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:30.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:30.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:30.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:30.269 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:30.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:30 smithi191 ceph-mon[102206]: pgmap v2119: 89 pgs: 89 active+clean; 447 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 394 op/s; 13 MiB/s, 208 objects/s recovering 2024-09-21T10:20:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:30 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2293998600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:30 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:20:30.370 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:30.420 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:30.522 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:30.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:30.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:30.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:30.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:30.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:30.825 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:30.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:30.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:30.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:30.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:31.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:31.107 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:31.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:31.245 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:31.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:31.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:31.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:31 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:31 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:20:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:31 smithi191 ceph-mon[102206]: osdmap e1341: 8 total, 8 up, 5 in 2024-09-21T10:20:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:31 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:31 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:20:31.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:31 smithi076 ceph-mon[106482]: osdmap e1341: 8 total, 8 up, 5 in 2024-09-21T10:20:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:31 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:31.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:31 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:20:31.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:31 smithi076 ceph-mon[110963]: osdmap e1341: 8 total, 8 up, 5 in 2024-09-21T10:20:31.380 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:31.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:31.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.522 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:31.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:31.623 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:31.654 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:31.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:31.797 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:31.899 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:31.920 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:31.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:32.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:32.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:32.160 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:32.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:32.290 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:32 smithi191 ceph-mon[102206]: pgmap v2121: 89 pgs: 89 active+clean; 447 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 379 op/s; 13 MiB/s, 200 objects/s recovering 2024-09-21T10:20:32.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:32 smithi076 ceph-mon[106482]: pgmap v2121: 89 pgs: 89 active+clean; 447 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 379 op/s; 13 MiB/s, 200 objects/s recovering 2024-09-21T10:20:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:32 smithi076 ceph-mon[110963]: pgmap v2121: 89 pgs: 89 active+clean; 447 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 379 op/s; 13 MiB/s, 200 objects/s recovering 2024-09-21T10:20:32.445 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:32.644 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:32.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:32.745 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:32.771 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:32.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:32.873 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:32.896 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:32.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:32.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:33.022 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:33.024 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:269712 10.e 10:78acd2a4:::benchmark_data_smithi076_299450_object134533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1341)", 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:33.016155+0000", 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0055348130000000004, 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0055658569999999996, 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 269712 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.025 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016155+0000", 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016155+0000", 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:33.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016154+0000", 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016160+0000", 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2820000000000002e-06 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016188+0000", 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8048e-05 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016196+0000", 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9500000000000001e-06 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016235+0000", 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8900999999999999e-05 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016351+0000", 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011649100000000001 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.016476+0000", 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012469299999999999 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.020140+0000", 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0036639649999999999 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.021179+0000", 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001039306 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:269714 10.0 10:0f5f6145:::benchmark_data_smithi076_299450_object134535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1341)", 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:33.019196+0000", 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.002493781, 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002609068, 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 269714 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:33.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019196+0000", 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019196+0000", 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019195+0000", 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019202+0000", 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0910000000000004e-06 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019216+0000", 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3526000000000001e-05 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019224+0000", 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0129999999999993e-06 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019339+0000", 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011547 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019482+0000", 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014331500000000001 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,4", 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.019611+0000", 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012827499999999999 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.033 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:269717 10.a 10:5260453e:::benchmark_data_smithi076_299450_object134538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1341)", 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:33.020992+0000", 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00069769299999999999, 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00087949600000000001, 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 269717 2024-09-21T10:20:33.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.020992+0000", 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.020992+0000", 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.020991+0000", 2024-09-21T10:20:33.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.020997+0000", 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2469999999999997e-06 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.021014+0000", 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6146000000000001e-05 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.021021+0000", 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8429999999999993e-06 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.021065+0000", 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3989000000000002e-05 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:20:33.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.021174+0000", 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108126 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:33.021282+0000", 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108782 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 3 2024-09-21T10:20:33.038 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:33.124 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:33.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:33.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:33.264 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:33.291 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:33.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:33.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:33.429 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:33.500 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:20:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:20:33] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.51.0" 2024-09-21T10:20:33.530 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:33.555 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:33.615 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Removed 135176 objects 2024-09-21T10:20:33.615 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Clean up completed and total clean up time :141.352 2024-09-21T10:20:33.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:33.677 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:33.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:33.779 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:33.801 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:33.835 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:20:33.896 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:33.903 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:33.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:33.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:33.979 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-21T10:20:33.979 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-21T10:20:34.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:34.032 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:34.048 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:34 smithi191 ceph-mon[102206]: pgmap v2122: 89 pgs: 89 active+clean; 118 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 971 op/s; 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:20:34.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:34 smithi076 ceph-mon[106482]: pgmap v2122: 89 pgs: 89 active+clean; 118 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 971 op/s; 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:20:34.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:34 smithi076 ceph-mon[110963]: pgmap v2122: 89 pgs: 89 active+clean; 118 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 971 op/s; 9.7 MiB/s, 154 objects/s recovering 2024-09-21T10:20:34.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:34.172 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:34.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:34.296 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:34.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:34.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:34.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:34.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:34.542 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:34.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:34.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:34.665 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:34.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:34.789 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:34.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:34.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:34.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:35.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:35.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:35.136 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:35.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:35.259 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:35.281 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:35.382 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:35.403 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:35.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:35.526 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:35.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:35.547 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:35.627 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:35.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:35.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:35.698 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:20:35.699 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:20:35.742 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:20:35.742 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:20:35.743 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:20:35.743 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:35.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:35.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:35.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:35.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:20:35.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:35.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:35.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:35.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.018 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:36.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:36.118 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:36.140 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:36.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:36.241 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:36.248 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:36 smithi076 ceph-mon[106482]: pgmap v2123: 89 pgs: 89 active+clean; 118 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 860 op/s; 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:20:36.248 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:36 smithi076 ceph-mon[110963]: pgmap v2123: 89 pgs: 89 active+clean; 118 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 860 op/s; 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:20:36.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:36.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:36.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:36 smithi191 ceph-mon[102206]: pgmap v2123: 89 pgs: 89 active+clean; 118 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 860 op/s; 8.6 MiB/s, 137 objects/s recovering 2024-09-21T10:20:36.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:36.386 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:36.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:36.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:20:36.488 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:36.551 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:36.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:36.652 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:36.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:36.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:36.684 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:36.776 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:36.797 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:36.898 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:36.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:36.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:37.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.043 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:37.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:37.061 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:37.062 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:37.062 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:37.062 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:20:37.062 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:20:37.062 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:37.062 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:37 smithi191 ceph-mon[102206]: from='client.36144 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2008054154' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:37.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:37.166 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:37.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:37.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:37.290 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:37.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:37 smithi076 ceph-mon[110963]: from='client.36144 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:37.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2008054154' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:37.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:37 smithi076 ceph-mon[106482]: from='client.36144 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:37.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2008054154' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:37.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:37.413 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.515 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:37.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:37.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:37.662 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:37.763 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:37.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:37.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:37.908 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:37.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:37.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:38.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:38.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:38.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:38.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:38.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:38.281 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:38.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:38 smithi191 ceph-mon[102206]: pgmap v2124: 89 pgs: 89 active+clean; 84 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 761 op/s; 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:20:38.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:38 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:20:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:38 smithi076 ceph-mon[106482]: pgmap v2124: 89 pgs: 89 active+clean; 84 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 761 op/s; 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:20:38.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:38 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:20:38.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:38 smithi076 ceph-mon[110963]: pgmap v2124: 89 pgs: 89 active+clean; 84 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 761 op/s; 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:20:38.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:38 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:20:38.383 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.406 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:38.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:38.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:38.530 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:38.631 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:38.656 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:38.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:38.758 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:38.783 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:38.883 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2123,"stamp":"2024-09-21T10:20:34.736271+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":124065789,"num_objects":1911,"num_object_clones":0,"num_object_copies":5733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1911,"num_whiteouts":0,"num_read":379,"num_read_kb":823,"num_write":268748,"num_write_kb":8653714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20994,"num_bytes_recovered":1373527901,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90777,"ondisk_log_size":90777,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4907880,"kb_used_data":780216,"kb_used_omap":94,"kb_used_meta":4127521,"kb_avail":463817880,"statfs":{"total":479975178240,"available":474949509120,"internally_reserved":0,"allocated":798941184,"data_stored":782581498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96735,"internal_metadata":4226582049},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,3,9,7,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-494272512,"num_objects":-7542,"num_object_clones":0,"num_object_copies":-22626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1316,"num_objects_unfound":0,"num_objects_dirty":-7542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7542,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1202,"num_bytes_recovered":78708736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.760025"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":499,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013945+0000","last_change":"2024-09-21T10:17:51.249951+0000","last_active":"2024-09-21T10:20:30.013945+0000","last_peered":"2024-09-21T10:20:30.013945+0000","last_clean":"2024-09-21T10:20:30.013945+0000","last_became_active":"2024-09-21T10:17:51.249158+0000","last_became_peered":"2024-09-21T10:17:51.249158+0000","last_unstale":"2024-09-21T10:20:30.013945+0000","last_undegraded":"2024-09-21T10:20:30.013945+0000","last_fullsized":"2024-09-21T10:20:30.013945+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:53.769122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":218,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980185+0000","last_change":"2024-09-21T10:15:19.141623+0000","last_active":"2024-09-21T10:20:25.980185+0000","last_peered":"2024-09-21T10:20:25.980185+0000","last_clean":"2024-09-21T10:20:25.980185+0000","last_became_active":"2024-09-21T10:15:19.111927+0000","last_became_peered":"2024-09-21T10:15:19.111927+0000","last_unstale":"2024-09-21T10:20:25.980185+0000","last_undegraded":"2024-09-21T10:20:25.980185+0000","last_fullsized":"2024-09-21T10:20:25.980185+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:10.426159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978302+0000","last_change":"2024-09-21T10:17:51.258024+0000","last_active":"2024-09-21T10:20:25.978302+0000","last_peered":"2024-09-21T10:20:25.978302+0000","last_clean":"2024-09-21T10:20:25.978302+0000","last_became_active":"2024-09-21T10:17:51.257639+0000","last_became_peered":"2024-09-21T10:17:51.257639+0000","last_unstale":"2024-09-21T10:20:25.978302+0000","last_undegraded":"2024-09-21T10:20:25.978302+0000","last_fullsized":"2024-09-21T10:20:25.978302+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:09.112335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1816,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.015683+0000","last_change":"2024-09-21T10:17:51.250405+0000","last_active":"2024-09-21T10:20:30.015683+0000","last_peered":"2024-09-21T10:20:30.015683+0000","last_clean":"2024-09-21T10:20:30.015683+0000","last_became_active":"2024-09-21T10:17:51.248030+0000","last_became_peered":"2024-09-21T10:17:51.248030+0000","last_unstale":"2024-09-21T10:20:30.015683+0000","last_undegraded":"2024-09-21T10:20:30.015683+0000","last_fullsized":"2024-09-21T10:20:30.015683+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:45.889214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1815,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.015013+0000","last_change":"2024-09-21T10:17:51.249710+0000","last_active":"2024-09-21T10:20:30.015013+0000","last_peered":"2024-09-21T10:20:30.015013+0000","last_clean":"2024-09-21T10:20:30.015013+0000","last_became_active":"2024-09-21T10:17:51.249284+0000","last_became_peered":"2024-09-21T10:17:51.249284+0000","last_unstale":"2024-09-21T10:20:30.015013+0000","last_undegraded":"2024-09-21T10:20:30.015013+0000","last_fullsized":"2024-09-21T10:20:30.015013+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:09.927717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":221,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980139+0000","last_change":"2024-09-21T10:16:38.451401+0000","last_active":"2024-09-21T10:20:25.980139+0000","last_peered":"2024-09-21T10:20:25.980139+0000","last_clean":"2024-09-21T10:20:25.980139+0000","last_became_active":"2024-09-21T10:16:38.450094+0000","last_became_peered":"2024-09-21T10:16:38.450094+0000","last_unstale":"2024-09-21T10:20:25.980139+0000","last_undegraded":"2024-09-21T10:20:25.980139+0000","last_fullsized":"2024-09-21T10:20:25.980139+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:22:50.798278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":492,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.015467+0000","last_change":"2024-09-21T10:16:38.451461+0000","last_active":"2024-09-21T10:20:30.015467+0000","last_peered":"2024-09-21T10:20:30.015467+0000","last_clean":"2024-09-21T10:20:30.015467+0000","last_became_active":"2024-09-21T10:16:38.451137+0000","last_became_peered":"2024-09-21T10:16:38.451137+0000","last_unstale":"2024-09-21T10:20:30.015467+0000","last_undegraded":"2024-09-21T10:20:30.015467+0000","last_fullsized":"2024-09-21T10:20:30.015467+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:40.847463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":279,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.979970+0000","last_change":"2024-09-21T10:17:51.269584+0000","last_active":"2024-09-21T10:20:25.979970+0000","last_peered":"2024-09-21T10:20:25.979970+0000","last_clean":"2024-09-21T10:20:25.979970+0000","last_became_active":"2024-09-21T10:17:51.260673+0000","last_became_peered":"2024-09-21T10:17:51.260673+0000","last_unstale":"2024-09-21T10:20:25.979970+0000","last_undegraded":"2024-09-21T10:20:25.979970+0000","last_fullsized":"2024-09-21T10:20:25.979970+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:16.649818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"1341'16615","reported_seq":17846,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.330919+0000","last_change":"2024-09-21T10:20:23.012354+0000","last_active":"2024-09-21T10:20:31.330919+0000","last_peered":"2024-09-21T10:20:31.330919+0000","last_clean":"2024-09-21T10:20:31.330919+0000","last_became_active":"2024-09-21T10:20:22.968366+0000","last_became_peered":"2024-09-21T10:20:22.968366+0000","last_unstale":"2024-09-21T10:20:31.330919+0000","last_undegraded":"2024-09-21T10:20:31.330919+0000","last_fullsized":"2024-09-21T10:20:31.330919+0000","mapping_epoch":1336,"log_start":"1318'10600","ondisk_log_start":"1318'10600","created":1296,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:09.938539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8847360,"num_objects":135,"num_object_clones":0,"num_object_copies":405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16615,"num_write_kb":536000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":559,"num_bytes_recovered":36634624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":402,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009999+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:20:30.009999+0000","last_peered":"2024-09-21T10:20:30.009999+0000","last_clean":"2024-09-21T10:20:30.009999+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:20:30.009999+0000","last_undegraded":"2024-09-21T10:20:30.009999+0000","last_fullsized":"2024-09-21T10:20:30.009999+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977373+0000","last_change":"2024-09-21T10:17:51.256335+0000","last_active":"2024-09-21T10:20:25.977373+0000","last_peered":"2024-09-21T10:20:25.977373+0000","last_clean":"2024-09-21T10:20:25.977373+0000","last_became_active":"2024-09-21T10:17:51.251630+0000","last_became_peered":"2024-09-21T10:17:51.251630+0000","last_unstale":"2024-09-21T10:20:25.977373+0000","last_undegraded":"2024-09-21T10:20:25.977373+0000","last_fullsized":"2024-09-21T10:20:25.977373+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:31.141409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"1341'16644","reported_seq":19025,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.990462+0000","last_change":"2024-09-21T10:20:25.006425+0000","last_active":"2024-09-21T10:20:31.990462+0000","last_peered":"2024-09-21T10:20:31.990462+0000","last_clean":"2024-09-21T10:20:31.990462+0000","last_became_active":"2024-09-21T10:20:24.980776+0000","last_became_peered":"2024-09-21T10:20:24.980776+0000","last_unstale":"2024-09-21T10:20:31.990462+0000","last_undegraded":"2024-09-21T10:20:31.990462+0000","last_fullsized":"2024-09-21T10:20:31.990462+0000","mapping_epoch":1338,"log_start":"1318'11000","ondisk_log_start":"1318'11000","created":1296,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:38:17.252879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8519680,"num_objects":130,"num_object_clones":0,"num_object_copies":390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16644,"num_write_kb":536768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1147,"num_bytes_recovered":75169792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":93,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980505+0000","last_change":"2024-09-21T10:17:51.281550+0000","last_active":"2024-09-21T10:20:25.980505+0000","last_peered":"2024-09-21T10:20:25.980505+0000","last_clean":"2024-09-21T10:20:25.980505+0000","last_became_active":"2024-09-21T10:17:51.262457+0000","last_became_peered":"2024-09-21T10:17:51.262457+0000","last_unstale":"2024-09-21T10:20:25.980505+0000","last_undegraded":"2024-09-21T10:20:25.980505+0000","last_fullsized":"2024-09-21T10:20:25.980505+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:49.212195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1476,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009821+0000","last_change":"2024-09-21T10:16:38.461867+0000","last_active":"2024-09-21T10:20:30.009821+0000","last_peered":"2024-09-21T10:20:30.009821+0000","last_clean":"2024-09-21T10:20:30.009821+0000","last_became_active":"2024-09-21T10:16:38.461416+0000","last_became_peered":"2024-09-21T10:16:38.461416+0000","last_unstale":"2024-09-21T10:20:30.009821+0000","last_undegraded":"2024-09-21T10:20:30.009821+0000","last_fullsized":"2024-09-21T10:20:30.009821+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:15.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"1341'16752","reported_seq":16843,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.711806+0000","last_change":"2024-09-21T10:16:38.451467+0000","last_active":"2024-09-21T10:20:31.711806+0000","last_peered":"2024-09-21T10:20:31.711806+0000","last_clean":"2024-09-21T10:20:31.711806+0000","last_became_active":"2024-09-21T10:16:38.446328+0000","last_became_peered":"2024-09-21T10:16:38.446328+0000","last_unstale":"2024-09-21T10:20:31.711806+0000","last_undegraded":"2024-09-21T10:20:31.711806+0000","last_fullsized":"2024-09-21T10:20:31.711806+0000","mapping_epoch":1301,"log_start":"1321'11500","ondisk_log_start":"1321'11500","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5252,"log_dups_size":0,"ondisk_log_size":5252,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:27.117612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6946816,"num_objects":106,"num_object_clones":0,"num_object_copies":318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16752,"num_write_kb":539456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":48,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978130+0000","last_change":"2024-09-21T10:17:51.283569+0000","last_active":"2024-09-21T10:20:25.978130+0000","last_peered":"2024-09-21T10:20:25.978130+0000","last_clean":"2024-09-21T10:20:25.978130+0000","last_became_active":"2024-09-21T10:17:51.259187+0000","last_became_peered":"2024-09-21T10:17:51.259187+0000","last_unstale":"2024-09-21T10:20:25.978130+0000","last_undegraded":"2024-09-21T10:20:25.978130+0000","last_fullsized":"2024-09-21T10:20:25.978130+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:55.002645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":231,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980835+0000","last_change":"2024-09-21T10:17:51.248805+0000","last_active":"2024-09-21T10:20:25.980835+0000","last_peered":"2024-09-21T10:20:25.980835+0000","last_clean":"2024-09-21T10:20:25.980835+0000","last_became_active":"2024-09-21T10:17:51.248157+0000","last_became_peered":"2024-09-21T10:17:51.248157+0000","last_unstale":"2024-09-21T10:20:25.980835+0000","last_undegraded":"2024-09-21T10:20:25.980835+0000","last_fullsized":"2024-09-21T10:20:25.980835+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:36.573842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"1341'16629","reported_seq":22620,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.742250+0000","last_change":"2024-09-21T10:19:51.830838+0000","last_active":"2024-09-21T10:20:31.742250+0000","last_peered":"2024-09-21T10:20:31.742250+0000","last_clean":"2024-09-21T10:20:31.742250+0000","last_became_active":"2024-09-21T10:19:51.804737+0000","last_became_peered":"2024-09-21T10:19:51.804737+0000","last_unstale":"2024-09-21T10:20:31.742250+0000","last_undegraded":"2024-09-21T10:20:31.742250+0000","last_fullsized":"2024-09-21T10:20:31.742250+0000","mapping_epoch":1328,"log_start":"1321'11400","ondisk_log_start":"1321'11400","created":1296,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5229,"log_dups_size":0,"ondisk_log_size":5229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:26:46.112314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929856,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16629,"num_write_kb":536000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2940,"num_bytes_recovered":192675840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977644+0000","last_change":"2024-09-21T10:17:51.259331+0000","last_active":"2024-09-21T10:20:25.977644+0000","last_peered":"2024-09-21T10:20:25.977644+0000","last_clean":"2024-09-21T10:20:25.977644+0000","last_became_active":"2024-09-21T10:17:51.258943+0000","last_became_peered":"2024-09-21T10:17:51.258943+0000","last_unstale":"2024-09-21T10:20:25.977644+0000","last_undegraded":"2024-09-21T10:20:25.977644+0000","last_fullsized":"2024-09-21T10:20:25.977644+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:59.304035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1862,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.011152+0000","last_change":"2024-09-21T10:16:38.461644+0000","last_active":"2024-09-21T10:20:30.011152+0000","last_peered":"2024-09-21T10:20:30.011152+0000","last_clean":"2024-09-21T10:20:30.011152+0000","last_became_active":"2024-09-21T10:16:38.461179+0000","last_became_peered":"2024-09-21T10:16:38.461179+0000","last_unstale":"2024-09-21T10:20:30.011152+0000","last_undegraded":"2024-09-21T10:20:30.011152+0000","last_fullsized":"2024-09-21T10:20:30.011152+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:43.638430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"1341'16687","reported_seq":16793,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.849105+0000","last_change":"2024-09-21T10:16:38.459692+0000","last_active":"2024-09-21T10:20:31.849105+0000","last_peered":"2024-09-21T10:20:31.849105+0000","last_clean":"2024-09-21T10:20:31.849105+0000","last_became_active":"2024-09-21T10:16:38.459132+0000","last_became_peered":"2024-09-21T10:16:38.459132+0000","last_unstale":"2024-09-21T10:20:31.849105+0000","last_undegraded":"2024-09-21T10:20:31.849105+0000","last_fullsized":"2024-09-21T10:20:31.849105+0000","mapping_epoch":1301,"log_start":"1318'10900","ondisk_log_start":"1318'10900","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:36:53.847003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7536640,"num_objects":115,"num_object_clones":0,"num_object_copies":345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16687,"num_write_kb":537664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":445,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013763+0000","last_change":"2024-09-21T10:16:38.459890+0000","last_active":"2024-09-21T10:20:30.013763+0000","last_peered":"2024-09-21T10:20:30.013763+0000","last_clean":"2024-09-21T10:20:30.013763+0000","last_became_active":"2024-09-21T10:16:38.454382+0000","last_became_peered":"2024-09-21T10:16:38.454382+0000","last_unstale":"2024-09-21T10:20:30.013763+0000","last_undegraded":"2024-09-21T10:20:30.013763+0000","last_fullsized":"2024-09-21T10:20:30.013763+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:02.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1489,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014231+0000","last_change":"2024-09-21T10:16:38.454504+0000","last_active":"2024-09-21T10:20:30.014231+0000","last_peered":"2024-09-21T10:20:30.014231+0000","last_clean":"2024-09-21T10:20:30.014231+0000","last_became_active":"2024-09-21T10:16:38.448833+0000","last_became_peered":"2024-09-21T10:16:38.448833+0000","last_unstale":"2024-09-21T10:20:30.014231+0000","last_undegraded":"2024-09-21T10:20:30.014231+0000","last_fullsized":"2024-09-21T10:20:30.014231+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:11.905645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"1341'16950","reported_seq":19076,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.384165+0000","last_change":"2024-09-21T10:20:26.039642+0000","last_active":"2024-09-21T10:20:31.384165+0000","last_peered":"2024-09-21T10:20:31.384165+0000","last_clean":"2024-09-21T10:20:31.384165+0000","last_became_active":"2024-09-21T10:20:25.980270+0000","last_became_peered":"2024-09-21T10:20:25.980270+0000","last_unstale":"2024-09-21T10:20:31.384165+0000","last_undegraded":"2024-09-21T10:20:31.384165+0000","last_fullsized":"2024-09-21T10:20:31.384165+0000","mapping_epoch":1339,"log_start":"1318'11000","ondisk_log_start":"1318'11000","created":1296,"last_epoch_clean":1340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5950,"log_dups_size":0,"ondisk_log_size":5950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:16:18.688950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830400,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16950,"num_write_kb":547200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1009,"num_bytes_recovered":66125824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":198,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980732+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:20:25.980732+0000","last_peered":"2024-09-21T10:20:25.980732+0000","last_clean":"2024-09-21T10:20:25.980732+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:20:25.980732+0000","last_undegraded":"2024-09-21T10:20:25.980732+0000","last_fullsized":"2024-09-21T10:20:25.980732+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":89,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014698+0000","last_change":"2024-09-21T10:16:39.439192+0000","last_active":"2024-09-21T10:20:30.014698+0000","last_peered":"2024-09-21T10:20:30.014698+0000","last_clean":"2024-09-21T10:20:30.014698+0000","last_became_active":"2024-09-21T10:16:39.438778+0000","last_became_peered":"2024-09-21T10:16:39.438778+0000","last_unstale":"2024-09-21T10:20:30.014698+0000","last_undegraded":"2024-09-21T10:20:30.014698+0000","last_fullsized":"2024-09-21T10:20:30.014698+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:49.643063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.0","version":"1341'16761","reported_seq":16817,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.969341+0000","last_change":"2024-09-21T10:16:23.239046+0000","last_active":"2024-09-21T10:20:31.969341+0000","last_peered":"2024-09-21T10:20:31.969341+0000","last_clean":"2024-09-21T10:20:31.969341+0000","last_became_active":"2024-09-21T10:16:23.238681+0000","last_became_peered":"2024-09-21T10:16:23.238681+0000","last_unstale":"2024-09-21T10:20:31.969341+0000","last_undegraded":"2024-09-21T10:20:31.969341+0000","last_fullsized":"2024-09-21T10:20:31.969341+0000","mapping_epoch":1296,"log_start":"1320'11200","ondisk_log_start":"1320'11200","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5561,"log_dups_size":0,"ondisk_log_size":5561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:15:42.666111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7405568,"num_objects":113,"num_object_clones":0,"num_object_copies":339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16761,"num_write_kb":539968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1521,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009676+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:20:30.009676+0000","last_peered":"2024-09-21T10:20:30.009676+0000","last_clean":"2024-09-21T10:20:30.009676+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:20:30.009676+0000","last_undegraded":"2024-09-21T10:20:30.009676+0000","last_fullsized":"2024-09-21T10:20:30.009676+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":197,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980785+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:20:25.980785+0000","last_peered":"2024-09-21T10:20:25.980785+0000","last_clean":"2024-09-21T10:20:25.980785+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:20:25.980785+0000","last_undegraded":"2024-09-21T10:20:25.980785+0000","last_fullsized":"2024-09-21T10:20:25.980785+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"1341'16763","reported_seq":18697,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.840693+0000","last_change":"2024-09-21T10:20:22.059554+0000","last_active":"2024-09-21T10:20:31.840693+0000","last_peered":"2024-09-21T10:20:31.840693+0000","last_clean":"2024-09-21T10:20:31.840693+0000","last_became_active":"2024-09-21T10:20:21.983838+0000","last_became_peered":"2024-09-21T10:20:21.983838+0000","last_unstale":"2024-09-21T10:20:31.840693+0000","last_undegraded":"2024-09-21T10:20:31.840693+0000","last_fullsized":"2024-09-21T10:20:31.840693+0000","mapping_epoch":1335,"log_start":"1318'10900","ondisk_log_start":"1318'10900","created":1296,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5863,"log_dups_size":0,"ondisk_log_size":5863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:38:00.800155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7536640,"num_objects":115,"num_object_clones":0,"num_object_copies":345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16763,"num_write_kb":540096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":905,"num_bytes_recovered":59244544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":470,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013231+0000","last_change":"2024-09-21T10:16:38.454278+0000","last_active":"2024-09-21T10:20:30.013231+0000","last_peered":"2024-09-21T10:20:30.013231+0000","last_clean":"2024-09-21T10:20:30.013231+0000","last_became_active":"2024-09-21T10:16:38.449058+0000","last_became_peered":"2024-09-21T10:16:38.449058+0000","last_unstale":"2024-09-21T10:20:30.013231+0000","last_undegraded":"2024-09-21T10:20:30.013231+0000","last_fullsized":"2024-09-21T10:20:30.013231+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:02.332428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1866,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.010127+0000","last_change":"2024-09-21T10:16:38.460495+0000","last_active":"2024-09-21T10:20:30.010127+0000","last_peered":"2024-09-21T10:20:30.010127+0000","last_clean":"2024-09-21T10:20:30.010127+0000","last_became_active":"2024-09-21T10:16:38.458759+0000","last_became_peered":"2024-09-21T10:16:38.458759+0000","last_unstale":"2024-09-21T10:20:30.010127+0000","last_undegraded":"2024-09-21T10:20:30.010127+0000","last_fullsized":"2024-09-21T10:20:30.010127+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:37.031677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"1341'16901","reported_seq":16956,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.953758+0000","last_change":"2024-09-21T10:16:23.237256+0000","last_active":"2024-09-21T10:20:31.953758+0000","last_peered":"2024-09-21T10:20:31.953758+0000","last_clean":"2024-09-21T10:20:31.953758+0000","last_became_active":"2024-09-21T10:16:23.236945+0000","last_became_peered":"2024-09-21T10:16:23.236945+0000","last_unstale":"2024-09-21T10:20:31.953758+0000","last_undegraded":"2024-09-21T10:20:31.953758+0000","last_fullsized":"2024-09-21T10:20:31.953758+0000","mapping_epoch":1296,"log_start":"1321'11300","ondisk_log_start":"1321'11300","created":1296,"last_epoch_clean":1297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5601,"log_dups_size":0,"ondisk_log_size":5601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:50.314871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7274496,"num_objects":111,"num_object_clones":0,"num_object_copies":333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16901,"num_write_kb":544384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":894,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.979867+0000","last_change":"2024-09-21T10:15:22.139134+0000","last_active":"2024-09-21T10:20:25.979867+0000","last_peered":"2024-09-21T10:20:25.979867+0000","last_clean":"2024-09-21T10:20:25.979867+0000","last_became_active":"2024-09-21T10:15:22.138788+0000","last_became_peered":"2024-09-21T10:15:22.138788+0000","last_unstale":"2024-09-21T10:20:25.979867+0000","last_undegraded":"2024-09-21T10:20:25.979867+0000","last_fullsized":"2024-09-21T10:20:25.979867+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:35.831693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":48,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977651+0000","last_change":"2024-09-21T10:17:51.287937+0000","last_active":"2024-09-21T10:20:25.977651+0000","last_peered":"2024-09-21T10:20:25.977651+0000","last_clean":"2024-09-21T10:20:25.977651+0000","last_became_active":"2024-09-21T10:17:51.256610+0000","last_became_peered":"2024-09-21T10:17:51.256610+0000","last_unstale":"2024-09-21T10:20:25.977651+0000","last_undegraded":"2024-09-21T10:20:25.977651+0000","last_fullsized":"2024-09-21T10:20:25.977651+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:15.137072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":492,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977966+0000","last_change":"2024-09-21T10:18:07.086426+0000","last_active":"2024-09-21T10:20:25.977966+0000","last_peered":"2024-09-21T10:20:25.977966+0000","last_clean":"2024-09-21T10:20:25.977966+0000","last_became_active":"2024-09-21T10:18:07.086157+0000","last_became_peered":"2024-09-21T10:18:07.086157+0000","last_unstale":"2024-09-21T10:20:25.977966+0000","last_undegraded":"2024-09-21T10:20:25.977966+0000","last_fullsized":"2024-09-21T10:20:25.977966+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:24:34.289605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":222,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980555+0000","last_change":"2024-09-21T10:16:38.450231+0000","last_active":"2024-09-21T10:20:25.980555+0000","last_peered":"2024-09-21T10:20:25.980555+0000","last_clean":"2024-09-21T10:20:25.980555+0000","last_became_active":"2024-09-21T10:16:38.449941+0000","last_became_peered":"2024-09-21T10:16:38.449941+0000","last_unstale":"2024-09-21T10:20:25.980555+0000","last_undegraded":"2024-09-21T10:20:25.980555+0000","last_fullsized":"2024-09-21T10:20:25.980555+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:51.559013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"1341'16553","reported_seq":25790,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.374419+0000","last_change":"2024-09-21T10:19:32.344481+0000","last_active":"2024-09-21T10:20:31.374419+0000","last_peered":"2024-09-21T10:20:31.374419+0000","last_clean":"2024-09-21T10:20:31.374419+0000","last_became_active":"2024-09-21T10:19:32.294085+0000","last_became_peered":"2024-09-21T10:19:32.294085+0000","last_unstale":"2024-09-21T10:20:31.374419+0000","last_undegraded":"2024-09-21T10:20:31.374419+0000","last_fullsized":"2024-09-21T10:20:31.374419+0000","mapping_epoch":1324,"log_start":"1318'10500","ondisk_log_start":"1318'10500","created":1296,"last_epoch_clean":1325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:49:58.316719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9109504,"num_objects":139,"num_object_clones":0,"num_object_copies":417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16553,"num_write_kb":534144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4575,"num_bytes_recovered":299827200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1158,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009553+0000","last_change":"2024-09-21T10:17:51.264078+0000","last_active":"2024-09-21T10:20:30.009553+0000","last_peered":"2024-09-21T10:20:30.009553+0000","last_clean":"2024-09-21T10:20:30.009553+0000","last_became_active":"2024-09-21T10:17:51.263721+0000","last_became_peered":"2024-09-21T10:17:51.263721+0000","last_unstale":"2024-09-21T10:20:30.009553+0000","last_undegraded":"2024-09-21T10:20:30.009553+0000","last_fullsized":"2024-09-21T10:20:30.009553+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:46:02.104778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1647,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013568+0000","last_change":"2024-09-21T10:17:51.251859+0000","last_active":"2024-09-21T10:20:30.013568+0000","last_peered":"2024-09-21T10:20:30.013568+0000","last_clean":"2024-09-21T10:20:30.013568+0000","last_became_active":"2024-09-21T10:17:51.251525+0000","last_became_peered":"2024-09-21T10:17:51.251525+0000","last_unstale":"2024-09-21T10:20:30.013568+0000","last_undegraded":"2024-09-21T10:20:30.013568+0000","last_fullsized":"2024-09-21T10:20:30.013568+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:24.106509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1151,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009895+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:20:30.009895+0000","last_peered":"2024-09-21T10:20:30.009895+0000","last_clean":"2024-09-21T10:20:30.009895+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:20:30.009895+0000","last_undegraded":"2024-09-21T10:20:30.009895+0000","last_fullsized":"2024-09-21T10:20:30.009895+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"1341'17016","reported_seq":31913,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.401194+0000","last_change":"2024-09-21T10:18:54.245834+0000","last_active":"2024-09-21T10:20:31.401194+0000","last_peered":"2024-09-21T10:20:31.401194+0000","last_clean":"2024-09-21T10:20:31.401194+0000","last_became_active":"2024-09-21T10:18:54.227580+0000","last_became_peered":"2024-09-21T10:18:54.227580+0000","last_unstale":"2024-09-21T10:20:31.401194+0000","last_undegraded":"2024-09-21T10:20:31.401194+0000","last_fullsized":"2024-09-21T10:20:31.401194+0000","mapping_epoch":1319,"log_start":"1318'11000","ondisk_log_start":"1318'11000","created":1296,"last_epoch_clean":1320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":6016,"log_dups_size":0,"ondisk_log_size":6016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:07.095924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8126464,"num_objects":124,"num_object_clones":0,"num_object_copies":372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17016,"num_write_kb":548480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7408,"num_bytes_recovered":485490688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1737,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014172+0000","last_change":"2024-09-21T10:16:38.456464+0000","last_active":"2024-09-21T10:20:30.014172+0000","last_peered":"2024-09-21T10:20:30.014172+0000","last_clean":"2024-09-21T10:20:30.014172+0000","last_became_active":"2024-09-21T10:16:38.455345+0000","last_became_peered":"2024-09-21T10:16:38.455345+0000","last_unstale":"2024-09-21T10:20:30.014172+0000","last_undegraded":"2024-09-21T10:20:30.014172+0000","last_fullsized":"2024-09-21T10:20:30.014172+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:39.393023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1882,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.010055+0000","last_change":"2024-09-21T10:17:51.254069+0000","last_active":"2024-09-21T10:20:30.010055+0000","last_peered":"2024-09-21T10:20:30.010055+0000","last_clean":"2024-09-21T10:20:30.010055+0000","last_became_active":"2024-09-21T10:17:51.253723+0000","last_became_peered":"2024-09-21T10:17:51.253723+0000","last_unstale":"2024-09-21T10:20:30.010055+0000","last_undegraded":"2024-09-21T10:20:30.010055+0000","last_fullsized":"2024-09-21T10:20:30.010055+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:22.160096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1882,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009990+0000","last_change":"2024-09-21T10:17:51.249147+0000","last_active":"2024-09-21T10:20:30.009990+0000","last_peered":"2024-09-21T10:20:30.009990+0000","last_clean":"2024-09-21T10:20:30.009990+0000","last_became_active":"2024-09-21T10:17:51.248713+0000","last_became_peered":"2024-09-21T10:17:51.248713+0000","last_unstale":"2024-09-21T10:20:30.009990+0000","last_undegraded":"2024-09-21T10:20:30.009990+0000","last_fullsized":"2024-09-21T10:20:30.009990+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:33.757896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"1341'16818","reported_seq":17934,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.978929+0000","last_change":"2024-09-21T10:20:26.011676+0000","last_active":"2024-09-21T10:20:31.978929+0000","last_peered":"2024-09-21T10:20:31.978929+0000","last_clean":"2024-09-21T10:20:31.978929+0000","last_became_active":"2024-09-21T10:20:25.981646+0000","last_became_peered":"2024-09-21T10:20:25.981646+0000","last_unstale":"2024-09-21T10:20:31.978929+0000","last_undegraded":"2024-09-21T10:20:31.978929+0000","last_fullsized":"2024-09-21T10:20:31.978929+0000","mapping_epoch":1339,"log_start":"1320'11200","ondisk_log_start":"1320'11200","created":1296,"last_epoch_clean":1340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5618,"log_dups_size":0,"ondisk_log_size":5618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:17:28.457402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6684672,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16818,"num_write_kb":541440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":513,"num_bytes_recovered":33619968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":48,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977958+0000","last_change":"2024-09-21T10:17:51.286472+0000","last_active":"2024-09-21T10:20:25.977958+0000","last_peered":"2024-09-21T10:20:25.977958+0000","last_clean":"2024-09-21T10:20:25.977958+0000","last_became_active":"2024-09-21T10:17:51.249949+0000","last_became_peered":"2024-09-21T10:17:51.249949+0000","last_unstale":"2024-09-21T10:20:25.977958+0000","last_undegraded":"2024-09-21T10:20:25.977958+0000","last_fullsized":"2024-09-21T10:20:25.977958+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:31.760578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1309,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980004+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:20:25.980004+0000","last_peered":"2024-09-21T10:20:25.980004+0000","last_clean":"2024-09-21T10:20:25.980004+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:20:25.980004+0000","last_undegraded":"2024-09-21T10:20:25.980004+0000","last_fullsized":"2024-09-21T10:20:25.980004+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1823,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014022+0000","last_change":"2024-09-21T10:16:39.440143+0000","last_active":"2024-09-21T10:20:30.014022+0000","last_peered":"2024-09-21T10:20:30.014022+0000","last_clean":"2024-09-21T10:20:30.014022+0000","last_became_active":"2024-09-21T10:16:39.439370+0000","last_became_peered":"2024-09-21T10:16:39.439370+0000","last_unstale":"2024-09-21T10:20:30.014022+0000","last_undegraded":"2024-09-21T10:20:30.014022+0000","last_fullsized":"2024-09-21T10:20:30.014022+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:36.417381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"1341'16711","reported_seq":16802,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.712794+0000","last_change":"2024-09-21T10:16:38.458835+0000","last_active":"2024-09-21T10:20:31.712794+0000","last_peered":"2024-09-21T10:20:31.712794+0000","last_clean":"2024-09-21T10:20:31.712794+0000","last_became_active":"2024-09-21T10:16:38.451145+0000","last_became_peered":"2024-09-21T10:16:38.451145+0000","last_unstale":"2024-09-21T10:20:31.712794+0000","last_undegraded":"2024-09-21T10:20:31.712794+0000","last_fullsized":"2024-09-21T10:20:31.712794+0000","mapping_epoch":1301,"log_start":"1321'11500","ondisk_log_start":"1321'11500","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5211,"log_dups_size":0,"ondisk_log_size":5211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:12:14.844302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7012352,"num_objects":107,"num_object_clones":0,"num_object_copies":321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16711,"num_write_kb":538176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978204+0000","last_change":"2024-09-21T10:17:51.259438+0000","last_active":"2024-09-21T10:20:25.978204+0000","last_peered":"2024-09-21T10:20:25.978204+0000","last_clean":"2024-09-21T10:20:25.978204+0000","last_became_active":"2024-09-21T10:17:51.258704+0000","last_became_peered":"2024-09-21T10:17:51.258704+0000","last_unstale":"2024-09-21T10:20:25.978204+0000","last_undegraded":"2024-09-21T10:20:25.978204+0000","last_fullsized":"2024-09-21T10:20:25.978204+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:16.479749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1896,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009679+0000","last_change":"2024-09-21T10:16:38.460671+0000","last_active":"2024-09-21T10:20:30.009679+0000","last_peered":"2024-09-21T10:20:30.009679+0000","last_clean":"2024-09-21T10:20:30.009679+0000","last_became_active":"2024-09-21T10:16:38.459939+0000","last_became_peered":"2024-09-21T10:16:38.459939+0000","last_unstale":"2024-09-21T10:20:30.009679+0000","last_undegraded":"2024-09-21T10:20:30.009679+0000","last_fullsized":"2024-09-21T10:20:30.009679+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:51.143325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":422,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.010241+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:20:30.010241+0000","last_peered":"2024-09-21T10:20:30.010241+0000","last_clean":"2024-09-21T10:20:30.010241+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:20:30.010241+0000","last_undegraded":"2024-09-21T10:20:30.010241+0000","last_fullsized":"2024-09-21T10:20:30.010241+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"1341'16830","reported_seq":16892,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.989273+0000","last_change":"2024-09-21T10:16:38.446821+0000","last_active":"2024-09-21T10:20:31.989273+0000","last_peered":"2024-09-21T10:20:31.989273+0000","last_clean":"2024-09-21T10:20:31.989273+0000","last_became_active":"2024-09-21T10:16:38.446505+0000","last_became_peered":"2024-09-21T10:16:38.446505+0000","last_unstale":"2024-09-21T10:20:31.989273+0000","last_undegraded":"2024-09-21T10:20:31.989273+0000","last_fullsized":"2024-09-21T10:20:31.989273+0000","mapping_epoch":1301,"log_start":"1320'11200","ondisk_log_start":"1320'11200","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:09:38.183784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6881304,"num_objects":106,"num_object_clones":0,"num_object_copies":318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16830,"num_write_kb":541889,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013741+0000","last_change":"2024-09-21T10:17:51.250313+0000","last_active":"2024-09-21T10:20:30.013741+0000","last_peered":"2024-09-21T10:20:30.013741+0000","last_clean":"2024-09-21T10:20:30.013741+0000","last_became_active":"2024-09-21T10:17:51.249931+0000","last_became_peered":"2024-09-21T10:17:51.249931+0000","last_unstale":"2024-09-21T10:20:30.013741+0000","last_undegraded":"2024-09-21T10:20:30.013741+0000","last_fullsized":"2024-09-21T10:20:30.013741+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:38:15.879964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1036,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009675+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:20:30.009675+0000","last_peered":"2024-09-21T10:20:30.009675+0000","last_clean":"2024-09-21T10:20:30.009675+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:20:30.009675+0000","last_undegraded":"2024-09-21T10:20:30.009675+0000","last_fullsized":"2024-09-21T10:20:30.009675+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977729+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:20:25.977729+0000","last_peered":"2024-09-21T10:20:25.977729+0000","last_clean":"2024-09-21T10:20:25.977729+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:20:25.977729+0000","last_undegraded":"2024-09-21T10:20:25.977729+0000","last_fullsized":"2024-09-21T10:20:25.977729+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"1341'17067","reported_seq":20853,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.978589+0000","last_change":"2024-09-21T10:20:10.240181+0000","last_active":"2024-09-21T10:20:31.978589+0000","last_peered":"2024-09-21T10:20:31.978589+0000","last_clean":"2024-09-21T10:20:31.978589+0000","last_became_active":"2024-09-21T10:20:10.136698+0000","last_became_peered":"2024-09-21T10:20:10.136698+0000","last_unstale":"2024-09-21T10:20:31.978589+0000","last_undegraded":"2024-09-21T10:20:31.978589+0000","last_fullsized":"2024-09-21T10:20:31.978589+0000","mapping_epoch":1332,"log_start":"1321'11500","ondisk_log_start":"1321'11500","created":1296,"last_epoch_clean":1333,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5567,"log_dups_size":0,"ondisk_log_size":5567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:44.765170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7667712,"num_objects":117,"num_object_clones":0,"num_object_copies":351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17067,"num_write_kb":549888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1850,"num_bytes_recovered":121176064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":440,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978391+0000","last_change":"2024-09-21T10:18:07.090592+0000","last_active":"2024-09-21T10:20:25.978391+0000","last_peered":"2024-09-21T10:20:25.978391+0000","last_clean":"2024-09-21T10:20:25.978391+0000","last_became_active":"2024-09-21T10:18:07.090188+0000","last_became_peered":"2024-09-21T10:18:07.090188+0000","last_unstale":"2024-09-21T10:20:25.978391+0000","last_undegraded":"2024-09-21T10:20:25.978391+0000","last_fullsized":"2024-09-21T10:20:25.978391+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.763114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013469+0000","last_change":"2024-09-21T10:16:39.439344+0000","last_active":"2024-09-21T10:20:30.013469+0000","last_peered":"2024-09-21T10:20:30.013469+0000","last_clean":"2024-09-21T10:20:30.013469+0000","last_became_active":"2024-09-21T10:16:39.438987+0000","last_became_peered":"2024-09-21T10:16:39.438987+0000","last_unstale":"2024-09-21T10:20:30.013469+0000","last_undegraded":"2024-09-21T10:20:30.013469+0000","last_fullsized":"2024-09-21T10:20:30.013469+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:17.163136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":451,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013125+0000","last_change":"2024-09-21T10:17:51.250156+0000","last_active":"2024-09-21T10:20:30.013125+0000","last_peered":"2024-09-21T10:20:30.013125+0000","last_clean":"2024-09-21T10:20:30.013125+0000","last_became_active":"2024-09-21T10:17:51.248282+0000","last_became_peered":"2024-09-21T10:17:51.248282+0000","last_unstale":"2024-09-21T10:20:30.013125+0000","last_undegraded":"2024-09-21T10:20:30.013125+0000","last_fullsized":"2024-09-21T10:20:30.013125+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:58.954423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"1341'16772","reported_seq":16833,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:31.986280+0000","last_change":"2024-09-21T10:16:38.454005+0000","last_active":"2024-09-21T10:20:31.986280+0000","last_peered":"2024-09-21T10:20:31.986280+0000","last_clean":"2024-09-21T10:20:31.986280+0000","last_became_active":"2024-09-21T10:16:38.453756+0000","last_became_peered":"2024-09-21T10:16:38.453756+0000","last_unstale":"2024-09-21T10:20:31.986280+0000","last_undegraded":"2024-09-21T10:20:31.986280+0000","last_fullsized":"2024-09-21T10:20:31.986280+0000","mapping_epoch":1301,"log_start":"1320'11200","ondisk_log_start":"1320'11200","created":1296,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:16:21.384119+0000","last_clean_scrub_stamp":"2024-09-21T10:16:21.384119+0000","objects_scrubbed":0,"log_size":5572,"log_dups_size":0,"ondisk_log_size":5572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:30.976913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16772,"num_write_kb":539712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":407,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.010117+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:20:30.010117+0000","last_peered":"2024-09-21T10:20:30.010117+0000","last_clean":"2024-09-21T10:20:30.010117+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:20:30.010117+0000","last_undegraded":"2024-09-21T10:20:30.010117+0000","last_fullsized":"2024-09-21T10:20:30.010117+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":222,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.981137+0000","last_change":"2024-09-21T10:16:38.455089+0000","last_active":"2024-09-21T10:20:25.981137+0000","last_peered":"2024-09-21T10:20:25.981137+0000","last_clean":"2024-09-21T10:20:25.981137+0000","last_became_active":"2024-09-21T10:16:38.454804+0000","last_became_peered":"2024-09-21T10:16:38.454804+0000","last_unstale":"2024-09-21T10:20:25.981137+0000","last_undegraded":"2024-09-21T10:20:25.981137+0000","last_fullsized":"2024-09-21T10:20:25.981137+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:16.448016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977388+0000","last_change":"2024-09-21T10:17:51.250324+0000","last_active":"2024-09-21T10:20:25.977388+0000","last_peered":"2024-09-21T10:20:25.977388+0000","last_clean":"2024-09-21T10:20:25.977388+0000","last_became_active":"2024-09-21T10:17:51.249933+0000","last_became_peered":"2024-09-21T10:17:51.249933+0000","last_unstale":"2024-09-21T10:20:25.977388+0000","last_undegraded":"2024-09-21T10:20:25.977388+0000","last_fullsized":"2024-09-21T10:20:25.977388+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:23.880577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":197,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980119+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:20:25.980119+0000","last_peered":"2024-09-21T10:20:25.980119+0000","last_clean":"2024-09-21T10:20:25.980119+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:20:25.980119+0000","last_undegraded":"2024-09-21T10:20:25.980119+0000","last_fullsized":"2024-09-21T10:20:25.980119+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":260,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980374+0000","last_change":"2024-09-21T10:17:51.277615+0000","last_active":"2024-09-21T10:20:25.980374+0000","last_peered":"2024-09-21T10:20:25.980374+0000","last_clean":"2024-09-21T10:20:25.980374+0000","last_became_active":"2024-09-21T10:17:51.261756+0000","last_became_peered":"2024-09-21T10:17:51.261756+0000","last_unstale":"2024-09-21T10:20:25.980374+0000","last_undegraded":"2024-09-21T10:20:25.980374+0000","last_fullsized":"2024-09-21T10:20:25.980374+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:05:35.608372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1869,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.010387+0000","last_change":"2024-09-21T10:17:51.251440+0000","last_active":"2024-09-21T10:20:30.010387+0000","last_peered":"2024-09-21T10:20:30.010387+0000","last_clean":"2024-09-21T10:20:30.010387+0000","last_became_active":"2024-09-21T10:17:51.249109+0000","last_became_peered":"2024-09-21T10:17:51.249109+0000","last_unstale":"2024-09-21T10:20:30.010387+0000","last_undegraded":"2024-09-21T10:20:30.010387+0000","last_fullsized":"2024-09-21T10:20:30.010387+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:59.577984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1379,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014937+0000","last_change":"2024-09-21T10:16:38.453215+0000","last_active":"2024-09-21T10:20:30.014937+0000","last_peered":"2024-09-21T10:20:30.014937+0000","last_clean":"2024-09-21T10:20:30.014937+0000","last_became_active":"2024-09-21T10:16:38.452464+0000","last_became_peered":"2024-09-21T10:16:38.452464+0000","last_unstale":"2024-09-21T10:20:30.014937+0000","last_undegraded":"2024-09-21T10:20:30.014937+0000","last_fullsized":"2024-09-21T10:20:30.014937+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:16.818394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978977+0000","last_change":"2024-09-21T10:17:51.248299+0000","last_active":"2024-09-21T10:20:25.978977+0000","last_peered":"2024-09-21T10:20:25.978977+0000","last_clean":"2024-09-21T10:20:25.978977+0000","last_became_active":"2024-09-21T10:17:51.247851+0000","last_became_peered":"2024-09-21T10:17:51.247851+0000","last_unstale":"2024-09-21T10:20:25.978977+0000","last_undegraded":"2024-09-21T10:20:25.978977+0000","last_fullsized":"2024-09-21T10:20:25.978977+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:04.319143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":527,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013649+0000","last_change":"2024-09-21T10:17:51.248187+0000","last_active":"2024-09-21T10:20:30.013649+0000","last_peered":"2024-09-21T10:20:30.013649+0000","last_clean":"2024-09-21T10:20:30.013649+0000","last_became_active":"2024-09-21T10:17:51.247436+0000","last_became_peered":"2024-09-21T10:17:51.247436+0000","last_unstale":"2024-09-21T10:20:30.013649+0000","last_undegraded":"2024-09-21T10:20:30.013649+0000","last_fullsized":"2024-09-21T10:20:30.013649+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:33.505149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":44,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978528+0000","last_change":"2024-09-21T10:17:51.258577+0000","last_active":"2024-09-21T10:20:25.978528+0000","last_peered":"2024-09-21T10:20:25.978528+0000","last_clean":"2024-09-21T10:20:25.978528+0000","last_became_active":"2024-09-21T10:17:51.257820+0000","last_became_peered":"2024-09-21T10:17:51.257820+0000","last_unstale":"2024-09-21T10:20:25.978528+0000","last_undegraded":"2024-09-21T10:20:25.978528+0000","last_fullsized":"2024-09-21T10:20:25.978528+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:00.922957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":511,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013112+0000","last_change":"2024-09-21T10:16:38.456905+0000","last_active":"2024-09-21T10:20:30.013112+0000","last_peered":"2024-09-21T10:20:30.013112+0000","last_clean":"2024-09-21T10:20:30.013112+0000","last_became_active":"2024-09-21T10:16:38.453276+0000","last_became_peered":"2024-09-21T10:16:38.453276+0000","last_unstale":"2024-09-21T10:20:30.013112+0000","last_undegraded":"2024-09-21T10:20:30.013112+0000","last_fullsized":"2024-09-21T10:20:30.013112+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:38.174569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":240,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.981032+0000","last_change":"2024-09-21T10:17:51.262911+0000","last_active":"2024-09-21T10:20:25.981032+0000","last_peered":"2024-09-21T10:20:25.981032+0000","last_clean":"2024-09-21T10:20:25.981032+0000","last_became_active":"2024-09-21T10:17:51.262314+0000","last_became_peered":"2024-09-21T10:17:51.262314+0000","last_unstale":"2024-09-21T10:20:25.981032+0000","last_undegraded":"2024-09-21T10:20:25.981032+0000","last_fullsized":"2024-09-21T10:20:25.981032+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:06.241585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":499,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013472+0000","last_change":"2024-09-21T10:17:51.254289+0000","last_active":"2024-09-21T10:20:30.013472+0000","last_peered":"2024-09-21T10:20:30.013472+0000","last_clean":"2024-09-21T10:20:30.013472+0000","last_became_active":"2024-09-21T10:17:51.253922+0000","last_became_peered":"2024-09-21T10:17:51.253922+0000","last_unstale":"2024-09-21T10:20:30.013472+0000","last_undegraded":"2024-09-21T10:20:30.013472+0000","last_fullsized":"2024-09-21T10:20:30.013472+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:57.111361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1762,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.015109+0000","last_change":"2024-09-21T10:17:51.249960+0000","last_active":"2024-09-21T10:20:30.015109+0000","last_peered":"2024-09-21T10:20:30.015109+0000","last_clean":"2024-09-21T10:20:30.015109+0000","last_became_active":"2024-09-21T10:17:51.248883+0000","last_became_peered":"2024-09-21T10:17:51.248883+0000","last_unstale":"2024-09-21T10:20:30.015109+0000","last_undegraded":"2024-09-21T10:20:30.015109+0000","last_fullsized":"2024-09-21T10:20:30.015109+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:20.076539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":155,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009725+0000","last_change":"2024-09-21T10:16:38.461877+0000","last_active":"2024-09-21T10:20:30.009725+0000","last_peered":"2024-09-21T10:20:30.009725+0000","last_clean":"2024-09-21T10:20:30.009725+0000","last_became_active":"2024-09-21T10:16:38.461554+0000","last_became_peered":"2024-09-21T10:16:38.461554+0000","last_unstale":"2024-09-21T10:20:30.009725+0000","last_undegraded":"2024-09-21T10:20:30.009725+0000","last_fullsized":"2024-09-21T10:20:30.009725+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:46.372637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":439,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014896+0000","last_change":"2024-09-21T10:16:38.447369+0000","last_active":"2024-09-21T10:20:30.014896+0000","last_peered":"2024-09-21T10:20:30.014896+0000","last_clean":"2024-09-21T10:20:30.014896+0000","last_became_active":"2024-09-21T10:16:38.447085+0000","last_became_peered":"2024-09-21T10:16:38.447085+0000","last_unstale":"2024-09-21T10:20:30.014896+0000","last_undegraded":"2024-09-21T10:20:30.014896+0000","last_fullsized":"2024-09-21T10:20:30.014896+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:59.110578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":426,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.010444+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:20:30.010444+0000","last_peered":"2024-09-21T10:20:30.010444+0000","last_clean":"2024-09-21T10:20:30.010444+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:20:30.010444+0000","last_undegraded":"2024-09-21T10:20:30.010444+0000","last_fullsized":"2024-09-21T10:20:30.010444+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1755,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.009820+0000","last_change":"2024-09-21T10:16:38.460957+0000","last_active":"2024-09-21T10:20:30.009820+0000","last_peered":"2024-09-21T10:20:30.009820+0000","last_clean":"2024-09-21T10:20:30.009820+0000","last_became_active":"2024-09-21T10:16:38.460325+0000","last_became_peered":"2024-09-21T10:16:38.460325+0000","last_unstale":"2024-09-21T10:20:30.009820+0000","last_undegraded":"2024-09-21T10:20:30.009820+0000","last_fullsized":"2024-09-21T10:20:30.009820+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:34.529895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1829,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.011189+0000","last_change":"2024-09-21T10:16:38.461576+0000","last_active":"2024-09-21T10:20:30.011189+0000","last_peered":"2024-09-21T10:20:30.011189+0000","last_clean":"2024-09-21T10:20:30.011189+0000","last_became_active":"2024-09-21T10:16:38.461205+0000","last_became_peered":"2024-09-21T10:16:38.461205+0000","last_unstale":"2024-09-21T10:20:30.011189+0000","last_undegraded":"2024-09-21T10:20:30.011189+0000","last_fullsized":"2024-09-21T10:20:30.011189+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:17.166658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1421,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.013684+0000","last_change":"2024-09-21T10:16:38.459378+0000","last_active":"2024-09-21T10:20:30.013684+0000","last_peered":"2024-09-21T10:20:30.013684+0000","last_clean":"2024-09-21T10:20:30.013684+0000","last_became_active":"2024-09-21T10:16:38.451582+0000","last_became_peered":"2024-09-21T10:16:38.451582+0000","last_unstale":"2024-09-21T10:20:30.013684+0000","last_undegraded":"2024-09-21T10:20:30.013684+0000","last_fullsized":"2024-09-21T10:20:30.013684+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:34.820762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":48,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.977561+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:20:25.977561+0000","last_peered":"2024-09-21T10:20:25.977561+0000","last_clean":"2024-09-21T10:20:25.977561+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:20:25.977561+0000","last_undegraded":"2024-09-21T10:20:25.977561+0000","last_fullsized":"2024-09-21T10:20:25.977561+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1438,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.015603+0000","last_change":"2024-09-21T10:16:38.453928+0000","last_active":"2024-09-21T10:20:30.015603+0000","last_peered":"2024-09-21T10:20:30.015603+0000","last_clean":"2024-09-21T10:20:30.015603+0000","last_became_active":"2024-09-21T10:16:38.450890+0000","last_became_peered":"2024-09-21T10:16:38.450890+0000","last_unstale":"2024-09-21T10:20:30.015603+0000","last_undegraded":"2024-09-21T10:20:30.015603+0000","last_fullsized":"2024-09-21T10:20:30.015603+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:13.500644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":48,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978053+0000","last_change":"2024-09-21T10:17:51.281734+0000","last_active":"2024-09-21T10:20:25.978053+0000","last_peered":"2024-09-21T10:20:25.978053+0000","last_clean":"2024-09-21T10:20:25.978053+0000","last_became_active":"2024-09-21T10:17:51.258206+0000","last_became_peered":"2024-09-21T10:17:51.258206+0000","last_unstale":"2024-09-21T10:20:25.978053+0000","last_undegraded":"2024-09-21T10:20:25.978053+0000","last_fullsized":"2024-09-21T10:20:25.978053+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:16.184203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":499,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.014076+0000","last_change":"2024-09-21T10:17:51.251289+0000","last_active":"2024-09-21T10:20:30.014076+0000","last_peered":"2024-09-21T10:20:30.014076+0000","last_clean":"2024-09-21T10:20:30.014076+0000","last_became_active":"2024-09-21T10:17:51.247868+0000","last_became_peered":"2024-09-21T10:17:51.247868+0000","last_unstale":"2024-09-21T10:20:30.014076+0000","last_undegraded":"2024-09-21T10:20:30.014076+0000","last_fullsized":"2024-09-21T10:20:30.014076+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:39:49.784298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":501,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.015195+0000","last_change":"2024-09-21T10:16:38.451567+0000","last_active":"2024-09-21T10:20:30.015195+0000","last_peered":"2024-09-21T10:20:30.015195+0000","last_clean":"2024-09-21T10:20:30.015195+0000","last_became_active":"2024-09-21T10:16:38.451245+0000","last_became_peered":"2024-09-21T10:16:38.451245+0000","last_unstale":"2024-09-21T10:20:30.015195+0000","last_undegraded":"2024-09-21T10:20:30.015195+0000","last_fullsized":"2024-09-21T10:20:30.015195+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:27.022083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":206,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.980004+0000","last_change":"2024-09-21T10:15:19.115911+0000","last_active":"2024-09-21T10:20:25.980004+0000","last_peered":"2024-09-21T10:20:25.980004+0000","last_clean":"2024-09-21T10:20:25.980004+0000","last_became_active":"2024-09-21T10:15:19.115615+0000","last_became_peered":"2024-09-21T10:15:19.115615+0000","last_unstale":"2024-09-21T10:20:25.980004+0000","last_undegraded":"2024-09-21T10:20:25.980004+0000","last_fullsized":"2024-09-21T10:20:25.980004+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":50,"reported_epoch":1340,"state":"active+clean","last_fresh":"2024-09-21T10:20:25.978576+0000","last_change":"2024-09-21T10:17:51.280012+0000","last_active":"2024-09-21T10:20:25.978576+0000","last_peered":"2024-09-21T10:20:25.978576+0000","last_clean":"2024-09-21T10:20:25.978576+0000","last_became_active":"2024-09-21T10:17:51.257359+0000","last_became_peered":"2024-09-21T10:17:51.257359+0000","last_unstale":"2024-09-21T10:20:25.978576+0000","last_undegraded":"2024-09-21T10:20:25.978576+0000","last_fullsized":"2024-09-21T10:20:25.978576+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:44.522556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":123469848,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":268469,"num_write_kb":8651265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20906,"num_bytes_recovered":1369964544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":632827904,"data_stored":632815688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90569,"ondisk_log_size":90569,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":21092,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17031,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":6}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234536,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496172,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1475544,"kb_used_data":306892,"kb_used_omap":1,"kb_used_meta":1168638,"kb_avail":92269608,"statfs":{"total":95995035648,"available":94484078592,"internally_reserved":0,"allocated":314257408,"data_stored":310973974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1196685767},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757811,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":527048,"kb_used_data":98944,"kb_used_omap":1,"kb_used_meta":428094,"kb_avail":93218104,"statfs":{"total":95995035648,"available":95455338496,"internally_reserved":0,"allocated":101318656,"data_stored":98046840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":438368710},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019447,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1225620,"kb_used_data":189576,"kb_used_omap":1,"kb_used_meta":1036030,"kb_avail":92519532,"statfs":{"total":95995035648,"available":94740000768,"internally_reserved":0,"allocated":194125824,"data_stored":190846419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1060895174},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,4,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":1301,"seq":5587752452144,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":507640,"kb_used_data":124720,"kb_used_omap":48,"kb_used_meta":382863,"kb_avail":93237512,"statfs":{"total":95995035648,"available":95475212288,"internally_reserved":0,"allocated":127713280,"data_stored":124452595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49883,"internal_metadata":392052005},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793967982,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047221,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1172028,"kb_used_data":60084,"kb_used_omap":41,"kb_used_meta":1111894,"kb_avail":92573124,"statfs":{"total":95995035648,"available":94794878976,"internally_reserved":0,"allocated":61526016,"data_stored":58261670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1138580393},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928855,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7559,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":43450368,"data_stored":43450368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":82644992,"data_stored":82640920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":134287360,"data_stored":134283288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":85524480,"data_stored":85524480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":286920704,"data_stored":286916632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:20:38.887 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:38.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:38.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:39.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:39.039 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:39.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:39.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:39.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:39.268 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:39.290 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:39.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:39.413 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:39.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:39.540 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:39.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:39.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:39.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:39.769 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:39.791 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:39.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:39.893 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:39.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:39.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:39.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:40.015 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:40.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:40 smithi076 ceph-mon[106482]: pgmap v2125: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 714 op/s 2024-09-21T10:20:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:40 smithi076 ceph-mon[110963]: pgmap v2125: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 714 op/s 2024-09-21T10:20:40.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:40.165 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:40.267 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:40.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:40.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:40 smithi191 ceph-mon[102206]: pgmap v2125: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 714 op/s 2024-09-21T10:20:40.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.404 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:40.416 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.517 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:40.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:40.645 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:40.666 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:40.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:40.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:40.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:40.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:40.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:41.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:41.040 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:41.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:41.162 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:41.264 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.286 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:41.388 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:41.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:41.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:41.534 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:41.580 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_5' removed 2024-09-21T10:20:41.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:41.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:41.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:41.760 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2188491155' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[106482]: pgmap v2126: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 666 op/s 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2188491155' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:20:41.761 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:41 smithi076 ceph-mon[110963]: pgmap v2126: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 666 op/s 2024-09-21T10:20:41.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:41 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2188491155' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:20:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:41 smithi191 ceph-mon[102206]: pgmap v2126: 89 pgs: 89 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 666 op/s 2024-09-21T10:20:41.881 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:20:41.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:41.887 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:20:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:41.912 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:41.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:42.013 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:42.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:42.129 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:42.129 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:42.141 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:42.162 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:42.264 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:42.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:42.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:42.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:42.408 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:42.509 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:42.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:42.633 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:42.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:42.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:42.776 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:42.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:42.850 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2188491155' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:20:42.850 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:42 smithi191 ceph-mon[102206]: osdmap e1342: 8 total, 8 up, 5 in 2024-09-21T10:20:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2188491155' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:20:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:42 smithi076 ceph-mon[106482]: osdmap e1342: 8 total, 8 up, 5 in 2024-09-21T10:20:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2188491155' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:20:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:42 smithi076 ceph-mon[110963]: osdmap e1342: 8 total, 8 up, 5 in 2024-09-21T10:20:42.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:20:42.878 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:42.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:42.901 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:42.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:43.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:43.023 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:43.043 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:43.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:43.148 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:43.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:43.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:43.271 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:43.280 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:43.372 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:43.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:43.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:43.497 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:43.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:43.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:43.586 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:20:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:20:43] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-21T10:20:43.616 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-21T10:20:43.617 INFO:tasks.radosbench:Beginning radosbench... 2024-09-21T10:20:43.617 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-09-21T10:20:43.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_6 16 2024-09-21T10:20:43.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:43.681 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:43.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:43.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:43.842 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:43 smithi076 ceph-mon[106482]: pgmap v2128: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:43.842 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:43 smithi076 ceph-mon[110963]: pgmap v2128: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:43.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:43 smithi191 ceph-mon[102206]: pgmap v2128: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:43.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:43.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:43.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:43.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:44.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:44.050 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:44.151 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:44.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:44.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:44.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:44.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:44.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:44.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:44.519 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:44.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:44.642 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:44.663 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:44.762 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:44.765 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:44.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:44.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:44.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:44.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:44.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:44.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:45.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.034 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.156 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:45.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.242 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:20:45.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:45.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:45.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:45.381 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:45.391 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:20:45.391 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:20:45.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:45.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:45.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:45.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3519086614' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:45.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3519086614' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:45.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3519086614' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:20:45.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:45.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:45.752 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:45.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:45.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:45.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:45.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:45.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:45.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:45.999 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:46.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:46.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:46.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:46.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:46.246 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:46.268 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:46.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:46.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:46.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:46.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:20:46.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:46.517 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:46.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:46.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[106482]: pgmap v2129: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:46.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:46.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3519086614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:46.528 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[106482]: osdmap e1343: 8 total, 8 up, 5 in 2024-09-21T10:20:46.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[110963]: pgmap v2129: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:46.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:46.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3519086614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:46.529 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:46 smithi076 ceph-mon[110963]: osdmap e1343: 8 total, 8 up, 5 in 2024-09-21T10:20:46.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:46 smithi191 ceph-mon[102206]: pgmap v2129: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:46.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:46 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:46.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:46 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3519086614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:20:46.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:46 smithi191 ceph-mon[102206]: osdmap e1343: 8 total, 8 up, 5 in 2024-09-21T10:20:46.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:46.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:46.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:46.743 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:46.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:46.866 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:46.889 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:46.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:46.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:47.002 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:47.012 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:47.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:47.136 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:47.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:47.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:47.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:47.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:47.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:47.505 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:47.607 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:47.628 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:47.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:47.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:47.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:47.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:47.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:47.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:20:47.997 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:47.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:47.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:48.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:48.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:48.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:48.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:48.347 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:48.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:48 smithi076 ceph-mon[106482]: pgmap v2131: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:48.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:48 smithi076 ceph-mon[110963]: pgmap v2131: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:48.353 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:20:48.353 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:48.368 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:20:48.368 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:48.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:48.384 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:20:48.384 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:48.400 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:20:48.400 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:48.417 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:20:48.417 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:48.433 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:20:48.433 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:48.449 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:20:48.449 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:48.465 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:20:48.465 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:48.472 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:48.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:48.594 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:48.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:48 smithi191 ceph-mon[102206]: pgmap v2131: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:48.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:48.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:48.738 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:48.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:48.840 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:48.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:48.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:48.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:48.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:49.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:49.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:49.209 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:49.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.231 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:49.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:49.332 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:49.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:49 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1632258793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:49.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.479 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:49.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:49.502 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:49 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1632258793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:49.502 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:49 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1632258793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:49.580 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:49.583 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:49.583 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:20:49.583 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:20:49.583 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:20:49.583 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:49.602 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:49.704 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:49.726 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:49.827 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:49.848 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:49.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:49.972 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:49.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:50.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:50.094 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:50.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:50.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:20:50.196 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:50.217 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:50.319 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:50.341 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:50.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:50.446 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:50 smithi076 ceph-mon[106482]: pgmap v2132: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:50.446 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:50 smithi076 ceph-mon[110963]: pgmap v2132: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:50.465 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:50.515 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:50 smithi191 ceph-mon[102206]: pgmap v2132: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:50.567 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:50.588 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:50.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:50.713 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:50.720 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:20:50.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:50.836 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:50.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:50.937 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:50.959 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:50.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:51.061 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:51.069 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:20:51.069 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:20:51.083 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:51.184 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:51.205 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:51.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:51.256 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_6' created 2024-09-21T10:20:51.307 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:51.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:51.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3633047368' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:20:51.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:20:51.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3633047368' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:20:51.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:20:51.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:51.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:51.553 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:51.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3633047368' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:20:51.554 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-21T10:20:51.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:51.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:51.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:51.698 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:51.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:51.822 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:51.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:51.923 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:51.944 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:51.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:52.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:52.067 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:52.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:52.168 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:52.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:52.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:52.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:52.314 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:52.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:52.416 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:52.437 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:52.525 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:52 smithi191 ceph-mon[102206]: pgmap v2133: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:52.525 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:52 smithi191 ceph-mon[102206]: from='client.46073 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:52.526 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:52 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-21T10:20:52.526 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:52 smithi191 ceph-mon[102206]: osdmap e1344: 8 total, 8 up, 5 in 2024-09-21T10:20:52.539 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:52.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[106482]: pgmap v2133: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[106482]: from='client.46073 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[106482]: osdmap e1344: 8 total, 8 up, 5 in 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[110963]: pgmap v2133: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[110963]: from='client.46073 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-21T10:20:52.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:52 smithi076 ceph-mon[110963]: osdmap e1344: 8 total, 8 up, 5 in 2024-09-21T10:20:52.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:52.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:52.785 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:52.799 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-09-21T10:20:52.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:52.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:52.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:52.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:52.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:52.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.030 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:53.052 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:53.175 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:53.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:53.299 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:53.398 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:53 smithi191 ceph-mon[102206]: osdmap e1345: 8 total, 8 up, 5 in 2024-09-21T10:20:53.401 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:53.422 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:53.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:20:53.523 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:53.542 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:53 smithi076 ceph-mon[106482]: osdmap e1345: 8 total, 8 up, 5 in 2024-09-21T10:20:53.542 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:20:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:20:53] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-09-21T10:20:53.542 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:53 smithi076 ceph-mon[110963]: osdmap e1345: 8 total, 8 up, 5 in 2024-09-21T10:20:53.545 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.632 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:53.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:53.652 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2133,"stamp":"2024-09-21T10:20:50.739695+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":3563357,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":228,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4342848,"kb_used_data":210772,"kb_used_omap":94,"kb_used_meta":4131745,"kb_avail":464382912,"statfs":{"total":479975178240,"available":475528101888,"internally_reserved":0,"allocated":215830528,"data_stored":199493039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96735,"internal_metadata":4230907425},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":32,"apply_latency_ms":32,"commit_latency_ns":32000000,"apply_latency_ns":32000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.159869"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":503,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332202+0000","last_change":"2024-09-21T10:17:51.249951+0000","last_active":"2024-09-21T10:20:45.332202+0000","last_peered":"2024-09-21T10:20:45.332202+0000","last_clean":"2024-09-21T10:20:45.332202+0000","last_became_active":"2024-09-21T10:17:51.249158+0000","last_became_peered":"2024-09-21T10:17:51.249158+0000","last_unstale":"2024-09-21T10:20:45.332202+0000","last_undegraded":"2024-09-21T10:20:45.332202+0000","last_fullsized":"2024-09-21T10:20:45.332202+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:53.769122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":220,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.846550+0000","last_change":"2024-09-21T10:15:19.141623+0000","last_active":"2024-09-21T10:20:41.846550+0000","last_peered":"2024-09-21T10:20:41.846550+0000","last_clean":"2024-09-21T10:20:41.846550+0000","last_became_active":"2024-09-21T10:15:19.111927+0000","last_became_peered":"2024-09-21T10:15:19.111927+0000","last_unstale":"2024-09-21T10:20:41.846550+0000","last_undegraded":"2024-09-21T10:20:41.846550+0000","last_fullsized":"2024-09-21T10:20:41.846550+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:10.426159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025725+0000","last_change":"2024-09-21T10:17:51.258024+0000","last_active":"2024-09-21T10:20:30.025725+0000","last_peered":"2024-09-21T10:20:30.025725+0000","last_clean":"2024-09-21T10:20:30.025725+0000","last_became_active":"2024-09-21T10:17:51.257639+0000","last_became_peered":"2024-09-21T10:17:51.257639+0000","last_unstale":"2024-09-21T10:20:30.025725+0000","last_undegraded":"2024-09-21T10:20:30.025725+0000","last_fullsized":"2024-09-21T10:20:30.025725+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:09.112335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1820,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331993+0000","last_change":"2024-09-21T10:17:51.250405+0000","last_active":"2024-09-21T10:20:45.331993+0000","last_peered":"2024-09-21T10:20:45.331993+0000","last_clean":"2024-09-21T10:20:45.331993+0000","last_became_active":"2024-09-21T10:17:51.248030+0000","last_became_peered":"2024-09-21T10:17:51.248030+0000","last_unstale":"2024-09-21T10:20:45.331993+0000","last_undegraded":"2024-09-21T10:20:45.331993+0000","last_fullsized":"2024-09-21T10:20:45.331993+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:45.889214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1819,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332451+0000","last_change":"2024-09-21T10:17:51.249710+0000","last_active":"2024-09-21T10:20:45.332451+0000","last_peered":"2024-09-21T10:20:45.332451+0000","last_clean":"2024-09-21T10:20:45.332451+0000","last_became_active":"2024-09-21T10:17:51.249284+0000","last_became_peered":"2024-09-21T10:17:51.249284+0000","last_unstale":"2024-09-21T10:20:45.332451+0000","last_undegraded":"2024-09-21T10:20:45.332451+0000","last_fullsized":"2024-09-21T10:20:45.332451+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:09.927717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":223,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.844970+0000","last_change":"2024-09-21T10:16:38.451401+0000","last_active":"2024-09-21T10:20:41.844970+0000","last_peered":"2024-09-21T10:20:41.844970+0000","last_clean":"2024-09-21T10:20:41.844970+0000","last_became_active":"2024-09-21T10:16:38.450094+0000","last_became_peered":"2024-09-21T10:16:38.450094+0000","last_unstale":"2024-09-21T10:20:41.844970+0000","last_undegraded":"2024-09-21T10:20:41.844970+0000","last_fullsized":"2024-09-21T10:20:41.844970+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:22:50.798278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":496,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332587+0000","last_change":"2024-09-21T10:16:38.451461+0000","last_active":"2024-09-21T10:20:45.332587+0000","last_peered":"2024-09-21T10:20:45.332587+0000","last_clean":"2024-09-21T10:20:45.332587+0000","last_became_active":"2024-09-21T10:16:38.451137+0000","last_became_peered":"2024-09-21T10:16:38.451137+0000","last_unstale":"2024-09-21T10:20:45.332587+0000","last_undegraded":"2024-09-21T10:20:45.332587+0000","last_fullsized":"2024-09-21T10:20:45.332587+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:40.847463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":281,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.844113+0000","last_change":"2024-09-21T10:17:51.269584+0000","last_active":"2024-09-21T10:20:41.844113+0000","last_peered":"2024-09-21T10:20:41.844113+0000","last_clean":"2024-09-21T10:20:41.844113+0000","last_became_active":"2024-09-21T10:17:51.260673+0000","last_became_peered":"2024-09-21T10:17:51.260673+0000","last_unstale":"2024-09-21T10:20:41.844113+0000","last_undegraded":"2024-09-21T10:20:41.844113+0000","last_fullsized":"2024-09-21T10:20:41.844113+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:16.649818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":406,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687232+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:20:46.687232+0000","last_peered":"2024-09-21T10:20:46.687232+0000","last_clean":"2024-09-21T10:20:46.687232+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:20:46.687232+0000","last_undegraded":"2024-09-21T10:20:46.687232+0000","last_fullsized":"2024-09-21T10:20:46.687232+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.022784+0000","last_change":"2024-09-21T10:17:51.256335+0000","last_active":"2024-09-21T10:20:30.022784+0000","last_peered":"2024-09-21T10:20:30.022784+0000","last_clean":"2024-09-21T10:20:30.022784+0000","last_became_active":"2024-09-21T10:17:51.251630+0000","last_became_peered":"2024-09-21T10:17:51.251630+0000","last_unstale":"2024-09-21T10:20:30.022784+0000","last_undegraded":"2024-09-21T10:20:30.022784+0000","last_fullsized":"2024-09-21T10:20:30.022784+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:31.141409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":95,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.859026+0000","last_change":"2024-09-21T10:17:51.281550+0000","last_active":"2024-09-21T10:20:41.859026+0000","last_peered":"2024-09-21T10:20:41.859026+0000","last_clean":"2024-09-21T10:20:41.859026+0000","last_became_active":"2024-09-21T10:17:51.262457+0000","last_became_peered":"2024-09-21T10:17:51.262457+0000","last_unstale":"2024-09-21T10:20:41.859026+0000","last_undegraded":"2024-09-21T10:20:41.859026+0000","last_fullsized":"2024-09-21T10:20:41.859026+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:49.212195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1480,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686288+0000","last_change":"2024-09-21T10:16:38.461867+0000","last_active":"2024-09-21T10:20:46.686288+0000","last_peered":"2024-09-21T10:20:46.686288+0000","last_clean":"2024-09-21T10:20:46.686288+0000","last_became_active":"2024-09-21T10:16:38.461416+0000","last_became_peered":"2024-09-21T10:16:38.461416+0000","last_unstale":"2024-09-21T10:20:46.686288+0000","last_undegraded":"2024-09-21T10:20:46.686288+0000","last_fullsized":"2024-09-21T10:20:46.686288+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:15.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":49,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025886+0000","last_change":"2024-09-21T10:17:51.283569+0000","last_active":"2024-09-21T10:20:30.025886+0000","last_peered":"2024-09-21T10:20:30.025886+0000","last_clean":"2024-09-21T10:20:30.025886+0000","last_became_active":"2024-09-21T10:17:51.259187+0000","last_became_peered":"2024-09-21T10:17:51.259187+0000","last_unstale":"2024-09-21T10:20:30.025886+0000","last_undegraded":"2024-09-21T10:20:30.025886+0000","last_fullsized":"2024-09-21T10:20:30.025886+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:55.002645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":233,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.845663+0000","last_change":"2024-09-21T10:17:51.248805+0000","last_active":"2024-09-21T10:20:41.845663+0000","last_peered":"2024-09-21T10:20:41.845663+0000","last_clean":"2024-09-21T10:20:41.845663+0000","last_became_active":"2024-09-21T10:17:51.248157+0000","last_became_peered":"2024-09-21T10:17:51.248157+0000","last_unstale":"2024-09-21T10:20:41.845663+0000","last_undegraded":"2024-09-21T10:20:41.845663+0000","last_fullsized":"2024-09-21T10:20:41.845663+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:36.573842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025537+0000","last_change":"2024-09-21T10:17:51.259331+0000","last_active":"2024-09-21T10:20:30.025537+0000","last_peered":"2024-09-21T10:20:30.025537+0000","last_clean":"2024-09-21T10:20:30.025537+0000","last_became_active":"2024-09-21T10:17:51.258943+0000","last_became_peered":"2024-09-21T10:17:51.258943+0000","last_unstale":"2024-09-21T10:20:30.025537+0000","last_undegraded":"2024-09-21T10:20:30.025537+0000","last_fullsized":"2024-09-21T10:20:30.025537+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:59.304035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1866,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686622+0000","last_change":"2024-09-21T10:16:38.461644+0000","last_active":"2024-09-21T10:20:46.686622+0000","last_peered":"2024-09-21T10:20:46.686622+0000","last_clean":"2024-09-21T10:20:46.686622+0000","last_became_active":"2024-09-21T10:16:38.461179+0000","last_became_peered":"2024-09-21T10:16:38.461179+0000","last_unstale":"2024-09-21T10:20:46.686622+0000","last_undegraded":"2024-09-21T10:20:46.686622+0000","last_fullsized":"2024-09-21T10:20:46.686622+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:43.638430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":449,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332118+0000","last_change":"2024-09-21T10:16:38.459890+0000","last_active":"2024-09-21T10:20:45.332118+0000","last_peered":"2024-09-21T10:20:45.332118+0000","last_clean":"2024-09-21T10:20:45.332118+0000","last_became_active":"2024-09-21T10:16:38.454382+0000","last_became_peered":"2024-09-21T10:16:38.454382+0000","last_unstale":"2024-09-21T10:20:45.332118+0000","last_undegraded":"2024-09-21T10:20:45.332118+0000","last_fullsized":"2024-09-21T10:20:45.332118+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:02.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1493,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332399+0000","last_change":"2024-09-21T10:16:38.454504+0000","last_active":"2024-09-21T10:20:45.332399+0000","last_peered":"2024-09-21T10:20:45.332399+0000","last_clean":"2024-09-21T10:20:45.332399+0000","last_became_active":"2024-09-21T10:16:38.448833+0000","last_became_peered":"2024-09-21T10:16:38.448833+0000","last_unstale":"2024-09-21T10:20:45.332399+0000","last_undegraded":"2024-09-21T10:20:45.332399+0000","last_fullsized":"2024-09-21T10:20:45.332399+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:11.905645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.845535+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:20:41.845535+0000","last_peered":"2024-09-21T10:20:41.845535+0000","last_clean":"2024-09-21T10:20:41.845535+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:20:41.845535+0000","last_undegraded":"2024-09-21T10:20:41.845535+0000","last_fullsized":"2024-09-21T10:20:41.845535+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":93,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332398+0000","last_change":"2024-09-21T10:16:39.439192+0000","last_active":"2024-09-21T10:20:45.332398+0000","last_peered":"2024-09-21T10:20:45.332398+0000","last_clean":"2024-09-21T10:20:45.332398+0000","last_became_active":"2024-09-21T10:16:39.438778+0000","last_became_peered":"2024-09-21T10:16:39.438778+0000","last_unstale":"2024-09-21T10:20:45.332398+0000","last_undegraded":"2024-09-21T10:20:45.332398+0000","last_fullsized":"2024-09-21T10:20:45.332398+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:49.643063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1525,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687065+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:20:46.687065+0000","last_peered":"2024-09-21T10:20:46.687065+0000","last_clean":"2024-09-21T10:20:46.687065+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:20:46.687065+0000","last_undegraded":"2024-09-21T10:20:46.687065+0000","last_fullsized":"2024-09-21T10:20:46.687065+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":200,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.581812+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:20:46.581812+0000","last_peered":"2024-09-21T10:20:46.581812+0000","last_clean":"2024-09-21T10:20:46.581812+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:20:46.581812+0000","last_undegraded":"2024-09-21T10:20:46.581812+0000","last_fullsized":"2024-09-21T10:20:46.581812+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":474,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331940+0000","last_change":"2024-09-21T10:16:38.454278+0000","last_active":"2024-09-21T10:20:45.331940+0000","last_peered":"2024-09-21T10:20:45.331940+0000","last_clean":"2024-09-21T10:20:45.331940+0000","last_became_active":"2024-09-21T10:16:38.449058+0000","last_became_peered":"2024-09-21T10:16:38.449058+0000","last_unstale":"2024-09-21T10:20:45.331940+0000","last_undegraded":"2024-09-21T10:20:45.331940+0000","last_fullsized":"2024-09-21T10:20:45.331940+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:02.332428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1870,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686441+0000","last_change":"2024-09-21T10:16:38.460495+0000","last_active":"2024-09-21T10:20:46.686441+0000","last_peered":"2024-09-21T10:20:46.686441+0000","last_clean":"2024-09-21T10:20:46.686441+0000","last_became_active":"2024-09-21T10:16:38.458759+0000","last_became_peered":"2024-09-21T10:16:38.458759+0000","last_unstale":"2024-09-21T10:20:46.686441+0000","last_undegraded":"2024-09-21T10:20:46.686441+0000","last_fullsized":"2024-09-21T10:20:46.686441+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:37.031677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":896,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.843958+0000","last_change":"2024-09-21T10:15:22.139134+0000","last_active":"2024-09-21T10:20:41.843958+0000","last_peered":"2024-09-21T10:20:41.843958+0000","last_clean":"2024-09-21T10:20:41.843958+0000","last_became_active":"2024-09-21T10:15:22.138788+0000","last_became_peered":"2024-09-21T10:15:22.138788+0000","last_unstale":"2024-09-21T10:20:41.843958+0000","last_undegraded":"2024-09-21T10:20:41.843958+0000","last_fullsized":"2024-09-21T10:20:41.843958+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:35.831693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":49,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.026539+0000","last_change":"2024-09-21T10:17:51.287937+0000","last_active":"2024-09-21T10:20:30.026539+0000","last_peered":"2024-09-21T10:20:30.026539+0000","last_clean":"2024-09-21T10:20:30.026539+0000","last_became_active":"2024-09-21T10:17:51.256610+0000","last_became_peered":"2024-09-21T10:17:51.256610+0000","last_unstale":"2024-09-21T10:20:30.026539+0000","last_undegraded":"2024-09-21T10:20:30.026539+0000","last_fullsized":"2024-09-21T10:20:30.026539+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:15.137072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":493,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025511+0000","last_change":"2024-09-21T10:18:07.086426+0000","last_active":"2024-09-21T10:20:30.025511+0000","last_peered":"2024-09-21T10:20:30.025511+0000","last_clean":"2024-09-21T10:20:30.025511+0000","last_became_active":"2024-09-21T10:18:07.086157+0000","last_became_peered":"2024-09-21T10:18:07.086157+0000","last_unstale":"2024-09-21T10:20:30.025511+0000","last_undegraded":"2024-09-21T10:20:30.025511+0000","last_fullsized":"2024-09-21T10:20:30.025511+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:24:34.289605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":224,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.847193+0000","last_change":"2024-09-21T10:16:38.450231+0000","last_active":"2024-09-21T10:20:41.847193+0000","last_peered":"2024-09-21T10:20:41.847193+0000","last_clean":"2024-09-21T10:20:41.847193+0000","last_became_active":"2024-09-21T10:16:38.449941+0000","last_became_peered":"2024-09-21T10:16:38.449941+0000","last_unstale":"2024-09-21T10:20:41.847193+0000","last_undegraded":"2024-09-21T10:20:41.847193+0000","last_fullsized":"2024-09-21T10:20:41.847193+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:51.559013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1162,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686606+0000","last_change":"2024-09-21T10:17:51.264078+0000","last_active":"2024-09-21T10:20:46.686606+0000","last_peered":"2024-09-21T10:20:46.686606+0000","last_clean":"2024-09-21T10:20:46.686606+0000","last_became_active":"2024-09-21T10:17:51.263721+0000","last_became_peered":"2024-09-21T10:17:51.263721+0000","last_unstale":"2024-09-21T10:20:46.686606+0000","last_undegraded":"2024-09-21T10:20:46.686606+0000","last_fullsized":"2024-09-21T10:20:46.686606+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:46:02.104778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1652,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331937+0000","last_change":"2024-09-21T10:17:51.251859+0000","last_active":"2024-09-21T10:20:45.331937+0000","last_peered":"2024-09-21T10:20:45.331937+0000","last_clean":"2024-09-21T10:20:45.331937+0000","last_became_active":"2024-09-21T10:17:51.251525+0000","last_became_peered":"2024-09-21T10:17:51.251525+0000","last_unstale":"2024-09-21T10:20:45.331937+0000","last_undegraded":"2024-09-21T10:20:45.331937+0000","last_fullsized":"2024-09-21T10:20:45.331937+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:24.106509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1155,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687052+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:20:46.687052+0000","last_peered":"2024-09-21T10:20:46.687052+0000","last_clean":"2024-09-21T10:20:46.687052+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:20:46.687052+0000","last_undegraded":"2024-09-21T10:20:46.687052+0000","last_fullsized":"2024-09-21T10:20:46.687052+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1742,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332048+0000","last_change":"2024-09-21T10:16:38.456464+0000","last_active":"2024-09-21T10:20:45.332048+0000","last_peered":"2024-09-21T10:20:45.332048+0000","last_clean":"2024-09-21T10:20:45.332048+0000","last_became_active":"2024-09-21T10:16:38.455345+0000","last_became_peered":"2024-09-21T10:16:38.455345+0000","last_unstale":"2024-09-21T10:20:45.332048+0000","last_undegraded":"2024-09-21T10:20:45.332048+0000","last_fullsized":"2024-09-21T10:20:45.332048+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:39.393023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1886,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686949+0000","last_change":"2024-09-21T10:17:51.254069+0000","last_active":"2024-09-21T10:20:46.686949+0000","last_peered":"2024-09-21T10:20:46.686949+0000","last_clean":"2024-09-21T10:20:46.686949+0000","last_became_active":"2024-09-21T10:17:51.253723+0000","last_became_peered":"2024-09-21T10:17:51.253723+0000","last_unstale":"2024-09-21T10:20:46.686949+0000","last_undegraded":"2024-09-21T10:20:46.686949+0000","last_fullsized":"2024-09-21T10:20:46.686949+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:22.160096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1886,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687240+0000","last_change":"2024-09-21T10:17:51.249147+0000","last_active":"2024-09-21T10:20:46.687240+0000","last_peered":"2024-09-21T10:20:46.687240+0000","last_clean":"2024-09-21T10:20:46.687240+0000","last_became_active":"2024-09-21T10:17:51.248713+0000","last_became_peered":"2024-09-21T10:17:51.248713+0000","last_unstale":"2024-09-21T10:20:46.687240+0000","last_undegraded":"2024-09-21T10:20:46.687240+0000","last_fullsized":"2024-09-21T10:20:46.687240+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:33.757896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":49,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.028032+0000","last_change":"2024-09-21T10:17:51.286472+0000","last_active":"2024-09-21T10:20:30.028032+0000","last_peered":"2024-09-21T10:20:30.028032+0000","last_clean":"2024-09-21T10:20:30.028032+0000","last_became_active":"2024-09-21T10:17:51.249949+0000","last_became_peered":"2024-09-21T10:17:51.249949+0000","last_unstale":"2024-09-21T10:20:30.028032+0000","last_undegraded":"2024-09-21T10:20:30.028032+0000","last_fullsized":"2024-09-21T10:20:30.028032+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:31.760578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1311,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.846049+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:20:41.846049+0000","last_peered":"2024-09-21T10:20:41.846049+0000","last_clean":"2024-09-21T10:20:41.846049+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:20:41.846049+0000","last_undegraded":"2024-09-21T10:20:41.846049+0000","last_fullsized":"2024-09-21T10:20:41.846049+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1827,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332310+0000","last_change":"2024-09-21T10:16:39.440143+0000","last_active":"2024-09-21T10:20:45.332310+0000","last_peered":"2024-09-21T10:20:45.332310+0000","last_clean":"2024-09-21T10:20:45.332310+0000","last_became_active":"2024-09-21T10:16:39.439370+0000","last_became_peered":"2024-09-21T10:16:39.439370+0000","last_unstale":"2024-09-21T10:20:45.332310+0000","last_undegraded":"2024-09-21T10:20:45.332310+0000","last_fullsized":"2024-09-21T10:20:45.332310+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:36.417381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025959+0000","last_change":"2024-09-21T10:17:51.259438+0000","last_active":"2024-09-21T10:20:30.025959+0000","last_peered":"2024-09-21T10:20:30.025959+0000","last_clean":"2024-09-21T10:20:30.025959+0000","last_became_active":"2024-09-21T10:17:51.258704+0000","last_became_peered":"2024-09-21T10:17:51.258704+0000","last_unstale":"2024-09-21T10:20:30.025959+0000","last_undegraded":"2024-09-21T10:20:30.025959+0000","last_fullsized":"2024-09-21T10:20:30.025959+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:16.479749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1900,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686492+0000","last_change":"2024-09-21T10:16:38.460671+0000","last_active":"2024-09-21T10:20:46.686492+0000","last_peered":"2024-09-21T10:20:46.686492+0000","last_clean":"2024-09-21T10:20:46.686492+0000","last_became_active":"2024-09-21T10:16:38.459939+0000","last_became_peered":"2024-09-21T10:16:38.459939+0000","last_unstale":"2024-09-21T10:20:46.686492+0000","last_undegraded":"2024-09-21T10:20:46.686492+0000","last_fullsized":"2024-09-21T10:20:46.686492+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:51.143325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":426,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686889+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:20:46.686889+0000","last_peered":"2024-09-21T10:20:46.686889+0000","last_clean":"2024-09-21T10:20:46.686889+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:20:46.686889+0000","last_undegraded":"2024-09-21T10:20:46.686889+0000","last_fullsized":"2024-09-21T10:20:46.686889+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332195+0000","last_change":"2024-09-21T10:17:51.250313+0000","last_active":"2024-09-21T10:20:45.332195+0000","last_peered":"2024-09-21T10:20:45.332195+0000","last_clean":"2024-09-21T10:20:45.332195+0000","last_became_active":"2024-09-21T10:17:51.249931+0000","last_became_peered":"2024-09-21T10:17:51.249931+0000","last_unstale":"2024-09-21T10:20:45.332195+0000","last_undegraded":"2024-09-21T10:20:45.332195+0000","last_fullsized":"2024-09-21T10:20:45.332195+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:38:15.879964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1040,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686899+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:20:46.686899+0000","last_peered":"2024-09-21T10:20:46.686899+0000","last_clean":"2024-09-21T10:20:46.686899+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:20:46.686899+0000","last_undegraded":"2024-09-21T10:20:46.686899+0000","last_fullsized":"2024-09-21T10:20:46.686899+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.026623+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:20:30.026623+0000","last_peered":"2024-09-21T10:20:30.026623+0000","last_clean":"2024-09-21T10:20:30.026623+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:20:30.026623+0000","last_undegraded":"2024-09-21T10:20:30.026623+0000","last_fullsized":"2024-09-21T10:20:30.026623+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":441,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.026151+0000","last_change":"2024-09-21T10:18:07.090592+0000","last_active":"2024-09-21T10:20:30.026151+0000","last_peered":"2024-09-21T10:20:30.026151+0000","last_clean":"2024-09-21T10:20:30.026151+0000","last_became_active":"2024-09-21T10:18:07.090188+0000","last_became_peered":"2024-09-21T10:18:07.090188+0000","last_unstale":"2024-09-21T10:20:30.026151+0000","last_undegraded":"2024-09-21T10:20:30.026151+0000","last_fullsized":"2024-09-21T10:20:30.026151+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.763114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":93,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332390+0000","last_change":"2024-09-21T10:16:39.439344+0000","last_active":"2024-09-21T10:20:45.332390+0000","last_peered":"2024-09-21T10:20:45.332390+0000","last_clean":"2024-09-21T10:20:45.332390+0000","last_became_active":"2024-09-21T10:16:39.438987+0000","last_became_peered":"2024-09-21T10:16:39.438987+0000","last_unstale":"2024-09-21T10:20:45.332390+0000","last_undegraded":"2024-09-21T10:20:45.332390+0000","last_fullsized":"2024-09-21T10:20:45.332390+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:17.163136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":456,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331836+0000","last_change":"2024-09-21T10:17:51.250156+0000","last_active":"2024-09-21T10:20:45.331836+0000","last_peered":"2024-09-21T10:20:45.331836+0000","last_clean":"2024-09-21T10:20:45.331836+0000","last_became_active":"2024-09-21T10:17:51.248282+0000","last_became_peered":"2024-09-21T10:17:51.248282+0000","last_unstale":"2024-09-21T10:20:45.331836+0000","last_undegraded":"2024-09-21T10:20:45.331836+0000","last_fullsized":"2024-09-21T10:20:45.331836+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:58.954423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":411,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687154+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:20:46.687154+0000","last_peered":"2024-09-21T10:20:46.687154+0000","last_clean":"2024-09-21T10:20:46.687154+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:20:46.687154+0000","last_undegraded":"2024-09-21T10:20:46.687154+0000","last_fullsized":"2024-09-21T10:20:46.687154+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":224,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.853796+0000","last_change":"2024-09-21T10:16:38.455089+0000","last_active":"2024-09-21T10:20:41.853796+0000","last_peered":"2024-09-21T10:20:41.853796+0000","last_clean":"2024-09-21T10:20:41.853796+0000","last_became_active":"2024-09-21T10:16:38.454804+0000","last_became_peered":"2024-09-21T10:16:38.454804+0000","last_unstale":"2024-09-21T10:20:41.853796+0000","last_undegraded":"2024-09-21T10:20:41.853796+0000","last_fullsized":"2024-09-21T10:20:41.853796+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:16.448016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.028306+0000","last_change":"2024-09-21T10:17:51.250324+0000","last_active":"2024-09-21T10:20:30.028306+0000","last_peered":"2024-09-21T10:20:30.028306+0000","last_clean":"2024-09-21T10:20:30.028306+0000","last_became_active":"2024-09-21T10:17:51.249933+0000","last_became_peered":"2024-09-21T10:17:51.249933+0000","last_unstale":"2024-09-21T10:20:30.028306+0000","last_undegraded":"2024-09-21T10:20:30.028306+0000","last_fullsized":"2024-09-21T10:20:30.028306+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:23.880577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":199,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.846185+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:20:41.846185+0000","last_peered":"2024-09-21T10:20:41.846185+0000","last_clean":"2024-09-21T10:20:41.846185+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:20:41.846185+0000","last_undegraded":"2024-09-21T10:20:41.846185+0000","last_fullsized":"2024-09-21T10:20:41.846185+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":262,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.844946+0000","last_change":"2024-09-21T10:17:51.277615+0000","last_active":"2024-09-21T10:20:41.844946+0000","last_peered":"2024-09-21T10:20:41.844946+0000","last_clean":"2024-09-21T10:20:41.844946+0000","last_became_active":"2024-09-21T10:17:51.261756+0000","last_became_peered":"2024-09-21T10:17:51.261756+0000","last_unstale":"2024-09-21T10:20:41.844946+0000","last_undegraded":"2024-09-21T10:20:41.844946+0000","last_fullsized":"2024-09-21T10:20:41.844946+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:05:35.608372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1873,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687308+0000","last_change":"2024-09-21T10:17:51.251440+0000","last_active":"2024-09-21T10:20:46.687308+0000","last_peered":"2024-09-21T10:20:46.687308+0000","last_clean":"2024-09-21T10:20:46.687308+0000","last_became_active":"2024-09-21T10:17:51.249109+0000","last_became_peered":"2024-09-21T10:17:51.249109+0000","last_unstale":"2024-09-21T10:20:46.687308+0000","last_undegraded":"2024-09-21T10:20:46.687308+0000","last_fullsized":"2024-09-21T10:20:46.687308+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:59.577984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1383,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332534+0000","last_change":"2024-09-21T10:16:38.453215+0000","last_active":"2024-09-21T10:20:45.332534+0000","last_peered":"2024-09-21T10:20:45.332534+0000","last_clean":"2024-09-21T10:20:45.332534+0000","last_became_active":"2024-09-21T10:16:38.452464+0000","last_became_peered":"2024-09-21T10:16:38.452464+0000","last_unstale":"2024-09-21T10:20:45.332534+0000","last_undegraded":"2024-09-21T10:20:45.332534+0000","last_fullsized":"2024-09-21T10:20:45.332534+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:16.818394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.026093+0000","last_change":"2024-09-21T10:17:51.248299+0000","last_active":"2024-09-21T10:20:30.026093+0000","last_peered":"2024-09-21T10:20:30.026093+0000","last_clean":"2024-09-21T10:20:30.026093+0000","last_became_active":"2024-09-21T10:17:51.247851+0000","last_became_peered":"2024-09-21T10:17:51.247851+0000","last_unstale":"2024-09-21T10:20:30.026093+0000","last_undegraded":"2024-09-21T10:20:30.026093+0000","last_fullsized":"2024-09-21T10:20:30.026093+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:04.319143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":531,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331885+0000","last_change":"2024-09-21T10:17:51.248187+0000","last_active":"2024-09-21T10:20:45.331885+0000","last_peered":"2024-09-21T10:20:45.331885+0000","last_clean":"2024-09-21T10:20:45.331885+0000","last_became_active":"2024-09-21T10:17:51.247436+0000","last_became_peered":"2024-09-21T10:17:51.247436+0000","last_unstale":"2024-09-21T10:20:45.331885+0000","last_undegraded":"2024-09-21T10:20:45.331885+0000","last_fullsized":"2024-09-21T10:20:45.331885+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:33.505149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":45,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.026426+0000","last_change":"2024-09-21T10:17:51.258577+0000","last_active":"2024-09-21T10:20:30.026426+0000","last_peered":"2024-09-21T10:20:30.026426+0000","last_clean":"2024-09-21T10:20:30.026426+0000","last_became_active":"2024-09-21T10:17:51.257820+0000","last_became_peered":"2024-09-21T10:17:51.257820+0000","last_unstale":"2024-09-21T10:20:30.026426+0000","last_undegraded":"2024-09-21T10:20:30.026426+0000","last_fullsized":"2024-09-21T10:20:30.026426+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:00.922957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":515,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331821+0000","last_change":"2024-09-21T10:16:38.456905+0000","last_active":"2024-09-21T10:20:45.331821+0000","last_peered":"2024-09-21T10:20:45.331821+0000","last_clean":"2024-09-21T10:20:45.331821+0000","last_became_active":"2024-09-21T10:16:38.453276+0000","last_became_peered":"2024-09-21T10:16:38.453276+0000","last_unstale":"2024-09-21T10:20:45.331821+0000","last_undegraded":"2024-09-21T10:20:45.331821+0000","last_fullsized":"2024-09-21T10:20:45.331821+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:38.174569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":242,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.848953+0000","last_change":"2024-09-21T10:17:51.262911+0000","last_active":"2024-09-21T10:20:41.848953+0000","last_peered":"2024-09-21T10:20:41.848953+0000","last_clean":"2024-09-21T10:20:41.848953+0000","last_became_active":"2024-09-21T10:17:51.262314+0000","last_became_peered":"2024-09-21T10:17:51.262314+0000","last_unstale":"2024-09-21T10:20:41.848953+0000","last_undegraded":"2024-09-21T10:20:41.848953+0000","last_fullsized":"2024-09-21T10:20:41.848953+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:06.241585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":503,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332074+0000","last_change":"2024-09-21T10:17:51.254289+0000","last_active":"2024-09-21T10:20:45.332074+0000","last_peered":"2024-09-21T10:20:45.332074+0000","last_clean":"2024-09-21T10:20:45.332074+0000","last_became_active":"2024-09-21T10:17:51.253922+0000","last_became_peered":"2024-09-21T10:17:51.253922+0000","last_unstale":"2024-09-21T10:20:45.332074+0000","last_undegraded":"2024-09-21T10:20:45.332074+0000","last_fullsized":"2024-09-21T10:20:45.332074+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:57.111361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1767,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332483+0000","last_change":"2024-09-21T10:17:51.249960+0000","last_active":"2024-09-21T10:20:45.332483+0000","last_peered":"2024-09-21T10:20:45.332483+0000","last_clean":"2024-09-21T10:20:45.332483+0000","last_became_active":"2024-09-21T10:17:51.248883+0000","last_became_peered":"2024-09-21T10:17:51.248883+0000","last_unstale":"2024-09-21T10:20:45.332483+0000","last_undegraded":"2024-09-21T10:20:45.332483+0000","last_fullsized":"2024-09-21T10:20:45.332483+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:20.076539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":159,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686474+0000","last_change":"2024-09-21T10:16:38.461877+0000","last_active":"2024-09-21T10:20:46.686474+0000","last_peered":"2024-09-21T10:20:46.686474+0000","last_clean":"2024-09-21T10:20:46.686474+0000","last_became_active":"2024-09-21T10:16:38.461554+0000","last_became_peered":"2024-09-21T10:16:38.461554+0000","last_unstale":"2024-09-21T10:20:46.686474+0000","last_undegraded":"2024-09-21T10:20:46.686474+0000","last_fullsized":"2024-09-21T10:20:46.686474+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:46.372637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":443,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332741+0000","last_change":"2024-09-21T10:16:38.447369+0000","last_active":"2024-09-21T10:20:45.332741+0000","last_peered":"2024-09-21T10:20:45.332741+0000","last_clean":"2024-09-21T10:20:45.332741+0000","last_became_active":"2024-09-21T10:16:38.447085+0000","last_became_peered":"2024-09-21T10:16:38.447085+0000","last_unstale":"2024-09-21T10:20:45.332741+0000","last_undegraded":"2024-09-21T10:20:45.332741+0000","last_fullsized":"2024-09-21T10:20:45.332741+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:59.110578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":430,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.687387+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:20:46.687387+0000","last_peered":"2024-09-21T10:20:46.687387+0000","last_clean":"2024-09-21T10:20:46.687387+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:20:46.687387+0000","last_undegraded":"2024-09-21T10:20:46.687387+0000","last_fullsized":"2024-09-21T10:20:46.687387+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1759,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686342+0000","last_change":"2024-09-21T10:16:38.460957+0000","last_active":"2024-09-21T10:20:46.686342+0000","last_peered":"2024-09-21T10:20:46.686342+0000","last_clean":"2024-09-21T10:20:46.686342+0000","last_became_active":"2024-09-21T10:16:38.460325+0000","last_became_peered":"2024-09-21T10:16:38.460325+0000","last_unstale":"2024-09-21T10:20:46.686342+0000","last_undegraded":"2024-09-21T10:20:46.686342+0000","last_fullsized":"2024-09-21T10:20:46.686342+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:34.529895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1833,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:46.686366+0000","last_change":"2024-09-21T10:16:38.461576+0000","last_active":"2024-09-21T10:20:46.686366+0000","last_peered":"2024-09-21T10:20:46.686366+0000","last_clean":"2024-09-21T10:20:46.686366+0000","last_became_active":"2024-09-21T10:16:38.461205+0000","last_became_peered":"2024-09-21T10:16:38.461205+0000","last_unstale":"2024-09-21T10:20:46.686366+0000","last_undegraded":"2024-09-21T10:20:46.686366+0000","last_fullsized":"2024-09-21T10:20:46.686366+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:17.166658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1425,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331978+0000","last_change":"2024-09-21T10:16:38.459378+0000","last_active":"2024-09-21T10:20:45.331978+0000","last_peered":"2024-09-21T10:20:45.331978+0000","last_clean":"2024-09-21T10:20:45.331978+0000","last_became_active":"2024-09-21T10:16:38.451582+0000","last_became_peered":"2024-09-21T10:16:38.451582+0000","last_unstale":"2024-09-21T10:20:45.331978+0000","last_undegraded":"2024-09-21T10:20:45.331978+0000","last_fullsized":"2024-09-21T10:20:45.331978+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:34.820762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":49,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025463+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:20:30.025463+0000","last_peered":"2024-09-21T10:20:30.025463+0000","last_clean":"2024-09-21T10:20:30.025463+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:20:30.025463+0000","last_undegraded":"2024-09-21T10:20:30.025463+0000","last_fullsized":"2024-09-21T10:20:30.025463+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1442,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.331946+0000","last_change":"2024-09-21T10:16:38.453928+0000","last_active":"2024-09-21T10:20:45.331946+0000","last_peered":"2024-09-21T10:20:45.331946+0000","last_clean":"2024-09-21T10:20:45.331946+0000","last_became_active":"2024-09-21T10:16:38.450890+0000","last_became_peered":"2024-09-21T10:16:38.450890+0000","last_unstale":"2024-09-21T10:20:45.331946+0000","last_undegraded":"2024-09-21T10:20:45.331946+0000","last_fullsized":"2024-09-21T10:20:45.331946+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:13.500644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":49,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.025809+0000","last_change":"2024-09-21T10:17:51.281734+0000","last_active":"2024-09-21T10:20:30.025809+0000","last_peered":"2024-09-21T10:20:30.025809+0000","last_clean":"2024-09-21T10:20:30.025809+0000","last_became_active":"2024-09-21T10:17:51.258206+0000","last_became_peered":"2024-09-21T10:17:51.258206+0000","last_unstale":"2024-09-21T10:20:30.025809+0000","last_undegraded":"2024-09-21T10:20:30.025809+0000","last_fullsized":"2024-09-21T10:20:30.025809+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:16.184203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":503,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332001+0000","last_change":"2024-09-21T10:17:51.251289+0000","last_active":"2024-09-21T10:20:45.332001+0000","last_peered":"2024-09-21T10:20:45.332001+0000","last_clean":"2024-09-21T10:20:45.332001+0000","last_became_active":"2024-09-21T10:17:51.247868+0000","last_became_peered":"2024-09-21T10:17:51.247868+0000","last_unstale":"2024-09-21T10:20:45.332001+0000","last_undegraded":"2024-09-21T10:20:45.332001+0000","last_fullsized":"2024-09-21T10:20:45.332001+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:39:49.784298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":506,"reported_epoch":1343,"state":"active+clean","last_fresh":"2024-09-21T10:20:45.332555+0000","last_change":"2024-09-21T10:16:38.451567+0000","last_active":"2024-09-21T10:20:45.332555+0000","last_peered":"2024-09-21T10:20:45.332555+0000","last_clean":"2024-09-21T10:20:45.332555+0000","last_became_active":"2024-09-21T10:16:38.451245+0000","last_became_peered":"2024-09-21T10:16:38.451245+0000","last_unstale":"2024-09-21T10:20:45.332555+0000","last_undegraded":"2024-09-21T10:20:45.332555+0000","last_fullsized":"2024-09-21T10:20:45.332555+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:27.022083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":208,"reported_epoch":1342,"state":"active+clean","last_fresh":"2024-09-21T10:20:41.844185+0000","last_change":"2024-09-21T10:15:19.115911+0000","last_active":"2024-09-21T10:20:41.844185+0000","last_peered":"2024-09-21T10:20:41.844185+0000","last_clean":"2024-09-21T10:20:41.844185+0000","last_became_active":"2024-09-21T10:15:19.115615+0000","last_became_peered":"2024-09-21T10:15:19.115615+0000","last_unstale":"2024-09-21T10:20:41.844185+0000","last_undegraded":"2024-09-21T10:20:41.844185+0000","last_fullsized":"2024-09-21T10:20:41.844185+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":51,"reported_epoch":1341,"state":"active+clean","last_fresh":"2024-09-21T10:20:30.026338+0000","last_change":"2024-09-21T10:17:51.280012+0000","last_active":"2024-09-21T10:20:30.026338+0000","last_peered":"2024-09-21T10:20:30.026338+0000","last_clean":"2024-09-21T10:20:30.026338+0000","last_became_active":"2024-09-21T10:17:51.257359+0000","last_became_peered":"2024-09-21T10:17:51.257359+0000","last_unstale":"2024-09-21T10:20:30.026338+0000","last_undegraded":"2024-09-21T10:20:30.026338+0000","last_fullsized":"2024-09-21T10:20:30.026338+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:44.522556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":21092,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17031,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":6}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234539,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496175,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1344096,"kb_used_data":171352,"kb_used_omap":1,"kb_used_meta":1172734,"kb_avail":92401056,"statfs":{"total":95995035648,"available":94618681344,"internally_reserved":0,"allocated":175464448,"data_stored":172177057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1200880071},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757814,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":441820,"kb_used_data":9616,"kb_used_omap":1,"kb_used_meta":432190,"kb_avail":93303332,"statfs":{"total":95995035648,"available":95542611968,"internally_reserved":0,"allocated":9846784,"data_stored":6566939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":442563014},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019450,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1033952,"kb_used_data":10196,"kb_used_omap":1,"kb_used_meta":1023742,"kb_avail":92711200,"statfs":{"total":95995035648,"available":94936268800,"internally_reserved":0,"allocated":10440704,"data_stored":7157342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1048312262},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[]},{"osd":3,"up_from":1301,"seq":5587752452147,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":396692,"kb_used_data":9428,"kb_used_omap":48,"kb_used_meta":387023,"kb_avail":93348460,"statfs":{"total":95995035648,"available":95588823040,"internally_reserved":0,"allocated":9654272,"data_stored":6439756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49883,"internal_metadata":396311845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793967986,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047224,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1126288,"kb_used_data":10180,"kb_used_omap":41,"kb_used_meta":1116054,"kb_avail":92618864,"statfs":{"total":95995035648,"available":94841716736,"internally_reserved":0,"allocated":10424320,"data_stored":7151945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1142840233},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928859,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7559,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:20:53.655 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:20:53.655 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:20:53.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:53.742 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:20:53.742 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:20:53.742 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:20:53.742 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [] 2024-09-21T10:20:53.742 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:20:53.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:53.789 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:53.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:53.891 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:53.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:53.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:53.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:54.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:54.038 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:54.139 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:54.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:54.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:54.263 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:54.284 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:54 smithi191 ceph-mon[102206]: pgmap v2136: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:54 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:20:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:54 smithi191 ceph-mon[102206]: osdmap e1346: 8 total, 8 up, 5 in 2024-09-21T10:20:54.386 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:54.397 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:54.400 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:20:54.409 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:54.503 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:20:54.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:54.519 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:54 smithi076 ceph-mon[106482]: pgmap v2136: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:54.519 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:54 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:20:54.520 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:54 smithi076 ceph-mon[106482]: osdmap e1346: 8 total, 8 up, 5 in 2024-09-21T10:20:54.520 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:54 smithi076 ceph-mon[110963]: pgmap v2136: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:54.520 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:54 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:20:54.520 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:54 smithi076 ceph-mon[110963]: osdmap e1346: 8 total, 8 up, 5 in 2024-09-21T10:20:54.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:54.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:54.658 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:54.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:54.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:54.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:54.884 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:54.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:54.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:54.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:55.006 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:55.028 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:55.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:55.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:55.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:55.252 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:55.273 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:55.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:55.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:55.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:55.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:55.520 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:55.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:55.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:55.642 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:55.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:55.765 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:55.867 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[106482]: pgmap v2138: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[110963]: pgmap v2138: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:55.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:55.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:55.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:55.991 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:56.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:55 smithi191 ceph-mon[102206]: pgmap v2138: 89 pgs: 16 creating+peering, 73 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:20:56.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:56.137 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:56.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:56.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:56.361 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:56.383 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:56.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:56.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:56.609 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:56.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:56.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:56.761 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:56.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:56.863 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:56.888 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:56.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:56.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:56.990 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:57.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:57.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:57.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:57.139 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:57.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:57.265 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:57.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:57.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:57.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:57.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:57.619 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:57.643 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:57.745 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.769 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:57.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:57.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:57.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:57.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:57.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:58.017 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:58.067 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:57 smithi076 ceph-mon[106482]: pgmap v2139: 89 pgs: 14 creating+peering, 75 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:58.067 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:57 smithi076 ceph-mon[110963]: pgmap v2139: 89 pgs: 14 creating+peering, 75 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:57 smithi191 ceph-mon[102206]: pgmap v2139: 89 pgs: 14 creating+peering, 75 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:58.119 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:58.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:58.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:58.257 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:20:58.257 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:20:58.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:58.275 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:20:58.275 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:20:58.291 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:20:58.291 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:20:58.308 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:20:58.308 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:20:58.324 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:20:58.325 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:20:58.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:20:58.342 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:20:58.359 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:20:58.359 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:20:58.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:58.376 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:20:58.376 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:20:58.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:58.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:58.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.517 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:58.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:58.618 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:58.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:58.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:58.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:20:58.767 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:58.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:20:58.808 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:20:58.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4102655814' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1404747897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4102655814' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:58.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:58.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1404747897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:58.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:58.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:58 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:58.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:58.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:58.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:20:58.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:20:59.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:59.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:20:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4102655814' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:58 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:20:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1404747897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:20:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:58 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:59.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:20:59.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:59.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:20:59.243 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:59.270 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:59.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:59.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:20:59.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:20:59.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:20:59.520 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:20:59.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:20:59.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:59.649 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:59.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:20:59.751 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:20:59.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:20:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[106482]: pgmap v2140: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[106482]: osdmap e1347: 8 total, 8 up, 5 in 2024-09-21T10:20:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[110963]: pgmap v2140: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[110963]: osdmap e1347: 8 total, 8 up, 5 in 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:20:59 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:20:59.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:20:59.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:20:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:20:59.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:00.000 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:00.021 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:00.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:59 smithi191 ceph-mon[102206]: pgmap v2140: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:59 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:59 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:21:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:59 smithi191 ceph-mon[102206]: osdmap e1347: 8 total, 8 up, 5 in 2024-09-21T10:21:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:20:59 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:00.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:00.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:00.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:00.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:00.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:00.372 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:00.395 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:00.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:00.497 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:00.520 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:00.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:00.643 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:00.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:00.745 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:00.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:00.868 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:00.891 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:00.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:00.968 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:00.993 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.016 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:01.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:01.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:01.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:01.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:01.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:01.264 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:01.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:01.387 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:01.488 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:01.509 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:01.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:21:01.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:01.646 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:01.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:01.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:01.808 INFO:teuthology.orchestra.run.smithi076.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-09-21T10:21:01.875 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:01.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:01.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:01.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:02.024 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:02.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:01 smithi191 ceph-mon[102206]: pgmap v2142: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/773630569' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:21:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:01 smithi076 ceph-mon[110963]: pgmap v2142: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:02.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/773630569' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:21:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:01 smithi076 ceph-mon[106482]: pgmap v2142: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:02.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/773630569' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:21:02.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:02.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:02.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:02.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:02.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:02.378 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:02.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:02.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:02.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:02.524 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:02.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:02.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:02.651 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:02.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:02.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:02.777 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:02.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:02.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:02.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:02.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:02.990 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:21:03.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:03.030 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/773630569' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:21:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:02 smithi191 ceph-mon[102206]: osdmap e1348: 8 total, 8 up, 5 in 2024-09-21T10:21:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/773630569' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:21:03.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:02 smithi076 ceph-mon[106482]: osdmap e1348: 8 total, 8 up, 5 in 2024-09-21T10:21:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/773630569' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:21:03.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:02 smithi076 ceph-mon[110963]: osdmap e1348: 8 total, 8 up, 5 in 2024-09-21T10:21:03.132 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:03.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:03.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:03.281 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:03.383 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:03.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:03.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:03.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:03.576 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:21:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:21:03] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.51.0" 2024-09-21T10:21:03.630 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:03.652 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:03.753 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:03.775 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:03.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:03.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:03 smithi076 ceph-mon[110963]: pgmap v2144: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:03.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:03 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:21:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:03 smithi076 ceph-mon[106482]: pgmap v2144: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:03 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:21:03.878 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:03.900 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:03.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:03.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:04.002 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:04.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:03 smithi191 ceph-mon[102206]: pgmap v2144: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:03 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:21:04.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:04.148 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:04.173 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:04.179 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:04.180 DEBUG:teuthology.orchestra.run.smithi076:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-09-21T10:21:04.181 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T10:21:04.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:21:04.183 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-21T10:21:04.183 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-21T10:21:04.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:hints = 1 2024-09-21T10:21:04.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-21T10:21:04.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object prefix: benchmark_data_smithi076_330857 2024-09-21T10:21:04.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:21:04.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 0 0 0 0 0 0 - 0 2024-09-21T10:21:04.250 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:04.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:04.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:04.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:04.526 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:04.562 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:04.664 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:04.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:04.833 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:04.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:04.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:04.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:04.982 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:04.982 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:05.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:05.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:05.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:05.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:05.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 1 16 3985 3969 248.048 248.062 0.00251045 0.00401589 2024-09-21T10:21:05.247 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:05.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:05.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:05.383 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:05.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:05.485 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:05.517 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:05.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:05.619 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:05.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:05.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:05.866 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:05.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:05.969 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.001 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:06.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:05 smithi191 ceph-mon[102206]: pgmap v2145: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:05 smithi076 ceph-mon[106482]: pgmap v2145: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:05 smithi076 ceph-mon[110963]: pgmap v2145: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:21:06.136 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:06.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:06.238 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:06.242 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 2 16 7877 7861 245.638 243.25 0.00493358 0.00406296 2024-09-21T10:21:06.271 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:06.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:06.373 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:06.408 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:06.510 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:06.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:06.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:06.812 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:06.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:06.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:06.950 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:06.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:07.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:07.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:07.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:07.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:07.220 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:07.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:07.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 3 16 11508 11492 239.398 226.938 0.00328852 0.00417009 2024-09-21T10:21:07.321 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:07.354 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:07.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:07.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:07.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:07.622 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:07.723 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:07.759 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:07.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:07.860 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:07.893 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:07.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:07.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:07.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:08.029 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:08.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:07 smithi191 ceph-mon[102206]: pgmap v2146: 89 pgs: 89 active+clean; 67 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 8.3 MiB/s wr, 133 op/s 2024-09-21T10:21:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:07 smithi076 ceph-mon[106482]: pgmap v2146: 89 pgs: 89 active+clean; 67 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 8.3 MiB/s wr, 133 op/s 2024-09-21T10:21:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:07 smithi076 ceph-mon[110963]: pgmap v2146: 89 pgs: 89 active+clean; 67 MiB data, 4.6 GiB used, 442 GiB / 447 GiB avail; 8.3 MiB/s wr, 133 op/s 2024-09-21T10:21:08.130 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:08.171 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:08.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 4 16 14828 14812 231.419 207.5 0.00516485 0.00431387 2024-09-21T10:21:08.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:08.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:08.409 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:08.440 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:08.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:08.575 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:08.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:08.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:08.811 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:08.846 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:08.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:08.947 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:08.983 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:08.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:09.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:09.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:09.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:09.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:09.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 5 16 17803 17787 222.319 185.938 0.00227506 0.00449201 2024-09-21T10:21:09.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:09.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:09.390 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:09.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.527 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:09.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:09.628 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:09.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:09.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:09.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:09.796 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:09.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:09.932 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:09.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:10.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:10.054 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:10.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:09 smithi191 ceph-mon[102206]: pgmap v2147: 89 pgs: 89 active+clean; 625 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-21T10:21:10.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:09 smithi076 ceph-mon[106482]: pgmap v2147: 89 pgs: 89 active+clean; 625 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-21T10:21:10.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:09 smithi076 ceph-mon[110963]: pgmap v2147: 89 pgs: 89 active+clean; 625 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-21T10:21:10.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:10.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:10.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:10.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 6 16 20438 20422 212.711 164.688 0.00358965 0.00469649 2024-09-21T10:21:10.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:10.325 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:10.427 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:10.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:10.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:10.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:10.588 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:10.690 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:10.722 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:10.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:10.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:10.857 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:10.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:10.992 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:10.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:11.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:11.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:11.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:11.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 7 16 22566 22550 201.322 133 0.00509264 0.0049615 2024-09-21T10:21:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:11.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:11.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:11.384 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:11.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:11.522 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:11.623 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:11.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:11.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:11.753 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:11.787 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:11.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:11.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:11 smithi191 ceph-mon[102206]: pgmap v2148: 89 pgs: 89 active+clean; 625 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-21T10:21:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:11 smithi076 ceph-mon[106482]: pgmap v2148: 89 pgs: 89 active+clean; 625 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-21T10:21:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:11 smithi076 ceph-mon[110963]: pgmap v2148: 89 pgs: 89 active+clean; 625 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-21T10:21:11.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:11.923 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:11.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:12.025 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:12.058 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:12.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:12.159 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 8 16 24768 24752 193.359 137.625 0.00675145 0.0051662 2024-09-21T10:21:12.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:12.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:12.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.468 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:12.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:12.569 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.605 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.706 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:12.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:12.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:12.846 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:12.878 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:12.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:12.979 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:13.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:13.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:13.153 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:13.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 9 16 26958 26942 187.081 136.875 0.00865534 0.00534033 2024-09-21T10:21:13.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:13.293 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:13.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:13.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:13.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:13.566 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:21:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:21:13] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.51.0" 2024-09-21T10:21:13.667 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:13.701 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:13.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:13.837 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:13.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:13.971 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:13.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:14.073 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.107 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:13 smithi191 ceph-mon[102206]: pgmap v2149: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-21T10:21:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:13 smithi076 ceph-mon[106482]: pgmap v2149: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-21T10:21:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:13 smithi076 ceph-mon[110963]: pgmap v2149: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-09-21T10:21:14.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:14.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 10 16 29085 29069 181.666 132.938 0.00640662 0.00549871 2024-09-21T10:21:14.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:14.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:14.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:14.484 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:14.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:14.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:14.643 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:14.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:14.778 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:14.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:14.916 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:14.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:14.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:15.017 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:15.040 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:15.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:15.141 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:15.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:15.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:15.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 11 16 31253 31237 177.468 135.5 0.00304713 0.00562969 2024-09-21T10:21:15.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:15.305 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:15.407 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:15.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.445 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:15.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:15.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:15.580 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:15.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:15.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:15.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:15.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:15.805 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:15 smithi076 ceph-mon[106482]: pgmap v2150: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:21:15.805 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:15 smithi076 ceph-mon[110963]: pgmap v2150: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:21:15.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:15.851 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:15.953 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:15.985 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:15.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:15.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:16.087 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:15 smithi191 ceph-mon[102206]: pgmap v2150: 89 pgs: 89 active+clean; 1.5 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-21T10:21:16.115 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:16.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:16.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:16.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 12 16 33179 33163 172.709 120.375 0.0063514 0.00578516 2024-09-21T10:21:16.251 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:16.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:16.353 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.373 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:16.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:16.475 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:16.510 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:16.612 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:16.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:16.747 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:16.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.773 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:16.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:16.813 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:21:16.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:16.909 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:16.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:17.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:17.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:17.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:17.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2187729505' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:17.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2187729505' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:17.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2187729505' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:17.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:17.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:17.243 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 13 16 35074 35058 168.534 118.438 0.00449107 0.00592888 2024-09-21T10:21:17.289 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:17.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:17.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:17.459 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:17.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:17.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:17.694 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:17.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:17.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:17.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:21:17.817 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:17.858 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:17.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:17.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:18.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:18.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:18.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:17 smithi191 ceph-mon[102206]: pgmap v2151: 89 pgs: 89 active+clean; 1.5 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-21T10:21:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:17 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:17 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:21:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:17 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2187729505' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:17 smithi191 ceph-mon[102206]: osdmap e1349: 8 total, 8 up, 5 in 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[106482]: pgmap v2151: 89 pgs: 89 active+clean; 1.5 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2187729505' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[106482]: osdmap e1349: 8 total, 8 up, 5 in 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[110963]: pgmap v2151: 89 pgs: 89 active+clean; 1.5 GiB data, 9.0 GiB used, 438 GiB / 447 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2187729505' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:17 smithi076 ceph-mon[110963]: osdmap e1349: 8 total, 8 up, 5 in 2024-09-21T10:21:18.133 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:18.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:18.234 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:18.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 14 16 36990 36974 165.048 119.75 0.00478687 0.00605387 2024-09-21T10:21:18.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:18.369 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.406 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:18.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:18.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:18.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:18.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:18.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:18.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:18.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:18.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:18.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:18.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:18.955 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:18.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:19.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:19.076 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:19.178 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:19.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.211 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:19.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:19.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 15 16 38753 38737 161.39 110.188 0.0099775 0.00618961 2024-09-21T10:21:19.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:19.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:19.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:19.449 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:19.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:19.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:19.606 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:19.708 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:19.741 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:19.843 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:19.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:19.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:19.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:19.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:20.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:20.104 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:19 smithi191 ceph-mon[102206]: pgmap v2153: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-21T10:21:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:19 smithi076 ceph-mon[106482]: pgmap v2153: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-21T10:21:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:19 smithi076 ceph-mon[110963]: pgmap v2153: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-21T10:21:20.138 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:20.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:20.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:20.244 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 16 16 40510 40494 158.166 109.812 0.00715732 0.0063182 2024-09-21T10:21:20.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:20.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:20.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:20.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:20.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:20.540 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:20.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:20.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:20.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:20.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:20.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:20.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:20.957 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:20.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:21.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:21.072 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:21:21.072 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:21.092 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:21.105 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:21:21.105 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:21.108 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:21.154 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:21:21.154 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:21.188 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:21:21.188 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:21.194 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:21.219 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:21:21.220 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:21.231 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:21.245 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 17 16 42435 42419 155.931 120.312 0.0063022 0.00640769 2024-09-21T10:21:21.252 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:21:21.253 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:21.285 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:21:21.285 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:21.319 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:21:21.319 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:21.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:21.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:21.470 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:21.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:21.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:21.611 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:21.645 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:21.746 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:21.785 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:21.886 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:21.922 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:21.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:22.024 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:22.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:22.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:22.112 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:21 smithi076 ceph-mon[106482]: pgmap v2154: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-21T10:21:22.113 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:21 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/738375730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:22.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:21 smithi076 ceph-mon[110963]: pgmap v2154: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-21T10:21:22.113 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:21 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/738375730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:21 smithi191 ceph-mon[102206]: pgmap v2154: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-21T10:21:22.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:21 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/738375730' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:22.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:22.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:22.245 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 18 16 44482 44466 154.376 127.938 0.00700925 0.00647321 2024-09-21T10:21:22.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:22.333 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:22.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:22.470 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:22.572 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:22.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:22.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:22.743 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:22.845 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:22.879 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:22.981 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:23.016 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:23.118 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:23.139 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:23.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:23.241 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:23.245 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 19 16 46403 46387 152.569 120.062 0.00828561 0.00654867 2024-09-21T10:21:23.275 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:21:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:21:23.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:21:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:21:23.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:21:23] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.51.0" 2024-09-21T10:21:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:21:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:21:23.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:23.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:23.409 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:23.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:23.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:23.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:23.641 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:23.676 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:23.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:23.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:21:23.777 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:23.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:23.828 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:23.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:23.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:23.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:23.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:24.074 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:24.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:24.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:24.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:21:24.244937+0000 min lat: 0.0014399 max lat: 0.112927 avg lat: 0.00662225 2024-09-21T10:21:24.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:21:24.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 20 16 48310 48294 150.894 119.188 0.0104157 0.00662225 2024-09-21T10:21:24.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:24.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:24.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:24.385 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:24.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:24.487 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:24.500 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:24 smithi076 ceph-mon[106482]: pgmap v2155: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-21T10:21:24.500 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:24 smithi076 ceph-mon[110963]: pgmap v2155: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-21T10:21:24.520 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:24.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:24.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:24 smithi191 ceph-mon[102206]: pgmap v2155: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-21T10:21:24.622 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:24.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:24.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:24.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:24.793 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:24.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:24.895 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:24.935 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:24.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:25.037 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:25.072 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:25.174 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:25.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:25.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:25.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 21 16 50196 50180 149.321 117.875 0.00645731 0.0066919 2024-09-21T10:21:25.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:25.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.355 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:25.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:25.456 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.492 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:25.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:25.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:25.622 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:25.723 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:25.761 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:25.863 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:25.899 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:25.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:25.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:25.988 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:21:26.002 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:26.064 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:26.165 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:26.199 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:26.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 22 16 51842 51826 147.21 102.875 0.00670621 0.00678798 2024-09-21T10:21:26.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:26.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:26.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:26 smithi076 ceph-mon[106482]: pgmap v2156: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-21T10:21:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:26 smithi076 ceph-mon[110963]: pgmap v2156: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-21T10:21:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:26.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:26.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:26.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:26.565 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:26.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:26.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:26.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:26 smithi191 ceph-mon[102206]: pgmap v2156: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-09-21T10:21:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:26.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:26.703 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:26.737 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:26.839 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:26.875 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:26.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:26.997 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:26.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:27.099 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.131 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:27.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.233 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 23 16 53382 53366 144.994 96.25 0.0506355 0.00688108 2024-09-21T10:21:27.263 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.365 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:27.400 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:27.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:27.502 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.540 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.642 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:27.680 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:27.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:27.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:27.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:27.910 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:27.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:27.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:28.048 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:28.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:28.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:28.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:28.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 24 16 54730 54714 142.463 84.25 0.00212045 0.00698523 2024-09-21T10:21:28.304 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:28.327 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:28.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:28.461 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:28.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:28.563 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:28.598 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:28.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:28.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:28 smithi191 ceph-mon[102206]: pgmap v2157: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:21:28.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:28 smithi076 ceph-mon[106482]: pgmap v2157: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:21:28.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:28 smithi076 ceph-mon[110963]: pgmap v2157: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-21T10:21:28.700 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:28.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:28.834 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:28.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:28.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:28.994 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:28.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:29.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 25 16 56027 56011 140.006 81.0625 0.00771352 0.00713784 2024-09-21T10:21:29.266 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:29.368 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:29.401 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:29.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:29.503 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:29.516 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:29.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:29.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:29.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.673 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.775 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:29.808 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:29.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:29.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:29.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:29.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:30.045 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:30.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.085 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:30.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:30.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:30.221 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:30.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 26 16 57699 57683 138.64 104.5 0.00430986 0.00720789 2024-09-21T10:21:30.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:30.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:30.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:30.427 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:30 smithi076 ceph-mon[106482]: pgmap v2158: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-09-21T10:21:30.427 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:30 smithi076 ceph-mon[110963]: pgmap v2158: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-09-21T10:21:30.453 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:30.497 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:30.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:30.599 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:30.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:30 smithi191 ceph-mon[102206]: pgmap v2158: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-09-21T10:21:30.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:30.730 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:30.760 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:30.836 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:21:30.837 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:21:30.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:30.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:30.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:30.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:31.023 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:31.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:31.163 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:31.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 27 16 59130 59114 136.817 89.4375 0.0125063 0.00730483 2024-09-21T10:21:31.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.303 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:31.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:31.404 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:31.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:31.538 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:31.575 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:31.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:31.714 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:31.811 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2159,"stamp":"2024-09-21T10:21:30.749974+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3460175845,"num_objects":52815,"num_object_clones":0,"num_object_copies":158445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52815,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":53068,"num_write_kb":3380945,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":3563357,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52997,"ondisk_log_size":52997,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":14394216,"kb_used_data":10082752,"kb_used_omap":94,"kb_used_meta":4311329,"kb_avail":454331544,"statfs":{"total":479975178240,"available":465235501056,"internally_reserved":0,"allocated":10324738048,"data_stored":10311235297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96735,"internal_metadata":4414801441},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,3,7,5,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1218445312,"num_objects":18592,"num_object_clones":0,"num_object_copies":55776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18592,"num_write_kb":1189888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003552"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":516,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818184+0000","last_change":"2024-09-21T10:17:51.249951+0000","last_active":"2024-09-21T10:21:16.818184+0000","last_peered":"2024-09-21T10:21:16.818184+0000","last_clean":"2024-09-21T10:21:16.818184+0000","last_became_active":"2024-09-21T10:17:51.249158+0000","last_became_peered":"2024-09-21T10:17:51.249158+0000","last_unstale":"2024-09-21T10:21:16.818184+0000","last_undegraded":"2024-09-21T10:21:16.818184+0000","last_fullsized":"2024-09-21T10:21:16.818184+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:19:53.769122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":227,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819165+0000","last_change":"2024-09-21T10:15:19.141623+0000","last_active":"2024-09-21T10:21:16.819165+0000","last_peered":"2024-09-21T10:21:16.819165+0000","last_clean":"2024-09-21T10:21:16.819165+0000","last_became_active":"2024-09-21T10:15:19.111927+0000","last_became_peered":"2024-09-21T10:15:19.111927+0000","last_unstale":"2024-09-21T10:21:16.819165+0000","last_undegraded":"2024-09-21T10:21:16.819165+0000","last_fullsized":"2024-09-21T10:21:16.819165+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:10.426159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849186+0000","last_change":"2024-09-21T10:17:51.258024+0000","last_active":"2024-09-21T10:21:01.849186+0000","last_peered":"2024-09-21T10:21:01.849186+0000","last_clean":"2024-09-21T10:21:01.849186+0000","last_became_active":"2024-09-21T10:17:51.257639+0000","last_became_peered":"2024-09-21T10:17:51.257639+0000","last_unstale":"2024-09-21T10:21:01.849186+0000","last_undegraded":"2024-09-21T10:21:01.849186+0000","last_fullsized":"2024-09-21T10:21:01.849186+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:04:09.112335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1833,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.820414+0000","last_change":"2024-09-21T10:17:51.250405+0000","last_active":"2024-09-21T10:21:16.820414+0000","last_peered":"2024-09-21T10:21:16.820414+0000","last_clean":"2024-09-21T10:21:16.820414+0000","last_became_active":"2024-09-21T10:17:51.248030+0000","last_became_peered":"2024-09-21T10:17:51.248030+0000","last_unstale":"2024-09-21T10:21:16.820414+0000","last_undegraded":"2024-09-21T10:21:16.820414+0000","last_fullsized":"2024-09-21T10:21:16.820414+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:03:45.889214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1832,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818454+0000","last_change":"2024-09-21T10:17:51.249710+0000","last_active":"2024-09-21T10:21:16.818454+0000","last_peered":"2024-09-21T10:21:16.818454+0000","last_clean":"2024-09-21T10:21:16.818454+0000","last_became_active":"2024-09-21T10:17:51.249284+0000","last_became_peered":"2024-09-21T10:17:51.249284+0000","last_unstale":"2024-09-21T10:21:16.818454+0000","last_undegraded":"2024-09-21T10:21:16.818454+0000","last_fullsized":"2024-09-21T10:21:16.818454+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:33:09.927717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":230,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.820675+0000","last_change":"2024-09-21T10:16:38.451401+0000","last_active":"2024-09-21T10:21:16.820675+0000","last_peered":"2024-09-21T10:21:16.820675+0000","last_clean":"2024-09-21T10:21:16.820675+0000","last_became_active":"2024-09-21T10:16:38.450094+0000","last_became_peered":"2024-09-21T10:16:38.450094+0000","last_unstale":"2024-09-21T10:21:16.820675+0000","last_undegraded":"2024-09-21T10:21:16.820675+0000","last_fullsized":"2024-09-21T10:21:16.820675+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:22:50.798278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":509,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818870+0000","last_change":"2024-09-21T10:16:38.451461+0000","last_active":"2024-09-21T10:21:16.818870+0000","last_peered":"2024-09-21T10:21:16.818870+0000","last_clean":"2024-09-21T10:21:16.818870+0000","last_became_active":"2024-09-21T10:16:38.451137+0000","last_became_peered":"2024-09-21T10:16:38.451137+0000","last_unstale":"2024-09-21T10:21:16.818870+0000","last_undegraded":"2024-09-21T10:21:16.818870+0000","last_fullsized":"2024-09-21T10:21:16.818870+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:40.847463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":288,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818579+0000","last_change":"2024-09-21T10:17:51.269584+0000","last_active":"2024-09-21T10:21:16.818579+0000","last_peered":"2024-09-21T10:21:16.818579+0000","last_clean":"2024-09-21T10:21:16.818579+0000","last_became_active":"2024-09-21T10:17:51.260673+0000","last_became_peered":"2024-09-21T10:17:51.260673+0000","last_unstale":"2024-09-21T10:21:16.818579+0000","last_undegraded":"2024-09-21T10:21:16.818579+0000","last_fullsized":"2024-09-21T10:21:16.818579+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:16.649818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"1349'3297","reported_seq":3310,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.405064+0000","last_change":"2024-09-21T10:20:52.262384+0000","last_active":"2024-09-21T10:21:26.405064+0000","last_peered":"2024-09-21T10:21:26.405064+0000","last_clean":"2024-09-21T10:21:26.405064+0000","last_became_active":"2024-09-21T10:20:52.262080+0000","last_became_peered":"2024-09-21T10:20:52.262080+0000","last_unstale":"2024-09-21T10:21:26.405064+0000","last_undegraded":"2024-09-21T10:21:26.405064+0000","last_fullsized":"2024-09-21T10:21:26.405064+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3297,"log_dups_size":0,"ondisk_log_size":3297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:48:52.162020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216072192,"num_objects":3297,"num_object_clones":0,"num_object_copies":9891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3297,"num_write_kb":211008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":418,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.824981+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:21:16.824981+0000","last_peered":"2024-09-21T10:21:16.824981+0000","last_clean":"2024-09-21T10:21:16.824981+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:21:16.824981+0000","last_undegraded":"2024-09-21T10:21:16.824981+0000","last_fullsized":"2024-09-21T10:21:16.824981+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.848900+0000","last_change":"2024-09-21T10:17:51.256335+0000","last_active":"2024-09-21T10:21:01.848900+0000","last_peered":"2024-09-21T10:21:01.848900+0000","last_clean":"2024-09-21T10:21:01.848900+0000","last_became_active":"2024-09-21T10:17:51.251630+0000","last_became_peered":"2024-09-21T10:17:51.251630+0000","last_unstale":"2024-09-21T10:21:01.848900+0000","last_undegraded":"2024-09-21T10:21:01.848900+0000","last_fullsized":"2024-09-21T10:21:01.848900+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:31.141409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"1349'3274","reported_seq":3295,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.877785+0000","last_change":"2024-09-21T10:20:52.259803+0000","last_active":"2024-09-21T10:21:26.877785+0000","last_peered":"2024-09-21T10:21:26.877785+0000","last_clean":"2024-09-21T10:21:26.877785+0000","last_became_active":"2024-09-21T10:20:52.259530+0000","last_became_peered":"2024-09-21T10:20:52.259530+0000","last_unstale":"2024-09-21T10:21:26.877785+0000","last_undegraded":"2024-09-21T10:21:26.877785+0000","last_fullsized":"2024-09-21T10:21:26.877785+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3274,"log_dups_size":0,"ondisk_log_size":3274,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:23.185046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214564864,"num_objects":3274,"num_object_clones":0,"num_object_copies":9822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3274,"num_write_kb":209536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":101,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.045918+0000","last_change":"2024-09-21T10:17:51.281550+0000","last_active":"2024-09-21T10:21:02.045918+0000","last_peered":"2024-09-21T10:21:02.045918+0000","last_clean":"2024-09-21T10:21:02.045918+0000","last_became_active":"2024-09-21T10:17:51.262457+0000","last_became_peered":"2024-09-21T10:17:51.262457+0000","last_unstale":"2024-09-21T10:21:02.045918+0000","last_undegraded":"2024-09-21T10:21:02.045918+0000","last_fullsized":"2024-09-21T10:21:02.045918+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:49.212195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1493,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823457+0000","last_change":"2024-09-21T10:16:38.461867+0000","last_active":"2024-09-21T10:21:16.823457+0000","last_peered":"2024-09-21T10:21:16.823457+0000","last_clean":"2024-09-21T10:21:16.823457+0000","last_became_active":"2024-09-21T10:16:38.461416+0000","last_became_peered":"2024-09-21T10:16:38.461416+0000","last_unstale":"2024-09-21T10:21:16.823457+0000","last_undegraded":"2024-09-21T10:21:16.823457+0000","last_fullsized":"2024-09-21T10:21:16.823457+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:15.872371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.5","version":"1349'3304","reported_seq":3316,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.415786+0000","last_change":"2024-09-21T10:20:52.262868+0000","last_active":"2024-09-21T10:21:26.415786+0000","last_peered":"2024-09-21T10:21:26.415786+0000","last_clean":"2024-09-21T10:21:26.415786+0000","last_became_active":"2024-09-21T10:20:52.262578+0000","last_became_peered":"2024-09-21T10:20:52.262578+0000","last_unstale":"2024-09-21T10:21:26.415786+0000","last_undegraded":"2024-09-21T10:21:26.415786+0000","last_fullsized":"2024-09-21T10:21:26.415786+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3304,"log_dups_size":0,"ondisk_log_size":3304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:45:57.347968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216530944,"num_objects":3304,"num_object_clones":0,"num_object_copies":9912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3304,"num_write_kb":211456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":56,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849430+0000","last_change":"2024-09-21T10:17:51.283569+0000","last_active":"2024-09-21T10:21:01.849430+0000","last_peered":"2024-09-21T10:21:01.849430+0000","last_clean":"2024-09-21T10:21:01.849430+0000","last_became_active":"2024-09-21T10:17:51.259187+0000","last_became_peered":"2024-09-21T10:17:51.259187+0000","last_unstale":"2024-09-21T10:21:01.849430+0000","last_undegraded":"2024-09-21T10:21:01.849430+0000","last_fullsized":"2024-09-21T10:21:01.849430+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:18:55.002645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":239,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.045606+0000","last_change":"2024-09-21T10:17:51.248805+0000","last_active":"2024-09-21T10:21:02.045606+0000","last_peered":"2024-09-21T10:21:02.045606+0000","last_clean":"2024-09-21T10:21:02.045606+0000","last_became_active":"2024-09-21T10:17:51.248157+0000","last_became_peered":"2024-09-21T10:17:51.248157+0000","last_unstale":"2024-09-21T10:21:02.045606+0000","last_undegraded":"2024-09-21T10:21:02.045606+0000","last_fullsized":"2024-09-21T10:21:02.045606+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:38:36.573842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"1349'3314","reported_seq":3326,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:27.040568+0000","last_change":"2024-09-21T10:20:52.266398+0000","last_active":"2024-09-21T10:21:27.040568+0000","last_peered":"2024-09-21T10:21:27.040568+0000","last_clean":"2024-09-21T10:21:27.040568+0000","last_became_active":"2024-09-21T10:20:52.265503+0000","last_became_peered":"2024-09-21T10:20:52.265503+0000","last_unstale":"2024-09-21T10:21:27.040568+0000","last_undegraded":"2024-09-21T10:21:27.040568+0000","last_fullsized":"2024-09-21T10:21:27.040568+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3314,"log_dups_size":0,"ondisk_log_size":3314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:52.953393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3314,"num_write_kb":212096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849267+0000","last_change":"2024-09-21T10:17:51.259331+0000","last_active":"2024-09-21T10:21:01.849267+0000","last_peered":"2024-09-21T10:21:01.849267+0000","last_clean":"2024-09-21T10:21:01.849267+0000","last_became_active":"2024-09-21T10:17:51.258943+0000","last_became_peered":"2024-09-21T10:17:51.258943+0000","last_unstale":"2024-09-21T10:21:01.849267+0000","last_undegraded":"2024-09-21T10:21:01.849267+0000","last_fullsized":"2024-09-21T10:21:01.849267+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:09:59.304035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1878,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.825845+0000","last_change":"2024-09-21T10:16:38.461644+0000","last_active":"2024-09-21T10:21:16.825845+0000","last_peered":"2024-09-21T10:21:16.825845+0000","last_clean":"2024-09-21T10:21:16.825845+0000","last_became_active":"2024-09-21T10:16:38.461179+0000","last_became_peered":"2024-09-21T10:16:38.461179+0000","last_unstale":"2024-09-21T10:21:16.825845+0000","last_undegraded":"2024-09-21T10:21:16.825845+0000","last_fullsized":"2024-09-21T10:21:16.825845+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:43.638430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.3","version":"1349'3255","reported_seq":3269,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:27.031919+0000","last_change":"2024-09-21T10:20:52.262575+0000","last_active":"2024-09-21T10:21:27.031919+0000","last_peered":"2024-09-21T10:21:27.031919+0000","last_clean":"2024-09-21T10:21:27.031919+0000","last_became_active":"2024-09-21T10:20:52.262291+0000","last_became_peered":"2024-09-21T10:20:52.262291+0000","last_unstale":"2024-09-21T10:21:27.031919+0000","last_undegraded":"2024-09-21T10:21:27.031919+0000","last_fullsized":"2024-09-21T10:21:27.031919+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3255,"log_dups_size":0,"ondisk_log_size":3255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:57.330139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213319680,"num_objects":3255,"num_object_clones":0,"num_object_copies":9765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3255,"num_write_kb":208320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":462,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818123+0000","last_change":"2024-09-21T10:16:38.459890+0000","last_active":"2024-09-21T10:21:16.818123+0000","last_peered":"2024-09-21T10:21:16.818123+0000","last_clean":"2024-09-21T10:21:16.818123+0000","last_became_active":"2024-09-21T10:16:38.454382+0000","last_became_peered":"2024-09-21T10:16:38.454382+0000","last_unstale":"2024-09-21T10:21:16.818123+0000","last_undegraded":"2024-09-21T10:21:16.818123+0000","last_fullsized":"2024-09-21T10:21:16.818123+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:02.741403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1506,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818485+0000","last_change":"2024-09-21T10:16:38.454504+0000","last_active":"2024-09-21T10:21:16.818485+0000","last_peered":"2024-09-21T10:21:16.818485+0000","last_clean":"2024-09-21T10:21:16.818485+0000","last_became_active":"2024-09-21T10:16:38.448833+0000","last_became_peered":"2024-09-21T10:16:38.448833+0000","last_unstale":"2024-09-21T10:21:16.818485+0000","last_undegraded":"2024-09-21T10:21:16.818485+0000","last_fullsized":"2024-09-21T10:21:16.818485+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:11:11.905645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"1349'3273","reported_seq":3294,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.777243+0000","last_change":"2024-09-21T10:20:52.263457+0000","last_active":"2024-09-21T10:21:26.777243+0000","last_peered":"2024-09-21T10:21:26.777243+0000","last_clean":"2024-09-21T10:21:26.777243+0000","last_became_active":"2024-09-21T10:20:52.262260+0000","last_became_peered":"2024-09-21T10:20:52.262260+0000","last_unstale":"2024-09-21T10:21:26.777243+0000","last_undegraded":"2024-09-21T10:21:26.777243+0000","last_fullsized":"2024-09-21T10:21:26.777243+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3273,"log_dups_size":0,"ondisk_log_size":3273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:50:59.675522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3273,"num_write_kb":209472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":206,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.045468+0000","last_change":"2024-09-21T10:02:38.994334+0000","last_active":"2024-09-21T10:21:02.045468+0000","last_peered":"2024-09-21T10:21:02.045468+0000","last_clean":"2024-09-21T10:21:02.045468+0000","last_became_active":"2024-09-21T10:02:38.993888+0000","last_became_peered":"2024-09-21T10:02:38.993888+0000","last_unstale":"2024-09-21T10:21:02.045468+0000","last_undegraded":"2024-09-21T10:21:02.045468+0000","last_fullsized":"2024-09-21T10:21:02.045468+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:12:50.573652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":106,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818238+0000","last_change":"2024-09-21T10:16:39.439192+0000","last_active":"2024-09-21T10:21:16.818238+0000","last_peered":"2024-09-21T10:21:16.818238+0000","last_clean":"2024-09-21T10:21:16.818238+0000","last_became_active":"2024-09-21T10:16:39.438778+0000","last_became_peered":"2024-09-21T10:16:39.438778+0000","last_unstale":"2024-09-21T10:21:16.818238+0000","last_undegraded":"2024-09-21T10:21:16.818238+0000","last_fullsized":"2024-09-21T10:21:16.818238+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:49.643063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.1","version":"1349'3367","reported_seq":3380,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:27.021983+0000","last_change":"2024-09-21T10:20:52.263910+0000","last_active":"2024-09-21T10:21:27.021983+0000","last_peered":"2024-09-21T10:21:27.021983+0000","last_clean":"2024-09-21T10:21:27.021983+0000","last_became_active":"2024-09-21T10:20:52.263363+0000","last_became_peered":"2024-09-21T10:20:52.263363+0000","last_unstale":"2024-09-21T10:21:27.021983+0000","last_undegraded":"2024-09-21T10:21:27.021983+0000","last_fullsized":"2024-09-21T10:21:27.021983+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3367,"log_dups_size":0,"ondisk_log_size":3367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:57:27.347679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220659712,"num_objects":3367,"num_object_clones":0,"num_object_copies":10101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3367,"num_write_kb":215488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1538,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823376+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:21:16.823376+0000","last_peered":"2024-09-21T10:21:16.823376+0000","last_clean":"2024-09-21T10:21:16.823376+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:21:16.823376+0000","last_undegraded":"2024-09-21T10:21:16.823376+0000","last_fullsized":"2024-09-21T10:21:16.823376+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":206,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.046060+0000","last_change":"2024-09-21T10:02:38.976956+0000","last_active":"2024-09-21T10:21:02.046060+0000","last_peered":"2024-09-21T10:21:02.046060+0000","last_clean":"2024-09-21T10:21:02.046060+0000","last_became_active":"2024-09-21T10:02:38.976579+0000","last_became_peered":"2024-09-21T10:02:38.976579+0000","last_unstale":"2024-09-21T10:21:02.046060+0000","last_undegraded":"2024-09-21T10:21:02.046060+0000","last_fullsized":"2024-09-21T10:21:02.046060+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:26.948945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"1349'3222","reported_seq":3243,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.876409+0000","last_change":"2024-09-21T10:20:52.265422+0000","last_active":"2024-09-21T10:21:26.876409+0000","last_peered":"2024-09-21T10:21:26.876409+0000","last_clean":"2024-09-21T10:21:26.876409+0000","last_became_active":"2024-09-21T10:20:52.265135+0000","last_became_peered":"2024-09-21T10:20:52.265135+0000","last_unstale":"2024-09-21T10:21:26.876409+0000","last_undegraded":"2024-09-21T10:21:26.876409+0000","last_fullsized":"2024-09-21T10:21:26.876409+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3222,"log_dups_size":0,"ondisk_log_size":3222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:34:00.471329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211156992,"num_objects":3222,"num_object_clones":0,"num_object_copies":9666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3222,"num_write_kb":206208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":487,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819047+0000","last_change":"2024-09-21T10:16:38.454278+0000","last_active":"2024-09-21T10:21:16.819047+0000","last_peered":"2024-09-21T10:21:16.819047+0000","last_clean":"2024-09-21T10:21:16.819047+0000","last_became_active":"2024-09-21T10:16:38.449058+0000","last_became_peered":"2024-09-21T10:16:38.449058+0000","last_unstale":"2024-09-21T10:21:16.819047+0000","last_undegraded":"2024-09-21T10:21:16.819047+0000","last_fullsized":"2024-09-21T10:21:16.819047+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:37:02.332428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1883,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.825207+0000","last_change":"2024-09-21T10:16:38.460495+0000","last_active":"2024-09-21T10:21:16.825207+0000","last_peered":"2024-09-21T10:21:16.825207+0000","last_clean":"2024-09-21T10:21:16.825207+0000","last_became_active":"2024-09-21T10:16:38.458759+0000","last_became_peered":"2024-09-21T10:16:38.458759+0000","last_unstale":"2024-09-21T10:21:16.825207+0000","last_undegraded":"2024-09-21T10:21:16.825207+0000","last_fullsized":"2024-09-21T10:21:16.825207+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:37.031677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"1349'3337","reported_seq":3358,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.789603+0000","last_change":"2024-09-21T10:20:52.263492+0000","last_active":"2024-09-21T10:21:26.789603+0000","last_peered":"2024-09-21T10:21:26.789603+0000","last_clean":"2024-09-21T10:21:26.789603+0000","last_became_active":"2024-09-21T10:20:52.263180+0000","last_became_peered":"2024-09-21T10:20:52.263180+0000","last_unstale":"2024-09-21T10:21:26.789603+0000","last_undegraded":"2024-09-21T10:21:26.789603+0000","last_fullsized":"2024-09-21T10:21:26.789603+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3337,"log_dups_size":0,"ondisk_log_size":3337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:25:38.672385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218693632,"num_objects":3337,"num_object_clones":0,"num_object_copies":10011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3337,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":902,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.044720+0000","last_change":"2024-09-21T10:15:22.139134+0000","last_active":"2024-09-21T10:21:02.044720+0000","last_peered":"2024-09-21T10:21:02.044720+0000","last_clean":"2024-09-21T10:21:02.044720+0000","last_became_active":"2024-09-21T10:15:22.138788+0000","last_became_peered":"2024-09-21T10:15:22.138788+0000","last_unstale":"2024-09-21T10:21:02.044720+0000","last_undegraded":"2024-09-21T10:21:02.044720+0000","last_fullsized":"2024-09-21T10:21:02.044720+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:52:35.831693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":56,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849135+0000","last_change":"2024-09-21T10:17:51.287937+0000","last_active":"2024-09-21T10:21:01.849135+0000","last_peered":"2024-09-21T10:21:01.849135+0000","last_clean":"2024-09-21T10:21:01.849135+0000","last_became_active":"2024-09-21T10:17:51.256610+0000","last_became_peered":"2024-09-21T10:17:51.256610+0000","last_unstale":"2024-09-21T10:21:01.849135+0000","last_undegraded":"2024-09-21T10:21:01.849135+0000","last_fullsized":"2024-09-21T10:21:01.849135+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:32:15.137072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":500,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849136+0000","last_change":"2024-09-21T10:18:07.086426+0000","last_active":"2024-09-21T10:21:01.849136+0000","last_peered":"2024-09-21T10:21:01.849136+0000","last_clean":"2024-09-21T10:21:01.849136+0000","last_became_active":"2024-09-21T10:18:07.086157+0000","last_became_peered":"2024-09-21T10:18:07.086157+0000","last_unstale":"2024-09-21T10:21:01.849136+0000","last_undegraded":"2024-09-21T10:21:01.849136+0000","last_fullsized":"2024-09-21T10:21:01.849136+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:24:34.289605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":230,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.046117+0000","last_change":"2024-09-21T10:16:38.450231+0000","last_active":"2024-09-21T10:21:02.046117+0000","last_peered":"2024-09-21T10:21:02.046117+0000","last_clean":"2024-09-21T10:21:02.046117+0000","last_became_active":"2024-09-21T10:16:38.449941+0000","last_became_peered":"2024-09-21T10:16:38.449941+0000","last_unstale":"2024-09-21T10:21:02.046117+0000","last_undegraded":"2024-09-21T10:21:02.046117+0000","last_fullsized":"2024-09-21T10:21:02.046117+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:51.559013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"1349'3315","reported_seq":3336,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.787225+0000","last_change":"2024-09-21T10:20:52.262741+0000","last_active":"2024-09-21T10:21:26.787225+0000","last_peered":"2024-09-21T10:21:26.787225+0000","last_clean":"2024-09-21T10:21:26.787225+0000","last_became_active":"2024-09-21T10:20:52.262490+0000","last_became_peered":"2024-09-21T10:20:52.262490+0000","last_unstale":"2024-09-21T10:21:26.787225+0000","last_undegraded":"2024-09-21T10:21:26.787225+0000","last_fullsized":"2024-09-21T10:21:26.787225+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3315,"log_dups_size":0,"ondisk_log_size":3315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:43:05.588475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217251840,"num_objects":3315,"num_object_clones":0,"num_object_copies":9945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3315,"num_write_kb":212160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1174,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823268+0000","last_change":"2024-09-21T10:17:51.264078+0000","last_active":"2024-09-21T10:21:16.823268+0000","last_peered":"2024-09-21T10:21:16.823268+0000","last_clean":"2024-09-21T10:21:16.823268+0000","last_became_active":"2024-09-21T10:17:51.263721+0000","last_became_peered":"2024-09-21T10:17:51.263721+0000","last_unstale":"2024-09-21T10:21:16.823268+0000","last_undegraded":"2024-09-21T10:21:16.823268+0000","last_fullsized":"2024-09-21T10:21:16.823268+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:46:02.104778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1664,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819526+0000","last_change":"2024-09-21T10:17:51.251859+0000","last_active":"2024-09-21T10:21:16.819526+0000","last_peered":"2024-09-21T10:21:16.819526+0000","last_clean":"2024-09-21T10:21:16.819526+0000","last_became_active":"2024-09-21T10:17:51.251525+0000","last_became_peered":"2024-09-21T10:17:51.251525+0000","last_unstale":"2024-09-21T10:21:16.819526+0000","last_undegraded":"2024-09-21T10:21:16.819526+0000","last_fullsized":"2024-09-21T10:21:16.819526+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:24.106509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1168,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823917+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:21:16.823917+0000","last_peered":"2024-09-21T10:21:16.823917+0000","last_clean":"2024-09-21T10:21:16.823917+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:21:16.823917+0000","last_undegraded":"2024-09-21T10:21:16.823917+0000","last_fullsized":"2024-09-21T10:21:16.823917+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"1349'3287","reported_seq":3308,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.784826+0000","last_change":"2024-09-21T10:20:52.263386+0000","last_active":"2024-09-21T10:21:26.784826+0000","last_peered":"2024-09-21T10:21:26.784826+0000","last_clean":"2024-09-21T10:21:26.784826+0000","last_became_active":"2024-09-21T10:20:52.263003+0000","last_became_peered":"2024-09-21T10:20:52.263003+0000","last_unstale":"2024-09-21T10:21:26.784826+0000","last_undegraded":"2024-09-21T10:21:26.784826+0000","last_fullsized":"2024-09-21T10:21:26.784826+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3287,"log_dups_size":0,"ondisk_log_size":3287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:48:03.599804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3287,"num_write_kb":210368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1754,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819745+0000","last_change":"2024-09-21T10:16:38.456464+0000","last_active":"2024-09-21T10:21:16.819745+0000","last_peered":"2024-09-21T10:21:16.819745+0000","last_clean":"2024-09-21T10:21:16.819745+0000","last_became_active":"2024-09-21T10:16:38.455345+0000","last_became_peered":"2024-09-21T10:16:38.455345+0000","last_unstale":"2024-09-21T10:21:16.819745+0000","last_undegraded":"2024-09-21T10:21:16.819745+0000","last_fullsized":"2024-09-21T10:21:16.819745+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:39.393023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1898,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823607+0000","last_change":"2024-09-21T10:17:51.254069+0000","last_active":"2024-09-21T10:21:16.823607+0000","last_peered":"2024-09-21T10:21:16.823607+0000","last_clean":"2024-09-21T10:21:16.823607+0000","last_became_active":"2024-09-21T10:17:51.253723+0000","last_became_peered":"2024-09-21T10:17:51.253723+0000","last_unstale":"2024-09-21T10:21:16.823607+0000","last_undegraded":"2024-09-21T10:21:16.823607+0000","last_fullsized":"2024-09-21T10:21:16.823607+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:22.160096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1898,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.825083+0000","last_change":"2024-09-21T10:17:51.249147+0000","last_active":"2024-09-21T10:21:16.825083+0000","last_peered":"2024-09-21T10:21:16.825083+0000","last_clean":"2024-09-21T10:21:16.825083+0000","last_became_active":"2024-09-21T10:17:51.248713+0000","last_became_peered":"2024-09-21T10:17:51.248713+0000","last_unstale":"2024-09-21T10:21:16.825083+0000","last_undegraded":"2024-09-21T10:21:16.825083+0000","last_fullsized":"2024-09-21T10:21:16.825083+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:22:33.757896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"1349'3215","reported_seq":3236,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.788858+0000","last_change":"2024-09-21T10:20:52.261920+0000","last_active":"2024-09-21T10:21:26.788858+0000","last_peered":"2024-09-21T10:21:26.788858+0000","last_clean":"2024-09-21T10:21:26.788858+0000","last_became_active":"2024-09-21T10:20:52.261632+0000","last_became_peered":"2024-09-21T10:20:52.261632+0000","last_unstale":"2024-09-21T10:21:26.788858+0000","last_undegraded":"2024-09-21T10:21:26.788858+0000","last_fullsized":"2024-09-21T10:21:26.788858+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3215,"log_dups_size":0,"ondisk_log_size":3215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:20:56.183818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210698240,"num_objects":3215,"num_object_clones":0,"num_object_copies":9645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3215,"num_write_kb":205760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":56,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849300+0000","last_change":"2024-09-21T10:17:51.286472+0000","last_active":"2024-09-21T10:21:01.849300+0000","last_peered":"2024-09-21T10:21:01.849300+0000","last_clean":"2024-09-21T10:21:01.849300+0000","last_became_active":"2024-09-21T10:17:51.249949+0000","last_became_peered":"2024-09-21T10:17:51.249949+0000","last_unstale":"2024-09-21T10:21:01.849300+0000","last_undegraded":"2024-09-21T10:21:01.849300+0000","last_fullsized":"2024-09-21T10:21:01.849300+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:31.760578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1317,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.045548+0000","last_change":"2024-09-21T10:03:28.853530+0000","last_active":"2024-09-21T10:21:02.045548+0000","last_peered":"2024-09-21T10:21:02.045548+0000","last_clean":"2024-09-21T10:21:02.045548+0000","last_became_active":"2024-09-21T10:03:28.853266+0000","last_became_peered":"2024-09-21T10:03:28.853266+0000","last_unstale":"2024-09-21T10:21:02.045548+0000","last_undegraded":"2024-09-21T10:21:02.045548+0000","last_fullsized":"2024-09-21T10:21:02.045548+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:15:13.333841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1840,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818265+0000","last_change":"2024-09-21T10:16:39.440143+0000","last_active":"2024-09-21T10:21:16.818265+0000","last_peered":"2024-09-21T10:21:16.818265+0000","last_clean":"2024-09-21T10:21:16.818265+0000","last_became_active":"2024-09-21T10:16:39.439370+0000","last_became_peered":"2024-09-21T10:16:39.439370+0000","last_unstale":"2024-09-21T10:21:16.818265+0000","last_undegraded":"2024-09-21T10:21:16.818265+0000","last_fullsized":"2024-09-21T10:21:16.818265+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:52:36.417381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"1349'3413","reported_seq":3427,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:27.036508+0000","last_change":"2024-09-21T10:20:52.264265+0000","last_active":"2024-09-21T10:21:27.036508+0000","last_peered":"2024-09-21T10:21:27.036508+0000","last_clean":"2024-09-21T10:21:27.036508+0000","last_became_active":"2024-09-21T10:20:52.264006+0000","last_became_peered":"2024-09-21T10:20:52.264006+0000","last_unstale":"2024-09-21T10:21:27.036508+0000","last_undegraded":"2024-09-21T10:21:27.036508+0000","last_fullsized":"2024-09-21T10:21:27.036508+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3413,"log_dups_size":0,"ondisk_log_size":3413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:55.611292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223674368,"num_objects":3413,"num_object_clones":0,"num_object_copies":10239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3413,"num_write_kb":218432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849446+0000","last_change":"2024-09-21T10:17:51.259438+0000","last_active":"2024-09-21T10:21:01.849446+0000","last_peered":"2024-09-21T10:21:01.849446+0000","last_clean":"2024-09-21T10:21:01.849446+0000","last_became_active":"2024-09-21T10:17:51.258704+0000","last_became_peered":"2024-09-21T10:17:51.258704+0000","last_unstale":"2024-09-21T10:21:01.849446+0000","last_undegraded":"2024-09-21T10:21:01.849446+0000","last_fullsized":"2024-09-21T10:21:01.849446+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:16.479749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1913,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823940+0000","last_change":"2024-09-21T10:16:38.460671+0000","last_active":"2024-09-21T10:21:16.823940+0000","last_peered":"2024-09-21T10:21:16.823940+0000","last_clean":"2024-09-21T10:21:16.823940+0000","last_became_active":"2024-09-21T10:16:38.459939+0000","last_became_peered":"2024-09-21T10:16:38.459939+0000","last_unstale":"2024-09-21T10:21:16.823940+0000","last_undegraded":"2024-09-21T10:21:16.823940+0000","last_fullsized":"2024-09-21T10:21:16.823940+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:51.143325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":439,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823510+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:21:16.823510+0000","last_peered":"2024-09-21T10:21:16.823510+0000","last_clean":"2024-09-21T10:21:16.823510+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:21:16.823510+0000","last_undegraded":"2024-09-21T10:21:16.823510+0000","last_fullsized":"2024-09-21T10:21:16.823510+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"1349'3388","reported_seq":3409,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.874181+0000","last_change":"2024-09-21T10:20:52.260323+0000","last_active":"2024-09-21T10:21:26.874181+0000","last_peered":"2024-09-21T10:21:26.874181+0000","last_clean":"2024-09-21T10:21:26.874181+0000","last_became_active":"2024-09-21T10:20:52.259616+0000","last_became_peered":"2024-09-21T10:20:52.259616+0000","last_unstale":"2024-09-21T10:21:26.874181+0000","last_undegraded":"2024-09-21T10:21:26.874181+0000","last_fullsized":"2024-09-21T10:21:26.874181+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3388,"log_dups_size":0,"ondisk_log_size":3388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:48:45.093720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":216832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":113,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818087+0000","last_change":"2024-09-21T10:17:51.250313+0000","last_active":"2024-09-21T10:21:16.818087+0000","last_peered":"2024-09-21T10:21:16.818087+0000","last_clean":"2024-09-21T10:21:16.818087+0000","last_became_active":"2024-09-21T10:17:51.249931+0000","last_became_peered":"2024-09-21T10:17:51.249931+0000","last_unstale":"2024-09-21T10:21:16.818087+0000","last_undegraded":"2024-09-21T10:21:16.818087+0000","last_fullsized":"2024-09-21T10:21:16.818087+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:38:15.879964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1052,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823997+0000","last_change":"2024-09-21T10:03:28.848851+0000","last_active":"2024-09-21T10:21:16.823997+0000","last_peered":"2024-09-21T10:21:16.823997+0000","last_clean":"2024-09-21T10:21:16.823997+0000","last_became_active":"2024-09-21T10:03:28.848543+0000","last_became_peered":"2024-09-21T10:03:28.848543+0000","last_unstale":"2024-09-21T10:21:16.823997+0000","last_undegraded":"2024-09-21T10:21:16.823997+0000","last_fullsized":"2024-09-21T10:21:16.823997+0000","mapping_epoch":1169,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:17.805494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849128+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:21:01.849128+0000","last_peered":"2024-09-21T10:21:01.849128+0000","last_clean":"2024-09-21T10:21:01.849128+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:21:01.849128+0000","last_undegraded":"2024-09-21T10:21:01.849128+0000","last_fullsized":"2024-09-21T10:21:01.849128+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.f","version":"1349'3255","reported_seq":3274,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.884815+0000","last_change":"2024-09-21T10:20:52.260187+0000","last_active":"2024-09-21T10:21:26.884815+0000","last_peered":"2024-09-21T10:21:26.884815+0000","last_clean":"2024-09-21T10:21:26.884815+0000","last_became_active":"2024-09-21T10:20:52.258903+0000","last_became_peered":"2024-09-21T10:20:52.258903+0000","last_unstale":"2024-09-21T10:21:26.884815+0000","last_undegraded":"2024-09-21T10:21:26.884815+0000","last_fullsized":"2024-09-21T10:21:26.884815+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3255,"log_dups_size":0,"ondisk_log_size":3255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:39:44.440093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213319680,"num_objects":3255,"num_object_clones":0,"num_object_copies":9765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3255,"num_write_kb":208320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":448,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849468+0000","last_change":"2024-09-21T10:18:07.090592+0000","last_active":"2024-09-21T10:21:01.849468+0000","last_peered":"2024-09-21T10:21:01.849468+0000","last_clean":"2024-09-21T10:21:01.849468+0000","last_became_active":"2024-09-21T10:18:07.090188+0000","last_became_peered":"2024-09-21T10:18:07.090188+0000","last_unstale":"2024-09-21T10:21:01.849468+0000","last_undegraded":"2024-09-21T10:21:01.849468+0000","last_fullsized":"2024-09-21T10:21:01.849468+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:16:54.763114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":106,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818213+0000","last_change":"2024-09-21T10:16:39.439344+0000","last_active":"2024-09-21T10:21:16.818213+0000","last_peered":"2024-09-21T10:21:16.818213+0000","last_clean":"2024-09-21T10:21:16.818213+0000","last_became_active":"2024-09-21T10:16:39.438987+0000","last_became_peered":"2024-09-21T10:16:39.438987+0000","last_unstale":"2024-09-21T10:21:16.818213+0000","last_undegraded":"2024-09-21T10:21:16.818213+0000","last_fullsized":"2024-09-21T10:21:16.818213+0000","mapping_epoch":1302,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1303,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:17.163136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":468,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818918+0000","last_change":"2024-09-21T10:17:51.250156+0000","last_active":"2024-09-21T10:21:16.818918+0000","last_peered":"2024-09-21T10:21:16.818918+0000","last_clean":"2024-09-21T10:21:16.818918+0000","last_became_active":"2024-09-21T10:17:51.248282+0000","last_became_peered":"2024-09-21T10:17:51.248282+0000","last_unstale":"2024-09-21T10:21:16.818918+0000","last_undegraded":"2024-09-21T10:21:16.818918+0000","last_fullsized":"2024-09-21T10:21:16.818918+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:58.954423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"1349'3273","reported_seq":3294,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:26.852010+0000","last_change":"2024-09-21T10:20:52.259785+0000","last_active":"2024-09-21T10:21:26.852010+0000","last_peered":"2024-09-21T10:21:26.852010+0000","last_clean":"2024-09-21T10:21:26.852010+0000","last_became_active":"2024-09-21T10:20:52.259403+0000","last_became_peered":"2024-09-21T10:20:52.259403+0000","last_unstale":"2024-09-21T10:21:26.852010+0000","last_undegraded":"2024-09-21T10:21:26.852010+0000","last_fullsized":"2024-09-21T10:21:26.852010+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":1344,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:20:51.245365+0000","last_clean_scrub_stamp":"2024-09-21T10:20:51.245365+0000","objects_scrubbed":0,"log_size":3273,"log_dups_size":0,"ondisk_log_size":3273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:24.197238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3273,"num_write_kb":209472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":424,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.824202+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:21:16.824202+0000","last_peered":"2024-09-21T10:21:16.824202+0000","last_clean":"2024-09-21T10:21:16.824202+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:21:16.824202+0000","last_undegraded":"2024-09-21T10:21:16.824202+0000","last_fullsized":"2024-09-21T10:21:16.824202+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":230,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.046089+0000","last_change":"2024-09-21T10:16:38.455089+0000","last_active":"2024-09-21T10:21:02.046089+0000","last_peered":"2024-09-21T10:21:02.046089+0000","last_clean":"2024-09-21T10:21:02.046089+0000","last_became_active":"2024-09-21T10:16:38.454804+0000","last_became_peered":"2024-09-21T10:16:38.454804+0000","last_unstale":"2024-09-21T10:21:02.046089+0000","last_undegraded":"2024-09-21T10:21:02.046089+0000","last_fullsized":"2024-09-21T10:21:02.046089+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:21:16.448016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.848917+0000","last_change":"2024-09-21T10:17:51.250324+0000","last_active":"2024-09-21T10:21:01.848917+0000","last_peered":"2024-09-21T10:21:01.848917+0000","last_clean":"2024-09-21T10:21:01.848917+0000","last_became_active":"2024-09-21T10:17:51.249933+0000","last_became_peered":"2024-09-21T10:17:51.249933+0000","last_unstale":"2024-09-21T10:21:01.848917+0000","last_undegraded":"2024-09-21T10:21:01.848917+0000","last_fullsized":"2024-09-21T10:21:01.848917+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:18:23.880577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":206,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819072+0000","last_change":"2024-09-21T10:02:39.020163+0000","last_active":"2024-09-21T10:21:16.819072+0000","last_peered":"2024-09-21T10:21:16.819072+0000","last_clean":"2024-09-21T10:21:16.819072+0000","last_became_active":"2024-09-21T10:02:39.019693+0000","last_became_peered":"2024-09-21T10:02:39.019693+0000","last_unstale":"2024-09-21T10:21:16.819072+0000","last_undegraded":"2024-09-21T10:21:16.819072+0000","last_fullsized":"2024-09-21T10:21:16.819072+0000","mapping_epoch":1161,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:32:44.532822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":269,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819017+0000","last_change":"2024-09-21T10:17:51.277615+0000","last_active":"2024-09-21T10:21:16.819017+0000","last_peered":"2024-09-21T10:21:16.819017+0000","last_clean":"2024-09-21T10:21:16.819017+0000","last_became_active":"2024-09-21T10:17:51.261756+0000","last_became_peered":"2024-09-21T10:17:51.261756+0000","last_unstale":"2024-09-21T10:21:16.819017+0000","last_undegraded":"2024-09-21T10:21:16.819017+0000","last_fullsized":"2024-09-21T10:21:16.819017+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:05:35.608372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1885,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823919+0000","last_change":"2024-09-21T10:17:51.251440+0000","last_active":"2024-09-21T10:21:16.823919+0000","last_peered":"2024-09-21T10:21:16.823919+0000","last_clean":"2024-09-21T10:21:16.823919+0000","last_became_active":"2024-09-21T10:17:51.249109+0000","last_became_peered":"2024-09-21T10:17:51.249109+0000","last_unstale":"2024-09-21T10:21:16.823919+0000","last_undegraded":"2024-09-21T10:21:16.823919+0000","last_fullsized":"2024-09-21T10:21:16.823919+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:59.577984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1396,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818313+0000","last_change":"2024-09-21T10:16:38.453215+0000","last_active":"2024-09-21T10:21:16.818313+0000","last_peered":"2024-09-21T10:21:16.818313+0000","last_clean":"2024-09-21T10:21:16.818313+0000","last_became_active":"2024-09-21T10:16:38.452464+0000","last_became_peered":"2024-09-21T10:16:38.452464+0000","last_unstale":"2024-09-21T10:21:16.818313+0000","last_undegraded":"2024-09-21T10:21:16.818313+0000","last_fullsized":"2024-09-21T10:21:16.818313+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:07:16.818394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849529+0000","last_change":"2024-09-21T10:17:51.248299+0000","last_active":"2024-09-21T10:21:01.849529+0000","last_peered":"2024-09-21T10:21:01.849529+0000","last_clean":"2024-09-21T10:21:01.849529+0000","last_became_active":"2024-09-21T10:17:51.247851+0000","last_became_peered":"2024-09-21T10:17:51.247851+0000","last_unstale":"2024-09-21T10:21:01.849529+0000","last_undegraded":"2024-09-21T10:21:01.849529+0000","last_fullsized":"2024-09-21T10:21:01.849529+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:41:04.319143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":544,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819601+0000","last_change":"2024-09-21T10:17:51.248187+0000","last_active":"2024-09-21T10:21:16.819601+0000","last_peered":"2024-09-21T10:21:16.819601+0000","last_clean":"2024-09-21T10:21:16.819601+0000","last_became_active":"2024-09-21T10:17:51.247436+0000","last_became_peered":"2024-09-21T10:17:51.247436+0000","last_unstale":"2024-09-21T10:21:16.819601+0000","last_undegraded":"2024-09-21T10:21:16.819601+0000","last_fullsized":"2024-09-21T10:21:16.819601+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:58:33.505149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849626+0000","last_change":"2024-09-21T10:17:51.258577+0000","last_active":"2024-09-21T10:21:01.849626+0000","last_peered":"2024-09-21T10:21:01.849626+0000","last_clean":"2024-09-21T10:21:01.849626+0000","last_became_active":"2024-09-21T10:17:51.257820+0000","last_became_peered":"2024-09-21T10:17:51.257820+0000","last_unstale":"2024-09-21T10:21:01.849626+0000","last_undegraded":"2024-09-21T10:21:01.849626+0000","last_fullsized":"2024-09-21T10:21:01.849626+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:28:00.922957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":528,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818367+0000","last_change":"2024-09-21T10:16:38.456905+0000","last_active":"2024-09-21T10:21:16.818367+0000","last_peered":"2024-09-21T10:21:16.818367+0000","last_clean":"2024-09-21T10:21:16.818367+0000","last_became_active":"2024-09-21T10:16:38.453276+0000","last_became_peered":"2024-09-21T10:16:38.453276+0000","last_unstale":"2024-09-21T10:21:16.818367+0000","last_undegraded":"2024-09-21T10:21:16.818367+0000","last_fullsized":"2024-09-21T10:21:16.818367+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:05:38.174569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":249,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819481+0000","last_change":"2024-09-21T10:17:51.262911+0000","last_active":"2024-09-21T10:21:16.819481+0000","last_peered":"2024-09-21T10:21:16.819481+0000","last_clean":"2024-09-21T10:21:16.819481+0000","last_became_active":"2024-09-21T10:17:51.262314+0000","last_became_peered":"2024-09-21T10:17:51.262314+0000","last_unstale":"2024-09-21T10:21:16.819481+0000","last_undegraded":"2024-09-21T10:21:16.819481+0000","last_fullsized":"2024-09-21T10:21:16.819481+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:46:06.241585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":516,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819257+0000","last_change":"2024-09-21T10:17:51.254289+0000","last_active":"2024-09-21T10:21:16.819257+0000","last_peered":"2024-09-21T10:21:16.819257+0000","last_clean":"2024-09-21T10:21:16.819257+0000","last_became_active":"2024-09-21T10:17:51.253922+0000","last_became_peered":"2024-09-21T10:17:51.253922+0000","last_unstale":"2024-09-21T10:21:16.819257+0000","last_undegraded":"2024-09-21T10:21:16.819257+0000","last_fullsized":"2024-09-21T10:21:16.819257+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:57.111361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1779,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818501+0000","last_change":"2024-09-21T10:17:51.249960+0000","last_active":"2024-09-21T10:21:16.818501+0000","last_peered":"2024-09-21T10:21:16.818501+0000","last_clean":"2024-09-21T10:21:16.818501+0000","last_became_active":"2024-09-21T10:17:51.248883+0000","last_became_peered":"2024-09-21T10:17:51.248883+0000","last_unstale":"2024-09-21T10:21:16.818501+0000","last_undegraded":"2024-09-21T10:21:16.818501+0000","last_fullsized":"2024-09-21T10:21:16.818501+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:32:20.076539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":171,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.823348+0000","last_change":"2024-09-21T10:16:38.461877+0000","last_active":"2024-09-21T10:21:16.823348+0000","last_peered":"2024-09-21T10:21:16.823348+0000","last_clean":"2024-09-21T10:21:16.823348+0000","last_became_active":"2024-09-21T10:16:38.461554+0000","last_became_peered":"2024-09-21T10:16:38.461554+0000","last_unstale":"2024-09-21T10:21:16.823348+0000","last_undegraded":"2024-09-21T10:21:16.823348+0000","last_fullsized":"2024-09-21T10:21:16.823348+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:02:46.372637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":455,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818426+0000","last_change":"2024-09-21T10:16:38.447369+0000","last_active":"2024-09-21T10:21:16.818426+0000","last_peered":"2024-09-21T10:21:16.818426+0000","last_clean":"2024-09-21T10:21:16.818426+0000","last_became_active":"2024-09-21T10:16:38.447085+0000","last_became_peered":"2024-09-21T10:16:38.447085+0000","last_unstale":"2024-09-21T10:21:16.818426+0000","last_undegraded":"2024-09-21T10:21:16.818426+0000","last_fullsized":"2024-09-21T10:21:16.818426+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:59.110578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":442,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.824024+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:21:16.824024+0000","last_peered":"2024-09-21T10:21:16.824024+0000","last_clean":"2024-09-21T10:21:16.824024+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:21:16.824024+0000","last_undegraded":"2024-09-21T10:21:16.824024+0000","last_fullsized":"2024-09-21T10:21:16.824024+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1772,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.824129+0000","last_change":"2024-09-21T10:16:38.460957+0000","last_active":"2024-09-21T10:21:16.824129+0000","last_peered":"2024-09-21T10:21:16.824129+0000","last_clean":"2024-09-21T10:21:16.824129+0000","last_became_active":"2024-09-21T10:16:38.460325+0000","last_became_peered":"2024-09-21T10:16:38.460325+0000","last_unstale":"2024-09-21T10:21:16.824129+0000","last_undegraded":"2024-09-21T10:21:16.824129+0000","last_fullsized":"2024-09-21T10:21:16.824129+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:02:34.529895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1846,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.825970+0000","last_change":"2024-09-21T10:16:38.461576+0000","last_active":"2024-09-21T10:21:16.825970+0000","last_peered":"2024-09-21T10:21:16.825970+0000","last_clean":"2024-09-21T10:21:16.825970+0000","last_became_active":"2024-09-21T10:16:38.461205+0000","last_became_peered":"2024-09-21T10:16:38.461205+0000","last_unstale":"2024-09-21T10:21:16.825970+0000","last_undegraded":"2024-09-21T10:21:16.825970+0000","last_fullsized":"2024-09-21T10:21:16.825970+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:36:17.166658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1437,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818046+0000","last_change":"2024-09-21T10:16:38.459378+0000","last_active":"2024-09-21T10:21:16.818046+0000","last_peered":"2024-09-21T10:21:16.818046+0000","last_clean":"2024-09-21T10:21:16.818046+0000","last_became_active":"2024-09-21T10:16:38.451582+0000","last_became_peered":"2024-09-21T10:16:38.451582+0000","last_unstale":"2024-09-21T10:21:16.818046+0000","last_undegraded":"2024-09-21T10:21:16.818046+0000","last_fullsized":"2024-09-21T10:21:16.818046+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:34.820762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":56,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849207+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:21:01.849207+0000","last_peered":"2024-09-21T10:21:01.849207+0000","last_clean":"2024-09-21T10:21:01.849207+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:21:01.849207+0000","last_undegraded":"2024-09-21T10:21:01.849207+0000","last_fullsized":"2024-09-21T10:21:01.849207+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1454,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.820333+0000","last_change":"2024-09-21T10:16:38.453928+0000","last_active":"2024-09-21T10:21:16.820333+0000","last_peered":"2024-09-21T10:21:16.820333+0000","last_clean":"2024-09-21T10:21:16.820333+0000","last_became_active":"2024-09-21T10:16:38.450890+0000","last_became_peered":"2024-09-21T10:16:38.450890+0000","last_unstale":"2024-09-21T10:21:16.820333+0000","last_undegraded":"2024-09-21T10:21:16.820333+0000","last_fullsized":"2024-09-21T10:21:16.820333+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:37:13.500644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":56,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849252+0000","last_change":"2024-09-21T10:17:51.281734+0000","last_active":"2024-09-21T10:21:01.849252+0000","last_peered":"2024-09-21T10:21:01.849252+0000","last_clean":"2024-09-21T10:21:01.849252+0000","last_became_active":"2024-09-21T10:17:51.258206+0000","last_became_peered":"2024-09-21T10:17:51.258206+0000","last_unstale":"2024-09-21T10:21:01.849252+0000","last_undegraded":"2024-09-21T10:21:01.849252+0000","last_fullsized":"2024-09-21T10:21:01.849252+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:16.184203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":516,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.819704+0000","last_change":"2024-09-21T10:17:51.251289+0000","last_active":"2024-09-21T10:21:16.819704+0000","last_peered":"2024-09-21T10:21:16.819704+0000","last_clean":"2024-09-21T10:21:16.819704+0000","last_became_active":"2024-09-21T10:17:51.247868+0000","last_became_peered":"2024-09-21T10:17:51.247868+0000","last_unstale":"2024-09-21T10:21:16.819704+0000","last_undegraded":"2024-09-21T10:21:16.819704+0000","last_fullsized":"2024-09-21T10:21:16.819704+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:39:49.784298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":518,"reported_epoch":1349,"state":"active+clean","last_fresh":"2024-09-21T10:21:16.818533+0000","last_change":"2024-09-21T10:16:38.451567+0000","last_active":"2024-09-21T10:21:16.818533+0000","last_peered":"2024-09-21T10:21:16.818533+0000","last_clean":"2024-09-21T10:21:16.818533+0000","last_became_active":"2024-09-21T10:16:38.451245+0000","last_became_peered":"2024-09-21T10:16:38.451245+0000","last_unstale":"2024-09-21T10:21:16.818533+0000","last_undegraded":"2024-09-21T10:21:16.818533+0000","last_fullsized":"2024-09-21T10:21:16.818533+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:27.022083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":214,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:02.044898+0000","last_change":"2024-09-21T10:15:19.115911+0000","last_active":"2024-09-21T10:21:02.044898+0000","last_peered":"2024-09-21T10:21:02.044898+0000","last_clean":"2024-09-21T10:21:02.044898+0000","last_became_active":"2024-09-21T10:15:19.115615+0000","last_became_peered":"2024-09-21T10:15:19.115615+0000","last_unstale":"2024-09-21T10:21:02.044898+0000","last_undegraded":"2024-09-21T10:21:02.044898+0000","last_fullsized":"2024-09-21T10:21:02.044898+0000","mapping_epoch":1275,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:15.500193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":58,"reported_epoch":1348,"state":"active+clean","last_fresh":"2024-09-21T10:21:01.849621+0000","last_change":"2024-09-21T10:17:51.280012+0000","last_active":"2024-09-21T10:21:01.849621+0000","last_peered":"2024-09-21T10:21:01.849621+0000","last_clean":"2024-09-21T10:21:01.849621+0000","last_became_active":"2024-09-21T10:17:51.257359+0000","last_became_peered":"2024-09-21T10:17:51.257359+0000","last_unstale":"2024-09-21T10:21:01.849621+0000","last_undegraded":"2024-09-21T10:21:01.849621+0000","last_fullsized":"2024-09-21T10:21:01.849621+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:44.522556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1746,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3459579904,"num_objects":52789,"num_object_clones":0,"num_object_copies":158367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":52789,"num_write_kb":3378496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10357833728,"data_stored":10357833728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52789,"ondisk_log_size":52789,"up":48,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":21092,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17031,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":378,"num_read_kb":822,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3563520,"data_stored":3542208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":6}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234547,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496183,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3189340,"kb_used_data":1978836,"kb_used_omap":1,"kb_used_meta":1210494,"kb_avail":90555812,"statfs":{"total":95995035648,"available":92729151488,"internally_reserved":0,"allocated":2026328064,"data_stored":2023766367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1239546311},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757822,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2126036,"kb_used_data":1665932,"kb_used_omap":1,"kb_used_meta":460094,"kb_avail":91619116,"statfs":{"total":95995035648,"available":93817974784,"internally_reserved":0,"allocated":1705914368,"data_stored":1703360217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":471136710},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019458,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2736920,"kb_used_data":1681168,"kb_used_omap":1,"kb_used_meta":1055742,"kb_avail":91008232,"statfs":{"total":95995035648,"available":93192429568,"internally_reserved":0,"allocated":1721516032,"data_stored":1718958364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1081080262},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1301,"seq":5587752452155,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2689516,"kb_used_data":2256612,"kb_used_omap":48,"kb_used_meta":432847,"kb_avail":91055636,"statfs":{"total":95995035648,"available":93240971264,"internally_reserved":0,"allocated":2310770688,"data_stored":2307859937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49883,"internal_metadata":443235621},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793967994,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047232,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3652404,"kb_used_data":2500204,"kb_used_omap":41,"kb_used_meta":1152150,"kb_avail":90092748,"statfs":{"total":95995035648,"available":92254973952,"internally_reserved":0,"allocated":2560208896,"data_stored":2557290412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1179802537},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928867,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7559,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2591227904,"data_stored":2591227904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2366963712,"data_stored":2366963712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1725038592,"data_stored":1725038592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1712586752,"data_stored":1712586752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1962016768,"data_stored":1962016768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:21:31.815 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:21:31.815 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 11 2024-09-21T10:21:31.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:31.847 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:31.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:31.982 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:31.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:32.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:32.118 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:32.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:32.220 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:32.246 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 28 15 60784 60769 135.625 103.438 0.0136565 0.00736818 2024-09-21T10:21:32.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:32.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:32 smithi076 ceph-mon[106482]: pgmap v2159: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:21:32.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:32 smithi076 ceph-mon[106482]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:21:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:32 smithi076 ceph-mon[110963]: pgmap v2159: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:21:32.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:32 smithi076 ceph-mon[110963]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:21:32.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:32.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.544 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:32.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:32 smithi191 ceph-mon[102206]: pgmap v2159: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-21T10:21:32.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:32 smithi191 ceph-mon[102206]: from='client.55651 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:21:32.644 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:32.673 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:32.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:32.775 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:32.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:32.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:32.939 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:32.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:32.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:33.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:33.211 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:33.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 29 16 62396 62380 134.42 100.688 0.0102448 0.00743501 2024-09-21T10:21:33.313 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:33.345 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:33.447 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.482 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:33.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:33.583 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:33.585 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:21:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:21:33] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.51.0" 2024-09-21T10:21:33.616 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:33.718 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:33.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:33.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:33.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:33.879 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:33.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:33.979 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:34.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:34.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:34.116 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:34.141 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:34.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:34.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 30 16 63830 63814 132.925 89.625 0.00501738 0.00751777 2024-09-21T10:21:34.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:34.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:34.400 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:34.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:34.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.539 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:34.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:34 smithi191 ceph-mon[102206]: pgmap v2160: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-21T10:21:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:34 smithi076 ceph-mon[106482]: pgmap v2160: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-21T10:21:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:34 smithi076 ceph-mon[110963]: pgmap v2160: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-21T10:21:34.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:34.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:34.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:34.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:34.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:34.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:34.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:34.935 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:34.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:34.948 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:21:34.948 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:21:34.948 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:21:34.948 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:21:34.948 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:21:35.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.069 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:35.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:35.170 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.204 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:35.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:35.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 31 16 64934 64918 130.863 69 0.0460724 0.00763336 2024-09-21T10:21:35.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:35.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:35.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:35.465 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:35.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.602 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:35.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:35.607 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:35.704 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:35.725 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:35.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:35.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:35.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:35.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:35.966 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:36.005 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:36.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:36.106 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:36.142 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:36.243 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:36.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 32 16 66217 66201 129.279 80.1875 0.0124919 0.00773026 2024-09-21T10:21:36.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:36.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:36.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:36 smithi076 ceph-mon[106482]: pgmap v2161: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:21:36.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:36 smithi076 ceph-mon[110963]: pgmap v2161: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:21:36.384 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.415 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:36.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:36.516 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:36.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:36.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:36.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:36 smithi191 ceph-mon[102206]: pgmap v2161: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-21T10:21:36.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:36.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:21:36.702 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:36.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:36.718 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:36.805 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:36.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:36.855 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:36.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:36.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:37.006 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:37.108 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:37.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:37.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:37.248 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 33 16 67682 67666 128.136 91.5625 0.00590409 0.00779919 2024-09-21T10:21:37.258 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:21:37.276 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:37.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:37.412 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:37.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:37.550 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:37.565 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:37 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2175500538' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:37.565 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:37 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:37.565 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:37 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2175500538' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:37.565 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:37 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:37.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:37 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2175500538' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:37.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:37 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:21:37.652 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:37.693 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:37.795 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:37.828 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:37.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:37.968 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:37.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:38.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.206 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:38.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:38.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:38.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 34 16 69233 69217 127.218 96.9375 0.00784958 0.00785582 2024-09-21T10:21:38.343 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:38.374 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:38.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:38.511 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:38.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[106482]: pgmap v2162: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[106482]: osdmap e1350: 8 total, 8 up, 5 in 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[110963]: pgmap v2162: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:38.530 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:38 smithi076 ceph-mon[110963]: osdmap e1350: 8 total, 8 up, 5 in 2024-09-21T10:21:38.612 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:38 smithi191 ceph-mon[102206]: pgmap v2162: 89 pgs: 89 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-21T10:21:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:38 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:38 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:21:38.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:38 smithi191 ceph-mon[102206]: osdmap e1350: 8 total, 8 up, 5 in 2024-09-21T10:21:38.646 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:38.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:38.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:38.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:38.919 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:38.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:38.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:39.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:39.056 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:39.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:39.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:39.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:39.247 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 35 16 70785 70769 126.355 97 0.00996004 0.00790806 2024-09-21T10:21:39.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:39.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:39.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:39.462 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:39.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:39.564 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:39.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:39.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:39.705 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:39.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:39.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:39.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:39.875 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:39.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:39.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:39.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:40.014 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:40.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:40.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:40.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:40.172 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:40.249 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 36 16 72305 72289 125.478 95 0.00700807 0.00796476 2024-09-21T10:21:40.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.289 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:40.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:40.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:40 smithi076 ceph-mon[106482]: pgmap v2164: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:21:40.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:40 smithi076 ceph-mon[110963]: pgmap v2164: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:21:40.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:40.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.425 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:40.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:40.526 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:40.566 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:40.567 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:21:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:40.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:40 smithi191 ceph-mon[102206]: pgmap v2164: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:21:40.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:40.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:40.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:40.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.855 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:40.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:40.956 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:40.988 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:40.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:41.086 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:21:41.086 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:41.089 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:41.095 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:21:41.096 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:41.130 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:41.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:41.139 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:21:41.139 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:41.172 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:21:41.173 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:41.200 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:21:41.200 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:41.226 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:21:41.226 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:41.232 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:41.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 37 16 73642 73626 124.345 83.5625 0.00804367 0.00803734 2024-09-21T10:21:41.259 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:21:41.260 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:41.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:41.292 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:21:41.292 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:41.369 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:41.401 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:41.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:41.503 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.538 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:41.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:41.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:41.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:41.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:41.802 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:41.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:41.842 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:41.842 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:41.843 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[106482]: pgmap v2165: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:21:41.843 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1161584981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:41.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:41.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:41.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[110963]: pgmap v2165: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:21:41.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:41 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1161584981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:21:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:41 smithi191 ceph-mon[102206]: pgmap v2165: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-21T10:21:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:41 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1161584981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:41.904 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:41.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:41.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:42.042 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:42.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:42.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 38 16 74790 74774 122.961 71.75 0.0058223 0.00812066 2024-09-21T10:21:42.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.475 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:42.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:42.711 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:42.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.747 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:42.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:42.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:42.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:42.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:42.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:42.985 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:43.018 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:43.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:43.120 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:43.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:43.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 39 16 76173 76157 122.024 86.4375 0.00896754 0.00818928 2024-09-21T10:21:43.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:43.288 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:43.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:43.390 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:43.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:43.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:43.518 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:21:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:21:43] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-21T10:21:43.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:43.564 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:43.666 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:43.704 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:43.806 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:43.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:43.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:43.981 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:43.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:44.083 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:43 smithi191 ceph-mon[102206]: pgmap v2166: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:21:44.117 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:43 smithi076 ceph-mon[106482]: pgmap v2166: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:21:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:43 smithi076 ceph-mon[110963]: pgmap v2166: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:21:44.219 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:44.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:21:44.249569+0000 min lat: 0.0014399 max lat: 0.280171 avg lat: 0.0082591 2024-09-21T10:21:44.250 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:21:44.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 40 16 77476 77460 121.007 81.4375 0.00734252 0.0082591 2024-09-21T10:21:44.252 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:44.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:44.353 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:44.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:44.494 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:44.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:44.557 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:44.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:44.667 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:44.769 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:44.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:44.828 INFO:teuthology.orchestra.run.smithi076.stderr:set pool 1 pg_num to 11 2024-09-21T10:21:44.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:44.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:44.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:44.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:45.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:45.077 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2283327312' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T10:21:45.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2283327312' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T10:21:45.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:45.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2283327312' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-21T10:21:45.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:45.218 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:45.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:45.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 41 15 78777 78762 120.041 81.375 0.00923078 0.00832591 2024-09-21T10:21:45.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:21:45.320 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.369 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:45.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:45.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:45.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:45.613 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:45.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:45.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:45.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:45.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:45.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:45.885 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:45.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:45.920 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:45.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:46.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.056 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:45 smithi191 ceph-mon[102206]: pgmap v2167: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:21:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2283327312' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T10:21:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:45 smithi191 ceph-mon[102206]: osdmap e1351: 8 total, 8 up, 5 in 2024-09-21T10:21:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:45 smithi076 ceph-mon[106482]: pgmap v2167: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:21:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2283327312' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T10:21:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:45 smithi076 ceph-mon[106482]: osdmap e1351: 8 total, 8 up, 5 in 2024-09-21T10:21:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:45 smithi076 ceph-mon[110963]: pgmap v2167: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-21T10:21:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2283327312' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-21T10:21:46.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:45 smithi076 ceph-mon[110963]: osdmap e1351: 8 total, 8 up, 5 in 2024-09-21T10:21:46.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:46.191 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:46.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 42 16 80147 80131 119.22 85.5625 0.010415 0.00838314 2024-09-21T10:21:46.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:46.468 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:46.553 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:21:46.553 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:21:46.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:46.602 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:46.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:46.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:46.835 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:46.873 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:46.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:46.975 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:47.009 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:47.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:47.110 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:46 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T10:21:47.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:46 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T10:21:47.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:46 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-21T10:21:47.143 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:47.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 43 16 81493 81477 118.403 84.125 0.012499 0.00844063 2024-09-21T10:21:47.277 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:47.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:47.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:21:47.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:47.521 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.567 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:47.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.668 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:47.704 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:47.805 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:47.846 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:47.948 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:47.979 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:47.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:48.014 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:48.080 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:47 smithi191 ceph-mon[102206]: pgmap v2169: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-21T10:21:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:47 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T10:21:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:47 smithi191 ceph-mon[102206]: osdmap e1352: 8 total, 8 up, 5 in 2024-09-21T10:21:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:48.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:47 smithi076 ceph-mon[106482]: pgmap v2169: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-21T10:21:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:47 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T10:21:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:47 smithi076 ceph-mon[106482]: osdmap e1352: 8 total, 8 up, 5 in 2024-09-21T10:21:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:47 smithi076 ceph-mon[110963]: pgmap v2169: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 53 MiB/s wr, 853 op/s 2024-09-21T10:21:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:47 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-21T10:21:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:47 smithi076 ceph-mon[110963]: osdmap e1352: 8 total, 8 up, 5 in 2024-09-21T10:21:48.217 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:48.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 44 15 82780 82765 117.542 80.5 0.0155684 0.00850268 2024-09-21T10:21:48.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:48.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:48.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:48.385 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:48.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:48.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:48.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:48.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:48.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:48.763 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:48.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:48.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:48.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:48.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:49.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:49.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:48 smithi191 ceph-mon[102206]: osdmap e1353: 8 total, 8 up, 5 in 2024-09-21T10:21:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:48 smithi076 ceph-mon[106482]: osdmap e1353: 8 total, 8 up, 5 in 2024-09-21T10:21:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:48 smithi076 ceph-mon[110963]: osdmap e1353: 8 total, 8 up, 5 in 2024-09-21T10:21:49.160 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:49.179 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:49.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.222 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:49.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:49.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 45 16 84208 84192 116.912 89.1875 0.00670709 0.0085488 2024-09-21T10:21:49.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:49.361 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:49.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:49.462 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.496 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:49.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:49.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:49.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:49.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:49.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:49.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:49.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:49.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.038 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:50.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:49 smithi191 ceph-mon[102206]: pgmap v2172: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:21:50.117 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:49 smithi076 ceph-mon[106482]: pgmap v2172: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:21:50.117 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:49 smithi076 ceph-mon[110963]: pgmap v2172: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:21:50.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.210 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:50.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:50.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 46 16 85603 85587 116.265 87.1875 0.00846961 0.00859621 2024-09-21T10:21:50.311 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:50.356 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:50.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.489 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:50.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:50.501 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:21:50.501 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:21:50.534 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:21:50.534 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:21:50.568 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:21:50.568 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:21:50.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:50.610 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:21:50.610 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:21:50.631 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:50.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:50.650 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:21:50.650 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:21:50.687 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:21:50.687 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:21:50.725 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:21:50.725 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:21:50.734 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.757 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:21:50.757 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:21:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.782 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:50.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:50.883 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:50.893 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6, 5] out_osds: [7, 0, 2] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 3, 0] 2024-09-21T10:21:50.893 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:21:50.893 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-09-21T10:21:50.893 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-21T10:21:50.893 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-21T10:21:50.893 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-09-21T10:21:50.917 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:50.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:50.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:51.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.057 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:51.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:51.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2727441680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:51.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2727441680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:51.158 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:51.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:51.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 47 16 86861 86845 115.464 78.625 0.00507699 0.00865567 2024-09-21T10:21:51.282 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:21:51.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.327 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2727441680' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:21:51.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.464 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:51.500 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:51.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.607 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:21:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=cleanup t=2024-09-21T10:21:51.324756871Z level=info msg="Completed cleanup jobs" duration=3.179751ms 2024-09-21T10:21:51.607 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:21:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=plugins.update.checker t=2024-09-21T10:21:51.467132704Z level=info msg="Update check succeeded" duration=59.381027ms 2024-09-21T10:21:51.640 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:51.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:51.881 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:21:51.887 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:51.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:51.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:52.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:51 smithi076 ceph-mon[106482]: pgmap v2173: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:21:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/579342611' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:51 smithi076 ceph-mon[110963]: pgmap v2173: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:21:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/579342611' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:52.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:52.161 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:52.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:52.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 48 16 88225 88209 114.835 85.25 0.0108581 0.00870349 2024-09-21T10:21:52.263 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.298 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:52.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.332 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:51 smithi191 ceph-mon[102206]: pgmap v2173: 99 pgs: 1 peering, 10 unknown, 88 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-21T10:21:52.332 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/579342611' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:52.332 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:21:52.399 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:52.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:52.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:52.584 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:52.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:52.686 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:52.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:52.870 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:52.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:52.971 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[106482]: osdmap e1354: 8 total, 8 up, 5 in 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[110963]: osdmap e1354: 8 total, 8 up, 5 in 2024-09-21T10:21:52.991 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:52 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T10:21:53.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.014 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:53.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:53.042 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:52 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:21:53.042 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:52 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:21:53.042 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:52 smithi191 ceph-mon[102206]: osdmap e1354: 8 total, 8 up, 5 in 2024-09-21T10:21:53.042 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:52 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-21T10:21:53.115 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:53.147 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:53.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:53.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 49 16 89602 89586 114.248 86.0625 0.00898393 0.00874816 2024-09-21T10:21:53.285 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:53.387 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:53.430 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:53.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:53.539 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:21:53.539 INFO:teuthology.orchestra.run.smithi191.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-21T10:21:53.540 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:21:53.565 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:53.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:21:53] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-21T10:21:53.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:53.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:53.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:53.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.849 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:53.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:53.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:53.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:53.984 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:53.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:54.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.124 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:54.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:54.125 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:21:54.125 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:21:54.125 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:21:54.125 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:21:54.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:54.251 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 50 16 90907 90891 113.594 81.5625 0.0154541 0.00879875 2024-09-21T10:21:54.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:54.361 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:53 smithi191 ceph-mon[102206]: pgmap v2175: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-21T10:21:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:53 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T10:21:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:53 smithi191 ceph-mon[102206]: osdmap e1355: 8 total, 8 up, 5 in 2024-09-21T10:21:54.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:53 smithi191 ceph-mon[102206]: osdmap e1356: 8 total, 8 up, 5 in 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[106482]: pgmap v2175: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[106482]: osdmap e1355: 8 total, 8 up, 5 in 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[106482]: osdmap e1356: 8 total, 8 up, 5 in 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[110963]: pgmap v2175: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[110963]: osdmap e1355: 8 total, 8 up, 5 in 2024-09-21T10:21:54.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:53 smithi076 ceph-mon[110963]: osdmap e1356: 8 total, 8 up, 5 in 2024-09-21T10:21:54.399 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:54.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:54.501 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:54.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:54.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:54.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:54.776 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:54.813 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:54.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:54.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:21:54.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:54.959 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:54.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:55.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:21:55.061 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:55.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.120 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:55.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:55.221 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:55.253 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 51 16 92223 92207 112.975 82.25 0.0104462 0.00884662 2024-09-21T10:21:55.259 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:55.299 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:55.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:55.392 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:55.485 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:21:55.493 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:55.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:55.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:55.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:55.812 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:55.862 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:55.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:55.995 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:55.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:56.096 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:55 smithi076 ceph-mon[106482]: pgmap v2178: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:21:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:55 smithi076 ceph-mon[106482]: osdmap e1357: 8 total, 8 up, 5 in 2024-09-21T10:21:56.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:55 smithi076 ceph-mon[110963]: pgmap v2178: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:21:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:55 smithi076 ceph-mon[110963]: osdmap e1357: 8 total, 8 up, 5 in 2024-09-21T10:21:56.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:56.133 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:56.138 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:21:56.235 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:56.253 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 52 16 93552 93536 112.4 83.0625 0.0134476 0.00889166 2024-09-21T10:21:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:56.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:55 smithi191 ceph-mon[102206]: pgmap v2178: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-21T10:21:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:55 smithi191 ceph-mon[102206]: osdmap e1357: 8 total, 8 up, 5 in 2024-09-21T10:21:56.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:21:56.370 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:56.403 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:56.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:56.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:56.640 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:56.672 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:56.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:56.774 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:56.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.818 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:56.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:56.919 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:56.953 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:56.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:56.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:57.055 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:57.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.092 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:57.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:57.193 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:57.228 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:57.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:57.253 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 53 16 95002 94986 111.989 90.625 0.0104218 0.008925 2024-09-21T10:21:57.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:57.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:56 smithi191 ceph-mon[102206]: osdmap e1358: 8 total, 8 up, 5 in 2024-09-21T10:21:57.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:56 smithi191 ceph-mon[102206]: osdmap e1359: 8 total, 8 up, 5 in 2024-09-21T10:21:57.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:57.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:57.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:56 smithi076 ceph-mon[110963]: osdmap e1358: 8 total, 8 up, 5 in 2024-09-21T10:21:57.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:56 smithi076 ceph-mon[110963]: osdmap e1359: 8 total, 8 up, 5 in 2024-09-21T10:21:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:56 smithi076 ceph-mon[106482]: osdmap e1358: 8 total, 8 up, 5 in 2024-09-21T10:21:57.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:56 smithi076 ceph-mon[106482]: osdmap e1359: 8 total, 8 up, 5 in 2024-09-21T10:21:57.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.509 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:57.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:57.610 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:57.642 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:57.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:21:57.780 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:21:57.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:57.909 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:57.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:58.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:58.048 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:58.150 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:58.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:58.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:58.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 54 16 96299 96283 111.416 81.0625 0.00809757 0.00897079 2024-09-21T10:21:58.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:21:58.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:58.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:21:58.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:57 smithi191 ceph-mon[102206]: pgmap v2181: 99 pgs: 4 peering, 95 active+clean; 5.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 1023 B/s rd, 13 MiB/s wr, 211 op/s 2024-09-21T10:21:58.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:57 smithi076 ceph-mon[106482]: pgmap v2181: 99 pgs: 4 peering, 95 active+clean; 5.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 1023 B/s rd, 13 MiB/s wr, 211 op/s 2024-09-21T10:21:58.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:57 smithi076 ceph-mon[110963]: pgmap v2181: 99 pgs: 4 peering, 95 active+clean; 5.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 1023 B/s rd, 13 MiB/s wr, 211 op/s 2024-09-21T10:21:58.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:21:58.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:58.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:21:58.559 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:58.583 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:58.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:58.685 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:58.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:58.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:58.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:58.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:58.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:58.992 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:58.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:59.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:21:59.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:59.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:21:59.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:21:59.253 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 55 16 97644 97628 110.919 84.0625 0.00928975 0.00901064 2024-09-21T10:21:59.267 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:21:59.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:21:59.371 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:21:59.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:59.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:21:59.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:59.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:59.647 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:59.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.683 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:59.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:21:59.784 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:21:59.805 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:21:59.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:21:59.942 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:21:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:59.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:00.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:00.073 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:21:59 smithi076 ceph-mon[106482]: pgmap v2183: 99 pgs: 6 peering, 93 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 1.6 KiB/s rd, 86 MiB/s wr, 1.38k op/s 2024-09-21T10:22:00.073 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:21:59 smithi076 ceph-mon[110963]: pgmap v2183: 99 pgs: 6 peering, 93 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 1.6 KiB/s rd, 86 MiB/s wr, 1.38k op/s 2024-09-21T10:22:00.079 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:00.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:00.181 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:00.219 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:00.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:00.249 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 6, 5] out_osds: [7, 0, 2] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 3, 0] 2024-09-21T10:22:00.249 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:22:00.249 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 4, 3, 6, 5] 2024-09-21T10:22:00.250 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd out 6 2024-09-21T10:22:00.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 56 16 98795 98779 110.222 71.9375 0.019141 0.00906774 2024-09-21T10:22:00.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:00.354 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:00.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:21:59 smithi191 ceph-mon[102206]: pgmap v2183: 99 pgs: 6 peering, 93 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 1.6 KiB/s rd, 86 MiB/s wr, 1.38k op/s 2024-09-21T10:22:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.367 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:00.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:00.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:00.499 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:00.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:00.601 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:00.634 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:00.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:00.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:00.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:00.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:00.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:00.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:00.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:01.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.048 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:01.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:01.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:01.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:01.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 57 16 99967 99951 109.574 73.25 0.0212537 0.0091215 2024-09-21T10:22:01.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:01.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:01.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:01.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:01.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:01.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:01.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:01.695 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:01.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:01.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:01.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:01.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:01.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:01.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:01.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:02.058 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:02.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:02.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:02.240 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:02.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:02.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 58 16 100965 100949 108.76 62.375 0.00853739 0.00918978 2024-09-21T10:22:02.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:02.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:01 smithi191 ceph-mon[102206]: pgmap v2184: 99 pgs: 6 peering, 93 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 1.3 KiB/s rd, 70 MiB/s wr, 1.12k op/s 2024-09-21T10:22:02.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:01 smithi076 ceph-mon[106482]: pgmap v2184: 99 pgs: 6 peering, 93 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 1.3 KiB/s rd, 70 MiB/s wr, 1.12k op/s 2024-09-21T10:22:02.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:01 smithi076 ceph-mon[110963]: pgmap v2184: 99 pgs: 6 peering, 93 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 1.3 KiB/s rd, 70 MiB/s wr, 1.12k op/s 2024-09-21T10:22:02.379 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:02.480 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:02.515 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:02.617 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:02.646 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:02.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:02.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:02.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:02.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:02.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:02.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:03.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:03.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:03.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:03.163 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:03.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:03.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:03.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 59 16 102335 102319 108.368 85.625 0.0185562 0.00922305 2024-09-21T10:22:03.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:03.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.342 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:03.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:03.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:03.479 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:03.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:03.581 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:03.614 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:03.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:22:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:22:03] "GET /metrics HTTP/1.1" 200 36616 "" "Prometheus/2.51.0" 2024-09-21T10:22:03.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:03.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:03.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:03.851 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:03.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:03.886 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:03.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:03.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:04.019 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:04.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:04.121 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:04.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:04.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:04.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:22:04.253103+0000 min lat: 0.0014399 max lat: 0.280171 avg lat: 0.00925155 2024-09-21T10:22:04.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:22:04.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 60 16 103748 103732 108.034 88.3125 0.00440892 0.00925155 2024-09-21T10:22:04.256 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.291 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:04.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:04.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:03 smithi191 ceph-mon[102206]: pgmap v2185: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 1.0 KiB/s rd, 101 MiB/s wr, 1.62k op/s 2024-09-21T10:22:04.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:03 smithi076 ceph-mon[106482]: pgmap v2185: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 1.0 KiB/s rd, 101 MiB/s wr, 1.62k op/s 2024-09-21T10:22:04.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:03 smithi076 ceph-mon[110963]: pgmap v2185: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 1.0 KiB/s rd, 101 MiB/s wr, 1.62k op/s 2024-09-21T10:22:04.392 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:04.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:04.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:04.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:04.660 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:04.695 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:04.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:04.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:04.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:04.931 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:04.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:04.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:05.070 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:05.104 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:05.125 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:22:05.125 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:22:05.125 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:22:05.125 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:22:05.126 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:22:05.206 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:05.244 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:05.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:05.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 61 16 104939 104923 107.483 74.4375 0.0256819 0.00929919 2024-09-21T10:22:05.346 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:05.367 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:05.469 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:05.506 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:05.608 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:05.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:05.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:05.749 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:05.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:05.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:05.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:05.917 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:05.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:06.019 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:06.054 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:06.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:06.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:06.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 62 16 106167 106151 106.987 76.75 0.00725098 0.00934254 2024-09-21T10:22:06.296 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.335 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:06.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:06.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:05 smithi191 ceph-mon[102206]: pgmap v2186: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 923 B/s rd, 90 MiB/s wr, 1.44k op/s 2024-09-21T10:22:06.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:05 smithi076 ceph-mon[106482]: pgmap v2186: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 923 B/s rd, 90 MiB/s wr, 1.44k op/s 2024-09-21T10:22:06.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:05 smithi076 ceph-mon[110963]: pgmap v2186: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 923 B/s rd, 90 MiB/s wr, 1.44k op/s 2024-09-21T10:22:06.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:06.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.571 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:06.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:06.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:06.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:06.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:06.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:06.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:06.991 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:07.025 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:07.126 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:07.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.165 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:07.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:07.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 63 16 107489 107473 106.6 82.625 0.00796245 0.00937642 2024-09-21T10:22:07.266 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:07.307 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:07.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:07.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:07.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:07.546 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:07.581 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:07.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:07.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:07.816 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.849 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:07.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:07.950 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:07.986 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:07.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:08.088 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:08.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:08.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:07 smithi076 ceph-mon[106482]: pgmap v2187: 99 pgs: 99 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 409 B/s rd, 79 MiB/s wr, 1.27k op/s 2024-09-21T10:22:08.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:07 smithi076 ceph-mon[110963]: pgmap v2187: 99 pgs: 99 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 409 B/s rd, 79 MiB/s wr, 1.27k op/s 2024-09-21T10:22:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:08.224 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:08.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:22:08.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 64 16 108913 108897 106.325 89 0.0194106 0.00940065 2024-09-21T10:22:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.256 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:08.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:08.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:07 smithi191 ceph-mon[102206]: pgmap v2187: 99 pgs: 99 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 409 B/s rd, 79 MiB/s wr, 1.27k op/s 2024-09-21T10:22:08.393 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:08.495 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:08.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.535 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:08.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:08.636 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:08.668 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:08.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:08.770 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:08.806 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:08.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:08.908 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:08.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:08.943 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:08.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.044 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:09.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:09.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:09.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.223 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:09.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:09.254 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 65 16 110263 110247 105.987 84.375 0.00763236 0.0094305 2024-09-21T10:22:09.324 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:09.361 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:09.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:09.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:09.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:09.603 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:09.646 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:09.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:22:09.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:22:09.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:22:09.717 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:22:09.717 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:22:09.718 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:09.719 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:09.719 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:09.719 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:09.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:09.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:09.882 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:09.915 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:09.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:09.939 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:09.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:09 smithi076 ceph-mon[106482]: pgmap v2188: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 86 B/s rd, 86 MiB/s wr, 1.38k op/s 2024-09-21T10:22:09.949 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:09 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1212416425' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:09.949 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:09 smithi076 ceph-mon[110963]: pgmap v2188: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 86 B/s rd, 86 MiB/s wr, 1.38k op/s 2024-09-21T10:22:10.017 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:10.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:10.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:10.182 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:10.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:10.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 66 16 111585 111569 105.633 82.625 0.0122734 0.00946222 2024-09-21T10:22:10.284 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:10.305 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:09 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1212416425' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:10.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:10.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:09 smithi191 ceph-mon[102206]: pgmap v2188: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 86 B/s rd, 86 MiB/s wr, 1.38k op/s 2024-09-21T10:22:10.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:09 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1212416425' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:10.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:10.459 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:10.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:10.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:10.696 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:10.731 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:10.833 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:10.867 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:10.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:10.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:10.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:10.964 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:22:10.976 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:11.002 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:11.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:11.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2316252648' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:11.109 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:11.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:11.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2316252648' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:11.110 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:11.140 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:11.242 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:11.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 67 16 112871 112855 105.255 80.375 0.00857433 0.0094965 2024-09-21T10:22:11.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:11.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2316252648' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:11.264 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:22:11.276 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:11.378 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:11.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.419 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:11.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:11.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:11.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:11.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:11.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:11.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:11.785 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:11.824 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:11.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:11.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:11.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:12.019 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[106482]: pgmap v2189: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-21T10:22:12.019 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[106482]: osdmap e1360: 8 total, 8 up, 5 in 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[106482]: 1.1 scrub starts 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[106482]: 1.1 scrub ok 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[110963]: pgmap v2189: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[110963]: osdmap e1360: 8 total, 8 up, 5 in 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[110963]: 1.1 scrub starts 2024-09-21T10:22:12.020 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:11 smithi076 ceph-mon[110963]: 1.1 scrub ok 2024-09-21T10:22:12.051 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:12.094 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:12.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:12.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 68 16 114037 114021 104.779 72.875 0.0217035 0.00953886 2024-09-21T10:22:12.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:11 smithi191 ceph-mon[102206]: pgmap v2189: 99 pgs: 99 active+clean; 6.5 GiB data, 24 GiB used, 423 GiB / 447 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-21T10:22:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:11 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:22:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:11 smithi191 ceph-mon[102206]: osdmap e1360: 8 total, 8 up, 5 in 2024-09-21T10:22:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:11 smithi191 ceph-mon[102206]: 1.1 scrub starts 2024-09-21T10:22:12.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:11 smithi191 ceph-mon[102206]: 1.1 scrub ok 2024-09-21T10:22:12.377 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:12.479 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:12.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.648 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:12.749 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.789 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:12.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:12.894 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:22:12.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:12.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:12.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:13.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.213 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:13.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 69 16 115300 115284 104.405 78.9375 0.00719032 0.00957377 2024-09-21T10:22:13.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:13.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:13.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:13.483 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:13.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.624 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:22:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:22:13] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-21T10:22:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:13.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.731 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:13.767 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:13.804 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:22:13.869 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:13.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:13.906 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:13.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:14.007 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:14.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:14.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:14.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:14.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:14.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 70 16 116596 116580 104.07 81 0.00611359 0.00960449 2024-09-21T10:22:14.302 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:13 smithi076 ceph-mon[106482]: pgmap v2191: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:22:14.303 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:13 smithi076 ceph-mon[110963]: pgmap v2191: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:22:14.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:14.334 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:14.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:13 smithi191 ceph-mon[102206]: pgmap v2191: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:22:14.436 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.477 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:14.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:14.578 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:14.619 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:14.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:14.752 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:14.854 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:14.889 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:14.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:14.990 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:15.020 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:15.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:15.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:15.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:15.184 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:22:15.184 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:22:15.255 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:15.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 71 16 117891 117875 103.744 80.9375 0.00825356 0.00963449 2024-09-21T10:22:15.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:15.397 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.432 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:15.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:15.532 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:15.533 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:22:15.371+0000 7f240edc2640 -1 mon.a@0(leader).osd e1360 definitely_dead 0 2024-09-21T10:22:15.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:15.674 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:15.708 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:15.809 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:15.844 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:15.946 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:15.974 INFO:teuthology.orchestra.run.smithi076.stderr:marked out osd.6. 2024-09-21T10:22:15.987 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:15.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:16.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-21T10:22:16.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:16.110 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:16.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:16.212 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:16.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:16.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:16.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 72 16 118913 118897 103.191 63.875 0.0165287 0.00965141 2024-09-21T10:22:16.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:16.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: pgmap v2192: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: 1.8 scrub starts 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: 1.8 scrub ok 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2178766318' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: 1.0 scrub starts 2024-09-21T10:22:16.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:15 smithi191 ceph-mon[102206]: 1.0 scrub ok 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: pgmap v2192: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: 1.8 scrub starts 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: 1.8 scrub ok 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2178766318' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: 1.0 scrub starts 2024-09-21T10:22:16.369 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[106482]: 1.0 scrub ok 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: pgmap v2192: 99 pgs: 99 active+clean; 6.9 GiB data, 25 GiB used, 422 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: 1.8 scrub starts 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: 1.8 scrub ok 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2178766318' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: Client client.admin marked osd.6 out, while it was still marked up 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: 1.0 scrub starts 2024-09-21T10:22:16.370 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:15 smithi076 ceph-mon[110963]: 1.0 scrub ok 2024-09-21T10:22:16.459 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.483 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:16.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:16.584 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.606 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:16.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:16.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:16.729 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:16.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:16.777 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:16.830 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:16.851 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:16.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:16.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:16.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:17.046 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[106482]: 1.2 scrub starts 2024-09-21T10:22:17.046 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[106482]: 1.2 scrub ok 2024-09-21T10:22:17.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2178766318' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:22:17.047 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[106482]: osdmap e1361: 8 total, 8 up, 4 in 2024-09-21T10:22:17.047 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[110963]: 1.2 scrub starts 2024-09-21T10:22:17.047 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[110963]: 1.2 scrub ok 2024-09-21T10:22:17.047 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2178766318' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:22:17.047 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:16 smithi076 ceph-mon[110963]: osdmap e1361: 8 total, 8 up, 4 in 2024-09-21T10:22:17.047 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:17.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:17.099 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:17.201 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:17.222 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:17.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:17.255 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 73 16 118913 118897 101.777 0 - 0.00965141 2024-09-21T10:22:17.323 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:17.344 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:17.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:16 smithi191 ceph-mon[102206]: 1.2 scrub starts 2024-09-21T10:22:17.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:16 smithi191 ceph-mon[102206]: 1.2 scrub ok 2024-09-21T10:22:17.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2178766318' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-09-21T10:22:17.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:16 smithi191 ceph-mon[102206]: osdmap e1361: 8 total, 8 up, 4 in 2024-09-21T10:22:17.446 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.470 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:17.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:17.571 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:17.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:17.694 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:17.716 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:17.766 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:17.818 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:17.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:17.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:17.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:17.962 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:17.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:18.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:18.098 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:18.139 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:17 smithi076 ceph-mon[106482]: pgmap v2194: 99 pgs: 1 unknown, 19 peering, 79 active+clean; 7.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-21T10:22:18.139 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:17 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:22:18.139 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:17 smithi076 ceph-mon[106482]: osdmap e1362: 8 total, 8 up, 4 in 2024-09-21T10:22:18.139 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:17 smithi076 ceph-mon[110963]: pgmap v2194: 99 pgs: 1 unknown, 19 peering, 79 active+clean; 7.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-21T10:22:18.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:17 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:22:18.140 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:17 smithi076 ceph-mon[110963]: osdmap e1362: 8 total, 8 up, 4 in 2024-09-21T10:22:18.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:18.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:18.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 74 16 119066 119050 100.531 4.78125 0.0096428 0.00994243 2024-09-21T10:22:18.339 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:18.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:17 smithi191 ceph-mon[102206]: pgmap v2194: 99 pgs: 1 unknown, 19 peering, 79 active+clean; 7.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-21T10:22:18.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:17 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:22:18.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:17 smithi191 ceph-mon[102206]: osdmap e1362: 8 total, 8 up, 4 in 2024-09-21T10:22:18.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:18.477 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:18.514 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:18.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:18.615 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:18.646 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:18.748 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:18.781 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:18.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:18.918 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:18.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:19.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:19.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:19.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:19.192 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:19.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 75 16 120427 120411 100.325 85.0625 0.0152617 0.00996352 2024-09-21T10:22:19.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:19.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:19.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: osdmap e1363: 8 total, 8 up, 4 in 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1360'7465 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.5 starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7507 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7425 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.3 starting backfill to osd.3 from (0'0,0'0] MAX to 1360'7302 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7590 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.4 starting backfill to osd.4 from (0'0,0'0] MAX to 1360'7580 2024-09-21T10:22:19.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7346 2024-09-21T10:22:19.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7438 2024-09-21T10:22:19.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:18 smithi191 ceph-mon[102206]: 11.9 starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7438 2024-09-21T10:22:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: osdmap e1363: 8 total, 8 up, 4 in 2024-09-21T10:22:19.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1360'7465 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.5 starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7507 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7425 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.3 starting backfill to osd.3 from (0'0,0'0] MAX to 1360'7302 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7590 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.4 starting backfill to osd.4 from (0'0,0'0] MAX to 1360'7580 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7346 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7438 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[106482]: 11.9 starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7438 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: osdmap e1363: 8 total, 8 up, 4 in 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1360'7465 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.5 starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7507 2024-09-21T10:22:19.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7425 2024-09-21T10:22:19.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.3 starting backfill to osd.3 from (0'0,0'0] MAX to 1360'7302 2024-09-21T10:22:19.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.d starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7590 2024-09-21T10:22:19.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.4 starting backfill to osd.4 from (0'0,0'0] MAX to 1360'7580 2024-09-21T10:22:19.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7346 2024-09-21T10:22:19.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1360'7438 2024-09-21T10:22:19.375 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:18 smithi076 ceph-mon[110963]: 11.9 starting backfill to osd.5 from (0'0,0'0] MAX to 1360'7438 2024-09-21T10:22:19.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:19.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.466 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:19.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:19.567 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:19.610 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:19.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:19.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:19.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:19.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:19.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:19.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:19.987 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:20.024 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:20 smithi076 ceph-mon[106482]: pgmap v2197: 99 pgs: 8 unknown, 53 peering, 38 active+clean; 7.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 55 MiB/s wr, 869 op/s 2024-09-21T10:22:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:20 smithi076 ceph-mon[106482]: osdmap e1364: 8 total, 8 up, 4 in 2024-09-21T10:22:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:20 smithi076 ceph-mon[110963]: pgmap v2197: 99 pgs: 8 unknown, 53 peering, 38 active+clean; 7.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 55 MiB/s wr, 869 op/s 2024-09-21T10:22:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:20 smithi076 ceph-mon[110963]: osdmap e1364: 8 total, 8 up, 4 in 2024-09-21T10:22:20.126 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:20.163 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:20.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:20.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 76 16 121781 121765 100.118 84.625 0.00517351 0.00998347 2024-09-21T10:22:20.265 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:20.297 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:20.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:20 smithi191 ceph-mon[102206]: pgmap v2197: 99 pgs: 8 unknown, 53 peering, 38 active+clean; 7.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 55 MiB/s wr, 869 op/s 2024-09-21T10:22:20.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:20 smithi191 ceph-mon[102206]: osdmap e1364: 8 total, 8 up, 4 in 2024-09-21T10:22:20.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:20.433 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:20.535 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:20.584 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:20.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:20.718 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:20.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:20.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:20.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:20.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:20.959 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:20.994 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:20.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:21.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.127 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:21.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:21.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:21.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 77 16 123108 123092 99.8949 82.9375 0.0188779 0.010006 2024-09-21T10:22:21.260 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:21.350 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:21 smithi076 ceph-mon[110963]: osdmap e1365: 8 total, 8 up, 4 in 2024-09-21T10:22:21.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:21 smithi076 ceph-mon[106482]: osdmap e1365: 8 total, 8 up, 4 in 2024-09-21T10:22:21.362 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:21.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:21 smithi191 ceph-mon[102206]: osdmap e1365: 8 total, 8 up, 4 in 2024-09-21T10:22:21.412 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:21.514 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:21.545 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:21.647 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:22:21.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:22:21.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:21.794 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 5] out_osds: [7, 0, 2, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 3, 0] 2024-09-21T10:22:21.794 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:22:21.794 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [1, 2, 4, 5, 6, 7, 3, 0] 2024-09-21T10:22:21.794 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:22:21.795 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:22:21.818 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:21.899 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:21 smithi076 systemd[1]: Stopping Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:22:21.919 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:21.967 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:22:21.967 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:22:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:21.970 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:21.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:22.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:22.104 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:22.205 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:22.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:22.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 78 16 124384 124368 99.6366 79.75 0.0114867 0.010032 2024-09-21T10:22:22.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:22.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:22 smithi191 ceph-mon[102206]: pgmap v2200: 99 pgs: 8 unknown, 53 peering, 38 active+clean; 7.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 255 B/s rd, 71 MiB/s wr, 1.13k op/s 2024-09-21T10:22:22.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:22 smithi076 ceph-mon[110963]: pgmap v2200: 99 pgs: 8 unknown, 53 peering, 38 active+clean; 7.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 255 B/s rd, 71 MiB/s wr, 1.13k op/s 2024-09-21T10:22:22.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:22 smithi076 ceph-mon[106482]: pgmap v2200: 99 pgs: 8 unknown, 53 peering, 38 active+clean; 7.3 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 255 B/s rd, 71 MiB/s wr, 1.13k op/s 2024-09-21T10:22:22.376 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:22.478 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:22.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:22.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:22.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:22.658 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:22.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:22.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:22.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:22.902 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:22.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:22.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:23.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.095 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:23.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:23.196 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:23.238 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:23.240 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:23 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:22:23.240 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:23 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:22:23.256 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 79 16 125583 125567 99.3239 74.9375 0.0150379 0.0100636 2024-09-21T10:22:23.340 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:23.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:23 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:22:23.386 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:23.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:23.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_blocked_ops 2024-09-21T10:22:23.488 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:23.494 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:22:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:22:23] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-21T10:22:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.558 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:23.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:23.659 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:23.710 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:23.711 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:22:23.711 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:23.712 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:22:23.713 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:23.713 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:23.713 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:23.713 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:23.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:23.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:23.852 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:23.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:23.953 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:23.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:23.990 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:23.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:24.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:24.132 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:24.234 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:24.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:2024-09-21T10:22:24.255940+0000 min lat: 0.0014399 max lat: 2.16199 avg lat: 0.0100929 2024-09-21T10:22:24.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-21T10:22:24.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 80 16 126803 126787 99.0347 76.25 0.0111278 0.0100929 2024-09-21T10:22:24.270 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:24.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:24 smithi191 ceph-mon[102206]: pgmap v2201: 99 pgs: 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 90 active+clean; 7.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 54 MiB/s wr, 857 op/s; 69021/371676 objects misplaced (18.570%); 9.6 MiB/s, 1 keys/s, 154 objects/s recovering 2024-09-21T10:22:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:24 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 14 pgs peering) 2024-09-21T10:22:24.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:24 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1445933538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:24.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-mon[106482]: pgmap v2201: 99 pgs: 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 90 active+clean; 7.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 54 MiB/s wr, 857 op/s; 69021/371676 objects misplaced (18.570%); 9.6 MiB/s, 1 keys/s, 154 objects/s recovering 2024-09-21T10:22:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 14 pgs peering) 2024-09-21T10:22:24.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1445933538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-mon[110963]: pgmap v2201: 99 pgs: 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 90 active+clean; 7.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 54 MiB/s wr, 857 op/s; 69021/371676 objects misplaced (18.570%); 9.6 MiB/s, 1 keys/s, 154 objects/s recovering 2024-09-21T10:22:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 14 pgs inactive, 14 pgs peering) 2024-09-21T10:22:24.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1445933538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:24.405 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:24.507 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:24.551 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:24.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:24.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:24.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:24.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:24.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:24.939 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:24.955 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:24.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:24.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:25.085 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:25.108 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:25.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:25.210 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:25.233 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:25.248 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:25 smithi076 ceph-mon[106482]: osd.3 marked itself down and dead 2024-09-21T10:22:25.249 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:25 smithi076 ceph-mon[110963]: osd.3 marked itself down and dead 2024-09-21T10:22:25.249 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[296572]: 2024-09-21T10:22:24.882+0000 7f57eb7ae640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:22:25.249 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[296572]: 2024-09-21T10:22:24.882+0000 7f57eb7ae640 -1 osd.3 1365 *** Got signal Terminated *** 2024-09-21T10:22:25.249 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:24 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[296572]: 2024-09-21T10:22:24.882+0000 7f57eb7ae640 -1 osd.3 1365 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:22:25.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 81 16 127686 127670 98.4929 55.1875 0.0198215 0.0101238 2024-09-21T10:22:25.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:25.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:25.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:25.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:25 smithi191 ceph-mon[102206]: osd.3 marked itself down and dead 2024-09-21T10:22:25.459 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:25.481 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:25.583 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:25.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:25.712 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:25.734 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:25.835 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:25.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.862 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:25.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:25.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:25.986 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:25.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:26.059 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:22:26.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:26.137 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:26.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:26.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:26.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: pgmap v2202: 99 pgs: 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 90 active+clean; 7.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 46 MiB/s wr, 731 op/s; 69021/371676 objects misplaced (18.570%); 8.2 MiB/s, 1 keys/s, 132 objects/s recovering 2024-09-21T10:22:26.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:22:26.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: osdmap e1366: 8 total, 7 up, 4 in 2024-09-21T10:22:26.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1309417328' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:26.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:26.253 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:26.254 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:26.254 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:25 smithi076 podman[341651]: 2024-09-21 10:22:25.924140476 +0000 UTC m=+1.224745432 container died 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b) 2024-09-21T10:22:26.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: pgmap v2202: 99 pgs: 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 90 active+clean; 7.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 46 MiB/s wr, 731 op/s; 69021/371676 objects misplaced (18.570%); 8.2 MiB/s, 1 keys/s, 132 objects/s recovering 2024-09-21T10:22:26.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:22:26.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: osdmap e1366: 8 total, 7 up, 4 in 2024-09-21T10:22:26.254 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1309417328' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:26.255 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:26.255 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:26.255 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:26.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 82 16 127909 127893 97.4619 13.9375 0.0107417 0.0102561 2024-09-21T10:22:26.268 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: pgmap v2202: 99 pgs: 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 90 active+clean; 7.6 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 46 MiB/s wr, 731 op/s; 69021/371676 objects misplaced (18.570%); 8.2 MiB/s, 1 keys/s, 132 objects/s recovering 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: osdmap e1366: 8 total, 7 up, 4 in 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1309417328' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:26.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:26.370 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:26.404 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:26.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:26.439 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:22:26.506 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:26.536 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:26.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:26.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:26 smithi076 podman[341651]: 2024-09-21 10:22:26.351739629 +0000 UTC m=+1.652344577 container cleanup 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:22:26.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:26 smithi076 bash[341651]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:22:26.638 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:26.682 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:26.784 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:26.810 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:26.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:26.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:26.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:26.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:27.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:27.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:27.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:27.240 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:27.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 83 16 129243 129227 97.2921 83.375 0.0221535 0.0102738 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: osdmap e1367: 8 total, 7 up, 4 in 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: 11.9 continuing backfill to osd.5 from (1349'4092,1365'7981] MIN to 1365'7981 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: 11.4 continuing backfill to osd.4 from (1349'4184,1365'8143] MIN to 1365'8143 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: 11.8 continuing backfill to osd.4 from (1349'4119,1365'8029] MIN to 1365'8029 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: 11.1 continuing backfill to osd.1 from (1349'4092,1365'8012] MIN to 1365'8012 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: 11.0 continuing backfill to osd.5 from (1349'3950,1365'7898] MIN to 1365'7898 2024-09-21T10:22:27.338 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[106482]: 11.5 continuing backfill to osd.1 from (1349'3841,1365'8032] MIN to 1365'8032 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: osdmap e1367: 8 total, 7 up, 4 in 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: 11.9 continuing backfill to osd.5 from (1349'4092,1365'7981] MIN to 1365'7981 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: 11.4 continuing backfill to osd.4 from (1349'4184,1365'8143] MIN to 1365'8143 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: 11.8 continuing backfill to osd.4 from (1349'4119,1365'8029] MIN to 1365'8029 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: 11.1 continuing backfill to osd.1 from (1349'4092,1365'8012] MIN to 1365'8012 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: 11.0 continuing backfill to osd.5 from (1349'3950,1365'7898] MIN to 1365'7898 2024-09-21T10:22:27.339 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:27 smithi076 ceph-mon[110963]: 11.5 continuing backfill to osd.1 from (1349'3841,1365'8032] MIN to 1365'8032 2024-09-21T10:22:27.342 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: osdmap e1367: 8 total, 7 up, 4 in 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: 11.9 continuing backfill to osd.5 from (1349'4092,1365'7981] MIN to 1365'7981 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: 11.4 continuing backfill to osd.4 from (1349'4184,1365'8143] MIN to 1365'8143 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: 11.8 continuing backfill to osd.4 from (1349'4119,1365'8029] MIN to 1365'8029 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: 11.1 continuing backfill to osd.1 from (1349'4092,1365'8012] MIN to 1365'8012 2024-09-21T10:22:27.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: 11.0 continuing backfill to osd.5 from (1349'3950,1365'7898] MIN to 1365'7898 2024-09-21T10:22:27.365 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:27 smithi191 ceph-mon[102206]: 11.5 continuing backfill to osd.1 from (1349'3841,1365'8032] MIN to 1365'8032 2024-09-21T10:22:27.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:27.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:27.499 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:27.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:27.602 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:27.630 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:27.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:27.732 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:27.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:27.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:27.915 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:27.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:28.016 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:28.148 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:28.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:28.257 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 84 16 130560 130544 97.1137 82.3125 0.00807555 0.0102928 2024-09-21T10:22:28.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:28 smithi191 ceph-mon[102206]: pgmap v2205: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 1 active+remapped+backfilling, 24 active+undersized, 3 active+remapped+backfill_wait, 10 active+undersized+degraded, 39 active+clean; 7.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s; 31240/376125 objects degraded (8.306%); 70620/376125 objects misplaced (18.776%); 13 MiB/s, 1 keys/s, 203 objects/s recovering 2024-09-21T10:22:28.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:28 smithi191 ceph-mon[102206]: Health check failed: Degraded data redundancy: 31240/376125 objects degraded (8.306%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:28 smithi076 ceph-mon[106482]: pgmap v2205: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 1 active+remapped+backfilling, 24 active+undersized, 3 active+remapped+backfill_wait, 10 active+undersized+degraded, 39 active+clean; 7.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s; 31240/376125 objects degraded (8.306%); 70620/376125 objects misplaced (18.776%); 13 MiB/s, 1 keys/s, 203 objects/s recovering 2024-09-21T10:22:28.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:28 smithi076 ceph-mon[106482]: Health check failed: Degraded data redundancy: 31240/376125 objects degraded (8.306%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:28 smithi076 ceph-mon[110963]: pgmap v2205: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 stale+active+remapped+backfill_wait, 17 stale+active+clean, 1 active+remapped+backfilling, 24 active+undersized, 3 active+remapped+backfill_wait, 10 active+undersized+degraded, 39 active+clean; 7.7 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s; 31240/376125 objects degraded (8.306%); 70620/376125 objects misplaced (18.776%); 13 MiB/s, 1 keys/s, 203 objects/s recovering 2024-09-21T10:22:28.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:28 smithi076 ceph-mon[110963]: Health check failed: Degraded data redundancy: 31240/376125 objects degraded (8.306%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:28.412 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:28.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:28.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:28.567 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:28.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:28.697 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:28.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:28.821 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:28.922 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:28.951 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:28.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:29.052 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:29.089 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:29.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:28 smithi076 podman[341846]: 2024-09-21 10:22:28.849073928 +0000 UTC m=+3.116967550 container remove 0fa3a3d245694ff11250c9e5803e08487c4b0bfa77b9a1629d7064230c9d8666 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:22:29.191 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:29.213 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:29.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 85 16 131663 131647 96.7822 68.9375 0.00891407 0.0103281 2024-09-21T10:22:29.315 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:29.348 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:29.450 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:29.477 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:29.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:29 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:29 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:29.618 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:29.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:29 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:29 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:29.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:29 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:29.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:29 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:29.692 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:22:29.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:29.755 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:29.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:29.858 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:29.895 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:29.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:29.997 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:30.025 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:30.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:30.127 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:30.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:30.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:30.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 86 16 132983 132967 96.616 82.5 0.0109368 0.0103455 2024-09-21T10:22:30.264 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:30.294 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:30.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:30.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:30.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:30.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:30.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:30.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:30.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:30 smithi191 ceph-mon[102206]: pgmap v2206: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 76 MiB/s wr, 1.22k op/s; 87346/386130 objects degraded (22.621%); 71087/386130 objects misplaced (18.410%); 19 MiB/s, 1 keys/s, 313 objects/s recovering 2024-09-21T10:22:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:30 smithi076 ceph-mon[106482]: pgmap v2206: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 76 MiB/s wr, 1.22k op/s; 87346/386130 objects degraded (22.621%); 71087/386130 objects misplaced (18.410%); 19 MiB/s, 1 keys/s, 313 objects/s recovering 2024-09-21T10:22:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:30 smithi076 ceph-mon[110963]: pgmap v2206: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 76 MiB/s wr, 1.22k op/s; 87346/386130 objects degraded (22.621%); 71087/386130 objects misplaced (18.410%); 19 MiB/s, 1 keys/s, 313 objects/s recovering 2024-09-21T10:22:30.656 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:30.692 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:30.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:30.794 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:30.824 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:30.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:30.926 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:30.962 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:30.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:30.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:31.064 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:31.087 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:31.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:31.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:31.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:31.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 87 16 134149 134133 96.3431 72.875 0.00854222 0.0103754 2024-09-21T10:22:31.332 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:31.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:31.471 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:31.511 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:31.612 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:31.641 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:31.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:31.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:31.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:31.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:31.917 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:31.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:32.019 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:32.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:32.152 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:32.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:32.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:32.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 88 16 135355 135339 96.1048 75.375 0.00848561 0.010401 2024-09-21T10:22:32.286 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:32.314 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:32.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:32.416 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:32.439 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:32.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:32.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:32.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:32 smithi191 ceph-mon[102206]: pgmap v2207: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 38 MiB/s wr, 601 op/s; 87346/386130 objects degraded (22.621%); 71087/386130 objects misplaced (18.410%); 13 MiB/s, 201 objects/s recovering 2024-09-21T10:22:32.621 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:32 smithi076 ceph-mon[106482]: pgmap v2207: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 38 MiB/s wr, 601 op/s; 87346/386130 objects degraded (22.621%); 71087/386130 objects misplaced (18.410%); 13 MiB/s, 201 objects/s recovering 2024-09-21T10:22:32.621 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:32 smithi076 ceph-mon[110963]: pgmap v2207: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 38 MiB/s wr, 601 op/s; 87346/386130 objects degraded (22.621%); 71087/386130 objects misplaced (18.410%); 13 MiB/s, 201 objects/s recovering 2024-09-21T10:22:32.657 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:32.677 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:32.709 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:32.811 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:32.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.840 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:32.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:32.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:32.980 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:32.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:33.082 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:33.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:33.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:33.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:33.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 89 16 136536 136520 95.8543 73.8125 0.00897466 0.0104181 2024-09-21T10:22:33.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:33.374 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:33.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:33.476 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:33.513 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:33.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:33.615 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:33.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:33 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 89413/404556 objects degraded (22.102%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:22:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:22:33] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.51.0" 2024-09-21T10:22:33.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:33 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 89413/404556 objects degraded (22.102%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:33.642 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:33.744 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:33.772 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:33.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:33.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:33 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 89413/404556 objects degraded (22.102%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:33.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:33.907 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:33.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:34.009 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:34.047 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:34.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:34.178 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:34.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:34.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout: 90 15 137766 137751 95.644 76.9375 0.0140646 0.0104496 2024-09-21T10:22:34.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:34.305 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:34.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:34.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:34.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:34.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:34.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:34.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:34 smithi076 ceph-mon[106482]: pgmap v2208: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.2 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s; 89413/404556 objects degraded (22.102%); 74478/404556 objects misplaced (18.410%); 29 MiB/s, 459 objects/s recovering 2024-09-21T10:22:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:34 smithi076 ceph-mon[110963]: pgmap v2208: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.2 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s; 89413/404556 objects degraded (22.102%); 74478/404556 objects misplaced (18.410%); 29 MiB/s, 459 objects/s recovering 2024-09-21T10:22:34.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:34.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:34.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:34.781 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:34.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:34.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:34 smithi191 ceph-mon[102206]: pgmap v2208: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.2 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s; 89413/404556 objects degraded (22.102%); 74478/404556 objects misplaced (18.410%); 29 MiB/s, 459 objects/s recovering 2024-09-21T10:22:34.905 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:34.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:34.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:34.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:35.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:35.051 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:35.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:35.153 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:35.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:35.246 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:35.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total time run: 90.0534 2024-09-21T10:22:35.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Total writes made: 137766 2024-09-21T10:22:35.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Write size: 65536 2024-09-21T10:22:35.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Object size: 65536 2024-09-21T10:22:35.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Bandwidth (MB/sec): 95.6141 2024-09-21T10:22:35.258 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Bandwidth: 39.9413 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max bandwidth (MB/sec): 248.062 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min bandwidth (MB/sec): 0 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average IOPS: 1529 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev IOPS: 639.074 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max IOPS: 3969 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min IOPS: 0 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Average Latency(s): 0.0104543 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Stddev Latency(s): 0.0267812 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Max latency(s): 2.16199 2024-09-21T10:22:35.259 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Min latency(s): 0.0014399 2024-09-21T10:22:35.266 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Cleaning up (deleting benchmark objects) 2024-09-21T10:22:35.280 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:35.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:35.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:35.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:35.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:35.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:35.578 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:35.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:35.679 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:35.702 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:35.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:35.803 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:35.836 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:35.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:35.938 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:35.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:35.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:35.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:36.064 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:36.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:36.200 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:36.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:36.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:36.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:36.362 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:36.463 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:36.495 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:36.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:36.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:36.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:36.727 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:36.757 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:36.859 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:36.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:36 smithi191 ceph-mon[102206]: pgmap v2209: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.2 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 70 MiB/s wr, 1.13k op/s; 89413/404556 objects degraded (22.102%); 74478/404556 objects misplaced (18.410%); 24 MiB/s, 378 objects/s recovering 2024-09-21T10:22:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:36 smithi076 ceph-mon[106482]: pgmap v2209: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.2 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 70 MiB/s wr, 1.13k op/s; 89413/404556 objects degraded (22.102%); 74478/404556 objects misplaced (18.410%); 24 MiB/s, 378 objects/s recovering 2024-09-21T10:22:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:36 smithi076 ceph-mon[110963]: pgmap v2209: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.2 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 70 MiB/s wr, 1.13k op/s; 89413/404556 objects degraded (22.102%); 74478/404556 objects misplaced (18.410%); 24 MiB/s, 378 objects/s recovering 2024-09-21T10:22:36.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:36.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:36.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:36.999 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:37.037 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:37.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:37.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:37.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:37.277 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:37.313 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:37.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:37.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:37.443 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:37.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:37.581 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:37.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:37.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:37 smithi076 podman[342533]: 2024-09-21 10:22:37.163043098 +0000 UTC m=+4.527248152 container died 9c8ee54234cf44919ed78b7e533905122d7a6035329a4815012182e099bef59a (image=quay.ceph.io/ceph-ci/ceph:reef, name=busy_taussig, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T10:22:37.682 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:37.719 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:37.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:37.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:37.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.858 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:37.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:37.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:37.987 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:37.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:37.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:38.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:38.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:38.227 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:38.268 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:38.370 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:38.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:38.500 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:38.528 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:38.629 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:38.666 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:38.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:38.768 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:38.799 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:38.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:38.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:38 smithi191 ceph-mon[102206]: pgmap v2210: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 70 MiB/s wr, 1.16k op/s; 87671/406308 objects degraded (21.577%); 74972/406308 objects misplaced (18.452%); 34 MiB/s, 549 objects/s recovering 2024-09-21T10:22:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:38 smithi076 ceph-mon[106482]: pgmap v2210: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 70 MiB/s wr, 1.16k op/s; 87671/406308 objects degraded (21.577%); 74972/406308 objects misplaced (18.452%); 34 MiB/s, 549 objects/s recovering 2024-09-21T10:22:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:38 smithi076 ceph-mon[110963]: pgmap v2210: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 70 MiB/s wr, 1.16k op/s; 87671/406308 objects degraded (21.577%); 74972/406308 objects misplaced (18.452%); 34 MiB/s, 549 objects/s recovering 2024-09-21T10:22:38.902 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:38.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:38.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:39.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.062 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:39.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:39.163 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:39.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:39.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:39.320 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:39.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:39.448 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:39.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:39.590 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:39.692 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:39.730 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:39.832 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 87671/406308 objects degraded (21.577%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 87671/406308 objects degraded (21.577%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:39.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 87671/406308 objects degraded (21.577%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:39.874 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:39.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:39.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:40.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:40.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:40.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:40.247 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:40.269 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:40.371 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:40.498 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:40.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:40.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:40.661 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:40 smithi076 ceph-mon[106482]: pgmap v2211: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 65 MiB/s wr, 1.19k op/s; 86995/407631 objects degraded (21.342%); 75137/407631 objects misplaced (18.433%); 33 MiB/s, 532 objects/s recovering 2024-09-21T10:22:40.661 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:40 smithi076 ceph-mon[110963]: pgmap v2211: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 65 MiB/s wr, 1.19k op/s; 86995/407631 objects degraded (21.342%); 75137/407631 objects misplaced (18.433%); 33 MiB/s, 532 objects/s recovering 2024-09-21T10:22:40.671 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:40.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:40.810 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:40.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:40 smithi191 ceph-mon[102206]: pgmap v2211: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 65 MiB/s wr, 1.19k op/s; 86995/407631 objects degraded (21.342%); 75137/407631 objects misplaced (18.433%); 33 MiB/s, 532 objects/s recovering 2024-09-21T10:22:40.912 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:40.950 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:40.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:41.052 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:41.073 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:41.175 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:41.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:41.303 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:41.339 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:41.441 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:41.462 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:41.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:41.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:41.607 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:41.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:41.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:41.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:41.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:41.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:41.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:41.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:41.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:41.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:42.012 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:42.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:42.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:42.147 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:42.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:42.249 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:42.293 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:42.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:42.417 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:42.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:42.519 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:42.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:42.659 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:42.685 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:42.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:42.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:42 smithi191 ceph-mon[102206]: pgmap v2212: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 47 MiB/s wr, 916 op/s; 86995/407631 objects degraded (21.342%); 75137/407631 objects misplaced (18.433%); 27 MiB/s, 438 objects/s recovering 2024-09-21T10:22:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:42 smithi076 ceph-mon[106482]: pgmap v2212: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 47 MiB/s wr, 916 op/s; 86995/407631 objects degraded (21.342%); 75137/407631 objects misplaced (18.433%); 27 MiB/s, 438 objects/s recovering 2024-09-21T10:22:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:42 smithi076 ceph-mon[110963]: pgmap v2212: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 8.3 GiB data, 24 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 47 MiB/s wr, 916 op/s; 86995/407631 objects degraded (21.342%); 75137/407631 objects misplaced (18.433%); 27 MiB/s, 438 objects/s recovering 2024-09-21T10:22:42.914 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:42.945 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:42.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:43.047 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:43.081 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:43.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:43.224 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:43.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:43.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:43.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:43.489 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:43.493 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:22:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:22:43] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-09-21T10:22:43.570 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:22:43.570 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:22:43.570 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:22:43.570 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:22:43.570 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:22:43.591 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:43.626 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:43.728 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:43.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:43.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:43.858 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:43.895 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:43.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:43.996 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:44.029 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:44.130 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:44.166 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:44.267 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:44.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:44.296 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:22:44.296 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:22:44.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:44.322 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:22:44.322 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:22:44.347 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:22:44.347 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:22:44.348 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:22:44.379 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:22:44.379 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:22:44.397 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:44.410 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:22:44.410 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:22:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.431 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:44.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:44.442 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:22:44.442 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:22:44.467 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:22:44.467 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:22:44.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:44.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:44.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:44.659 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:44.686 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:44.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:44.788 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:44.830 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:44 smithi191 ceph-mon[102206]: pgmap v2213: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 47 MiB/s wr, 1.46k op/s; 80670/388089 objects degraded (20.786%); 71515/388089 objects misplaced (18.427%); 39 MiB/s, 619 objects/s recovering 2024-09-21T10:22:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/950313327' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 80670/388089 objects degraded (20.786%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:44 smithi076 ceph-mon[106482]: pgmap v2213: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 47 MiB/s wr, 1.46k op/s; 80670/388089 objects degraded (20.786%); 71515/388089 objects misplaced (18.427%); 39 MiB/s, 619 objects/s recovering 2024-09-21T10:22:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/950313327' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 80670/388089 objects degraded (20.786%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:44 smithi076 ceph-mon[110963]: pgmap v2213: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 47 MiB/s wr, 1.46k op/s; 80670/388089 objects degraded (20.786%); 71515/388089 objects misplaced (18.427%); 39 MiB/s, 619 objects/s recovering 2024-09-21T10:22:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/950313327' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:22:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 80670/388089 objects degraded (20.786%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:44.932 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:44.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:44.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:44.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:45.060 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.097 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:45.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:45.198 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:45.237 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:45.338 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:45.349 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:44 smithi076 podman[343455]: 2024-09-21 10:22:44.77741031 +0000 UTC m=+0.758376631 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:22:45.349 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:44 smithi076 podman[343455]: 2024-09-21 10:22:44.910699967 +0000 UTC m=+0.891666278 container create 0bfb64f0857470f322a15a1bc71e14d2f4f8419c4dc441d01fcf33f4687cd80a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:22:45.349 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:45 smithi076 podman[343455]: 2024-09-21 10:22:45.347628905 +0000 UTC m=+1.328595228 container init 0bfb64f0857470f322a15a1bc71e14d2f4f8419c4dc441d01fcf33f4687cd80a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:22:45.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:45.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:45.508 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:45.610 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:45.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:45 smithi076 podman[343455]: 2024-09-21 10:22:45.356074212 +0000 UTC m=+1.337040528 container start 0bfb64f0857470f322a15a1bc71e14d2f4f8419c4dc441d01fcf33f4687cd80a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:22:45.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:45 smithi076 podman[343455]: 2024-09-21 10:22:45.488898157 +0000 UTC m=+1.469864483 container attach 0bfb64f0857470f322a15a1bc71e14d2f4f8419c4dc441d01fcf33f4687cd80a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T10:22:45.639 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:45.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:45.774 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:45.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:45.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:45.914 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:45.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:45.954 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:45 smithi076 ceph-mon[106482]: pgmap v2214: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 947 op/s; 80670/388089 objects degraded (20.786%); 71515/388089 objects misplaced (18.427%); 28 MiB/s, 447 objects/s recovering 2024-09-21T10:22:45.954 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:45 smithi076 ceph-mon[110963]: pgmap v2214: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 947 op/s; 80670/388089 objects degraded (20.786%); 71515/388089 objects misplaced (18.427%); 28 MiB/s, 447 objects/s recovering 2024-09-21T10:22:46.017 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:46.044 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:46.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:45 smithi191 ceph-mon[102206]: pgmap v2214: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.9 GiB data, 23 GiB used, 334 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 947 op/s; 80670/388089 objects degraded (20.786%); 71515/388089 objects misplaced (18.427%); 28 MiB/s, 447 objects/s recovering 2024-09-21T10:22:46.146 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:46.180 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:46.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:46.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:46.411 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:46.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:46.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:46.489 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.1 dump_historic_ops 2024-09-21T10:22:46.554 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:46.595 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:46.697 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:46.736 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:46.838 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:46.872 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:46.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:46.973 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:47.009 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:47.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:22:47.088 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:22:47.111 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:47.144 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:47.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:47.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:47.382 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:47.410 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:47.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:47.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:47.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:47.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:47.675 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:47.777 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:47.781 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:47.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:47.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:47 smithi076 ceph-mon[106482]: pgmap v2215: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.8 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 1.13k op/s; 78008/381522 objects degraded (20.447%); 69751/381522 objects misplaced (18.282%); 35 MiB/s, 552 objects/s recovering 2024-09-21T10:22:47.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:47 smithi076 ceph-mon[110963]: pgmap v2215: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.8 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 1.13k op/s; 78008/381522 objects degraded (20.447%); 69751/381522 objects misplaced (18.282%); 35 MiB/s, 552 objects/s recovering 2024-09-21T10:22:47.914 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:47.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:47.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:48.054 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:48.080 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:47 smithi191 ceph-mon[102206]: pgmap v2215: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.8 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 1.13k op/s; 78008/381522 objects degraded (20.447%); 69751/381522 objects misplaced (18.282%); 35 MiB/s, 552 objects/s recovering 2024-09-21T10:22:48.181 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:48.209 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:48.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:48.311 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:48.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:48.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:48.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:48.480 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:48.582 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:48.619 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:48.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:48.720 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.747 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:48.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:48.848 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:48.880 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:48.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:48.981 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:49.003 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:49.089 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:22:49.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:49.144 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:49.245 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:49.282 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:49.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:49.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:49.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:49.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:49.513 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:49.541 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:49.643 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:49.681 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:49.783 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:49.807 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:49.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:49.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:49.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:50.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:50.080 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:49 smithi191 ceph-mon[102206]: pgmap v2216: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.5 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 1.39k op/s; 74481/367545 objects degraded (20.264%); 67729/367545 objects misplaced (18.427%); 27 MiB/s, 428 objects/s recovering 2024-09-21T10:22:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 74481/367545 objects degraded (20.264%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:49 smithi076 ceph-mon[106482]: pgmap v2216: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.5 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 1.39k op/s; 74481/367545 objects degraded (20.264%); 67729/367545 objects misplaced (18.427%); 27 MiB/s, 428 objects/s recovering 2024-09-21T10:22:50.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 74481/367545 objects degraded (20.264%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:49 smithi076 ceph-mon[110963]: pgmap v2216: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.5 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 9.9 MiB/s wr, 1.39k op/s; 74481/367545 objects degraded (20.264%); 67729/367545 objects misplaced (18.427%); 27 MiB/s, 428 objects/s recovering 2024-09-21T10:22:50.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 74481/367545 objects degraded (20.264%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:50.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:50.217 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:50.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.359 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:50.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:50.460 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:50.487 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:50.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:50.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:50.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:50.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:50.717 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:50.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:50.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:50.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:50.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:50.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:50.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:51.014 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:51.115 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:51.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:51.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:51.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:51.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:51.393 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:51.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:51.532 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:51.561 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:51.664 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:51.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:51.806 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:51.835 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:51 smithi076 podman[343455]: 2024-09-21 10:22:51.470187334 +0000 UTC m=+7.451153642 container died 0bfb64f0857470f322a15a1bc71e14d2f4f8419c4dc441d01fcf33f4687cd80a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T10:22:51.844 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:51.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:51.945 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:51.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:51.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:52.074 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:52.109 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:52.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:52.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:51 smithi191 ceph-mon[102206]: pgmap v2217: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.5 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 0 B/s wr, 1.11k op/s; 74481/367545 objects degraded (20.264%); 67729/367545 objects misplaced (18.427%); 22 MiB/s, 346 objects/s recovering 2024-09-21T10:22:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:51 smithi076 ceph-mon[106482]: pgmap v2217: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.5 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 0 B/s wr, 1.11k op/s; 74481/367545 objects degraded (20.264%); 67729/367545 objects misplaced (18.427%); 22 MiB/s, 346 objects/s recovering 2024-09-21T10:22:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:51 smithi076 ceph-mon[110963]: pgmap v2217: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.5 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 0 B/s wr, 1.11k op/s; 74481/367545 objects degraded (20.264%); 67729/367545 objects misplaced (18.427%); 22 MiB/s, 346 objects/s recovering 2024-09-21T10:22:52.211 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:52.239 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:52.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:52.382 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:52.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:52.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:52.516 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:52.617 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:52.648 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:52.750 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:52.787 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:52.847 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:22:52.865 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:52 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/995275172' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:52.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:52 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/995275172' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:52.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:52.926 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:52.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:53.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:53.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:53.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:53.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:52 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/995275172' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:22:53.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:53.194 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:53.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:53.333 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:53.435 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:53.474 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:53.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:53.576 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:53.602 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:53.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:22:53] "GET /metrics HTTP/1.1" 200 36652 "" "Prometheus/2.51.0" 2024-09-21T10:22:53.704 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:53.741 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:53.843 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:53.891 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:53.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:53.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:54.000 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[106482]: pgmap v2218: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.68k op/s; 68329/347124 objects degraded (19.684%); 63961/347124 objects misplaced (18.426%); 33 MiB/s, 520 objects/s recovering 2024-09-21T10:22:54.000 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:54.000 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/995275172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:54.000 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[106482]: osdmap e1368: 8 total, 7 up, 4 in 2024-09-21T10:22:54.000 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[110963]: pgmap v2218: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.68k op/s; 68329/347124 objects degraded (19.684%); 63961/347124 objects misplaced (18.426%); 33 MiB/s, 520 objects/s recovering 2024-09-21T10:22:54.001 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:54.001 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/995275172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:54.001 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:53 smithi076 ceph-mon[110963]: osdmap e1368: 8 total, 7 up, 4 in 2024-09-21T10:22:54.020 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:53 smithi191 ceph-mon[102206]: pgmap v2218: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.68k op/s; 68329/347124 objects degraded (19.684%); 63961/347124 objects misplaced (18.426%); 33 MiB/s, 520 objects/s recovering 2024-09-21T10:22:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:53 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:22:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:53 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/995275172' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:22:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:53 smithi191 ceph-mon[102206]: osdmap e1368: 8 total, 7 up, 4 in 2024-09-21T10:22:54.121 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:54.156 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:54.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:54.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.297 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:54.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:54.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:54.425 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:54.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:54.567 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:54.668 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:54.696 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:54.767 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:22:54.797 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:54.819 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:22:54.819 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:22:54.827 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:54.929 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:54.958 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:54.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:55.060 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:55.091 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 68329/347124 objects degraded (19.684%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:55.092 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 68329/347124 objects degraded (19.684%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:55.101 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:55.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 68329/347124 objects degraded (19.684%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:55.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:55.232 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:55.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:55.333 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:55.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:55.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:55.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:55.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:55.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:55.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:55.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:55.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:55.822 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:22:55.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:55.907 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:55.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:56.009 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:56.035 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:56.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:56.137 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:56.177 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:56.279 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:56.317 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:56.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:56.454 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:56.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:56.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:56.591 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:56.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:56.692 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:56.719 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:56.821 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:56.850 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:56.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:56 smithi191 ceph-mon[102206]: pgmap v2220: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 68329/347124 objects degraded (19.684%); 63961/347124 objects misplaced (18.426%); 25 MiB/s, 405 objects/s recovering 2024-09-21T10:22:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:56 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:56 smithi076 ceph-mon[106482]: pgmap v2220: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 68329/347124 objects degraded (19.684%); 63961/347124 objects misplaced (18.426%); 25 MiB/s, 405 objects/s recovering 2024-09-21T10:22:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:56 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:56 smithi076 ceph-mon[110963]: pgmap v2220: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 7.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 68329/347124 objects degraded (19.684%); 63961/347124 objects misplaced (18.426%); 25 MiB/s, 405 objects/s recovering 2024-09-21T10:22:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:22:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:56 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:22:56.952 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:56.955 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:22:56.990 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:56.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:57.092 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.128 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:57.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:57.229 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:57.256 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:57.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:22:57.357 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:57.391 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:57.493 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:57.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:57.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:57.635 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:57.659 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:57.760 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:57.787 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:57.888 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:57.929 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:57.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:57.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:58.031 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:58.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:58.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:58.193 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:58.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:58.295 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:58.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:58.428 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:58.467 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:58.569 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:22:58.601 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:58.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:22:58.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:22:58.743 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:22:58.753 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:58 smithi076 ceph-mon[106482]: pgmap v2221: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.38k op/s; 65716/340248 objects degraded (19.314%); 62114/340248 objects misplaced (18.256%); 25 MiB/s, 407 objects/s recovering 2024-09-21T10:22:58.753 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:58 smithi076 ceph-mon[110963]: pgmap v2221: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.38k op/s; 65716/340248 objects degraded (19.314%); 62114/340248 objects misplaced (18.256%); 25 MiB/s, 407 objects/s recovering 2024-09-21T10:22:58.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:58.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:58 smithi191 ceph-mon[102206]: pgmap v2221: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.9 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.38k op/s; 65716/340248 objects degraded (19.314%); 62114/340248 objects misplaced (18.256%); 25 MiB/s, 407 objects/s recovering 2024-09-21T10:22:58.876 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:58.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:58.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:58.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:59.012 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:59.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:22:59.146 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:22:59.248 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:59.275 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:59.377 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:22:59.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.411 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:59.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:22:59.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:59.539 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:59.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:59.641 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:22:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:22:59.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:22:59.779 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:22:59.810 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:59.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:22:59.820 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 5] out_osds: [7, 0, 2, 6] dead_osds: [3] live_osds: [1, 2, 4, 5, 6, 7, 0] 2024-09-21T10:22:59.820 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:22:59.820 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2024-09-21T10:22:59.820 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2024-09-21T10:22:59.820 DEBUG:teuthology.orchestra.run.smithi076:osd.3> sudo journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T10:22:59.843 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:22:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 65716/340248 objects degraded (19.314%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:59.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:22:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 65716/340248 objects degraded (19.314%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:59.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl start ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:22:59.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:22:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 65716/340248 objects degraded (19.314%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:22:59.912 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:22:59.957 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:22:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:22:59.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:00.059 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:00.093 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:00.123 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:22:59 smithi076 systemd[1]: Starting Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:23:00.194 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:00.233 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:00.310 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:23:00.310 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:23:00.335 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:00.374 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:00.476 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:00.507 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:00.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:00.609 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:00.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:00.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:00.738 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.767 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:00.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:00.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:00 smithi191 ceph-mon[102206]: pgmap v2222: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.6 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 62050/325314 objects degraded (19.074%); 59933/325314 objects misplaced (18.423%); 26 MiB/s, 410 objects/s recovering 2024-09-21T10:23:00.868 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:00 smithi076 ceph-mon[106482]: pgmap v2222: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.6 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 62050/325314 objects degraded (19.074%); 59933/325314 objects misplaced (18.423%); 26 MiB/s, 410 objects/s recovering 2024-09-21T10:23:00.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:00 smithi076 ceph-mon[110963]: pgmap v2222: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.6 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 62050/325314 objects degraded (19.074%); 59933/325314 objects misplaced (18.423%); 26 MiB/s, 410 objects/s recovering 2024-09-21T10:23:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:00.897 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:00.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:00.998 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:01.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:01.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:01.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:01.237 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:01.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:01.295 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:01.398 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:01.438 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:01.540 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:01.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:01.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:01.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.711 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:01.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:01.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:01.848 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:01.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:01.988 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:01.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:01.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:02.090 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:02.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:02.223 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:02.249 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:02.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:02.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.391 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:02.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:02.492 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.528 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:02.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:02.629 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:02.657 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:02.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:02.759 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:02.794 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:02.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:02 smithi191 ceph-mon[102206]: pgmap v2223: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.6 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 62050/325314 objects degraded (19.074%); 59933/325314 objects misplaced (18.423%); 26 MiB/s, 410 objects/s recovering 2024-09-21T10:23:02.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:02 smithi076 ceph-mon[106482]: pgmap v2223: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.6 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 62050/325314 objects degraded (19.074%); 59933/325314 objects misplaced (18.423%); 26 MiB/s, 410 objects/s recovering 2024-09-21T10:23:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:02 smithi076 ceph-mon[110963]: pgmap v2223: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.6 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 62050/325314 objects degraded (19.074%); 59933/325314 objects misplaced (18.423%); 26 MiB/s, 410 objects/s recovering 2024-09-21T10:23:02.897 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:02.935 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:02.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:03.036 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:03.065 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:03.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:03.166 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:03.194 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:03.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:03.325 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:03.427 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:03.457 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:03.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:03.559 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:03.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:03.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:23:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:23:03] "GET /metrics HTTP/1.1" 200 36647 "" "Prometheus/2.51.0" 2024-09-21T10:23:03.689 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:03.724 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:03.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:03.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-21T10:23:03.860 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:03.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:03.962 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:04.001 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:04.103 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:04.139 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:04.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:04.240 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.277 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:04.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:04.378 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:04.407 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:04.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:04.508 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:04.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:04.650 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:04.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:04.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:04.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:04.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:04 smithi191 ceph-mon[102206]: pgmap v2224: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 26 MiB/s, 412 objects/s recovering 2024-09-21T10:23:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 56164/304791 objects degraded (18.427%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:04 smithi076 ceph-mon[106482]: pgmap v2224: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 26 MiB/s, 412 objects/s recovering 2024-09-21T10:23:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 56164/304791 objects degraded (18.427%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:04 smithi076 ceph-mon[110963]: pgmap v2224: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.41k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 26 MiB/s, 412 objects/s recovering 2024-09-21T10:23:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 56164/304791 objects degraded (18.427%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:04.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:04.955 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:04.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:05.057 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.094 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:05.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:05.195 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:05.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:05.328 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:05.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:05.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:05.467 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:05.504 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:05.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:05.636 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:05.738 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:05.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:05.875 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:05.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:05.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:06.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:06.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:06.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:06.183 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: pgmap v2225: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.18k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 22 MiB/s, 345 objects/s recovering 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: pgmap v2226: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.31k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 24 MiB/s, 384 objects/s recovering 2024-09-21T10:23:06.235 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[106482]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: pgmap v2225: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.18k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 22 MiB/s, 345 objects/s recovering 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: pgmap v2226: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.31k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 24 MiB/s, 384 objects/s recovering 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.236 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:05 smithi076 ceph-mon[110963]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T10:23:06.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:06.319 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: pgmap v2225: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.18k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 22 MiB/s, 345 objects/s recovering 2024-09-21T10:23:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:23:06.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:23:06.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: pgmap v2226: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 6.2 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.31k op/s; 56164/304791 objects degraded (18.427%); 56171/304791 objects misplaced (18.429%); 24 MiB/s, 384 objects/s recovering 2024-09-21T10:23:06.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:06.364 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:05 smithi191 ceph-mon[102206]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-21T10:23:06.420 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:06.456 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:06.558 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:06.587 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:06.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:06.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:06.718 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:06.820 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:06.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:06.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:06.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:06.947 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:23:06.947 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:06.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:06.983 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:06.984 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:23:06.984 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:07.011 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:07.016 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:23:07.016 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:07.018 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:23:07.046 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:23:07.046 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:07.073 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:23:07.073 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:07.103 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:23:07.103 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:07.113 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:07.126 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:23:07.126 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:07.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:07.258 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:07.293 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:07.358 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3252109283' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:07.358 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3252109283' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:07.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3252109283' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:07.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:07.422 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:07.524 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:07.549 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:07.651 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:07.675 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:07.777 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:07.814 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:07.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:07.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:07.956 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:07.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:07.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:08.057 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:08.084 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:08.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:08.228 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:08.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:08.232 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:07 smithi076 ceph-mon[106482]: pgmap v2227: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.77k op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 28 MiB/s, 448 objects/s recovering 2024-09-21T10:23:08.232 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:07 smithi076 ceph-mon[110963]: pgmap v2227: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.77k op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 28 MiB/s, 448 objects/s recovering 2024-09-21T10:23:08.330 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:08.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:07 smithi191 ceph-mon[102206]: pgmap v2227: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.77k op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 28 MiB/s, 448 objects/s recovering 2024-09-21T10:23:08.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:08.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:08.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:08.490 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:08.592 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:08.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:08 smithi076 podman[345975]: 2024-09-21 10:23:08.253616071 +0000 UTC m=+1.267011832 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:23:08.629 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:08.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:08.730 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:08.758 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:08.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:08.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:08.888 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:08.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:08.990 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:09.017 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:09.119 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:09.152 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:09.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:09.280 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:09.382 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:09.410 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:09.511 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:09.543 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:09.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 50303/283302 objects degraded (17.756%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:09.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 50303/283302 objects degraded (17.756%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:09.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 50303/283302 objects degraded (17.756%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:09.645 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:09.684 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:09.786 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:09.814 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:09.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:09.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:09.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:10.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:10.080 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:10.182 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:10.216 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:10.318 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:10.340 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:23:10.357 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:10.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:10.458 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.506 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:10.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:10.607 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:10.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1529154373' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:10.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:10.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:10 smithi191 ceph-mon[102206]: pgmap v2228: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 24 MiB/s, 379 objects/s recovering 2024-09-21T10:23:10.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1529154373' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:10.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:10.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:10 smithi076 ceph-mon[106482]: pgmap v2228: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 24 MiB/s, 379 objects/s recovering 2024-09-21T10:23:10.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1529154373' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:10.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:10.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:10 smithi076 ceph-mon[110963]: pgmap v2228: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 24 MiB/s, 379 objects/s recovering 2024-09-21T10:23:10.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:10.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:10.746 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:10.787 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:10.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:10.930 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:10.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:11.032 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:11.065 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:10 smithi076 podman[345975]: 2024-09-21 10:23:10.699472041 +0000 UTC m=+3.712867792 container create 92737152044c0198e861dbddba998644fc9655f98763a2c7cbfd1f89d8efcf80 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T10:23:11.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:11.173 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.200 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:11.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:11.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:11.329 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 podman[345975]: 2024-09-21 10:23:11.195864155 +0000 UTC m=+4.209259907 container init 92737152044c0198e861dbddba998644fc9655f98763a2c7cbfd1f89d8efcf80 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T10:23:11.329 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 podman[345975]: 2024-09-21 10:23:11.202066302 +0000 UTC m=+4.215462054 container start 92737152044c0198e861dbddba998644fc9655f98763a2c7cbfd1f89d8efcf80 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T10:23:11.329 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 podman[345975]: 2024-09-21 10:23:11.304337031 +0000 UTC m=+4.317732782 container attach 92737152044c0198e861dbddba998644fc9655f98763a2c7cbfd1f89d8efcf80 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T10:23:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:11.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:11.443 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:11.474 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:11.576 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:11.590 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:11 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:11.590 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:11.590 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:11 smithi191 ceph-mon[102206]: osdmap e1369: 8 total, 7 up, 4 in 2024-09-21T10:23:11.590 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:11.591 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:11.615 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[110963]: osdmap e1369: 8 total, 7 up, 4 in 2024-09-21T10:23:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:11.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:11.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:11.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:11.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[106482]: osdmap e1369: 8 total, 7 up, 4 in 2024-09-21T10:23:11.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:11.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:11.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:23:11.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 bash[345975]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:23:11.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:23:11.624 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:11 smithi076 bash[345975]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:23:11.717 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:11.745 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:11.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:11.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:11.875 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:11.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:11.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:12.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:12.132 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.161 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:12.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:12.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:12.287 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:12.334 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:23:12.334 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:23:12.334 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T10:23:12.335 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T10:23:12.389 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.422 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:12.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:12.523 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:12.567 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:12.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:12.597 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-mon[106482]: pgmap v2230: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 14 MiB/s, 226 objects/s recovering 2024-09-21T10:23:12.597 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-mon[110963]: pgmap v2230: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 14 MiB/s, 226 objects/s recovering 2024-09-21T10:23:12.597 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate[346279]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T10:23:12.598 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 bash[345975]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T10:23:12.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:12 smithi191 ceph-mon[102206]: pgmap v2230: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.8 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 50303/283302 objects degraded (17.756%); 52221/283302 objects misplaced (18.433%); 14 MiB/s, 226 objects/s recovering 2024-09-21T10:23:12.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:12.710 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:12.812 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.853 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:12.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:12.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:12 smithi076 podman[345975]: 2024-09-21 10:23:12.734129622 +0000 UTC m=+5.747525380 container died 92737152044c0198e861dbddba998644fc9655f98763a2c7cbfd1f89d8efcf80 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-21T10:23:12.954 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:12.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:12.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:13.090 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:13.118 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:13.219 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:13.253 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:13.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:13.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:13.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:13.519 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:13.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:13.564 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:23:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:23:13] "GET /metrics HTTP/1.1" 200 36650 "" "Prometheus/2.51.0" 2024-09-21T10:23:13.620 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:13.653 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:13.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:13.755 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:13.795 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:13.897 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:13.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:13.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:23:14.014 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:23:14.026 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:14.060 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:14.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:14.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:14.198 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:14.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:14.215 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:23:14.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:14.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:14.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:14.480 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:14.582 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:14.611 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:14.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:14.713 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:14.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:14.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:14.849 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:14 smithi191 ceph-mon[102206]: pgmap v2231: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.3 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.60k op/s; 44770/262845 objects degraded (17.033%); 48404/262845 objects misplaced (18.415%); 28 MiB/s, 455 objects/s recovering 2024-09-21T10:23:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 44770/262845 objects degraded (17.033%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:14 smithi076 ceph-mon[106482]: pgmap v2231: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.3 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.60k op/s; 44770/262845 objects degraded (17.033%); 48404/262845 objects misplaced (18.415%); 28 MiB/s, 455 objects/s recovering 2024-09-21T10:23:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 44770/262845 objects degraded (17.033%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:14 smithi076 ceph-mon[110963]: pgmap v2231: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.3 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.60k op/s; 44770/262845 objects degraded (17.033%); 48404/262845 objects misplaced (18.415%); 28 MiB/s, 455 objects/s recovering 2024-09-21T10:23:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 44770/262845 objects degraded (17.033%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:14.885 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:14.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:14.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:14.986 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:15.013 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:15.114 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:15.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:15.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:15.284 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:15.386 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:15.414 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:15.515 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:15.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:15.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:15.672 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:15.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:15.796 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:15.898 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:15.921 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:15.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:16.015 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:23:16.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:16.045 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:16.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:16.147 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:16.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:16.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:16.297 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:16.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:16.398 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:16.420 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:16.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:16.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:16.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:16.646 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:16.670 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:16.772 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:16.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:16.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:16.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:16 smithi191 ceph-mon[102206]: pgmap v2232: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.3 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.40k op/s; 44770/262845 objects degraded (17.033%); 48404/262845 objects misplaced (18.415%); 25 MiB/s, 398 objects/s recovering 2024-09-21T10:23:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:16 smithi191 ceph-mon[102206]: osdmap e1370: 8 total, 7 up, 4 in 2024-09-21T10:23:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:16 smithi076 ceph-mon[106482]: pgmap v2232: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.3 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.40k op/s; 44770/262845 objects degraded (17.033%); 48404/262845 objects misplaced (18.415%); 25 MiB/s, 398 objects/s recovering 2024-09-21T10:23:16.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:16 smithi076 ceph-mon[106482]: osdmap e1370: 8 total, 7 up, 4 in 2024-09-21T10:23:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:16 smithi076 ceph-mon[110963]: pgmap v2232: 99 pgs: 1 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.3 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.40k op/s; 44770/262845 objects degraded (17.033%); 48404/262845 objects misplaced (18.415%); 25 MiB/s, 398 objects/s recovering 2024-09-21T10:23:16.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:16 smithi076 ceph-mon[110963]: osdmap e1370: 8 total, 7 up, 4 in 2024-09-21T10:23:16.906 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:16.935 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:16.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:17.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:17.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:17.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:17.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:17.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:17.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:17.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:17.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:17.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:17.556 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:17.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:17.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:17.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:17.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:17.826 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:17 smithi191 ceph-mon[102206]: osdmap e1371: 8 total, 7 up, 4 in 2024-09-21T10:23:17.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:17 smithi076 ceph-mon[106482]: osdmap e1371: 8 total, 7 up, 4 in 2024-09-21T10:23:17.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:17 smithi076 podman[346560]: 2024-09-21 10:23:17.817143507 +0000 UTC m=+5.306129447 container remove 92737152044c0198e861dbddba998644fc9655f98763a2c7cbfd1f89d8efcf80 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:23:17.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:17 smithi076 ceph-mon[110963]: osdmap e1371: 8 total, 7 up, 4 in 2024-09-21T10:23:17.911 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:17.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:17.969 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:17.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:17.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:18.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:18.096 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:18.197 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:18.236 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:18.338 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:18.368 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:18.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:18.470 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:18.498 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:18.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:18.641 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:18.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:18.715 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:18.727 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:23:18.727 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:23:18.727 INFO:teuthology.orchestra.run.smithi076.stdout: "num_ops": 0 2024-09-21T10:23:18.727 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:23:18.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:18.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:18.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:18.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:18 smithi191 ceph-mon[102206]: pgmap v2235: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.69k op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 38 MiB/s, 610 objects/s recovering 2024-09-21T10:23:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:18 smithi191 ceph-mon[102206]: osdmap e1372: 8 total, 7 up, 4 in 2024-09-21T10:23:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:18 smithi076 ceph-mon[106482]: pgmap v2235: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.69k op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 38 MiB/s, 610 objects/s recovering 2024-09-21T10:23:18.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:18 smithi076 ceph-mon[106482]: osdmap e1372: 8 total, 7 up, 4 in 2024-09-21T10:23:18.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:18 smithi076 ceph-mon[110963]: pgmap v2235: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.69k op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 38 MiB/s, 610 objects/s recovering 2024-09-21T10:23:18.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:18 smithi076 ceph-mon[110963]: osdmap e1372: 8 total, 7 up, 4 in 2024-09-21T10:23:18.879 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:18.910 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:18.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:19.049 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:19.151 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:19.187 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:19.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:19.324 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:19.426 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:19.457 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:19.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:19.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:19.592 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:19.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.693 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:19.728 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:19.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 39709/246795 objects degraded (16.090%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:19.757 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:19 smithi076 ceph-mon[106482]: pgmap v2237: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 891 op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 25 MiB/s, 396 objects/s recovering 2024-09-21T10:23:19.757 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 39709/246795 objects degraded (16.090%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:19.757 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:19 smithi076 ceph-mon[110963]: pgmap v2237: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 891 op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 25 MiB/s, 396 objects/s recovering 2024-09-21T10:23:19.757 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:19 smithi076 podman[347127]: 2024-09-21 10:23:19.396621572 +0000 UTC m=+0.658322239 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:23:19.830 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:19.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 39709/246795 objects degraded (16.090%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:19.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:19 smithi191 ceph-mon[102206]: pgmap v2237: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 891 op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 25 MiB/s, 396 objects/s recovering 2024-09-21T10:23:19.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:19.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:19.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:20.003 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:20.027 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:19 smithi076 podman[347127]: 2024-09-21 10:23:19.755066519 +0000 UTC m=+1.016767163 container create 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T10:23:20.105 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:20.143 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:20.245 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:20.275 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:20.377 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:20.408 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:20.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:20.537 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:20.639 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:20.672 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:20.773 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:20.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:20.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:20.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:20.937 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:20.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:20.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:21.039 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:21.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:21.170 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:21.208 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:21.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:21.310 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:21.341 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:21.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:21.443 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:21.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:21.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:21.585 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:21.612 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:21.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:21.714 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:21.749 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:21.850 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:21.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:21.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:21.995 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:22.031 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:22.133 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.161 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:22.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:22.262 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:22.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:22.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:22.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:22.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:22.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:22.554 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:22.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:22.684 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:22.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:22.786 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:22.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:22.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:22 smithi191 ceph-mon[102206]: pgmap v2238: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 891 op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 25 MiB/s, 396 objects/s recovering 2024-09-21T10:23:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:22 smithi076 ceph-mon[106482]: pgmap v2238: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 891 op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 25 MiB/s, 396 objects/s recovering 2024-09-21T10:23:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:22 smithi076 ceph-mon[110963]: pgmap v2238: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 891 op/s; 39709/246795 objects degraded (16.090%); 40291/246795 objects misplaced (16.326%); 25 MiB/s, 396 objects/s recovering 2024-09-21T10:23:22.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:22.949 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:22.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:23.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:23.078 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:23.180 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:23.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:23.310 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:23.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.345 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:23.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:23.351 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_blocked_ops 2024-09-21T10:23:23.445 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:23.477 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:23.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:23.534 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:23:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:23:23] "GET /metrics HTTP/1.1" 200 36650 "" "Prometheus/2.51.0" 2024-09-21T10:23:23.534 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:23 smithi076 podman[347127]: 2024-09-21 10:23:23.486399299 +0000 UTC m=+4.748099946 container init 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-21T10:23:23.534 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:23 smithi076 podman[347127]: 2024-09-21 10:23:23.495129799 +0000 UTC m=+4.756830441 container start 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:23:23.579 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.614 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:23.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:23.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:23:23.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:23.744 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:23.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:23.793 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:23 smithi076 ceph-osd[347496]: -- 172.21.15.76:0/1741278713 <== mon.1 v2:172.21.15.191:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e0e2c95a40 con 0x55e0e3a62000 2024-09-21T10:23:23.793 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:23 smithi076 bash[347127]: 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f 2024-09-21T10:23:23.793 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:23 smithi076 systemd[1]: Started Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:23:23.846 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:23.876 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:23.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:23.978 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:24.007 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:24.109 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:24.137 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:24.239 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:24.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:24.367 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:24.404 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:24.506 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:24.547 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:24.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:24.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:24.685 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:24.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:24.787 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:24.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:24 smithi191 ceph-mon[102206]: pgmap v2239: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.6 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.47k op/s; 35065/227886 objects degraded (15.387%); 41960/227886 objects misplaced (18.413%); 32 MiB/s, 516 objects/s recovering 2024-09-21T10:23:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 35065/227886 objects degraded (15.387%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:24 smithi076 ceph-mon[106482]: pgmap v2239: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.6 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.47k op/s; 35065/227886 objects degraded (15.387%); 41960/227886 objects misplaced (18.413%); 32 MiB/s, 516 objects/s recovering 2024-09-21T10:23:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 35065/227886 objects degraded (15.387%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:24 smithi076 ceph-mon[110963]: pgmap v2239: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.6 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.47k op/s; 35065/227886 objects degraded (15.387%); 41960/227886 objects misplaced (18.413%); 32 MiB/s, 516 objects/s recovering 2024-09-21T10:23:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 35065/227886 objects degraded (15.387%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:24.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:24.960 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:24.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:25.062 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:25.089 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:25.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:25.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:25.274 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:23:25.125+0000 7fcb430c7640 -1 Falling back to public interface 2024-09-21T10:23:25.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:25.348 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:25.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:25.450 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:25.487 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:25.589 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:25.590 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:25.635 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:25.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:25.767 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:23:25.768 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:25.778 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:25.794 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:23:25.794 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:25.820 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:23:25.820 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:25.822 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:23:25.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:25.855 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:23:25.855 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:25.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:25.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:25.888 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:23:25.888 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:25.914 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:25.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:25.920 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:23:25.920 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:25.921 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:25.952 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:23:25.952 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:26.016 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:26.053 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:26.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:26.155 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:26.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:26.293 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:26.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:26.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:26.456 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:26.493 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:26.558 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:26.592 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:23:26.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:26.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:26.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:26.723 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:26.824 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:26.861 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:26.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:26 smithi191 ceph-mon[102206]: pgmap v2240: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.6 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 35065/227886 objects degraded (15.387%); 41960/227886 objects misplaced (18.413%); 29 MiB/s, 458 objects/s recovering 2024-09-21T10:23:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3391764718' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2571271723' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:26.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:26 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[106482]: pgmap v2240: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.6 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 35065/227886 objects degraded (15.387%); 41960/227886 objects misplaced (18.413%); 29 MiB/s, 458 objects/s recovering 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3391764718' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2571271723' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[110963]: pgmap v2240: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.6 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 35065/227886 objects degraded (15.387%); 41960/227886 objects misplaced (18.413%); 29 MiB/s, 458 objects/s recovering 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3391764718' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2571271723' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:26 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:23:26.963 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:26.993 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:26.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:26.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:27.095 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:27.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:27.225 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:27.254 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:27.356 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:27.385 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:27.487 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:27.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:27.621 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.656 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:27.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:27.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:27.786 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:27.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:27.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:27 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:27.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:27 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:23:27.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:27 smithi191 ceph-mon[102206]: osdmap e1373: 8 total, 7 up, 4 in 2024-09-21T10:23:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:27 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:27 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:23:27.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:27 smithi076 ceph-mon[106482]: osdmap e1373: 8 total, 7 up, 4 in 2024-09-21T10:23:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:27 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:23:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:27 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:23:27.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:27 smithi076 ceph-mon[110963]: osdmap e1373: 8 total, 7 up, 4 in 2024-09-21T10:23:27.888 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:27.916 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:27.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:27.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:28.018 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:28.057 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:28.158 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:28.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:28.294 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:28.333 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:28.434 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.472 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:28.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:28.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:28.604 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:28.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:28.705 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:28.746 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:28.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:28.816 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:28 smithi076 ceph-mon[106482]: pgmap v2242: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.37k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 19 MiB/s, 297 objects/s recovering 2024-09-21T10:23:28.817 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:28 smithi076 ceph-mon[110963]: pgmap v2242: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.37k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 19 MiB/s, 297 objects/s recovering 2024-09-21T10:23:28.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:28.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:28 smithi191 ceph-mon[102206]: pgmap v2242: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.37k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 19 MiB/s, 297 objects/s recovering 2024-09-21T10:23:28.887 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:28.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:28.926 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:23:28.926 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:23:28.989 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:29.023 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:29.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:29.125 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:29.154 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:29.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:29.293 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:29.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:29.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:29.529 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:29.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:29.671 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:29.702 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:29.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:29.845 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:29.862 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 30495/206094 objects degraded (14.797%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:29.862 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:29 smithi076 ceph-mon[106482]: pgmap v2243: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 18 MiB/s, 294 objects/s recovering 2024-09-21T10:23:29.862 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 30495/206094 objects degraded (14.797%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:29.862 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:29 smithi076 ceph-mon[110963]: pgmap v2243: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 18 MiB/s, 294 objects/s recovering 2024-09-21T10:23:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 30495/206094 objects degraded (14.797%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:29.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:29 smithi191 ceph-mon[102206]: pgmap v2243: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 18 MiB/s, 294 objects/s recovering 2024-09-21T10:23:29.947 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:29.983 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:29.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:23:30.033 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:23:30.084 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:30.126 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:30.228 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:30.259 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:30.361 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:30.402 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:30.504 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.538 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:30.640 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:30.677 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:30.722 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.2 dump_historic_ops 2024-09-21T10:23:30.778 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:30.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:30.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:30.951 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:30.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:30.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:31.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:31.088 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:31.190 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:31.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:31.325 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:31.361 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:31.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:31.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:31.497 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:31.599 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.640 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:31.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:31.741 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:31.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.779 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:31.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:31.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:31.910 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:31.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:32.011 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:32.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:23:32.043 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:32.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:32.145 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:32.179 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:32.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:32.281 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:32.313 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:32.414 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:32.442 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:32.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:32.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:32.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:32.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:32.681 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:32.715 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:32.807 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:32 smithi076 ceph-mon[106482]: pgmap v2244: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 18 MiB/s, 294 objects/s recovering 2024-09-21T10:23:32.807 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:32 smithi076 ceph-mon[110963]: pgmap v2244: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 18 MiB/s, 294 objects/s recovering 2024-09-21T10:23:32.817 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:32.857 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:32.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:32.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:32 smithi191 ceph-mon[102206]: pgmap v2244: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 4.2 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.36k op/s; 30495/206094 objects degraded (14.797%); 37954/206094 objects misplaced (18.416%); 18 MiB/s, 294 objects/s recovering 2024-09-21T10:23:32.959 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:32.962 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:32.966 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:no valid command found; 10 closest matches: 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:0 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:1 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:2 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:abort 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:assert 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs debug_inject_read_zeros 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs files list 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:bluefs stats 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator dump block 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:bluestore allocator fragmentation block 2024-09-21T10:23:32.968 INFO:teuthology.orchestra.run.smithi076.stderr:admin_socket: invalid command 2024-09-21T10:23:33.015 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:33.117 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:33.146 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:33.248 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:33.276 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:33.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:33.378 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:33.409 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:33.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:33.511 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:33.517 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:23:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:23:33] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.51.0" 2024-09-21T10:23:33.547 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:33.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:33.649 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.686 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:33.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:33.787 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:33.824 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:33.926 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:33.957 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:33.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:34.032 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-21T10:23:34.033 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2024-09-21T10:23:34.059 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:34.088 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:34.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:34.190 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:34.227 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:34.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:34.324 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:23:34.329 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:34.365 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:34.467 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.499 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:34.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:34.600 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:34.639 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:34.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:34.741 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:34.788 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:34 smithi191 ceph-mon[102206]: pgmap v2245: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.49k op/s; 25851/183180 objects degraded (14.112%); 33742/183180 objects misplaced (18.420%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:23:34.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:34 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 25851/183180 objects degraded (14.112%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:34 smithi076 ceph-mon[106482]: pgmap v2245: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.49k op/s; 25851/183180 objects degraded (14.112%); 33742/183180 objects misplaced (18.420%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:23:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:34 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 25851/183180 objects degraded (14.112%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:34 smithi076 ceph-mon[110963]: pgmap v2245: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.49k op/s; 25851/183180 objects degraded (14.112%); 33742/183180 objects misplaced (18.420%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:23:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:34 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 25851/183180 objects degraded (14.112%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:34.889 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:34.932 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:34.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:35.033 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:35.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:35.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:35.209 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:35.310 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:35.342 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:35.444 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:35.471 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:35.574 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:35.607 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:35.709 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.744 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:35.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:35.845 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:35.878 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:35.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:35.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:35.980 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:36.016 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:36.117 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:36.157 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:36.259 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:36.293 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:36.394 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:36.426 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:36.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:36.528 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:36.568 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:36.670 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.697 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:36.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:36.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:36.835 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:36.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:36 smithi191 ceph-mon[102206]: pgmap v2246: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.49k op/s; 25851/183180 objects degraded (14.112%); 33742/183180 objects misplaced (18.420%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:23:36.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:36 smithi076 ceph-mon[106482]: pgmap v2246: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.49k op/s; 25851/183180 objects degraded (14.112%); 33742/183180 objects misplaced (18.420%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:23:36.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:36 smithi076 ceph-mon[110963]: pgmap v2246: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.49k op/s; 25851/183180 objects degraded (14.112%); 33742/183180 objects misplaced (18.420%); 15 MiB/s, 247 objects/s recovering 2024-09-21T10:23:36.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:36 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:23:36.627+0000 7fcb430c7640 -1 osd.3 1365 log_to_monitors true 2024-09-21T10:23:36.937 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:36.974 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:36.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:36.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:37.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:37.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.106 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:37.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:37.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:37.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:37.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:37.364 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:37.465 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:37.493 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:37.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:37.616 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:37.717 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:37.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:37.853 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:37.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:37 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:23:37.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:37 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:23:37.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:23:37 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:23:37.704+0000 7fcb36e69640 -1 osd.3 1374 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:23:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:37 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:23:37.881 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:37.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:37.983 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:38.009 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:38.111 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:38.149 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:38.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:38.251 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:38.279 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:38.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:38.381 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:38.418 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:38.520 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:38.558 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:38.659 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:38.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.688 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:38.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:38.789 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:38.811 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:38.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:38 smithi191 ceph-mon[102206]: pgmap v2247: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 2.04k op/s; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%); 21 MiB/s, 335 objects/s recovering 2024-09-21T10:23:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:38 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:23:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:38 smithi191 ceph-mon[102206]: osdmap e1374: 8 total, 7 up, 4 in 2024-09-21T10:23:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:38 smithi191 ceph-mon[102206]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:23:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3170916717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[110963]: pgmap v2247: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 2.04k op/s; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%); 21 MiB/s, 335 objects/s recovering 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[110963]: osdmap e1374: 8 total, 7 up, 4 in 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[110963]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3170916717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[106482]: pgmap v2247: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 2.04k op/s; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%); 21 MiB/s, 335 objects/s recovering 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:23:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[106482]: osdmap e1374: 8 total, 7 up, 4 in 2024-09-21T10:23:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[106482]: from='osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi076", "root=default"]} : dispatch 2024-09-21T10:23:38.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3170916717' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:38.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:38.936 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:38.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:39.034 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:23:39.037 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:39.101 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:39.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:39.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:39.225 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:39.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:39.361 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:39.463 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:39.484 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:39.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:39.586 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:39.608 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:39.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:39.709 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:39.732 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:39.833 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:39.854 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515] boot 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: osdmap e1375: 8 total, 8 up, 4 in 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 21542/160758 objects degraded (13.400%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: osdmap e1376: 8 total, 8 up, 4 in 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.a continuing backfill to osd.1 from (1367'9779,1374'14035] 11:5e119f9b:::benchmark_data_smithi076_330857_object93282:head to 1374'14035 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.a starting backfill to osd.3 from (1349'4079,1365'7964] MAX to 1374'14035 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.8 starting backfill to osd.3 from (1349'4119,1365'8029] MAX to 1374'14014 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.5 starting backfill to osd.1 from (1367'9841,1374'14092] MIN to 1374'14093 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.1 starting backfill to osd.3 from (1349'4092,1365'8012] MAX to 1374'14140 2024-09-21T10:23:39.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.4 starting backfill to osd.3 from (1349'4184,1365'8143] MAX to 1374'14340 2024-09-21T10:23:39.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.5 starting backfill to osd.3 from (1349'3841,1365'8032] MAX to 1374'14093 2024-09-21T10:23:39.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.3 starting backfill to osd.3 from (1349'3906,1365'7861] MIN to 1374'13763 2024-09-21T10:23:39.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.0 starting backfill to osd.3 from (1349'3950,1365'7898] MAX to 1374'13836 2024-09-21T10:23:39.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.9 starting backfill to osd.3 from (1349'4092,1365'7981] MAX to 1374'14066 2024-09-21T10:23:39.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.4 continuing backfill to osd.4 from (1368'10384,1374'14340] 11:2c74882f:::benchmark_data_smithi076_330857_object92721:head to 1374'14340 2024-09-21T10:23:39.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:39 smithi191 ceph-mon[102206]: 11.0 continuing backfill to osd.5 from (1367'9850,1374'13836] MIN to 1374'13836 2024-09-21T10:23:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:23:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515] boot 2024-09-21T10:23:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: osdmap e1375: 8 total, 8 up, 4 in 2024-09-21T10:23:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:23:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 21542/160758 objects degraded (13.400%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:39.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: osdmap e1376: 8 total, 8 up, 4 in 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.a continuing backfill to osd.1 from (1367'9779,1374'14035] 11:5e119f9b:::benchmark_data_smithi076_330857_object93282:head to 1374'14035 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.a starting backfill to osd.3 from (1349'4079,1365'7964] MAX to 1374'14035 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.8 starting backfill to osd.3 from (1349'4119,1365'8029] MAX to 1374'14014 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.5 starting backfill to osd.1 from (1367'9841,1374'14092] MIN to 1374'14093 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.1 starting backfill to osd.3 from (1349'4092,1365'8012] MAX to 1374'14140 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.4 starting backfill to osd.3 from (1349'4184,1365'8143] MAX to 1374'14340 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.5 starting backfill to osd.3 from (1349'3841,1365'8032] MAX to 1374'14093 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.3 starting backfill to osd.3 from (1349'3906,1365'7861] MIN to 1374'13763 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.0 starting backfill to osd.3 from (1349'3950,1365'7898] MAX to 1374'13836 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.9 starting backfill to osd.3 from (1349'4092,1365'7981] MAX to 1374'14066 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.4 continuing backfill to osd.4 from (1368'10384,1374'14340] 11:2c74882f:::benchmark_data_smithi076_330857_object92721:head to 1374'14340 2024-09-21T10:23:39.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[106482]: 11.0 continuing backfill to osd.5 from (1367'9850,1374'13836] MIN to 1374'13836 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: osd.3 [v2:172.21.15.76:6802/1408876515,v1:172.21.15.76:6803/1408876515] boot 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: osdmap e1375: 8 total, 8 up, 4 in 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 21542/160758 objects degraded (13.400%), 19 pgs degraded, 74 pgs undersized (PG_DEGRADED) 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: osdmap e1376: 8 total, 8 up, 4 in 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.a continuing backfill to osd.1 from (1367'9779,1374'14035] 11:5e119f9b:::benchmark_data_smithi076_330857_object93282:head to 1374'14035 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.a starting backfill to osd.3 from (1349'4079,1365'7964] MAX to 1374'14035 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.8 starting backfill to osd.3 from (1349'4119,1365'8029] MAX to 1374'14014 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.5 starting backfill to osd.1 from (1367'9841,1374'14092] MIN to 1374'14093 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.1 starting backfill to osd.3 from (1349'4092,1365'8012] MAX to 1374'14140 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.4 starting backfill to osd.3 from (1349'4184,1365'8143] MAX to 1374'14340 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.5 starting backfill to osd.3 from (1349'3841,1365'8032] MAX to 1374'14093 2024-09-21T10:23:39.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.3 starting backfill to osd.3 from (1349'3906,1365'7861] MIN to 1374'13763 2024-09-21T10:23:39.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.0 starting backfill to osd.3 from (1349'3950,1365'7898] MAX to 1374'13836 2024-09-21T10:23:39.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.9 starting backfill to osd.3 from (1349'4092,1365'7981] MAX to 1374'14066 2024-09-21T10:23:39.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.4 continuing backfill to osd.4 from (1368'10384,1374'14340] 11:2c74882f:::benchmark_data_smithi076_330857_object92721:head to 1374'14340 2024-09-21T10:23:39.876 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:39 smithi076 ceph-mon[110963]: 11.0 continuing backfill to osd.5 from (1367'9850,1374'13836] MIN to 1374'13836 2024-09-21T10:23:39.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:39.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:39.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:39.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:40.080 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:40.102 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:40.200 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:23:40.200 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:23:40.200 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:23:40.201 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:23:40.202 INFO:teuthology.orchestra.run.smithi076.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-09-21T10:23:40.202 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:40.202 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:40.202 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:40.202 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:40.203 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:40.228 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:40.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:40.330 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:40.358 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:40.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:40.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:40.486 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:40.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:40.599 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: pgmap v2251: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 1.25k op/s; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:23:40.599 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: osdmap e1377: 8 total, 8 up, 4 in 2024-09-21T10:23:40.622 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:40.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:40.724 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:40.758 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:40.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:40.860 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: pgmap v2251: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 1.25k op/s; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:23:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: osdmap e1377: 8 total, 8 up, 4 in 2024-09-21T10:23:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: 11.2 starting backfill to osd.3 from (1349'3976,1365'7907] MAX to 1374'13999 2024-09-21T10:23:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: 11.b starting backfill to osd.3 from (1349'4079,1365'7999] MAX to 1374'14001 2024-09-21T10:23:40.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: 11.e starting backfill to osd.3 from (1349'3938,1365'7873] MAX to 1374'13898 2024-09-21T10:23:40.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: 11.f starting backfill to osd.3 from (1349'3957,1365'7894] MAX to 1374'13924 2024-09-21T10:23:40.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: 11.2 starting backfill to osd.3 from (1349'3976,1365'7907] MAX to 1374'13999 2024-09-21T10:23:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: 11.b starting backfill to osd.3 from (1349'4079,1365'7999] MAX to 1374'14001 2024-09-21T10:23:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: 11.e starting backfill to osd.3 from (1349'3938,1365'7873] MAX to 1374'13898 2024-09-21T10:23:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: 11.f starting backfill to osd.3 from (1349'3957,1365'7894] MAX to 1374'13924 2024-09-21T10:23:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:40.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: pgmap v2251: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 1.25k op/s; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%); 12 MiB/s, 199 objects/s recovering 2024-09-21T10:23:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: osdmap e1377: 8 total, 8 up, 4 in 2024-09-21T10:23:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: 11.2 starting backfill to osd.3 from (1349'3976,1365'7907] MAX to 1374'13999 2024-09-21T10:23:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: 11.b starting backfill to osd.3 from (1349'4079,1365'7999] MAX to 1374'14001 2024-09-21T10:23:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: 11.e starting backfill to osd.3 from (1349'3938,1365'7873] MAX to 1374'13898 2024-09-21T10:23:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: 11.f starting backfill to osd.3 from (1349'3957,1365'7894] MAX to 1374'13924 2024-09-21T10:23:40.875 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:23:40.896 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:40.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:40.997 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:41.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:41.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:41.135 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:41.170 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:41.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:41.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:41.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:41.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:41.442 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:41.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:41.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:41.680 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:41.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:41.818 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:41.854 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:41.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:41.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:41.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:41.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:41.956 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:41.979 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:41.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:42.022 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:42.081 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:42.116 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:42.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:42.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:42.252 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:42.354 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:42.377 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:42.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:42.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:42.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:42.614 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:42.647 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:42.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.779 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:42.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:42.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:42 smithi191 ceph-mon[102206]: pgmap v2253: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%) 2024-09-21T10:23:42.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:42 smithi076 ceph-mon[106482]: pgmap v2253: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%) 2024-09-21T10:23:42.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:42 smithi076 ceph-mon[110963]: pgmap v2253: 99 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 55 active+undersized, 15 active+undersized+degraded, 20 active+clean; 3.3 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 21542/160758 objects degraded (13.400%); 29617/160758 objects misplaced (18.423%) 2024-09-21T10:23:42.880 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:42.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:42.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:43.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:43.047 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:43.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:43.149 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:43.184 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:43.286 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:43.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:43.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:43.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:43.454 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:43.556 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:43.593 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:43.616 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:23:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:23:43] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-21T10:23:43.694 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:43.720 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:43.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:43.821 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:43.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:43 smithi076 ceph-mon[106482]: pgmap v2254: 99 pgs: 7 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 763 op/s; 19115/147276 objects degraded (12.979%); 26118/147276 objects misplaced (17.734%); 12 MiB/s, 187 objects/s recovering 2024-09-21T10:23:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:43 smithi076 ceph-mon[110963]: pgmap v2254: 99 pgs: 7 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 763 op/s; 19115/147276 objects degraded (12.979%); 26118/147276 objects misplaced (17.734%); 12 MiB/s, 187 objects/s recovering 2024-09-21T10:23:43.944 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:43.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:43.977 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:43.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:44.078 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:44.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:43 smithi191 ceph-mon[102206]: pgmap v2254: 99 pgs: 7 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 763 op/s; 19115/147276 objects degraded (12.979%); 26118/147276 objects misplaced (17.734%); 12 MiB/s, 187 objects/s recovering 2024-09-21T10:23:44.214 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:44.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:44.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:44.347 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:44.380 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:44.481 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:44.515 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:44.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:44.618 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:44.654 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:44.756 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:44.777 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:44.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:44.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:44 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 19115/147276 objects degraded (12.979%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:44.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:44 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 19115/147276 objects degraded (12.979%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:44.879 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:44.913 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:44.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:45.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:45.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.053 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:45.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:44 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 19115/147276 objects degraded (12.979%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:45.154 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:45.182 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:45.284 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:45.309 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:45.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:45.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:45.447 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:45.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:45.583 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:45.685 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:45.723 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:45.825 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:45.860 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:45.961 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:45.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:45.993 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:45.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:46.094 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:45 smithi191 ceph-mon[102206]: pgmap v2255: 99 pgs: 7 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 652 op/s; 19115/147276 objects degraded (12.979%); 26118/147276 objects misplaced (17.734%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:23:46.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:45 smithi076 ceph-mon[106482]: pgmap v2255: 99 pgs: 7 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 652 op/s; 19115/147276 objects degraded (12.979%); 26118/147276 objects misplaced (17.734%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:23:46.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:45 smithi076 ceph-mon[110963]: pgmap v2255: 99 pgs: 7 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 active+remapped+backfill_wait, 86 active+clean; 3.0 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 652 op/s; 19115/147276 objects degraded (12.979%); 26118/147276 objects misplaced (17.734%); 10 MiB/s, 160 objects/s recovering 2024-09-21T10:23:46.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:46.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.265 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:46.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:46.366 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:46.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.398 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:46.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:46.499 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:46.535 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:46.637 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:46.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.661 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:46.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:46.762 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:46.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.795 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:46.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:46.896 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:46.931 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:46.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:46.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:47.032 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:47.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:47.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:47.172 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.218 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:47.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:47.319 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:47.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:47.460 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:47.497 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:47.598 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:47.628 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:47.731 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:47.751 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:47.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:47.853 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:47.884 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:47.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:47.986 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:48.020 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:48.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:48.122 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:48.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-21T10:23:48.240 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-21T10:23:48.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:48.291 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:48.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:48.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:48.427 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:48.529 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:48.561 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:48.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:48.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:48.702 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:48.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:48.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:48.839 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:48.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:48 smithi191 ceph-mon[102206]: pgmap v2256: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 1.29k op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:23:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:48 smithi076 ceph-mon[106482]: pgmap v2256: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 1.29k op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:23:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:48 smithi076 ceph-mon[110963]: pgmap v2256: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 1.29k op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 12 MiB/s, 189 objects/s recovering 2024-09-21T10:23:48.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:48.973 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:48.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:49.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.111 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:49.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:49.213 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:49.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.241 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:49.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:49.342 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:49.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:49.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:49.520 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:49.622 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:49.655 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:49.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:49.789 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:49.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:49 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 16260/128709 objects degraded (12.633%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:49.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:49 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 16260/128709 objects degraded (12.633%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:49.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:49 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 16260/128709 objects degraded (12.633%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:49.890 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:49.926 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:49.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:50.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:50.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:50.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:50.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:50.229 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:23:50.230 INFO:teuthology.orchestra.run.smithi076.stdout: "size": 20, 2024-09-21T10:23:50.230 INFO:teuthology.orchestra.run.smithi076.stdout: "duration": 600, 2024-09-21T10:23:50.230 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [] 2024-09-21T10:23:50.230 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:23:50.240 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:23:50.294 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:50.336 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:50.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:50.486 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:50.587 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:50.608 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:50.711 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.746 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:50.847 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:50.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:50 smithi191 ceph-mon[102206]: pgmap v2257: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 1.07k op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:23:50.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:50 smithi076 ceph-mon[106482]: pgmap v2257: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 1.07k op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:23:50.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:50 smithi076 ceph-mon[110963]: pgmap v2257: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 1.07k op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 9.8 MiB/s, 156 objects/s recovering 2024-09-21T10:23:50.878 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:50.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:50.979 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:51.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.015 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:51.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:51.116 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:51.151 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:51.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:51.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:51.391 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:51.423 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:51.525 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:51.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:51.648 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:51.669 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:51.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:51.771 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.807 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:51.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:51.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:51.945 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:51.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:52.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:52.083 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:52.185 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:52.220 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:52.321 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:52.354 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:52.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:52.456 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:52.501 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:52.602 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:52.638 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:52.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:52.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:52.772 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:52.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:52 smithi191 ceph-mon[102206]: pgmap v2258: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 949 op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 8.7 MiB/s, 139 objects/s recovering 2024-09-21T10:23:52.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:52 smithi076 ceph-mon[106482]: pgmap v2258: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 949 op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 8.7 MiB/s, 139 objects/s recovering 2024-09-21T10:23:52.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:52 smithi076 ceph-mon[110963]: pgmap v2258: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.6 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 949 op/s; 16260/128709 objects degraded (12.633%); 22391/128709 objects misplaced (17.397%); 8.7 MiB/s, 139 objects/s recovering 2024-09-21T10:23:52.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:52.909 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:52.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:53.011 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:53.046 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:53.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-21T10:23:53.149 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:53.183 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:53.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:53.285 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:53.347 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:53.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:23:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:23:53] "GET /metrics HTTP/1.1" 200 36642 "" "Prometheus/2.51.0" 2024-09-21T10:23:53.448 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:53.473 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:23:53.488 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:53.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:53.521 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:53.590 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:53.626 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:53.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:53.728 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:53.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.764 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:53.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:53.865 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:53.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:53.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:54.005 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:54.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:54.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:54.052 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:23:54.143 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:54.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:54.275 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:54.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:54.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:54.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:54.550 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:54.586 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:54.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:54 smithi076 ceph-mon[106482]: pgmap v2259: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.42k op/s; 13499/109608 objects degraded (12.316%); 18713/109608 objects misplaced (17.073%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:23:54.641 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:54 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 13499/109608 objects degraded (12.316%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:54.641 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:54 smithi076 ceph-mon[110963]: pgmap v2259: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.42k op/s; 13499/109608 objects degraded (12.316%); 18713/109608 objects misplaced (17.073%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:23:54.641 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:54 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 13499/109608 objects degraded (12.316%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:54.689 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:54.727 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:54.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:54.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:54.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:54 smithi191 ceph-mon[102206]: pgmap v2259: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.42k op/s; 13499/109608 objects degraded (12.316%); 18713/109608 objects misplaced (17.073%); 10 MiB/s, 161 objects/s recovering 2024-09-21T10:23:54.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:54 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 13499/109608 objects degraded (12.316%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:54.864 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:54.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:54.965 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:55.001 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:55.102 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.145 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:55.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:55.246 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:55.286 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:55.388 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.425 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:55.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:55.525 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:55.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:55.614 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:55.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.688 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:55.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:55.751 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:55.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:55.826 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:55.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:55.870 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:23:55.928 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:55.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:55.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:56.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:56.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:56.202 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.237 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:56.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:56.338 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:56.378 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:56.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:245127 11.b e1377/1376 11:da586378:::benchmark_data_smithi076_330857_object107358:head v 1377'15307, mlcod=1377'15305)", 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:56.392987+0000", 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.0018033960000000001, 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001822403, 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:23:56.397 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.392987+0000", 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.392987+0000", 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.398 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.392986+0000", 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.392992+0000", 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.156e-06 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.393002+0000", 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1330000000000005e-06 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.399 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.393008+0000", 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0279999999999997e-06 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.393042+0000", 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4341000000000002e-05 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.393059+0000", 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6891000000000001e-05 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:23:56.400 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:245125 11.d e1377/1362 11:b10a668b:::benchmark_data_smithi076_330857_object107356:head v 1377'15608, mlcod=1377'15605)", 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:56.394186+0000", 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00060466000000000003, 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00066080100000000003, 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:23:56.401 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394186+0000", 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394186+0000", 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:23:56.402 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394184+0000", 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394190+0000", 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9479999999999996e-06 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394199+0000", 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7660000000000004e-06 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.403 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394205+0000", 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9800000000000003e-06 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394264+0000", 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9898000000000001e-05 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:56.394289+0000", 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4266000000000001e-05 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:23:56.404 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:23:56.405 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:23:56.405 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2024-09-21T10:23:56.405 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 2 2024-09-21T10:23:56.405 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:23:56.479 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:56.512 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:56.614 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:56.645 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:56.747 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:56.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:56 smithi191 ceph-mon[102206]: pgmap v2260: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.04k op/s; 13499/109608 objects degraded (12.316%); 18713/109608 objects misplaced (17.073%); 4.3 MiB/s, 69 objects/s recovering 2024-09-21T10:23:56.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:56 smithi076 ceph-mon[106482]: pgmap v2260: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.04k op/s; 13499/109608 objects degraded (12.316%); 18713/109608 objects misplaced (17.073%); 4.3 MiB/s, 69 objects/s recovering 2024-09-21T10:23:56.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:56 smithi076 ceph-mon[110963]: pgmap v2260: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 2.2 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 1.04k op/s; 13499/109608 objects degraded (12.316%); 18713/109608 objects misplaced (17.073%); 4.3 MiB/s, 69 objects/s recovering 2024-09-21T10:23:56.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:56.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:56.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:57.027 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:57.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:57.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:57.214 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:57.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:57.316 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:57.351 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:57.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:57.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:57.496 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:57.598 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:57.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:57.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_blocked_ops 2024-09-21T10:23:57.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:57.753 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:57 smithi076 ceph-mon[106482]: pgmap v2261: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.62k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:23:57.753 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:57 smithi076 ceph-mon[110963]: pgmap v2261: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.62k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:23:57.771 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:57.873 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:57.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:57.922 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:57.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:58.005 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:23:58.023 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:23:58.070 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:58.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:23:58.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:57 smithi191 ceph-mon[102206]: pgmap v2261: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.62k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 6.3 MiB/s, 101 objects/s recovering 2024-09-21T10:23:58.172 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:58.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.220 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:58.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:58.302 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:23:58.303 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:23:58.320 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:58.341 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:23:58.342 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:23:58.352 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:58.377 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:23:58.377 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:23:58.411 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:23:58.411 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-21T10:23:58.448 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:23:58.448 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:23:58.454 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:58.485 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:23:58.485 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:23:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.496 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:58.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:58.526 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:23:58.527 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:23:58.543 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:23:58.543 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:23:58.561 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:23:58.562 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:23:58.597 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:23:58.634 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:23:58.639 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:23:58.736 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.773 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:58.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:58.833 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2416363543' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:58.833 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4047644577' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:58.833 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2416363543' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:58.833 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4047644577' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:58.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2416363543' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:23:58.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4047644577' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:23:58.874 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:58.911 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:58.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:58.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:59.013 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:23:59.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.052 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:59.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:23:59.152 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:59.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:59.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:59.329 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:59.431 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:23:59.464 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:23:59.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:23:59.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:23:59.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:23:59.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:23:59.733 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:23:59.844 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:23:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:23:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2416363543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[106482]: osdmap e1378: 8 total, 8 up, 4 in 2024-09-21T10:23:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 10583/88731 objects degraded (11.927%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:59.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[106482]: pgmap v2263: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:23:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:23:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2416363543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[110963]: osdmap e1378: 8 total, 8 up, 4 in 2024-09-21T10:23:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 10583/88731 objects degraded (11.927%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:59.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:23:59 smithi076 ceph-mon[110963]: pgmap v2263: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:23:59.892 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:23:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:23:59.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:23:59.943 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:59 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:23:59.943 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:59 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2416363543' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:23:59.943 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:59 smithi191 ceph-mon[102206]: osdmap e1378: 8 total, 8 up, 4 in 2024-09-21T10:23:59.943 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:59 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 10583/88731 objects degraded (11.927%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:23:59.943 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:23:59 smithi191 ceph-mon[102206]: pgmap v2263: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:23:59.994 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.027 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:00.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:00.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:00.163 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:00.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:00.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.298 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:00.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:00.399 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:00.434 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:00.536 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:00.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:00.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:00.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:00.807 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:00.837 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:00.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:00.939 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:00.972 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:00.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:01.074 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:01.110 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:01.212 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:01.248 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:01.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:01.350 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:01.384 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:01.486 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:01.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.519 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:01.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:01.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:01.642 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:01.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:01.744 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:01.784 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:01.886 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:01.908 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:01.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:01.964 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:24:02.010 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:02.044 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:02.147 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:02.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.182 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:02.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:02.283 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:02.286 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:02.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:02.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:02.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:02.448 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:02.549 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.586 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:02.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:02.687 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:02.726 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:02.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:02.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:02.859 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:02.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:02.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:02 smithi191 ceph-mon[102206]: pgmap v2264: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:24:02.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:02 smithi076 ceph-mon[106482]: pgmap v2264: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:24:02.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:02 smithi076 ceph-mon[110963]: pgmap v2264: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 10583/88731 objects degraded (11.927%); 14824/88731 objects misplaced (16.707%); 4.7 MiB/s, 75 objects/s recovering 2024-09-21T10:24:02.962 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:02.991 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:02.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:03.077 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set noscrub 2024-09-21T10:24:03.093 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:03.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:03.266 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:03.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.309 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:03.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:03.410 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:03.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:03.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:03.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.582 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:03.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:24:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:24:03] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.51.0" 2024-09-21T10:24:03.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:03.712 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:03.814 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:03.850 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:03.851 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:03.858 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:03.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:03.960 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:04.002 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:04.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:04.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:04.137 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:04.238 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:04.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:04.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:04.373 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:04.384 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:04.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:04.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:04.440 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:24:04.440 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:24:04.477 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:24:04.477 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:04.508 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:24:04.508 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:04.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:04.542 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:24:04.543 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:04.579 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:04.579 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:24:04.580 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:04.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:04.585 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.4 dump_historic_ops 2024-09-21T10:24:04.615 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:24:04.615 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:04 smithi076 ceph-mon[106482]: pgmap v2265: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 8020/69687 objects degraded (11.509%); 11315/69687 objects misplaced (16.237%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:24:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:04 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 8020/69687 objects degraded (11.509%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1039522759' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:04 smithi076 ceph-mon[110963]: pgmap v2265: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 8020/69687 objects degraded (11.509%); 11315/69687 objects misplaced (16.237%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:24:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:04 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 8020/69687 objects degraded (11.509%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1039522759' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:04.647 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:24:04.647 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:04.681 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:24:04.682 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:04.682 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:04.724 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:04.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:04.826 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:04.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:04 smithi191 ceph-mon[102206]: pgmap v2265: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 8020/69687 objects degraded (11.509%); 11315/69687 objects misplaced (16.237%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:24:04.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:04 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 8020/69687 objects degraded (11.509%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:04.864 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1039522759' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:04.865 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:04.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:04.966 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:04.980 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:05.016 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:05.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:05.118 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:05.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:05.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:05.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.289 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:05.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:05.390 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.425 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:05.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:05.526 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:05.562 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:05.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:05.566 INFO:tasks.thrashosds.thrasher:in_osds: [1, 4, 3, 5] out_osds: [7, 0, 2, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2024-09-21T10:24:05.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:24:05.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:05.707 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:05.809 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:05.850 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:05.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:05.951 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:06.041 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:06.143 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:06.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.180 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:06.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:06.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:06.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:06.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:06.412 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:06.457 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:06.559 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:06.600 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:06.702 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:06.748 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:06.843 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:06 smithi191 ceph-mon[102206]: pgmap v2266: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 8020/69687 objects degraded (11.509%); 11315/69687 objects misplaced (16.237%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:24:06.843 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:06.851 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:06 smithi076 ceph-mon[106482]: pgmap v2266: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 8020/69687 objects degraded (11.509%); 11315/69687 objects misplaced (16.237%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:24:06.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:06 smithi076 ceph-mon[110963]: pgmap v2266: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 1.33k op/s; 8020/69687 objects degraded (11.509%); 11315/69687 objects misplaced (16.237%); 4.5 MiB/s, 71 objects/s recovering 2024-09-21T10:24:06.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:24:06.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:06.897 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:06.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:06.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27517 10.d e1303/1301 10:b3f91c28:::benchmark_data_smithi076_299450_object27516:head v 1303'1716, mlcod=1303'1714)", 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414166+0000", 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.59025073999999, 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18228551500000001, 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.011 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414166+0000", 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414166+0000", 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.012 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414165+0000", 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414187+0000", 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1362999999999999e-05 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414197+0000", 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0569e-05 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.013 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414203+0000", 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8239999999999996e-06 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414243+0000", 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.002e-05 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414263+0000", 2024-09-21T10:24:07.014 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0213000000000001e-05 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596375+0000", 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.182111997 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596452+0000", 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6447999999999997e-05 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.015 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27522 10.5 e1303/1301 10:a9158b13:::benchmark_data_smithi076_299450_object27521:head v 1303'1686, mlcod=1303'1684)", 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414175+0000", 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.59024175600001, 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18425534199999999, 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414175+0000", 2024-09-21T10:24:07.016 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414175+0000", 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414174+0000", 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.017 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414238+0000", 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3662000000000001e-05 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414248+0000", 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0324e-05 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414254+0000", 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1349999999999997e-06 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.018 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414395+0000", 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000140345 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414419+0000", 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.4470000000000001e-05 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598374+0000", 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.183954642 2024-09-21T10:24:07.019 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598430+0000", 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6586999999999999e-05 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27520 10.b e1303/1296 10:de5038e7:::benchmark_data_smithi076_299450_object27519:head v 1303'1709, mlcod=1303'1707)", 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.590202428, 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.184107515, 2024-09-21T10:24:07.020 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.021 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414214+0000", 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414233+0000", 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8686e-05 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414239+0000", 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2450000000000003e-06 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.022 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414243+0000", 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3560000000000003e-06 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414334+0000", 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0291999999999995e-05 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414351+0000", 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6980999999999999e-05 2024-09-21T10:24:07.023 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.597786+0000", 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18343555 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598322+0000", 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00053571599999999999 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.024 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27519 10.0 e1303/1296 10:003e95b4:::benchmark_data_smithi076_299450_object27518:head v 1303'1711, mlcod=1303'1708)", 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.59016340300002, 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18245903899999999, 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.025 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414253+0000", 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.026 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414271+0000", 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7893e-05 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414277+0000", 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.48e-06 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414280+0000", 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8399999999999997e-06 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.027 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414310+0000", 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0008000000000001e-05 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414326+0000", 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5532e-05 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596671+0000", 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18234499400000001 2024-09-21T10:24:07.028 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596713+0000", 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1530999999999999e-05 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27523 10.c e1303/1301 10:3b9534aa:::benchmark_data_smithi076_299450_object27522:head v 1303'1837, mlcod=1303'1835)", 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.59015598799999, 2024-09-21T10:24:07.029 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.182033742, 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.030 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414261+0000", 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414305+0000", 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4935000000000003e-05 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414315+0000", 2024-09-21T10:24:07.031 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3649999999999993e-06 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414320+0000", 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6949999999999999e-06 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414374+0000", 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4620999999999998e-05 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.032 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414393+0000", 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9202e-05 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596251+0000", 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18185779899999999 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.033 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.596295+0000", 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3504999999999998e-05 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.034 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27518 10.6 e1303/1296 10:65ba96f9:::benchmark_data_smithi076_299450_object27517:head v 1303'1721, mlcod=1303'1718)", 2024-09-21T10:24:07.035 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.414731+0000", 2024-09-21T10:24:07.035 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.589686196, 2024-09-21T10:24:07.035 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.183657557, 2024-09-21T10:24:07.035 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.035 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414731+0000", 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414731+0000", 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.036 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414730+0000", 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.414818+0000", 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8442999999999993e-05 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595101+0000", 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18028290699999999 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.037 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595114+0000", 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2323999999999999e-05 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595257+0000", 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014315899999999999 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595278+0000", 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1328e-05 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.038 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598304+0000", 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003025537 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598388+0000", 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4610999999999993e-05 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.039 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27524 10.3 e1303/1301 10:cb011d2f:::benchmark_data_smithi076_299450_object27523:head v 1303'1719, mlcod=1303'1716)", 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.58939554300002, 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18332594199999999, 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.040 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.041 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:24:07.041 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.041 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.041 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415021+0000", 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.415082+0000", 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1815000000000004e-05 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.042 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595106+0000", 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.180023974 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595118+0000", 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2024999999999999e-05 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595186+0000", 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7844999999999998e-05 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.043 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595212+0000", 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5483999999999999e-05 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598327+0000", 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0031148769999999998 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598347+0000", 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0596999999999999e-05 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.044 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.35901.0:27521 10.d e1303/1301 10:b20dfcd0:::benchmark_data_smithi076_299450_object27520:head v 1303'1717, mlcod=1303'1714)", 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:16:50.416052+0000", 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 436.58836498400001, 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.18245150199999999, 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.045 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416052+0000", 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416052+0000", 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416051+0000", 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.046 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.416072+0000", 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0950000000000001e-05 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595128+0000", 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.17905605899999999 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595134+0000", 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.4970000000000001e-06 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.047 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595327+0000", 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00019367000000000001 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.595345+0000", 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.785e-05 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.048 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598435+0000", 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003089699 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:16:50.598503+0000", 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8544e-05 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.049 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:136523 11.1 11:812d0cd2:::benchmark_data_smithi076_330857_object136522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1367)", 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:33.164325+0000", 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 93.840091686999997, 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15543122500000001, 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136523 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.050 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164325+0000", 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164325+0000", 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164324+0000", 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.051 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164392+0000", 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8758999999999997e-05 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164405+0000", 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2284e-05 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164419+0000", 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4469e-05 2024-09-21T10:24:07.052 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166314+0000", 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001894319 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166469+0000", 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015566199999999999 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,6", 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166624+0000", 2024-09-21T10:24:07.053 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000155215 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.176269+0000", 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096450189999999995 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.176323+0000", 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3143999999999998e-05 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.054 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.319592+0000", 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.14326926600000001 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.319647+0000", 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5067e-05 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.319756+0000", 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109497 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.055 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:136524 11.1 11:8f80917d:::benchmark_data_smithi076_330857_object136523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1367)", 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:33.164430+0000", 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 93.839987281999996, 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15655691299999999, 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:07.056 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136524 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164430+0000", 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164430+0000", 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.057 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164429+0000", 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164461+0000", 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1936000000000001e-05 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164469+0000", 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9400000000000002e-06 2024-09-21T10:24:07.058 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.164474+0000", 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5759999999999996e-06 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.169051+0000", 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.004577065 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.059 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.169197+0000", 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014537600000000001 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,6", 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.169354+0000", 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015679600000000001 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.176314+0000", 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069598810000000002 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.060 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.176330+0000", 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6952999999999998e-05 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.320833+0000", 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.144502303 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.320882+0000", 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9342000000000003e-05 2024-09-21T10:24:07.061 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.320987+0000", 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104381 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:136527 11.1 11:8dc8b445:::benchmark_data_smithi076_330857_object136526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1367)", 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:33.166097+0000", 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 93.838320187999997, 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.15515474400000001, 2024-09-21T10:24:07.062 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 136527 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166097+0000", 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.063 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166097+0000", 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166095+0000", 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166163+0000", 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7014999999999994e-05 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166172+0000", 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7440000000000002e-06 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.166178+0000", 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9540000000000001e-06 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.173877+0000", 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0076987230000000002 2024-09-21T10:24:07.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.174035+0000", 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015789299999999999 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,6", 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.174219+0000", 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000184358 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.176336+0000", 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0021164819999999998 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.176740+0000", 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00040464199999999998 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.321086+0000", 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.144345267 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.321144+0000", 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8130000000000001e-05 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:33.321251+0000", 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107738 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228652 11.e e1377/1376 11:7afcd56e:::benchmark_data_smithi076_330857_object90883:head v 1377'14166, mlcod=1377'14164)", 2024-09-21T10:24:07.068 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.675324+0000", 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.329092810999999, 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23816593599999999, 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675324+0000", 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675324+0000", 2024-09-21T10:24:07.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675324+0000", 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675328+0000", 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9700000000000001e-06 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675336+0000", 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7579999999999998e-06 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675341+0000", 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3000000000000003e-06 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675375+0000", 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4629999999999999e-05 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.675395+0000", 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.942e-05 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683520+0000", 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081252119999999997 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913490+0000", 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.229970118 2024-09-21T10:24:07.072 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228643 11.8 e1377/1375 11:10936de5:::benchmark_data_smithi076_330857_object90874:head v 1377'14278, mlcod=1377'14276)", 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.676323+0000", 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.328094092000001, 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.237112557, 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676323+0000", 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676323+0000", 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676322+0000", 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:07.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676328+0000", 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7619999999999998e-06 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676340+0000", 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1273e-05 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676345+0000", 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8340000000000004e-06 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676426+0000", 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0529000000000001e-05 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.676446+0000", 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9799000000000001e-05 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683487+0000", 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0070408329999999998 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913435+0000", 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22994874300000001 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228654 11.4 e1377/1375 11:27a4a35c:::benchmark_data_smithi076_330857_object90885:head v 1377'14617, mlcod=1377'14614)", 2024-09-21T10:24:07.077 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.677423+0000", 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.326993386000002, 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23600643499999999, 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677423+0000", 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677423+0000", 2024-09-21T10:24:07.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677423+0000", 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677430+0000", 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8210000000000004e-06 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.079 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677441+0000", 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1287999999999999e-05 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677447+0000", 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3459999999999999e-06 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677490+0000", 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2635000000000001e-05 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677510+0000", 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.0414000000000001e-05 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683496+0000", 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059856120000000004 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913430+0000", 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22993411799999999 2024-09-21T10:24:07.081 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228656 11.1 e1377/1375 11:8ea430ca:::benchmark_data_smithi076_330857_object90887:head v 1377'14389, mlcod=1377'14387)", 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.677462+0000", 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.326954387000001, 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.236085503, 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677462+0000", 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677462+0000", 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677462+0000", 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677466+0000", 2024-09-21T10:24:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.085 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:07.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:07.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7220000000000001e-06 2024-09-21T10:24:07.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677473+0000", 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6370000000000002e-06 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677477+0000", 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.899e-06 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677562+0000", 2024-09-21T10:24:07.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4343000000000002e-05 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677584+0000", 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2226999999999999e-05 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913426+0000", 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.235842158 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913548+0000", 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121818 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228657 11.6 e1377/1344 11:6ca7b321:::benchmark_data_smithi076_330857_object90888:head v 1377'14152, mlcod=1377'14149)", 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.677488+0000", 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.326928730999999, 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23602809899999999, 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.088 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677488+0000", 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677488+0000", 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677488+0000", 2024-09-21T10:24:07.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677496+0000", 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.892e-06 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677502+0000", 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6529999999999997e-06 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.090 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.677507+0000", 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.194e-06 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.678569+0000", 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001062105 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.678587+0000", 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7989e-05 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913494+0000", 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.234907119 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913516+0000", 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.2442e-05 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.092 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228658 11.4 e1377/1375 11:252d2359:::benchmark_data_smithi076_330857_object90889:head v 1377'14618, mlcod=1377'14615)", 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.680790+0000", 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.323627299999998, 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23267536599999999, 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680790+0000", 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680790+0000", 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680789+0000", 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680797+0000", 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9650000000000008e-06 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680809+0000", 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2213e-05 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680815+0000", 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6599999999999998e-06 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680856+0000", 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0092999999999997e-05 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.680875+0000", 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9891e-05 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913437+0000", 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23256120299999999 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913465+0000", 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8300999999999999e-05 2024-09-21T10:24:07.096 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228655 11.8 e1377/1375 11:1668038a:::benchmark_data_smithi076_330857_object90886:head v 1377'14279, mlcod=1377'14276)", 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.681148+0000", 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.323268656, 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.232324327, 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681148+0000", 2024-09-21T10:24:07.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681148+0000", 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681147+0000", 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681155+0000", 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.481e-06 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681166+0000", 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1079e-05 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681172+0000", 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3149999999999997e-06 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681229+0000", 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6384000000000001e-05 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.681250+0000", 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1342999999999999e-05 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913441+0000", 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23219146099999999 2024-09-21T10:24:07.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913473+0000", 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1053000000000002e-05 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228660 11.c e1377/1344 11:3787252f:::benchmark_data_smithi076_330857_object90891:head v 1377'14268, mlcod=1377'14266)", 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.683899+0000", 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.320518065999998, 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.232080697, 2024-09-21T10:24:07.101 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683899+0000", 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683899+0000", 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683898+0000", 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.683904+0000", 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.485e-06 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913433+0000", 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.22952910500000001 2024-09-21T10:24:07.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913441+0000", 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7250000000000003e-06 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913493+0000", 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1338999999999998e-05 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913513+0000", 2024-09-21T10:24:07.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9763999999999999e-05 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.915945+0000", 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0024321 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.915980+0000", 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4851999999999998e-05 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:228667 11.7 e1377/1344 11:e52313c1:::benchmark_data_smithi076_330857_object90898:head v 1377'14360, mlcod=1377'14358)", 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:23:43.686111+0000", 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 23.318306321000001, 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.23394173400000001, 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.686111+0000", 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.686111+0000", 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.686110+0000", 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.686117+0000", 2024-09-21T10:24:07.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8499999999999996e-06 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913451+0000", 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.227333962 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913456+0000", 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5799999999999999e-06 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913494+0000", 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8130000000000003e-05 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.913515+0000", 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1212e-05 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.920021+0000", 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00650594 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:07.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:23:43.920052+0000", 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0874999999999998e-05 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:07.110 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:07.186 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:07.226 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:07.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:07.327 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:07.363 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:07.364 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:07.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:07.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:07.502 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:07.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:07.604 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:07.632 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is set 2024-09-21T10:24:07.650 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:07.751 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:07.775 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:07.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:07.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:07 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3501426355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:07.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:07 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:07 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3501426355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:07.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:07 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:07 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3501426355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:07.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:07 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-21T10:24:07.876 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:07.898 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:07.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:07.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-21T10:24:08.001 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:08.032 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:08.134 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.155 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:08.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:08.256 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:08.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:08.379 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:08.403 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:08.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:08.455 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:08.505 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:08.527 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:08.628 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:08.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:08.719 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[106482]: pgmap v2267: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 1.26k op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[106482]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[106482]: osdmap e1379: 8 total, 8 up, 4 in 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[110963]: pgmap v2267: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 1.26k op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[110963]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:24:08.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:08 smithi076 ceph-mon[110963]: osdmap e1379: 8 total, 8 up, 4 in 2024-09-21T10:24:08.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:08.804 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:08 smithi191 ceph-mon[102206]: pgmap v2267: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 1.26k op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 4.2 MiB/s, 66 objects/s recovering 2024-09-21T10:24:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:08 smithi191 ceph-mon[102206]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:08 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-21T10:24:08.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:08 smithi191 ceph-mon[102206]: osdmap e1379: 8 total, 8 up, 4 in 2024-09-21T10:24:08.905 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:08.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:08.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:09.042 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:09.074 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:09.176 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:09.210 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:09.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:09.297 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:24:09.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:09.382 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:09.484 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:09.522 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:09.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:09.624 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.661 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:09.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:09.761 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.788 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:09.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:09.889 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:09.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:09 smithi076 ceph-mon[106482]: osdmap e1380: 8 total, 8 up, 4 in 2024-09-21T10:24:09.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:09 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 5594/50754 objects degraded (11.022%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:09.895 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:09 smithi076 ceph-mon[106482]: pgmap v2270: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 1.58k op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:24:09.895 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:09 smithi076 ceph-mon[110963]: osdmap e1380: 8 total, 8 up, 4 in 2024-09-21T10:24:09.895 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:09 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 5594/50754 objects degraded (11.022%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:09.896 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:09 smithi076 ceph-mon[110963]: pgmap v2270: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 1.58k op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:24:09.925 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:09.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:09.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:10.008 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:09 smithi191 ceph-mon[102206]: osdmap e1380: 8 total, 8 up, 4 in 2024-09-21T10:24:10.009 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:09 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 5594/50754 objects degraded (11.022%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:10.009 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:09 smithi191 ceph-mon[102206]: pgmap v2270: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 1.58k op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 5.2 MiB/s, 83 objects/s recovering 2024-09-21T10:24:10.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:10.061 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:10.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.206 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:10.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:10.306 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:10.338 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:10.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:10.440 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:10.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.476 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:10.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:10.577 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:10.613 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:10.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:10.715 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:10.762 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:10.843 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:10.843 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:10.864 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:10.894 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:10.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:10.996 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:11.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:11.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:11.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:11.128 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:11.175 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:11.227 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:11.277 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:11.328 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:11.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:11.463 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:11.564 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:11.596 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:11.651 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:24:11.698 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:11.741 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:11.744 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2347187414' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:24:11.744 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:11 smithi191 ceph-mon[102206]: pgmap v2271: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:24:11.777 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:11.777 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:263568 11.a 11:507c731a:::benchmark_data_smithi076_330857_object125799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1381)", 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:24:11.750163+0000", 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.023633477, 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023666406000000001, 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 263568 2024-09-21T10:24:11.778 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.750163+0000", 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.750163+0000", 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.750163+0000", 2024-09-21T10:24:11.779 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.750166+0000", 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.473e-06 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.750173+0000", 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6109999999999999e-06 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:11.780 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.750178+0000", 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8160000000000001e-06 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762397+0000", 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012219211000000001 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762541+0000", 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000143769 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.781 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,6", 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762679+0000", 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000138281 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.771498+0000", 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0088190230000000005 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.771530+0000", 2024-09-21T10:24:11.782 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2030999999999999e-05 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.772269+0000", 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00073855000000000004 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773811+0000", 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001542664 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.783 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.36207.0:263568 11.a e1381/1375 ondisk, result = 0)", 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:24:11.773364+0000", 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00043232600000000002, 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00057136000000000003, 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "started", 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773364+0000", 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.784 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773364+0000", 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773364+0000", 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.785 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773369+0000", 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9300000000000002e-06 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773377+0000", 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5930000000000006e-06 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773383+0000", 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1279999999999996e-06 2024-09-21T10:24:11.786 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773632+0000", 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00024908299999999999 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773805+0000", 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017242900000000001 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.787 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop_reply(client.36207.0:263579 11.a e1381/1375)", 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:24:11.773395+0000", 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00040126499999999999, 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00059654100000000002, 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.788 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773395+0000", 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773395+0000", 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773395+0000", 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.789 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773398+0000", 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.6740000000000001e-06 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773404+0000", 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5560000000000004e-06 2024-09-21T10:24:11.790 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773413+0000", 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0569999999999999e-06 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_repop(client.36207.0:263585 11.2 e1381/1376)", 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:24:11.773427+0000", 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.00036955100000000003, 2024-09-21T10:24:11.791 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00060886800000000004, 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "queued for pg", 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773427+0000", 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.792 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773427+0000", 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773427+0000", 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:11.793 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773432+0000", 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0549999999999999e-06 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773439+0000", 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9410000000000001e-06 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:11.794 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.773444+0000", 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0350000000000001e-06 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:263579 11.a 11:5bce0dfb:::benchmark_data_smithi076_330857_object125810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1381)", 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:24:11.762159+0000", 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 0.011638153, 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.011929472, 2024-09-21T10:24:11.795 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "waiting for sub ops", 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 263579 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:11.796 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762159+0000", 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762159+0000", 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:11.797 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762157+0000", 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762163+0000", 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0589999999999999e-06 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762173+0000", 2024-09-21T10:24:11.798 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5699999999999999e-06 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.762180+0000", 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7220000000000002e-06 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.765241+0000", 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030616509999999999 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:11.799 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.765459+0000", 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021779899999999999 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,6", 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.765617+0000", 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000157804 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.771578+0000", 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059614710000000003 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:11.800 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.772131+0000", 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000552339 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:24:11.772275+0000", 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014459100000000001 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: ], 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 5 2024-09-21T10:24:11.801 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:11.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:11.883 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:11.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:11.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:11.896 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd set nodeep-scrub 2024-09-21T10:24:11.960 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2347187414' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:24:11.960 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:11 smithi076 ceph-mon[106482]: pgmap v2271: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:24:11.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2347187414' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:24:11.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:11 smithi076 ceph-mon[110963]: pgmap v2271: 99 pgs: 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 86 active+clean; 1.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 5594/50754 objects degraded (11.022%); 7937/50754 objects misplaced (15.638%); 2.6 MiB/s, 41 objects/s recovering 2024-09-21T10:24:11.985 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:12.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.034 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:12.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:12.136 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:12.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.174 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:12.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:12.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:12.312 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:12.414 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:12.441 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:12.543 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:12.573 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:12.675 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:12.714 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:12.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:12.816 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:12.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.843 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:12.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:12 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2347187414' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:24:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:12 smithi191 ceph-mon[102206]: osdmap e1381: 8 total, 8 up, 4 in 2024-09-21T10:24:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:12 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2347187414' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:24:12.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:12 smithi076 ceph-mon[106482]: osdmap e1381: 8 total, 8 up, 4 in 2024-09-21T10:24:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:12 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2347187414' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:24:12.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:12 smithi076 ceph-mon[110963]: osdmap e1381: 8 total, 8 up, 4 in 2024-09-21T10:24:12.944 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:12.968 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_blocked_ops 2024-09-21T10:24:12.977 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:12.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:13.079 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:13.116 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:13.218 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.254 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:13.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:13.355 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:13.398 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:13.459 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:13.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.548 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:13.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:13.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:24:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:24:13] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.51.0" 2024-09-21T10:24:13.649 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:13.690 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:13.792 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:13.828 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:13.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:13 smithi191 ceph-mon[102206]: pgmap v2273: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 727 MiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 877 op/s; 3856/34956 objects degraded (11.031%); 5004/34956 objects misplaced (14.315%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:24:13.930 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:13.965 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:13.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:14.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:14.098 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:13 smithi076 ceph-mon[106482]: pgmap v2273: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 727 MiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 877 op/s; 3856/34956 objects degraded (11.031%); 5004/34956 objects misplaced (14.315%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:24:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:13 smithi076 ceph-mon[110963]: pgmap v2273: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 727 MiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 877 op/s; 3856/34956 objects degraded (11.031%); 5004/34956 objects misplaced (14.315%); 4.3 MiB/s, 68 objects/s recovering 2024-09-21T10:24:14.200 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:14.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:14.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:14.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:14.368 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:14.469 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:14.505 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:14.606 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.641 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:14.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:14.742 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:14.767 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:14.869 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:14.872 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:14 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 3856/34956 objects degraded (11.031%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:14 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 3856/34956 objects degraded (11.031%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:14.903 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:14.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:15.005 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:15.042 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:15.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:14 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 3856/34956 objects degraded (11.031%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:15.144 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:15.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.181 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:15.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:15.282 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:15.313 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:15.415 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:15.437 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:15.538 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:15.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:15.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:15.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:15.799 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:15.840 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:15.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:15 smithi191 ceph-mon[102206]: pgmap v2274: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 727 MiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 3856/34956 objects degraded (11.031%); 5004/34956 objects misplaced (14.315%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:24:15.941 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:15.975 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:15.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:16.076 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:16.105 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:16.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:15 smithi076 ceph-mon[106482]: pgmap v2274: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 727 MiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 3856/34956 objects degraded (11.031%); 5004/34956 objects misplaced (14.315%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:24:16.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:15 smithi076 ceph-mon[110963]: pgmap v2274: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 727 MiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 3856/34956 objects degraded (11.031%); 5004/34956 objects misplaced (14.315%); 3.2 MiB/s, 51 objects/s recovering 2024-09-21T10:24:16.207 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:16.227 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:16.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:16.328 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:16.351 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:16.452 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:16.487 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:16.588 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:16.623 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:16.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:16.725 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.756 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:16.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:16.857 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:16.881 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:16.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:16.982 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:17.003 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:17.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:17.105 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:17.126 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:17.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:17.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.250 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:17.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:17.351 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:17.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.373 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:17.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:17.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:17.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:17.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:17.617 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:17.719 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:17.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:17.842 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:17.863 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:17.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:17.964 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:17.986 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:17.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:17.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:18.088 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:18.112 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:17 smithi191 ceph-mon[102206]: osdmap e1382: 8 total, 8 up, 4 in 2024-09-21T10:24:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:17 smithi191 ceph-mon[102206]: pgmap v2276: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:24:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:17 smithi076 ceph-mon[106482]: osdmap e1382: 8 total, 8 up, 4 in 2024-09-21T10:24:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:17 smithi076 ceph-mon[106482]: pgmap v2276: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:24:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:17 smithi076 ceph-mon[110963]: osdmap e1382: 8 total, 8 up, 4 in 2024-09-21T10:24:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:17 smithi076 ceph-mon[110963]: pgmap v2276: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 1.30k op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 5.1 MiB/s, 81 objects/s recovering 2024-09-21T10:24:18.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:18.235 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:18.337 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:18.359 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:18.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:18.461 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:18.495 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:18.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:18.596 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:18.627 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:18.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:18.729 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:18.762 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:18.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:18.864 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:18.892 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:18.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:18.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:18.994 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:19.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:19.082 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:18 smithi191 ceph-mon[102206]: osdmap e1383: 8 total, 8 up, 4 in 2024-09-21T10:24:19.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:18 smithi076 ceph-mon[106482]: osdmap e1383: 8 total, 8 up, 4 in 2024-09-21T10:24:19.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:18 smithi076 ceph-mon[110963]: osdmap e1383: 8 total, 8 up, 4 in 2024-09-21T10:24:19.129 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:19.152 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:19.254 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:19.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.290 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:19.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:19.391 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:19.421 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:19.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:19.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:19.555 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:19.657 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:19.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:19.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:19.793 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:19.793 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:24:19.793 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:24:19.793 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:24:19.793 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:19.794 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:19.834 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:19.935 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:19.969 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:19.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:20.071 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:20.097 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:20.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:19 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 2012/19419 objects degraded (10.361%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:19 smithi191 ceph-mon[102206]: pgmap v2278: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 8.7 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 1.32k op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:24:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:19 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 2012/19419 objects degraded (10.361%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:20.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:19 smithi076 ceph-mon[106482]: pgmap v2278: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 8.7 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 1.32k op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:24:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:19 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 2012/19419 objects degraded (10.361%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:20.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:19 smithi076 ceph-mon[110963]: pgmap v2278: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 8.7 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 1.32k op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 5.1 MiB/s, 82 objects/s recovering 2024-09-21T10:24:20.199 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:20.230 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:20.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:20.332 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.364 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:20.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:20.465 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:20.500 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.5 dump_historic_ops 2024-09-21T10:24:20.503 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:20.605 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:20.638 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:20.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:20.740 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:20.764 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:20.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:20.866 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:20.904 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:20.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:21.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.041 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:21.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:21.142 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:21.174 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:21.275 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.311 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:21.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:21.339 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:21.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:21.441 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:21.543 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:21.578 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:21.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:21.726 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:21.828 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:21.852 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:21.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:21.954 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:21.992 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:21.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:21.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:22.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:22.131 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:22.232 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:22.258 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:22.273 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:24:22.360 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:22.396 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:22.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:22.498 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:22.520 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:22.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:22.621 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:22.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.655 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:22.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:22.756 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:22.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.780 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:22.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:22 smithi191 ceph-mon[102206]: pgmap v2279: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 8.7 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 647 op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:24:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:22 smithi076 ceph-mon[106482]: pgmap v2279: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 8.7 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 647 op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:24:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:22 smithi076 ceph-mon[110963]: pgmap v2279: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 87 active+clean; 404 MiB data, 8.7 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 647 op/s; 2012/19419 objects degraded (10.361%); 2575/19419 objects misplaced (13.260%); 1.9 MiB/s, 30 objects/s recovering 2024-09-21T10:24:22.881 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:22.912 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:22.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:22.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:23.014 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:23.044 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:23.146 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:23.177 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:23.201 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:23.303 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:23.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:23.375 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:24:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:24:23] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.51.0" 2024-09-21T10:24:23.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:23.469 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:23.570 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:23.603 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:23.705 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:23.739 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:23.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:23.841 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:23.862 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:23.964 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:23.989 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:23.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:24.062 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:24.062 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177657 10.9 10:9662158c:::benchmark_data_smithi076_299450_object42478:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223326+0000", 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.83382353000002, 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0223126890000001, 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.063 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177657 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223326+0000", 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223326+0000", 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.064 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223324+0000", 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223328+0000", 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6950000000000002e-06 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.065 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223336+0000", 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2190000000000004e-06 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223342+0000", 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0689999999999998e-06 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.066 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223487+0000", 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014458400000000001 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223495+0000", 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8650000000000001e-06 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245576+0000", 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.022081561 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.067 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245638+0000", 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2071999999999998e-05 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177665 10.9 10:9b6f0096:::benchmark_data_smithi076_299450_object42486:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.83379876700002, 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.022590052, 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.068 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177665 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.069 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223350+0000", 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223352+0000", 2024-09-21T10:24:24.070 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8759999999999999e-06 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223357+0000", 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.135e-06 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223362+0000", 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5129999999999998e-06 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.071 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223513+0000", 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015170400000000001 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223519+0000", 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3199999999999999e-06 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245899+0000", 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.022380165 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.072 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245940+0000", 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1669999999999999e-05 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177668 10.9 10:9f5283a7:::benchmark_data_smithi076_299450_object42489:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.83378392200001, 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.02263262, 2024-09-21T10:24:24.073 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177668 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.074 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223365+0000", 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223367+0000", 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8330000000000001e-06 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.075 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223372+0000", 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9099999999999996e-06 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223376+0000", 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1840000000000001e-06 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223541+0000", 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016516 2024-09-21T10:24:24.076 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223548+0000", 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3440000000000003e-06 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.077 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245963+0000", 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0224149629999999 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.245998+0000", 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4524999999999999e-05 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.078 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177671 10.9 10:91d97354:::benchmark_data_smithi076_299450_object42492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.83377027099999, 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0226838979999999, 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177671 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.079 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223379+0000", 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.080 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223380+0000", 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.751e-06 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223385+0000", 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6759999999999998e-06 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223389+0000", 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0249999999999996e-06 2024-09-21T10:24:24.081 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223568+0000", 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000178627 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223573+0000", 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5690000000000004e-06 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246018+0000", 2024-09-21T10:24:24.082 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0224451210000001 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246063+0000", 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4397999999999997e-05 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177675 10.9 10:94de0b4c:::benchmark_data_smithi076_299450_object42496:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.83374583099999, 2024-09-21T10:24:24.083 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0379785429999999, 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177675 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.084 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223403+0000", 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223405+0000", 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.26e-06 2024-09-21T10:24:24.085 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223419+0000", 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3491e-05 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223424+0000", 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0769999999999998e-06 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.086 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223593+0000", 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016949200000000001 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223600+0000", 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.6189999999999996e-06 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246083+0000", 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0224835459999999 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.087 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246159+0000", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6076999999999997e-05 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.246280+0000", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000120429 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256305+0000", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010025552 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.088 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256340+0000", 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4144999999999999e-05 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261293+0000", 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0049532669999999999 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261340+0000", 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6830999999999999e-05 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.089 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261382+0000", 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2138999999999998e-05 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177677 10.9 10:95d800f0:::benchmark_data_smithi076_299450_object42498:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.83372072899999, 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0381621590000001, 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.090 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177677 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:24:24.091 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223428+0000", 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223430+0000", 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.3779999999999999e-06 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.092 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223436+0000", 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5709999999999997e-06 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223440+0000", 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.177e-06 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223620+0000", 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000179741 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.093 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.223625+0000", 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3550000000000004e-06 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.248686+0000", 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.025060804 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.248795+0000", 2024-09-21T10:24:24.094 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108602 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.248903+0000", 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000108515 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256371+0000", 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0074677620000000002 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.095 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256433+0000", 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1599000000000004e-05 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261503+0000", 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050703149999999997 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261555+0000", 2024-09-21T10:24:24.096 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1925000000000002e-05 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261591+0000", 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5796000000000001e-05 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177687 10.9 10:9283661d:::benchmark_data_smithi076_299450_object42508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.232288+0000", 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.82486158, 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0293759579999999, 2024-09-21T10:24:24.097 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177687 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232288+0000", 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.098 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232288+0000", 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232286+0000", 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232292+0000", 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.241e-06 2024-09-21T10:24:24.099 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232309+0000", 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6951000000000001e-05 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232316+0000", 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8619999999999997e-06 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232359+0000", 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2562999999999999e-05 2024-09-21T10:24:24.100 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.232366+0000", 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.7519999999999996e-06 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.251387+0000", 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.019020714 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.251505+0000", 2024-09-21T10:24:24.101 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011822799999999999 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.251606+0000", 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010082 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256399+0000", 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0047931110000000001 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.102 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256465+0000", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5335999999999999e-05 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261599+0000", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051342990000000002 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261634+0000", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5091e-05 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.103 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261664+0000", 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9558000000000001e-05 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177694 10.9 10:9c545240:::benchmark_data_smithi076_299450_object42515:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.239830+0000", 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.817319295, 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021904913, 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.104 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177694 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239830+0000", 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239830+0000", 2024-09-21T10:24:24.105 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239828+0000", 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239846+0000", 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.7907999999999999e-05 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.106 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239858+0000", 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1528999999999999e-05 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239865+0000", 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7819999999999996e-06 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239908+0000", 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3652000000000002e-05 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.107 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.239916+0000", 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2840000000000006e-06 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.253937+0000", 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0140201680000001 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.254041+0000", 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104066 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.108 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.254152+0000", 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111322 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261447+0000", 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0072946529999999999 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261495+0000", 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8523999999999998e-05 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.109 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261673+0000", 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017774699999999999 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261706+0000", 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2935999999999998e-05 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261735+0000", 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.887e-05 2024-09-21T10:24:24.110 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177696 10.9 10:9bab1062:::benchmark_data_smithi076_299450_object42517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.240479+0000", 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.81667042700002, 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0300966309999999, 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.111 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177696 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240479+0000", 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.112 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240479+0000", 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240478+0000", 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240482+0000", 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6600000000000001e-06 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.113 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240489+0000", 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2840000000000001e-06 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240494+0000", 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6759999999999998e-06 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240529+0000", 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5552000000000001e-05 2024-09-21T10:24:24.114 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.240537+0000", 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9049999999999997e-06 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256486+0000", 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0159486790000001 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.115 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256597+0000", 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111092 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.256702+0000", 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000105289 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261745+0000", 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0050424700000000003 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.116 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261803+0000", 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8353999999999997e-05 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270493+0000", 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086903450000000004 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270539+0000", 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5556000000000002e-05 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.117 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270575+0000", 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6482000000000001e-05 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177703 10.9 10:9803cc08:::benchmark_data_smithi076_299450_object42524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.247427+0000", 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.80972261900001, 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034124346, 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.118 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177703 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.119 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247427+0000", 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247427+0000", 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247425+0000", 2024-09-21T10:24:24.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247439+0000", 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3509000000000001e-05 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247450+0000", 2024-09-21T10:24:24.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1494999999999999e-05 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247457+0000", 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0670000000000003e-06 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247549+0000", 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1928999999999993e-05 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.247557+0000", 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8029999999999997e-06 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.258917+0000", 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0113603289999999 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.259032+0000", 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114831 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.259114+0000", 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1827000000000001e-05 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261861+0000", 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027469 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270614+0000", 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0087532370000000005 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.281470+0000", 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010855492 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.281512+0000", 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1931999999999998e-05 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.281551+0000", 2024-09-21T10:24:24.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9348999999999999e-05 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177707 10.9 10:91aa2076:::benchmark_data_smithi076_299450_object42528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.249447+0000", 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.80770244000001, 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0387998350000001, 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.127 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177707 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249447+0000", 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249447+0000", 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249445+0000", 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249452+0000", 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.443e-06 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249468+0000", 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.6738000000000001e-05 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249476+0000", 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9359999999999999e-06 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249886+0000", 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00041004700000000003 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.249894+0000", 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4359999999999996e-06 2024-09-21T10:24:24.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261889+0000", 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0119947090000001 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.261992+0000", 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010364100000000001 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.262098+0000", 2024-09-21T10:24:24.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010531600000000001 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270646+0000", 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085488430000000004 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270678+0000", 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1634000000000002e-05 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288170+0000", 2024-09-21T10:24:24.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.017492167999999999 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288211+0000", 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0664000000000002e-05 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288247+0000", 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5704999999999998e-05 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177709 10.9 10:9ef77a4b:::benchmark_data_smithi076_299450_object42530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.253684+0000", 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.80346549500001, 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.034637657, 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177709 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253684+0000", 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253684+0000", 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253682+0000", 2024-09-21T10:24:24.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253689+0000", 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4110000000000001e-06 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253701+0000", 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2897e-05 2024-09-21T10:24:24.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253708+0000", 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.9909999999999996e-06 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253755+0000", 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6978999999999999e-05 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.253762+0000", 2024-09-21T10:24:24.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8600000000000004e-06 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.267340+0000", 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0135781589999999 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.267450+0000", 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109311 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.267709+0000", 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00025922800000000002 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.276075+0000", 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0083659670000000002 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.276104+0000", 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8580000000000001e-05 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288254+0000", 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012150564000000001 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288291+0000", 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6832000000000002e-05 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288321+0000", 2024-09-21T10:24:24.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0388000000000001e-05 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:177726 10.9 10:9d8bfc47:::benchmark_data_smithi076_299450_object42547:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:18:53.275006+0000", 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 330.782143512, 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0134493389999999, 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.141 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 177726 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275006+0000", 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275006+0000", 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275004+0000", 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275009+0000", 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.259e-06 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275021+0000", 2024-09-21T10:24:24.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1382999999999999e-05 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275027+0000", 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9000000000000003e-06 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275069+0000", 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1944999999999999e-05 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:53.275077+0000", 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.7539999999999995e-06 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270703+0000", 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99562528800000005 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270841+0000", 2024-09-21T10:24:24.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013829900000000001 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.270967+0000", 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012629699999999999 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.276132+0000", 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051644009999999999 2024-09-21T10:24:24.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288328+0000", 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.012195869 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288385+0000", 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7873e-05 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288425+0000", 2024-09-21T10:24:24.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9493000000000003e-05 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:18:54.288455+0000", 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.0040999999999999e-05 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:214746 10.8 10:1a04690b:::benchmark_data_smithi076_299450_object79567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1324)", 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:19:31.325536+0000", 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 292.73161293300001, 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0216948720000001, 2024-09-21T10:24:24.148 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 214746 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325536+0000", 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325536+0000", 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325534+0000", 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325541+0000", 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.4420000000000003e-06 2024-09-21T10:24:24.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325553+0000", 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2490999999999999e-05 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325560+0000", 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8190000000000003e-06 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325609+0000", 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8715999999999999e-05 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:31.325617+0000", 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8560000000000007e-06 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.341627+0000", 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.016009951 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.341745+0000", 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0001188 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.341863+0000", 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000117589 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347084+0000", 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0052205990000000002 2024-09-21T10:24:24.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347129+0000", 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5185000000000002e-05 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347164+0000", 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5385000000000001e-05 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347193+0000", 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.8297e-05 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:19:32.347231+0000", 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.8546e-05 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262305 10.3 10:c1fd82fe:::benchmark_data_smithi076_299450_object127126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.038678+0000", 2024-09-21T10:24:24.155 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 239.018471511, 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0040169240000001, 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262305 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.156 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038678+0000", 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038678+0000", 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038676+0000", 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038683+0000", 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5610000000000004e-06 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038695+0000", 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2646999999999999e-05 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038703+0000", 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.306e-06 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038749+0000", 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6431999999999999e-05 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.038758+0000", 2024-09-21T10:24:24.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6060000000000003e-06 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.036325+0000", 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99756721800000003 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.036437+0000", 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111712 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4", 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.036526+0000", 2024-09-21T10:24:24.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9777000000000005e-05 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042505+0000", 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0059781929999999997 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042513+0000", 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1859999999999992e-06 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042626+0000", 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000112977 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042662+0000", 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6607999999999999e-05 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.042695+0000", 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2162000000000001e-05 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.162 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:127672 11.5 11:a752c71e:::benchmark_data_smithi076_330857_object127671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1365)", 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:25.047972+0000", 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 119.00917719900001, 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0354526669999999, 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 127672 2024-09-21T10:24:24.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.047972+0000", 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.047972+0000", 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.047970+0000", 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.048024+0000", 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.3910999999999999e-05 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.048036+0000", 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2318e-05 2024-09-21T10:24:24.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.048044+0000", 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2090000000000005e-06 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.048089+0000", 2024-09-21T10:24:24.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4855999999999999e-05 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.048207+0000", 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011756000000000001 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,6", 2024-09-21T10:24:24.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.048320+0000", 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000113579 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.052919+0000", 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045982319999999998 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.053236+0000", 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031696099999999999 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.056737+0000", 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0035015900000000002 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.077021+0000", 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.020283314 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.077032+0000", 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1209000000000001e-05 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.083343+0000", 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0063109649999999 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.083425+0000", 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1890000000000004e-05 2024-09-21T10:24:24.170 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:127675 11.5 11:ad992f0f:::benchmark_data_smithi076_330857_object127674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1365)", 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:25.049503+0000", 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 119.00764571400001, 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0343645050000001, 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.171 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 127675 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.049503+0000", 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.049503+0000", 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.049503+0000", 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.049571+0000", 2024-09-21T10:24:24.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8379000000000004e-05 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.049581+0000", 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0301e-05 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.049588+0000", 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.2790000000000004e-06 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.050498+0000", 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091021099999999996 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.050655+0000", 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000157572 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,3,6", 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.050822+0000", 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016643100000000001 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.053023+0000", 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002200975 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.053719+0000", 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00069607800000000004 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.077088+0000", 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.023369418999999999 2024-09-21T10:24:24.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.077094+0000", 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5199999999999997e-06 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.083810+0000", 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0067158979999999 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.083868+0000", 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8301000000000003e-05 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:127674 11.a 11:5ac9c677:::benchmark_data_smithi076_330857_object127673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1366)", 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:25.064288+0000", 2024-09-21T10:24:24.178 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 118.992861148, 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0194926710000001, 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 127674 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.180 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064288+0000", 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064288+0000", 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064287+0000", 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064323+0000", 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6748999999999997e-05 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064385+0000", 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1390999999999995e-05 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064395+0000", 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0489000000000001e-05 2024-09-21T10:24:24.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.073091+0000", 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086962649999999999 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.073097+0000", 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2190000000000003e-06 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.082021+0000", 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.008924726 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.083781+0000", 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001759233 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:127683 11.a 11:57f7275c:::benchmark_data_smithi076_330857_object127682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1366)", 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:25.064406+0000", 2024-09-21T10:24:24.184 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 118.992742761, 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.029338691, 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 127683 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064406+0000", 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064406+0000", 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064406+0000", 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:24.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064447+0000", 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.1071e-05 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064454+0000", 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4749999999999996e-06 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.064460+0000", 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7230000000000001e-06 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.073113+0000", 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0086531920000000005 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.073117+0000", 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.304e-06 2024-09-21T10:24:24.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.083809+0000", 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0106921820000001 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.084276+0000", 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00046701199999999999 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 1,6", 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.084449+0000", 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017320000000000001 2024-09-21T10:24:24.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.093518+0000", 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0090688620000000005 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.093526+0000", 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.1559999999999995e-06 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.093592+0000", 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5607999999999999e-05 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.093643+0000", 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.1480999999999997e-05 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.093745+0000", 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000101992 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.191 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.36207.0:127686 11.8 11:1040ba70:::benchmark_data_smithi076_330857_object127685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1366)", 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:22:25.068056+0000", 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 118.989092837, 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.021740748, 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.36207", 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/1273400410", 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 127686 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:24.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.068056+0000", 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.068056+0000", 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.068055+0000", 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:24.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.068089+0000", 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.4165000000000002e-05 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.068099+0000", 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.5030000000000003e-06 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.194 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.068106+0000", 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4610000000000002e-06 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.072680+0000", 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0045738970000000004 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:25.072690+0000", 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9410000000000002e-06 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.074376+0000", 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0016864640000001 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.077787+0000", 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034110730000000001 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,6", 2024-09-21T10:24:24.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.078185+0000", 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00039749000000000002 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.086352+0000", 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0081670089999999994 2024-09-21T10:24:24.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.089369+0000", 2024-09-21T10:24:24.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0030173520000000001 2024-09-21T10:24:24.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.089601+0000", 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000231728 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.089664+0000", 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3162000000000003e-05 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:24.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:22:26.089797+0000", 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000132819 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:24.200 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.201 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:24.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:24.233 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Removed 137766 objects 2024-09-21T10:24:24.233 INFO:tasks.radosbench.radosbench.0.smithi076.stdout:Clean up completed and total clean up time :108.97 2024-09-21T10:24:24.302 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:24.323 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:24.424 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:24.445 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:24.547 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:24.572 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:24.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:24.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:24.696 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:24.798 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:24.819 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:24 smithi191 ceph-mon[102206]: pgmap v2280: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 88 active+clean; 134 MiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 1.19k op/s; 516/6468 objects degraded (7.978%); 839/6468 objects misplaced (12.972%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:24:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:24 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 516/6468 objects degraded (7.978%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:24 smithi191 ceph-mon[102206]: osdmap e1384: 8 total, 8 up, 4 in 2024-09-21T10:24:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:24 smithi076 ceph-mon[106482]: pgmap v2280: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 88 active+clean; 134 MiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 1.19k op/s; 516/6468 objects degraded (7.978%); 839/6468 objects misplaced (12.972%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:24:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:24 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 516/6468 objects degraded (7.978%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:24 smithi076 ceph-mon[106482]: osdmap e1384: 8 total, 8 up, 4 in 2024-09-21T10:24:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:24 smithi076 ceph-mon[110963]: pgmap v2280: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 88 active+clean; 134 MiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 1.19k op/s; 516/6468 objects degraded (7.978%); 839/6468 objects misplaced (12.972%); 4.1 MiB/s, 65 objects/s recovering 2024-09-21T10:24:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:24 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 516/6468 objects degraded (7.978%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:24:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:24 smithi076 ceph-mon[110963]: osdmap e1384: 8 total, 8 up, 4 in 2024-09-21T10:24:24.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:24.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:24.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:25.048 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:25.071 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:25.173 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.195 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:25.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:25.296 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:25.320 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:25.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:25.441 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:25.447 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:25.525 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-09-21T10:24:25.525 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-09-21T10:24:25.549 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:25.593 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:25.695 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:25.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:25.735 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-21T10:24:25.820 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:25.841 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:25.937 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:25.943 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:25.963 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:25.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:25.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:26.065 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:26.087 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:26.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:26.189 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:26.211 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:26.312 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:26.336 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:26.344 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:26 smithi191 ceph-mon[102206]: osdmap e1385: 8 total, 8 up, 4 in 2024-09-21T10:24:26.344 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:26.344 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:26.344 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:26 smithi191 ceph-mon[102206]: pgmap v2283: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 88 active+clean; 134 MiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 548 op/s; 516/6468 objects degraded (7.978%); 839/6468 objects misplaced (12.972%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:24:26.344 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:26 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[110963]: osdmap e1385: 8 total, 8 up, 4 in 2024-09-21T10:24:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[110963]: pgmap v2283: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 88 active+clean; 134 MiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 548 op/s; 516/6468 objects degraded (7.978%); 839/6468 objects misplaced (12.972%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:24:26.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:26.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[106482]: osdmap e1385: 8 total, 8 up, 4 in 2024-09-21T10:24:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[106482]: pgmap v2283: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 88 active+clean; 134 MiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 548 op/s; 516/6468 objects degraded (7.978%); 839/6468 objects misplaced (12.972%); 2.3 MiB/s, 36 objects/s recovering 2024-09-21T10:24:26.374 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:26 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:26.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:26.458 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:26.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:26.560 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:26.582 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:26.683 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:26.705 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:26.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:26.807 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:26.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:26.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:26.935 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:26.956 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:26.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:27.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:27.078 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:27.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:27.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:27.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:27.305 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:27.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:27.429 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:27.449 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:27 smithi191 ceph-mon[102206]: osdmap e1386: 8 total, 8 up, 4 in 2024-09-21T10:24:27.450 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:27.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:27.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:27.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:27 smithi076 ceph-mon[106482]: osdmap e1386: 8 total, 8 up, 4 in 2024-09-21T10:24:27.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:27 smithi076 ceph-mon[110963]: osdmap e1386: 8 total, 8 up, 4 in 2024-09-21T10:24:27.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:27.691 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:27.691 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:24:27.691 INFO:teuthology.orchestra.run.smithi191.stdout: "num_ops": 0 2024-09-21T10:24:27.691 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:27.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.699 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:27.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:27.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:27.821 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:27.923 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:27.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:27.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:27.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:28.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:28.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.068 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:28.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:28.169 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:28.190 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:28.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:28.309 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:28 smithi076 ceph-mon[106482]: osdmap e1387: 8 total, 8 up, 4 in 2024-09-21T10:24:28.318 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:28.349 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_blocked_ops 2024-09-21T10:24:28.420 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:28.443 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:28.544 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:28.545 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:28.567 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:28 smithi191 ceph-mon[102206]: osdmap e1387: 8 total, 8 up, 4 in 2024-09-21T10:24:28.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:28 smithi191 ceph-mon[102206]: pgmap v2286: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 89 active+clean; 11 MiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 490 op/s; 17/579 objects degraded (2.936%); 131/579 objects misplaced (22.625%); 448 KiB/s, 6 objects/s recovering 2024-09-21T10:24:28.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:28 smithi076 ceph-mon[106482]: pgmap v2286: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 89 active+clean; 11 MiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 490 op/s; 17/579 objects degraded (2.936%); 131/579 objects misplaced (22.625%); 448 KiB/s, 6 objects/s recovering 2024-09-21T10:24:28.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:28 smithi076 ceph-mon[110963]: osdmap e1387: 8 total, 8 up, 4 in 2024-09-21T10:24:28.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:28 smithi076 ceph-mon[110963]: pgmap v2286: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 89 active+clean; 11 MiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 490 op/s; 17/579 objects degraded (2.936%); 131/579 objects misplaced (22.625%); 448 KiB/s, 6 objects/s recovering 2024-09-21T10:24:28.669 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:28.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:28.791 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:28.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.814 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:28.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:28.915 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:28.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:28.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:29.038 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:29.059 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:29.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:29.161 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:29.183 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:29.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:29.284 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:29.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:29.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:29.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:29.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:29.534 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:29.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.557 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:29.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:29 smithi191 ceph-mon[102206]: osdmap e1388: 8 total, 8 up, 4 in 2024-09-21T10:24:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:29 smithi191 ceph-mon[102206]: Health check update: Degraded data redundancy: 17/579 objects degraded (2.936%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:24:29.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:29 smithi191 ceph-mon[102206]: osdmap e1389: 8 total, 8 up, 4 in 2024-09-21T10:24:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:29 smithi076 ceph-mon[106482]: osdmap e1388: 8 total, 8 up, 4 in 2024-09-21T10:24:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:29 smithi076 ceph-mon[106482]: Health check update: Degraded data redundancy: 17/579 objects degraded (2.936%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:24:29.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:29 smithi076 ceph-mon[106482]: osdmap e1389: 8 total, 8 up, 4 in 2024-09-21T10:24:29.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:29 smithi076 ceph-mon[110963]: osdmap e1388: 8 total, 8 up, 4 in 2024-09-21T10:24:29.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:29 smithi076 ceph-mon[110963]: Health check update: Degraded data redundancy: 17/579 objects degraded (2.936%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:24:29.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:29 smithi076 ceph-mon[110963]: osdmap e1389: 8 total, 8 up, 4 in 2024-09-21T10:24:29.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:29.679 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:29.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:29.801 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:29.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:29.924 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:29.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:30.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:30.046 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:30.148 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:30.170 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:30.272 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:30.294 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:30 smithi191 ceph-mon[102206]: pgmap v2289: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 90 active+clean; 681 KiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 532 op/s; 448 KiB/s, 6 objects/s recovering 2024-09-21T10:24:30.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:30 smithi191 ceph-mon[102206]: osdmap e1390: 8 total, 8 up, 4 in 2024-09-21T10:24:30.395 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:30.412 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:30.412 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [], 2024-09-21T10:24:30.412 INFO:teuthology.orchestra.run.smithi191.stdout: "complaint_time": 30, 2024-09-21T10:24:30.412 INFO:teuthology.orchestra.run.smithi191.stdout: "num_blocked_ops": 0 2024-09-21T10:24:30.412 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:30.417 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:30.518 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.546 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:30.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:30 smithi076 ceph-mon[106482]: pgmap v2289: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 90 active+clean; 681 KiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 532 op/s; 448 KiB/s, 6 objects/s recovering 2024-09-21T10:24:30.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:30 smithi076 ceph-mon[106482]: osdmap e1390: 8 total, 8 up, 4 in 2024-09-21T10:24:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:30 smithi076 ceph-mon[110963]: pgmap v2289: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 90 active+clean; 681 KiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 532 op/s; 448 KiB/s, 6 objects/s recovering 2024-09-21T10:24:30.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:30 smithi076 ceph-mon[110963]: osdmap e1390: 8 total, 8 up, 4 in 2024-09-21T10:24:30.647 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:30.679 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:30.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:30.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.810 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:30.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:30.910 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:30.936 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:30.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:31.037 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:31.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:31.123 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.6 dump_historic_ops 2024-09-21T10:24:31.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:31.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.191 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:31.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:31.292 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:31.317 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:31.327 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:31.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:31 smithi191 ceph-mon[102206]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/579 objects degraded (2.936%), 1 pg degraded) 2024-09-21T10:24:31.419 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:31.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.443 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:31.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:31.544 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.566 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:31.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:31.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:31 smithi076 ceph-mon[106482]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/579 objects degraded (2.936%), 1 pg degraded) 2024-09-21T10:24:31.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:31 smithi076 ceph-mon[110963]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/579 objects degraded (2.936%), 1 pg degraded) 2024-09-21T10:24:31.667 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:31.689 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:31.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:31.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:31.811 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:31.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:31.935 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:31.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:32.037 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:32.059 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:32.161 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.183 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:32.285 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:32.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.306 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:32.407 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:32.429 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:32.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:32.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:32.552 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:32 smithi191 ceph-mon[102206]: osdmap e1391: 8 total, 8 up, 4 in 2024-09-21T10:24:32.553 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:32 smithi191 ceph-mon[102206]: pgmap v2292: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 90 active+clean; 681 KiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 41 op/s 2024-09-21T10:24:32.554 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:32 smithi076 ceph-mon[106482]: osdmap e1391: 8 total, 8 up, 4 in 2024-09-21T10:24:32.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:32 smithi076 ceph-mon[106482]: pgmap v2292: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 90 active+clean; 681 KiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 41 op/s 2024-09-21T10:24:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:32 smithi076 ceph-mon[110963]: osdmap e1391: 8 total, 8 up, 4 in 2024-09-21T10:24:32.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:32 smithi076 ceph-mon[110963]: pgmap v2292: 99 pgs: 1 active+undersized+remapped+backfill_wait, 2 active+undersized+remapped+backfilling, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 90 active+clean; 681 KiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 41 op/s 2024-09-21T10:24:32.656 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:32.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:32.779 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:32.801 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:32.902 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:32.924 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:32.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:32.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:33.026 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:33.048 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:33.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:33.117 INFO:teuthology.orchestra.run.smithi191.stdout:{ 2024-09-21T10:24:33.117 INFO:teuthology.orchestra.run.smithi191.stdout: "size": 20, 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 600, 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "ops": [ 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115559 10.e 10:74245ea0:::benchmark_data_smithi076_299450_object115558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1307)", 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.252563+0000", 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 402.86093924300002, 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9920921419999997, 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.118 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115559 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.252563+0000", 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.252563+0000", 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.119 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.252561+0000", 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.252601+0000", 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0055000000000002e-05 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.252612+0000", 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0176e-05 2024-09-21T10:24:33.120 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.252620+0000", 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.9279999999999993e-06 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.253794+0000", 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0011746630000000001 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.121 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.253806+0000", 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1345999999999999e-05 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.243631+0000", 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98982572400000002 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.243645+0000", 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.377e-05 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.122 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.233429+0000", 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.989784376 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.233575+0000", 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000145771 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.233734+0000", 2024-09-21T10:24:33.123 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000158632 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.239719+0000", 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005985003 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.241201+0000", 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.001482506 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.243745+0000", 2024-09-21T10:24:33.124 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002544087 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.244511+0000", 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00076535399999999999 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.244554+0000", 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2744000000000003e-05 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.125 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.244655+0000", 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010147800000000001 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115566 10.e 10:7de8eb9b:::benchmark_data_smithi076_299450_object115565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.257163+0000", 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 402.85633873400002, 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9919416669999999, 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.126 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115566 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257163+0000", 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257163+0000", 2024-09-21T10:24:33.127 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257162+0000", 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257201+0000", 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9363000000000002e-05 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.128 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257211+0000", 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6350000000000001e-06 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257219+0000", 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8730000000000007e-06 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257534+0000", 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00031476099999999999 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.129 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.257545+0000", 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1613e-05 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246779+0000", 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98923410599999995 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246790+0000", 2024-09-21T10:24:33.130 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0645000000000001e-05 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.236591+0000", 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.989800703 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.236754+0000", 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016363099999999999 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:24:33.131 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.236936+0000", 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00018115000000000001 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.240420+0000", 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0034843719999999999 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.241238+0000", 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00081846700000000002 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.132 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.243827+0000", 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00258837 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.248958+0000", 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0051310440000000004 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.249006+0000", 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.8658000000000001e-05 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.133 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.249105+0000", 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8648999999999997e-05 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115569 10.7 10:e3a4b5a0:::benchmark_data_smithi076_299450_object115568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.262594+0000", 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 402.85090806800002, 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.000576669, 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.134 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115569 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262594+0000", 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.135 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262594+0000", 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262592+0000", 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262633+0000", 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.0967000000000003e-05 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.136 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262649+0000", 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5801999999999999e-05 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262657+0000", 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.515e-06 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262709+0000", 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2896000000000003e-05 2024-09-21T10:24:33.137 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.262722+0000", 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2293000000000001e-05 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246260+0000", 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.983538723 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.138 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246273+0000", 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2828e-05 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.225959+0000", 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9796856900000002 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.226107+0000", 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000147947 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:24:33.139 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.226266+0000", 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00015944200000000001 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.235756+0000", 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0094895099999999996 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.242713+0000", 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0069574229999999999 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.140 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.249233+0000", 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065198649999999997 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.262911+0000", 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013677879 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.262960+0000", 2024-09-21T10:24:33.141 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.901e-05 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263171+0000", 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00021070799999999999 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115572 10.7 10:e9d58661:::benchmark_data_smithi076_299450_object115571:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.265976+0000", 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 402.84752632099998, 2024-09-21T10:24:33.142 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9975414530000002, 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115572 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.265976+0000", 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.143 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.265976+0000", 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.265975+0000", 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.266033+0000", 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7901e-05 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.144 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.145 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.145 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.266042+0000", 2024-09-21T10:24:33.145 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8470000000000002e-06 2024-09-21T10:24:33.145 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.145 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.266046+0000", 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5120000000000002e-06 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.266096+0000", 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9416000000000003e-05 2024-09-21T10:24:33.146 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.266108+0000", 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2243e-05 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246305+0000", 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.98019729099999997 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.147 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246314+0000", 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8310000000000007e-06 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.229276+0000", 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9829625480000002 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.229426+0000", 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014931699999999999 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.148 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.229564+0000", 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013806300000000001 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.238563+0000", 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0089994410000000004 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.242753+0000", 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0041898860000000003 2024-09-21T10:24:33.149 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.249313+0000", 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065593919999999998 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263271+0000", 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.013958724 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.150 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263316+0000", 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4740999999999997e-05 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263517+0000", 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00020124800000000001 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:115578 10.7 10:e93fb7f9:::benchmark_data_smithi076_299450_object115577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1308)", 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:17:50.276054+0000", 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 402.83744791200002, 2024-09-21T10:24:33.151 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9878803249999999, 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 115578 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276054+0000", 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.152 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276054+0000", 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276052+0000", 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276112+0000", 2024-09-21T10:24:33.153 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9351999999999997e-05 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276139+0000", 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7172999999999999e-05 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276151+0000", 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.152e-05 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.154 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276235+0000", 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4058000000000001e-05 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:50.276253+0000", 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.8397e-05 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246675+0000", 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97042209400000001 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.155 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:51.246685+0000", 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7489999999999993e-06 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.232220+0000", 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9855347590000001 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.232385+0000", 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000165119 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.156 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.232554+0000", 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016895500000000001 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.242644+0000", 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.010090007999999999 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.242791+0000", 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014739399999999999 2024-09-21T10:24:33.157 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.249347+0000", 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0065561229999999996 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263656+0000", 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014308346 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.158 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263705+0000", 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9549999999999998e-05 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:17:53.263935+0000", 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00022947399999999999 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:250718 10.e 10:72f6f440:::benchmark_data_smithi076_299450_object115539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1331)", 2024-09-21T10:24:33.159 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.896216+0000", 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 264.21728565799998, 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2779539, 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "reached pg", 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 250718 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.896216+0000", 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.160 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.896216+0000", 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.896215+0000", 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.161 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.896222+0000", 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.3600000000000001e-06 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.896231+0000", 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8649999999999996e-06 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.896238+0000", 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.1999999999999999e-06 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.162 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.902548+0000", 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063101540000000001 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.902664+0000", 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115732 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,4,5", 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.902782+0000", 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011877799999999999 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.163 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916899+0000", 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.014116449999999999 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916909+0000", 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7499999999999998e-06 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.165373+0000", 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2484648110000001 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.164 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for missing object", 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.165485+0000", 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011208 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.174083+0000", 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0085979990000000003 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.174170+0000", 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.6985999999999997e-05 2024-09-21T10:24:33.165 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:250737 10.e 10:74245ea0:::benchmark_data_smithi076_299450_object115558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1332)", 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.916450+0000", 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 264.19705201599999, 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.263389629, 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.166 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 250737 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916450+0000", 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916450+0000", 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.167 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916449+0000", 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916454+0000", 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2490000000000001e-06 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916464+0000", 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4839999999999993e-06 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.168 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.916470+0000", 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.9440000000000002e-06 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.917381+0000", 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00091104599999999999 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.917388+0000", 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8450000000000003e-06 2024-09-21T10:24:33.169 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.165693+0000", 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2483048450000001 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.165773+0000", 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0177000000000002e-05 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:24:33.170 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.165863+0000", 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9907000000000005e-05 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.173838+0000", 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0079743990000000001 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.173885+0000", 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7210999999999999e-05 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.171 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.179686+0000", 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0058011879999999997 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.179736+0000", 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9835e-05 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.179840+0000", 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0001038 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.172 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:250744 10.e 10:7de8eb9b:::benchmark_data_smithi076_299450_object115565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1332)", 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.923329+0000", 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 264.19017339999999, 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2567533580000001, 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.173 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 250744 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923329+0000", 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923329+0000", 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.174 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923328+0000", 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923332+0000", 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3189999999999996e-06 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923342+0000", 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6439999999999995e-06 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.175 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923349+0000", 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.43e-06 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923399+0000", 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9827000000000003e-05 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.923415+0000", 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5976000000000001e-05 2024-09-21T10:24:33.176 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.168467+0000", 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2450513999999999 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.168543+0000", 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6817999999999995e-05 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.168615+0000", 2024-09-21T10:24:33.177 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1953999999999996e-05 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.174049+0000", 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0054332720000000003 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.174210+0000", 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00016106200000000001 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.178 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.179904+0000", 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0056944559999999996 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.179983+0000", 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8491000000000005e-05 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180082+0000", 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9431000000000004e-05 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.179 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:250791 10.e 10:7a9c6f0d:::benchmark_data_smithi076_299450_object115612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1332)", 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:08.983720+0000", 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 264.129781751, 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1968013179999999, 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 250791 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.180 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983720+0000", 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983720+0000", 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983719+0000", 2024-09-21T10:24:33.181 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983725+0000", 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6829999999999996e-06 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983738+0000", 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2719e-05 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.182 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983745+0000", 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3000000000000004e-06 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983922+0000", 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017681700000000001 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:08.983930+0000", 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8960000000000003e-06 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.183 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.171209+0000", 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.187279124 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.171325+0000", 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000116168 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.171469+0000", 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00014425000000000001 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.184 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.174215+0000", 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0027456939999999999 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.174259+0000", 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4495e-05 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180339+0000", 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0060795149999999997 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.185 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180420+0000", 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0845000000000003e-05 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180522+0000", 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010187300000000001 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.186 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:250834 10.e 10:7b56b181:::benchmark_data_smithi076_299450_object115655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1332)", 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:09.020543+0000", 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 264.09295896100002, 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1603708610000001, 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 250834 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.187 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020543+0000", 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020543+0000", 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020543+0000", 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999995 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.188 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020546+0000", 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.881e-06 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020552+0000", 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5950000000000004e-06 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020557+0000", 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7999999999999998e-06 2024-09-21T10:24:33.189 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020671+0000", 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000114225 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.020678+0000", 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.007e-06 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.190 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.176704+0000", 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1560260019999999 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.176816+0000", 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011129999999999999 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.176950+0000", 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000134914 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.191 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180151+0000", 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032006819999999998 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180252+0000", 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000100511 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180794+0000", 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00054268000000000003 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.192 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180832+0000", 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7503000000000003e-05 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.180914+0000", 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.2247e-05 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.193 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:250844 10.e 10:77e1e357:::benchmark_data_smithi076_299450_object115665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1332)", 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:09.027780+0000", 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 264.08572187999999, 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.163365738, 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 250844 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.194 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.027780+0000", 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.027780+0000", 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.027779+0000", 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.195 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.027784+0000", 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.5669999999999996e-06 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.027792+0000", 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.4339999999999992e-06 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.027799+0000", 2024-09-21T10:24:33.196 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0560000000000001e-06 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.028479+0000", 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00068061300000000005 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:09.028487+0000", 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8590000000000005e-06 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.197 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.181039+0000", 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1525524899999999 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.181130+0000", 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.0675999999999997e-05 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 4,5", 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.181230+0000", 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.9994999999999994e-05 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.198 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.190896+0000", 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0096656580000000006 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.190936+0000", 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9684000000000002e-05 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.191008+0000", 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.2200999999999995e-05 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.199 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.191052+0000", 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3955999999999997e-05 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:10.191146+0000", 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.4338000000000004e-05 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.200 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262288 10.a 10:5e1e6343:::benchmark_data_smithi076_299450_object127109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.017850+0000", 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.09565166499999, 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99670415199999995, 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262288 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.201 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017850+0000", 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017850+0000", 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017849+0000", 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.202 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017854+0000", 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.826e-06 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017863+0000", 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1330000000000005e-06 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017868+0000", 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9300000000000002e-06 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.203 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017914+0000", 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.6195000000000001e-05 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.017923+0000", 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.9169999999999995e-06 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.008541+0000", 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.99061761999999998 2024-09-21T10:24:33.204 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.008632+0000", 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.1379999999999996e-05 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.008726+0000", 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3282000000000006e-05 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.205 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014357+0000", 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.005630955 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014399+0000", 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2821000000000002e-05 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014498+0000", 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8981999999999995e-05 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.206 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014528+0000", 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9099999999999999e-05 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014555+0000", 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.7115e-05 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262311 10.a 10:511d81fc:::benchmark_data_smithi076_299450_object127132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.207 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.043763+0000", 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.069739357, 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.97399511800000005, 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262311 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043763+0000", 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.208 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043763+0000", 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043762+0000", 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.209 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043767+0000", 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0370000000000002e-06 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043778+0000", 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1297e-05 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043785+0000", 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.8159999999999996e-06 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.210 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043882+0000", 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.6978000000000006e-05 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.043890+0000", 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0909999999999993e-06 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.011752+0000", 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.96786218099999999 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.211 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.011863+0000", 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000110538 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.011984+0000", 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000121125 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014493+0000", 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0025086470000000001 2024-09-21T10:24:33.212 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017559+0000", 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003066184 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017657+0000", 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.8228999999999998e-05 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017716+0000", 2024-09-21T10:24:33.213 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.8570999999999999e-05 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017758+0000", 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2289000000000001e-05 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262337 10.a 10:51944d4e:::benchmark_data_smithi076_299450_object127158:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.068552+0000", 2024-09-21T10:24:33.214 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.04495032700001, 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.95975563500000005, 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262337 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068552+0000", 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.215 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068552+0000", 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068551+0000", 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068556+0000", 2024-09-21T10:24:33.216 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6799999999999998e-06 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068568+0000", 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.1618e-05 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068574+0000", 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.5969999999999997e-06 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.217 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068624+0000", 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9635999999999997e-05 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.068633+0000", 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.5799999999999992e-06 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014584+0000", 2024-09-21T10:24:33.218 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.945951129 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014662+0000", 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.8028000000000006e-05 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.014732+0000", 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.0271000000000005e-05 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.219 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.021512+0000", 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0067802829999999998 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.021545+0000", 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2922000000000001e-05 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028221+0000", 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0066758 2024-09-21T10:24:33.220 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028270+0000", 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.9129999999999999e-05 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028307+0000", 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.7218999999999997e-05 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.221 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262356 10.a 10:5bdcf2aa:::benchmark_data_smithi076_299450_object127177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.082284+0000", 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.03121856000001, 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.94626855600000004, 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262356 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.222 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082284+0000", 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082284+0000", 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082282+0000", 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.223 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082288+0000", 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.5489999999999997e-06 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082303+0000", 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.4919999999999999e-05 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082310+0000", 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.798e-06 2024-09-21T10:24:33.224 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082443+0000", 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000132953 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.082452+0000", 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.8149999999999995e-06 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.225 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017789+0000", 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.93533716899999997 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017865+0000", 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.6100999999999995e-05 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.017969+0000", 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000104157 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.226 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.021579+0000", 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003609573 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028381+0000", 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0068022940000000004 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028438+0000", 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.7324999999999997e-05 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.227 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028513+0000", 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.4406000000000006e-05 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028552+0000", 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9603e-05 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.228 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.228 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:33.229 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262390 10.a 10:51f94198:::benchmark_data_smithi076_299450_object127211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.111190+0000", 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.00231256199999, 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91746708399999999, 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.230 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262390 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111190+0000", 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111190+0000", 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.231 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111188+0000", 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111194+0000", 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0159999999999999e-06 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111213+0000", 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.9304999999999999e-05 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.232 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111220+0000", 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.221e-06 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111271+0000", 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.0767999999999999e-05 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.111279+0000", 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.3999999999999992e-06 2024-09-21T10:24:33.233 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.021601+0000", 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91032158799999996 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.021710+0000", 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000109263 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.234 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.021820+0000", 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00011016800000000001 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028208+0000", 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0063873020000000001 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028562+0000", 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00035417599999999999 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.235 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028598+0000", 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.6426e-05 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028623+0000", 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.5043000000000001e-05 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028657+0000", 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3497000000000001e-05 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.236 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262392 10.a 10:5a003bf4:::benchmark_data_smithi076_299450_object127213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.112989+0000", 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.00051309200001, 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91894940000000003, 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262392 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.237 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.112989+0000", 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.112989+0000", 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.112988+0000", 2024-09-21T10:24:33.238 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999986 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.112994+0000", 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.0580000000000003e-06 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113007+0000", 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.2897999999999999e-05 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.239 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113014+0000", 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.1189999999999999e-06 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113056+0000", 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.2119000000000001e-05 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113065+0000", 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.3840000000000003e-06 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.240 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.024905+0000", 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91184008800000005 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.025017+0000", 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000111894 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.025148+0000", 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00013058300000000001 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.241 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028428+0000", 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0032802700000000001 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.031828+0000", 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0033995729999999999 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.031863+0000", 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.5308000000000001e-05 2024-09-21T10:24:33.242 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.031903+0000", 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9539999999999998e-05 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.031938+0000", 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.587e-05 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.243 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262394 10.a 10:5c5827e4:::benchmark_data_smithi076_299450_object127215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.113433+0000", 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 248.000069193, 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92787085199999997, 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262394 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.244 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113433+0000", 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113433+0000", 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113432+0000", 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.245 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113440+0000", 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 8.0120000000000005e-06 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113462+0000", 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.1549999999999999e-05 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113469+0000", 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3479999999999998e-06 2024-09-21T10:24:33.246 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113499+0000", 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9893e-05 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.113504+0000", 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.2769999999999996e-06 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028705+0000", 2024-09-21T10:24:33.247 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91520131599999999 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028820+0000", 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000115109 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.028921+0000", 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00010052600000000001 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.248 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.038199+0000", 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0092775779999999999 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041199+0000", 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.002999945 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041232+0000", 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3469000000000001e-05 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.249 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041272+0000", 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.9928999999999997e-05 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041304+0000", 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.1863999999999998e-05 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.250 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262401 10.a 10:52b78093:::benchmark_data_smithi076_299450_object127222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.121001+0000", 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 247.992501336, 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.92044451900000002, 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262401 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.251 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121001+0000", 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121001+0000", 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.120999+0000", 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.9999981 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.252 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121009+0000", 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.0373999999999999e-05 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121025+0000", 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.5284000000000001e-05 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121032+0000", 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 7.3370000000000002e-06 2024-09-21T10:24:33.253 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121080+0000", 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7713000000000002e-05 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.121093+0000", 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 1.3387000000000001e-05 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.031964+0000", 2024-09-21T10:24:33.254 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91087115799999996 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.032067+0000", 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000102722 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.032174+0000", 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000107274 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.255 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041165+0000", 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.008990629 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041339+0000", 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00017405400000000001 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041368+0000", 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 2.9252999999999999e-05 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.256 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041413+0000", 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4391000000000002e-05 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041445+0000", 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.2676999999999999e-05 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.257 INFO:teuthology.orchestra.run.smithi191.stdout: "description": "osd_op(client.35901.0:262421 10.a 10:5c2f71bf:::benchmark_data_smithi076_299450_object127242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "initiated_at": "2024-09-21T10:20:25.147789+0000", 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "age": 247.96571286400001, 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.91963135900000004, 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "type_data": { 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "client_info": { 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "client": "client.35901", 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "client_addr": "172.21.15.76:0/3707940031", 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "tid": 262421 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "events": [ 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "initiated", 2024-09-21T10:24:33.258 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147789+0000", 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "throttled", 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147789+0000", 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "header_read", 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147789+0000", 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4294967295.999999 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.259 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "all_read", 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147793+0000", 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.3030000000000001e-06 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "dispatched", 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147803+0000", 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 9.7559999999999994e-06 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "queued_for_pg", 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147809+0000", 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 6.7209999999999997e-06 2024-09-21T10:24:33.260 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147842+0000", 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 3.3083000000000002e-05 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for peered", 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:25.147848+0000", 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 5.6019999999999999e-06 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "reached_pg", 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.038228+0000", 2024-09-21T10:24:33.261 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.89038037999999997 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "started", 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.038335+0000", 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.000106289 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "waiting for subops from 3,5", 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.038460+0000", 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.00012492199999999999 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.262 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.041540+0000", 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.003079951 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "op_commit", 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.060078+0000", 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.018538522000000002 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "sub_op_commit_rec", 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.067328+0000", 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 0.0072500020000000002 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.263 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "commit_sent", 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.067373+0000", 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.4709999999999997e-05 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: }, 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: { 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: "event": "done", 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: "time": "2024-09-21T10:20:26.067421+0000", 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: "duration": 4.7846000000000002e-05 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: } 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout: ] 2024-09-21T10:24:33.264 INFO:teuthology.orchestra.run.smithi191.stdout:} 2024-09-21T10:24:33.288 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:33.362 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:33 smithi191 ceph-mon[102206]: osdmap e1392: 8 total, 8 up, 4 in 2024-09-21T10:24:33.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:33.411 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:33.512 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:33.533 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:33.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:33 smithi076 ceph-mon[106482]: osdmap e1392: 8 total, 8 up, 4 in 2024-09-21T10:24:33.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:33 smithi076 ceph-mon[110963]: osdmap e1392: 8 total, 8 up, 4 in 2024-09-21T10:24:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:24:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:24:33] "GET /metrics HTTP/1.1" 200 36634 "" "Prometheus/2.51.0" 2024-09-21T10:24:33.635 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:33.656 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:33.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:33.757 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:33.781 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:33.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:33.883 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:33.905 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:33.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:34.006 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:34.027 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:34.096 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-21T10:24:34.129 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:34.155 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:34.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:34.257 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:34.282 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:34.290 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:34.383 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:34.406 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:34.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:34.508 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:34.529 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:34 smithi191 ceph-mon[102206]: osdmap e1393: 8 total, 8 up, 4 in 2024-09-21T10:24:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:34 smithi191 ceph-mon[102206]: pgmap v2295: 99 pgs: 1 active+clean+remapped, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:34.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:34 smithi191 ceph-mon[102206]: osdmap e1394: 8 total, 8 up, 4 in 2024-09-21T10:24:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:34 smithi076 ceph-mon[106482]: osdmap e1393: 8 total, 8 up, 4 in 2024-09-21T10:24:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:34 smithi076 ceph-mon[106482]: pgmap v2295: 99 pgs: 1 active+clean+remapped, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:34.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:34 smithi076 ceph-mon[106482]: osdmap e1394: 8 total, 8 up, 4 in 2024-09-21T10:24:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:34 smithi076 ceph-mon[110963]: osdmap e1393: 8 total, 8 up, 4 in 2024-09-21T10:24:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:34 smithi076 ceph-mon[110963]: pgmap v2295: 99 pgs: 1 active+clean+remapped, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:34.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:34 smithi076 ceph-mon[110963]: osdmap e1394: 8 total, 8 up, 4 in 2024-09-21T10:24:34.631 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:34.653 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:34.755 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:34.776 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:34.877 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:34.901 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:34.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:35.002 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.023 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:35.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:35.124 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:35.150 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:35.251 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:35.274 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:35.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:35.303 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:35.304 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:35.304 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:35.304 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:35.304 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:35.304 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:35.375 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:35.397 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:35.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:35.499 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:35.523 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:35.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:35.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:35 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1122650105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:35.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:35 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1122650105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:35.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:35 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1122650105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:24:35.625 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:35.647 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:35.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:35.774 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:35.876 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:35.900 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:35.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:35.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:36.002 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:36.026 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:36.127 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:36.151 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:36.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:36.253 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:36.278 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:36.380 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:36.405 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:36.507 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:36.531 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:36.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:36.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:36 smithi191 ceph-mon[102206]: pgmap v2297: 99 pgs: 1 active+clean+remapped, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:36.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:36 smithi076 ceph-mon[106482]: pgmap v2297: 99 pgs: 1 active+clean+remapped, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:36.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:36 smithi076 ceph-mon[110963]: pgmap v2297: 99 pgs: 1 active+clean+remapped, 1 peering, 1 active+remapped+backfilling, 96 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:36.633 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:36.658 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:36.760 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:36.786 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:36.888 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:36.913 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:36.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:37.014 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:37.038 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:37.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:37.164 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:37.265 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:37.291 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:37.392 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:37.418 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:37.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:37.520 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:37.545 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:37.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:37.647 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:37.673 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:37.774 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:37.800 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:37.901 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:37.926 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:37.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:37.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:38.028 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:38.053 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:38.155 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:38.180 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:38.281 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:38.308 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:38.410 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:38.435 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:38.537 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:38.561 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:38.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:38.663 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:38.689 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:38.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:38.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:38.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:38 smithi191 ceph-mon[102206]: pgmap v2298: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 97 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:38.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:38 smithi076 ceph-mon[106482]: pgmap v2298: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 97 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:38.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:38 smithi076 ceph-mon[110963]: pgmap v2298: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 97 active+clean; 681 KiB data, 3.9 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:38.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:38.943 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:38.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:39.045 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:39.070 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:39.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:39.196 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:39.298 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:39.322 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:39.424 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:39.449 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:39.536 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:39.536 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:39.550 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:39.572 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:39.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:39.673 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:39.694 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:39.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:39.796 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:39.822 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:39.924 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:39.948 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:39.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:40.050 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:40.075 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:40.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:40.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_blocked_ops 2024-09-21T10:24:40.177 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:40.234 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:40.336 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:40.360 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:40.369 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:40.462 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:40.488 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:40.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:40.589 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:40.613 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:40.715 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:40.740 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:40.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:40.830 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:40 smithi191 ceph-mon[102206]: pgmap v2299: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:40.830 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:40.830 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:40.841 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:40.866 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:40 smithi076 ceph-mon[106482]: pgmap v2299: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:40 smithi076 ceph-mon[110963]: pgmap v2299: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:24:40.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:40.968 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:40.995 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:40.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:41.097 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.122 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:41.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:41.224 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:41.246 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:41.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.372 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:41.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:41.473 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:41.495 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:41.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:41.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.620 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:41.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:41.722 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:41.747 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:41.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:41.849 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:41.874 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:41.976 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:41.998 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:41.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:42.100 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:42.123 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:42.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:42.165 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:42.165 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:42.185 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:42.225 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:42.246 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:42.348 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:42.370 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:42.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:42.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.494 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:42.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:42.595 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:42.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.620 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:42.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:42.652 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:42 smithi076 ceph-mon[106482]: pgmap v2300: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:42.652 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:42 smithi076 ceph-mon[110963]: pgmap v2300: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:42.721 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:42.742 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:42.844 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:42.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:42 smithi191 ceph-mon[102206]: pgmap v2300: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:42.865 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:42.967 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:42.988 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:42.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:43.090 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:43.093 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.7 dump_historic_ops 2024-09-21T10:24:43.111 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:43.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:43.234 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:43.324 INFO:teuthology.orchestra.run.smithi191.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T10:24:43.336 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.360 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:43.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:43.461 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:43.482 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:43.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:43.584 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:43.605 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:43.614 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:24:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:24:43] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.51.0" 2024-09-21T10:24:43.616 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is set 2024-09-21T10:24:43.646 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1676199660' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:43.646 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:43 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:43.707 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:43.730 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:43.831 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:43.853 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1676199660' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:43.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:43 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1676199660' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:43.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:43 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:24:43.955 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:43.975 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:43.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:44.077 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:44.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.100 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:44.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:44.201 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:44.208 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-21T10:24:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.224 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:44.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:44.326 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:44.347 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:44.449 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:44.470 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:44.572 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:44.594 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:44.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:44.696 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:44.717 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[106482]: pgmap v2301: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[106482]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[106482]: osdmap e1395: 8 total, 8 up, 4 in 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[110963]: pgmap v2301: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[110963]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:44.810 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:44 smithi076 ceph-mon[110963]: osdmap e1395: 8 total, 8 up, 4 in 2024-09-21T10:24:44.819 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:44.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.842 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:44.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:44.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:44 smithi191 ceph-mon[102206]: pgmap v2301: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:44 smithi191 ceph-mon[102206]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:44 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-21T10:24:44.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:44 smithi191 ceph-mon[102206]: osdmap e1395: 8 total, 8 up, 4 in 2024-09-21T10:24:44.942 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:44.965 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:44.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:45.066 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:45.087 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:45.121 INFO:teuthology.orchestra.run.smithi191.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:45.121 INFO:teuthology.orchestra.run.smithi191.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:45.189 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:45.211 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:45.312 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:45.335 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:45.437 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:45.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.460 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:45.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:45.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:45.582 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:45.684 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:45.706 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:45.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:45.808 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:45.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:45.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:45 smithi191 ceph-mon[102206]: pgmap v2303: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:45.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:45 smithi076 ceph-mon[106482]: pgmap v2303: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:45.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:45 smithi076 ceph-mon[110963]: pgmap v2303: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:45.931 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:45.939 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-21T10:24:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:45.953 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:45.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:46.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:46.077 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:46.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:46.179 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:46.199 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:46.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:46.301 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:46.324 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:46.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:46.327 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:46.425 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:46.449 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:46.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:46.575 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:46.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:46.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:46.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:46.800 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:46.825 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:46.832 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:24:46.927 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:46.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:46.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:47.053 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:47.077 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:47.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:46 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3901336504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:24:47.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:46 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3901336504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:24:47.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:46 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3901336504' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:24:47.179 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:47.203 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:47.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:47.305 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:47.328 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:47.430 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:47.455 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:47.557 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:47.579 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:47.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:47.680 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:47.706 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:47.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:47.810 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:47.832 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:47.851 INFO:teuthology.orchestra.run.smithi076.stderr:pool 'unique_pool_6' removed 2024-09-21T10:24:47.933 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:47.955 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:47.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:48.056 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:48.078 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:48.081 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:24:48.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3901336504' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:24:48.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[106482]: osdmap e1396: 8 total, 8 up, 4 in 2024-09-21T10:24:48.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[106482]: pgmap v2305: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:48.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3897850896' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:24:48.104 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3901336504' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:24:48.104 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[110963]: osdmap e1396: 8 total, 8 up, 4 in 2024-09-21T10:24:48.104 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[110963]: pgmap v2305: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:48.104 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3897850896' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:24:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:47 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3901336504' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:24:48.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:47 smithi191 ceph-mon[102206]: osdmap e1396: 8 total, 8 up, 4 in 2024-09-21T10:24:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:47 smithi191 ceph-mon[102206]: pgmap v2305: 99 pgs: 99 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:47 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3897850896' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:24:48.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:48.207 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:48.309 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:48.330 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:48.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:48.432 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:48.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:48.555 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:48.577 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:48.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:48.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:48.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:48.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.829 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:48.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:48.930 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:48.952 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:48.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:49.054 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:49.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:49.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3897850896' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:24:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:48 smithi191 ceph-mon[102206]: osdmap e1397: 8 total, 8 up, 4 in 2024-09-21T10:24:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3897850896' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:24:49.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:48 smithi076 ceph-mon[110963]: osdmap e1397: 8 total, 8 up, 4 in 2024-09-21T10:24:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3897850896' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:24:49.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:48 smithi076 ceph-mon[106482]: osdmap e1397: 8 total, 8 up, 4 in 2024-09-21T10:24:49.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.199 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:49.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:49.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:49.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:49.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:49.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:49.444 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:49.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:49.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:49.568 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:49.670 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:49.692 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:49.793 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:49.816 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:49.918 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:49.941 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:49.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:49.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:49.966 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:49 smithi076 ceph-mon[106482]: pgmap v2307: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:49.966 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:49 smithi076 ceph-mon[110963]: pgmap v2307: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:49.991 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:24:50.043 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:50.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:50.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:49 smithi191 ceph-mon[102206]: pgmap v2307: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:50.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:50.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:50.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:50.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.314 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:50.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:50.415 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:50.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:50.448 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T10:24:50.448 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-21T10:24:50.448 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-21T10:24:50.453 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-21T10:24:50.453 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T10:24:50.454 DEBUG:teuthology.orchestra.run.smithi076:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:24:50.469 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.504 INFO:teuthology.orchestra.run.smithi076.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T10:24:50.504 DEBUG:teuthology.orchestra.run.smithi076:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:24:50.541 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:50.566 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:50.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:50.569 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T10:24:50.569 DEBUG:teuthology.orchestra.run.smithi076:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T10:24:50.624 INFO:tasks.workunit:timeout=3h 2024-09-21T10:24:50.624 INFO:tasks.workunit:cleanup=True 2024-09-21T10:24:50.625 DEBUG:teuthology.orchestra.run.smithi076:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:24:50.667 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:50.690 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:50.708 INFO:tasks.workunit.client.0.smithi076.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T10:24:50.792 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:50.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:50.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:50.916 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:50.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:50.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:50.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:51.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:51.062 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:51.164 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:51.185 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:51.287 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.310 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:51.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:51.411 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:51.433 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:51.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:51.535 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:51.557 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:51.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:51.659 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:51.680 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:51.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:51.782 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:51.803 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:51.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:51.905 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:51.927 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:51.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:51.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:52.029 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:52.051 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:52.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:52.125 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:52.153 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:52.175 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:52.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:52.276 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:52.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:52.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:52.402 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:52.425 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:52.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:52.527 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:52.552 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:52.653 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:52.678 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:52.780 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:52.784 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:52 smithi076 ceph-mon[106482]: pgmap v2308: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:52.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:52 smithi076 ceph-mon[110963]: pgmap v2308: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:52.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:52.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:52 smithi191 ceph-mon[102206]: pgmap v2308: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:52.906 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:52.928 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:52.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:53.030 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:53.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.054 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:53.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:53.071 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:24:53.071 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:24:53.156 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:53.178 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:53.280 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:24:53.304 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:24:53.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:24:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:24:53] "GET /metrics HTTP/1.1" 200 36606 "" "Prometheus/2.51.0" 2024-09-21T10:24:53.405 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:53.431 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:53.533 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:53.555 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:53.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:53.657 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:53.678 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:53.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:53.722 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:24:53.722 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:53 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2991038015' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:53.723 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:53 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2991038015' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:53.780 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:53.802 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:53.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:53 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2991038015' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:24:53.903 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:53.928 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:53.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:53.955 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:24:54.029 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:54.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.051 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:54.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:54.152 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:54.173 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:54.275 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:54.301 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:54.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:54.403 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:54.424 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:54.492 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_blocked_ops 2024-09-21T10:24:54.526 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:54.556 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:54.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:54.658 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:54.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:54.734 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[106482]: pgmap v2309: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:54.734 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[106482]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:54.734 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2991038015' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:24:54.734 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[106482]: osdmap e1398: 8 total, 8 up, 4 in 2024-09-21T10:24:54.735 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[110963]: pgmap v2309: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:54.735 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[110963]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:54.735 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2991038015' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:24:54.735 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:54 smithi076 ceph-mon[110963]: osdmap e1398: 8 total, 8 up, 4 in 2024-09-21T10:24:54.789 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:54.809 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:54.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:54.911 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:54.937 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:54.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:55.039 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:55.064 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:55.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:54 smithi191 ceph-mon[102206]: pgmap v2309: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:54 smithi191 ceph-mon[102206]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-21T10:24:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2991038015' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:24:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:54 smithi191 ceph-mon[102206]: osdmap e1398: 8 total, 8 up, 4 in 2024-09-21T10:24:55.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:55.187 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:55.289 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:55.315 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:55.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:55.438 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:55.540 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:55.566 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:55.668 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:55.689 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:55.790 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:55.816 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[106482]: 1.a scrub starts 2024-09-21T10:24:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[106482]: 1.a scrub ok 2024-09-21T10:24:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[106482]: pgmap v2311: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[110963]: 1.a scrub starts 2024-09-21T10:24:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[110963]: 1.a scrub ok 2024-09-21T10:24:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[110963]: pgmap v2311: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:55.918 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:55.944 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:55.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:56.046 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:55 smithi191 ceph-mon[102206]: 1.a scrub starts 2024-09-21T10:24:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:55 smithi191 ceph-mon[102206]: 1.a scrub ok 2024-09-21T10:24:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:55 smithi191 ceph-mon[102206]: pgmap v2311: 83 pgs: 83 active+clean; 681 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:24:56.168 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:56.189 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:56.291 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:56.314 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:56.428 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:56.449 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:56.551 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:56.576 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:56.678 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:56.700 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:56.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:56.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:56.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:56.947 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:56.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:56.967 DEBUG:teuthology.orchestra.run.smithi076:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T10:24:57.049 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:57.071 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:57.083 INFO:tasks.workunit.client.0.smithi076.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T10:24:57.085 INFO:tasks.workunit.client.0.smithi076.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:24:57.085 INFO:tasks.workunit.client.0.smithi076.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T10:24:57.174 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:57.200 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:57.213 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:24:57.301 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:57.327 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:57.429 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:57.451 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:57.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:57.552 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:57.574 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:57.676 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:57.700 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:57.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:57.802 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:57.823 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:57.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:57.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:57 smithi191 ceph-mon[102206]: 1.5 scrub starts 2024-09-21T10:24:57.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:57 smithi191 ceph-mon[102206]: 1.5 scrub ok 2024-09-21T10:24:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:57 smithi076 ceph-mon[106482]: 1.5 scrub starts 2024-09-21T10:24:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:57 smithi076 ceph-mon[106482]: 1.5 scrub ok 2024-09-21T10:24:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:57 smithi076 ceph-mon[110963]: 1.5 scrub starts 2024-09-21T10:24:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:57 smithi076 ceph-mon[110963]: 1.5 scrub ok 2024-09-21T10:24:57.925 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:57.948 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:57.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:58.049 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:58.075 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:58.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:58.177 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:58.199 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:58.300 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:58.326 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:58.427 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:24:58.453 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:24:58.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:58.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:58.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:24:58 smithi076 ceph-mon[106482]: pgmap v2312: 83 pgs: 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:58.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:24:58 smithi076 ceph-mon[110963]: pgmap v2312: 83 pgs: 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:58.679 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:58.703 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:58.804 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:24:58.830 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:58.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:24:58.862 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:24:58 smithi191 ceph-mon[102206]: pgmap v2312: 83 pgs: 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:24:58.932 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:24:58.957 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:24:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:58.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:24:59.058 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:24:59.081 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:24:59.183 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:59.204 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:59.306 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:59.328 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:59.430 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:59.452 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:59.554 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:59.576 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:59.677 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:24:59.699 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:24:59.784 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:24:59.785 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:24:59.788 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:24:59.788 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:24:59.801 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:24:59.804 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:24:59.805 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:24:59.822 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:24:59.822 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:24:59.822 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:24:59.839 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:24:59.839 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:24:59.855 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:24:59.855 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:24:59.872 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:24:59.872 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:24:59.889 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:24:59.889 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:24:59.924 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:24:59.946 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:24:59.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:59.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:24:59.984 INFO:tasks.workunit.client.0.smithi076.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T10:25:00.047 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:00.068 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:00.116 INFO:tasks.workunit.client.0.smithi076.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T10:25:00.156 INFO:tasks.workunit.client.0.smithi076.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:25:00.158 INFO:tasks.workunit.client.0.smithi076.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:25:00.158 INFO:tasks.workunit.client.0.smithi076.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T10:25:00.171 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:00.193 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:00.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:00.200 INFO:tasks.workunit.client.0.smithi076.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:25:00.220 DEBUG:teuthology.orchestra.run.smithi076:> set -ex 2024-09-21T10:25:00.220 DEBUG:teuthology.orchestra.run.smithi076:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T10:25:00.291 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-21T10:25:00.292 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-21T10:25:00.292 DEBUG:teuthology.orchestra.run.smithi076:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-21T10:25:00.295 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:00.321 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:00.423 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.445 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:00.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:00.500 INFO:tasks.workunit.client.0.smithi076.stdout:Running main() from gmock_main.cc 2024-09-21T10:25:00.500 INFO:tasks.workunit.client.0.smithi076.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-21T10:25:00.500 INFO:tasks.workunit.client.0.smithi076.stdout:[----------] Global test environment set-up. 2024-09-21T10:25:00.500 INFO:tasks.workunit.client.0.smithi076.stdout:[----------] 70 tests from TestClsRbd 2024-09-21T10:25:00.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:00.571 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:00.611 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:25:00.621 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:00 smithi076 ceph-mon[106482]: pgmap v2313: 83 pgs: 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:00.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4117599643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:25:00.622 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:00 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2947862417' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:00.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:00 smithi076 ceph-mon[110963]: pgmap v2313: 83 pgs: 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:00.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4117599643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:25:00.622 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:00 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2947862417' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:00.673 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:25:00.696 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:00.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:25:00.797 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:25:00.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:25:00.920 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:00.946 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:00.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:01.048 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:01.069 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:00 smithi191 ceph-mon[102206]: pgmap v2313: 83 pgs: 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4117599643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:25:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:00 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2947862417' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:01.171 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:01.197 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:01.299 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:01.320 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:01.422 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:01.444 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:01.546 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:01.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.573 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:01.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:01.627 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-21T10:25:01.651 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_all_features (24 ms) 2024-09-21T10:25:01.651 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.copyup 2024-09-21T10:25:01.674 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:01.697 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:01.784 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4117599643' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:25:01.784 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: osdmap e1399: 8 total, 8 up, 4 in 2024-09-21T10:25:01.784 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/112399901' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi076-358804-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:25:01.784 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: pgmap v2315: 115 pgs: 32 unknown, 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: pgmap v2316: 115 pgs: 32 unknown, 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4117599643' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: osdmap e1399: 8 total, 8 up, 4 in 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/112399901' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi076-358804-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:01.785 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:01.786 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: pgmap v2315: 115 pgs: 32 unknown, 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:01.786 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:25:01.786 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:25:01.786 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: pgmap v2316: 115 pgs: 32 unknown, 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:01.786 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:01 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:01.798 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:01.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:01.844 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.copyup (194 ms) 2024-09-21T10:25:01.844 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-21T10:25:01.887 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.sparse_copyup (42 ms) 2024-09-21T10:25:01.887 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-21T10:25:01.896 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_and_set_id (10 ms) 2024-09-21T10:25:01.896 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-21T10:25:01.906 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-09-21T10:25:01.907 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-21T10:25:01.921 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:01.933 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.directory_methods (27 ms) 2024-09-21T10:25:01.933 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.create 2024-09-21T10:25:01.942 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:01.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:01.958 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.create (24 ms) 2024-09-21T10:25:01.958 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_features 2024-09-21T10:25:01.959 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-21T10:25:01.959 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-21T10:25:01.961 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-21T10:25:01.961 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-21T10:25:01.962 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-21T10:25:01.962 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-21T10:25:01.963 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-09-21T10:25:01.963 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-21T10:25:01.964 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-21T10:25:01.964 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-21T10:25:01.967 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-21T10:25:01.968 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.get_size 2024-09-21T10:25:01.971 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-21T10:25:01.972 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.set_size 2024-09-21T10:25:01.976 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-09-21T10:25:01.980 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-21T10:25:01.994 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2024-09-21T10:25:01.994 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-21T10:25:02.000 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-21T10:25:02.001 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-21T10:25:02.042 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.parents_v1 (42 ms) 2024-09-21T10:25:02.042 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-21T10:25:02.044 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:02.065 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:02.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:02.070 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.parents_v2 (28 ms) 2024-09-21T10:25:02.070 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-21T10:25:02.091 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2024-09-21T10:25:02.091 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-21T10:25:02.097 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2024-09-21T10:25:02.097 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-21T10:25:02.106 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-21T10:25:02.106 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-21T10:25:02.110 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-09-21T10:25:02.110 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-21T10:25:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4117599643' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: osdmap e1399: 8 total, 8 up, 4 in 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/112399901' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi076-358804-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: pgmap v2315: 115 pgs: 32 unknown, 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: pgmap v2316: 115 pgs: 32 unknown, 83 active+clean; 629 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:01 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:02.118 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-09-21T10:25:02.118 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-21T10:25:02.127 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2024-09-21T10:25:02.127 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-21T10:25:02.127 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2024-09-21T10:25:02.127 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-21T10:25:02.134 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2024-09-21T10:25:02.134 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-21T10:25:02.142 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2024-09-21T10:25:02.142 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.flags 2024-09-21T10:25:02.148 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.flags (6 ms) 2024-09-21T10:25:02.148 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.metadata 2024-09-21T10:25:02.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:25:02.167 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:02.170 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.metadata (22 ms) 2024-09-21T10:25:02.170 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.set_features 2024-09-21T10:25:02.177 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.set_features (7 ms) 2024-09-21T10:25:02.177 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.mirror 2024-09-21T10:25:02.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:02.195 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:02.196 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:02.220 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.mirror (42 ms) 2024-09-21T10:25:02.220 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-21T10:25:02.241 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.mirror_image (22 ms) 2024-09-21T10:25:02.241 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-21T10:25:02.290 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:02.316 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:02.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:02.417 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:02.439 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:02.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:02.541 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:02.561 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:02.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:02.663 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:02.690 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:02.791 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:25:02.792 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/112399901' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi076-358804-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:25:02.792 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[106482]: osdmap e1400: 8 total, 8 up, 4 in 2024-09-21T10:25:02.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[106482]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:25:02.793 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[106482]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T10:25:02.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/112399901' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi076-358804-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:25:02.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[110963]: osdmap e1400: 8 total, 8 up, 4 in 2024-09-21T10:25:02.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[110963]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:25:02.793 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:02 smithi076 ceph-mon[110963]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T10:25:02.819 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:02.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:25:02.921 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:02.944 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:02.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:03.046 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:03.067 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:03.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/112399901' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi076-358804-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:25:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:02 smithi191 ceph-mon[102206]: osdmap e1400: 8 total, 8 up, 4 in 2024-09-21T10:25:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:02 smithi191 ceph-mon[102206]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:25:03.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:02 smithi191 ceph-mon[102206]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-21T10:25:03.168 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:03.189 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:03.291 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:03.311 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:03.413 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:03.417 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-21T10:25:03.436 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:03.537 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:03.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.560 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:03.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:25:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:25:03] "GET /metrics HTTP/1.1" 200 34206 "" "Prometheus/2.51.0" 2024-09-21T10:25:03.662 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:03.683 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:03.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:03.805 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:03 smithi076 ceph-mon[106482]: osdmap e1401: 8 total, 8 up, 4 in 2024-09-21T10:25:03.885 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:03 smithi076 ceph-mon[106482]: pgmap v2319: 115 pgs: 115 active+clean; 4.6 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 1.1 MiB/s rd, 1010 KiB/s wr, 218 op/s 2024-09-21T10:25:03.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:03 smithi076 ceph-mon[110963]: osdmap e1401: 8 total, 8 up, 4 in 2024-09-21T10:25:03.885 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:03 smithi076 ceph-mon[110963]: pgmap v2319: 115 pgs: 115 active+clean; 4.6 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 1.1 MiB/s rd, 1010 KiB/s wr, 218 op/s 2024-09-21T10:25:03.907 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:03.932 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:03.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:03.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:04.033 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:04.060 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:03 smithi191 ceph-mon[102206]: osdmap e1401: 8 total, 8 up, 4 in 2024-09-21T10:25:04.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:03 smithi191 ceph-mon[102206]: pgmap v2319: 115 pgs: 115 active+clean; 4.6 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 1.1 MiB/s rd, 1010 KiB/s wr, 218 op/s 2024-09-21T10:25:04.162 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:04.186 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:04.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:04.288 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:04.316 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.418 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:04.442 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:04.544 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:04.567 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.669 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.692 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:04.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:04.743 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.mirror_image_status (2501 ms) 2024-09-21T10:25:04.743 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-21T10:25:04.793 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:04.815 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:04.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:04.820 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.mirror_image_map (77 ms) 2024-09-21T10:25:04.820 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-21T10:25:04.827 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2024-09-21T10:25:04.827 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-21T10:25:04.838 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2024-09-21T10:25:04.838 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-21T10:25:04.840 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-09-21T10:25:04.840 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-21T10:25:04.842 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-21T10:25:04.842 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-21T10:25:04.845 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-21T10:25:04.845 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-21T10:25:04.850 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-09-21T10:25:04.850 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-21T10:25:04.853 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-21T10:25:04.853 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-21T10:25:04.856 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (3 ms) 2024-09-21T10:25:04.856 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-21T10:25:04.858 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-21T10:25:04.859 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-21T10:25:04.861 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-21T10:25:04.862 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-21T10:25:04.864 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-21T10:25:04.864 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-21T10:25:04.868 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-21T10:25:04.868 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-21T10:25:04.870 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-21T10:25:04.870 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-21T10:25:04.872 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.image_group_remove (2 ms) 2024-09-21T10:25:04.872 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-21T10:25:04.874 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-21T10:25:04.875 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-21T10:25:04.876 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-21T10:25:04.876 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-21T10:25:04.878 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-21T10:25:04.878 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-21T10:25:04.881 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-21T10:25:04.881 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-21T10:25:04.883 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2024-09-21T10:25:04.883 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-21T10:25:04.885 INFO:tasks.workunit.client.0.smithi076.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-871-g6a1bdf85/rpm/el9/BUILD/ceph-18.2.4-871-g6a1bdf85/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-21T10:25:04.885 INFO:tasks.workunit.client.0.smithi076.stdout:Expected equality of these values: 2024-09-21T10:25:04.885 INFO:tasks.workunit.client.0.smithi076.stdout: 1U 2024-09-21T10:25:04.885 INFO:tasks.workunit.client.0.smithi076.stdout: Which is: 1 2024-09-21T10:25:04.885 INFO:tasks.workunit.client.0.smithi076.stdout: keys.size() 2024-09-21T10:25:04.886 INFO:tasks.workunit.client.0.smithi076.stdout: Which is: 3 2024-09-21T10:25:04.886 INFO:tasks.workunit.client.0.smithi076.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-09-21T10:25:04.886 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-21T10:25:04.888 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-21T10:25:04.888 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-21T10:25:04.908 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (19 ms) 2024-09-21T10:25:04.908 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-21T10:25:04.917 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:04.938 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:04.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:04.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:04.941 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:04 smithi076 ceph-mon[106482]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:25:04.941 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/935066759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:04.941 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:04 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:04.942 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:04 smithi076 ceph-mon[110963]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:25:04.942 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/935066759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:04.942 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:04 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:05.040 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:05.063 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:05.102 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (195 ms) 2024-09-21T10:25:05.102 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-21T10:25:05.105 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-09-21T10:25:05.105 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-21T10:25:05.107 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-21T10:25:05.107 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-21T10:25:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:04 smithi191 ceph-mon[102206]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:25:05.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/935066759' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:04 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:05.114 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-21T10:25:05.114 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.op_features 2024-09-21T10:25:05.122 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-09-21T10:25:05.122 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-21T10:25:05.139 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2024-09-21T10:25:05.139 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-21T10:25:05.147 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-21T10:25:05.147 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-21T10:25:05.157 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-09-21T10:25:05.157 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-21T10:25:05.165 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3 2024-09-21T10:25:05.165 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.namespace_methods (8 ms) 2024-09-21T10:25:05.165 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.migration 2024-09-21T10:25:05.178 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.migration (13 ms) 2024-09-21T10:25:05.179 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-21T10:25:05.195 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:05.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.3' 2024-09-21T10:25:05.199 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.migration_v1 (21 ms) 2024-09-21T10:25:05.199 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-21T10:25:05.297 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:05.308 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:25:05.320 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:05.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:05.422 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:05.442 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:05.544 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:05.566 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:05.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:05.668 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:05.689 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:05.790 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:05.812 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:05.913 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:05.934 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:05.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:06.036 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6 2024-09-21T10:25:06.058 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.6' 2024-09-21T10:25:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:05 smithi191 ceph-mon[102206]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:25:06.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:05 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:25:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:05 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:05 smithi191 ceph-mon[102206]: osdmap e1402: 8 total, 8 up, 4 in 2024-09-21T10:25:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:05 smithi191 ceph-mon[102206]: pgmap v2321: 115 pgs: 115 active+clean; 4.6 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 1.1 MiB/s rd, 1010 KiB/s wr, 218 op/s 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[106482]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[106482]: osdmap e1402: 8 total, 8 up, 4 in 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[106482]: pgmap v2321: 115 pgs: 115 active+clean; 4.6 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 1.1 MiB/s rd, 1010 KiB/s wr, 218 op/s 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[110963]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:25:06.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[110963]: osdmap e1402: 8 total, 8 up, 4 in 2024-09-21T10:25:06.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:05 smithi076 ceph-mon[110963]: pgmap v2321: 115 pgs: 115 active+clean; 4.6 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 1.1 MiB/s rd, 1010 KiB/s wr, 218 op/s 2024-09-21T10:25:06.160 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:06.183 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:06.284 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1 2024-09-21T10:25:06.306 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.1' 2024-09-21T10:25:06.311 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1112 ms) 2024-09-21T10:25:06.311 INFO:tasks.workunit.client.0.smithi076.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-21T10:25:06.352 INFO:tasks.workunit.client.0.smithi076.stdout:[ OK ] TestClsRbd.sparsify (41 ms) 2024-09-21T10:25:06.407 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:06.429 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:06.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:06.552 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:06.654 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:06.676 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:06.740 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:06.741 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:06.779 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:06.801 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:06.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:06.902 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:06.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:06.926 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:06.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:07.027 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0 2024-09-21T10:25:07.049 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.0' 2024-09-21T10:25:07.151 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7 2024-09-21T10:25:07.172 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.7' 2024-09-21T10:25:07.274 DEBUG:teuthology.orchestra.run.smithi076:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2 2024-09-21T10:25:07.277 INFO:teuthology.orchestra.run.smithi076.stdout:{ 2024-09-21T10:25:07.278 INFO:teuthology.orchestra.run.smithi076.stdout: "ops": [], 2024-09-21T10:25:07.278 INFO:teuthology.orchestra.run.smithi076.stdout: "complaint_time": 30, 2024-09-21T10:25:07.278 INFO:teuthology.orchestra.run.smithi076.stdout: "num_blocked_ops": 0 2024-09-21T10:25:07.278 INFO:teuthology.orchestra.run.smithi076.stdout:} 2024-09-21T10:25:07.297 INFO:teuthology.orchestra.run.smithi076.stderr:sudo: docker: command not found 2024-09-21T10:25:07.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi076 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.2' 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout:[----------] 70 tests from TestClsRbd (4726 ms total) 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout: 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout:[----------] Global test environment tear-down 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout:[==========] 70 tests from 1 test suite ran. (6828 ms total) 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout:[ PASSED ] 69 tests. 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout:[ FAILED ] 1 test, listed below: 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout: 2024-09-21T10:25:07.329 INFO:tasks.workunit.client.0.smithi076.stdout: 1 FAILED TEST 2024-09-21T10:25:07.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.354 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-21T10:25:07.355 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:25:07.399 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4 2024-09-21T10:25:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.421 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:07.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.4' 2024-09-21T10:25:07.522 DEBUG:teuthology.orchestra.run.smithi191:> sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5 2024-09-21T10:25:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:25:07.544 INFO:teuthology.orchestra.run.smithi191.stderr:sudo: docker: command not found 2024-09-21T10:25:07.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi191 with status 1: 'sudo docker kill -s 1 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd.5' 2024-09-21T10:25:07.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:07 smithi191 ceph-mon[102206]: osdmap e1403: 8 total, 8 up, 4 in 2024-09-21T10:25:07.616 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:07 smithi076 ceph-mon[106482]: osdmap e1403: 8 total, 8 up, 4 in 2024-09-21T10:25:07.616 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:07 smithi076 ceph-mon[110963]: osdmap e1403: 8 total, 8 up, 4 in 2024-09-21T10:25:07.617 INFO:tasks.thrashosds:joining thrashosds 2024-09-21T10:25:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:08 smithi191 ceph-mon[102206]: osdmap e1404: 8 total, 8 up, 4 in 2024-09-21T10:25:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:08 smithi191 ceph-mon[102206]: pgmap v2324: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:08 smithi076 ceph-mon[106482]: osdmap e1404: 8 total, 8 up, 4 in 2024-09-21T10:25:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:08 smithi076 ceph-mon[106482]: pgmap v2324: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:08 smithi076 ceph-mon[110963]: osdmap e1404: 8 total, 8 up, 4 in 2024-09-21T10:25:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:08 smithi076 ceph-mon[110963]: pgmap v2324: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:08.676 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset noscrub 2024-09-21T10:25:09.496 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.0 dump_historic_ops 2024-09-21T10:25:09.767 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:09 smithi076 ceph-mon[106482]: pgmap v2325: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:09.768 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:09 smithi076 ceph-mon[110963]: pgmap v2325: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:09 smithi191 ceph-mon[102206]: pgmap v2325: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:11.085 INFO:teuthology.orchestra.run.smithi076.stdout:osd.0: {} 2024-09-21T10:25:11.086 INFO:teuthology.orchestra.run.smithi076.stdout:osd.1: {} 2024-09-21T10:25:11.086 INFO:teuthology.orchestra.run.smithi076.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-21T10:25:11.086 INFO:teuthology.orchestra.run.smithi076.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-21T10:25:11.094 INFO:teuthology.orchestra.run.smithi076.stdout:osd.2: {} 2024-09-21T10:25:11.094 INFO:teuthology.orchestra.run.smithi076.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-21T10:25:11.104 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:11.105 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:11.113 INFO:teuthology.orchestra.run.smithi076.stdout:osd.3: {} 2024-09-21T10:25:11.114 INFO:teuthology.orchestra.run.smithi076.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-21T10:25:11.136 INFO:teuthology.orchestra.run.smithi076.stdout:osd.4: {} 2024-09-21T10:25:11.137 INFO:teuthology.orchestra.run.smithi076.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-21T10:25:11.153 INFO:teuthology.orchestra.run.smithi076.stdout:osd.5: {} 2024-09-21T10:25:11.153 INFO:teuthology.orchestra.run.smithi076.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-21T10:25:11.169 INFO:teuthology.orchestra.run.smithi076.stdout:osd.6: {} 2024-09-21T10:25:11.170 INFO:teuthology.orchestra.run.smithi076.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-21T10:25:11.186 INFO:teuthology.orchestra.run.smithi076.stdout:osd.7: {} 2024-09-21T10:25:11.187 INFO:teuthology.orchestra.run.smithi076.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-21T10:25:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:11.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:11 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1832160595' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:11.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:11 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1832160595' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:11.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:11 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1832160595' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-21T10:25:12.707 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:12 smithi076 ceph-mon[106482]: pgmap v2326: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:12.707 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:12 smithi076 ceph-mon[110963]: pgmap v2326: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:12.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:12 smithi191 ceph-mon[102206]: pgmap v2326: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:13.578 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:25:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:25:13] "GET /metrics HTTP/1.1" 200 34202 "" "Prometheus/2.51.0" 2024-09-21T10:25:14.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:14 smithi191 ceph-mon[102206]: pgmap v2327: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:14.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:14 smithi076 ceph-mon[106482]: pgmap v2327: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:14.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:14 smithi076 ceph-mon[110963]: pgmap v2327: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:16.386 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:16.386 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:16.847 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:16 smithi076 ceph-mon[106482]: pgmap v2328: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:16.847 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:16 smithi076 ceph-mon[110963]: pgmap v2328: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:16 smithi191 ceph-mon[102206]: pgmap v2328: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:17.573 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-21T10:25:17.607 INFO:teuthology.orchestra.run.smithi076.stderr:noscrub is unset 2024-09-21T10:25:17.607 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2544858613' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2414256528' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2544858613' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2414256528' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:25:17.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:17 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:25:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:17 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2544858613' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:17 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-21T10:25:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:17 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2414256528' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:25:17.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:17 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:25:18.455 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:25:18.455 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:25:18.719 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[110963]: pgmap v2329: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[110963]: osdmap e1405: 8 total, 8 up, 4 in 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[106482]: pgmap v2329: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:25:18.720 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:18 smithi076 ceph-mon[106482]: osdmap e1405: 8 total, 8 up, 4 in 2024-09-21T10:25:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:18 smithi191 ceph-mon[102206]: pgmap v2329: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:18 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-21T10:25:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:18 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:25:18.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:18 smithi191 ceph-mon[102206]: osdmap e1405: 8 total, 8 up, 4 in 2024-09-21T10:25:19.043 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd unset nodeep-scrub 2024-09-21T10:25:19.055 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:19.765 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:19.799 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:20.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:20 smithi076 ceph-mon[106482]: pgmap v2331: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:20.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:20 smithi076 ceph-mon[110963]: pgmap v2331: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:20.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:20 smithi191 ceph-mon[102206]: pgmap v2331: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:21.742 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:21.743 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:22.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:22 smithi191 ceph-mon[102206]: pgmap v2332: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:22.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:22 smithi076 ceph-mon[110963]: pgmap v2332: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:22.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:22 smithi076 ceph-mon[106482]: pgmap v2332: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:23.505 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:25:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:25:23] "GET /metrics HTTP/1.1" 200 34202 "" "Prometheus/2.51.0" 2024-09-21T10:25:23.874 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_blocked_ops 2024-09-21T10:25:24.362 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:24.520 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:25:24.521 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:25:24.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:24 smithi191 ceph-mon[102206]: pgmap v2333: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:24.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:24 smithi076 ceph-mon[106482]: pgmap v2333: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:24.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:24 smithi076 ceph-mon[110963]: pgmap v2333: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:25.622 INFO:teuthology.orchestra.run.smithi076.stderr:nodeep-scrub is unset 2024-09-21T10:25:25.630 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2333,"stamp":"2024-09-21T10:25:23.574891+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590280,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":181,"num_write_kb":1351,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":1846594,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3750648,"kb_used_data":31368,"kb_used_omap":97,"kb_used_meta":3719134,"kb_avail":371229960,"statfs":{"total":383980142592,"available":380139479040,"internally_reserved":0,"allocated":32120832,"data_stored":21219799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":99354,"internal_metadata":3808394214},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002276"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":591,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591797+0000","last_change":"2024-09-21T10:23:39.397937+0000","last_active":"2024-09-21T10:25:18.591797+0000","last_peered":"2024-09-21T10:25:18.591797+0000","last_clean":"2024-09-21T10:25:18.591797+0000","last_became_active":"2024-09-21T10:23:39.397625+0000","last_became_peered":"2024-09-21T10:23:39.397625+0000","last_unstale":"2024-09-21T10:25:18.591797+0000","last_undegraded":"2024-09-21T10:25:18.591797+0000","last_fullsized":"2024-09-21T10:25:18.591797+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:27.988977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":242,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916943+0000","last_change":"2024-09-21T10:22:17.003755+0000","last_active":"2024-09-21T10:25:17.916943+0000","last_peered":"2024-09-21T10:25:17.916943+0000","last_clean":"2024-09-21T10:25:17.916943+0000","last_became_active":"2024-09-21T10:22:16.991551+0000","last_became_peered":"2024-09-21T10:22:16.991551+0000","last_unstale":"2024-09-21T10:25:17.916943+0000","last_undegraded":"2024-09-21T10:25:17.916943+0000","last_fullsized":"2024-09-21T10:25:17.916943+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:54.135786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":133,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579781+0000","last_change":"2024-09-21T10:23:39.376137+0000","last_active":"2024-09-21T10:25:07.579781+0000","last_peered":"2024-09-21T10:25:07.579781+0000","last_clean":"2024-09-21T10:25:07.579781+0000","last_became_active":"2024-09-21T10:23:39.375735+0000","last_became_peered":"2024-09-21T10:23:39.375735+0000","last_unstale":"2024-09-21T10:25:07.579781+0000","last_undegraded":"2024-09-21T10:25:07.579781+0000","last_fullsized":"2024-09-21T10:25:07.579781+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:14.995856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1908,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592143+0000","last_change":"2024-09-21T10:23:39.362192+0000","last_active":"2024-09-21T10:25:18.592143+0000","last_peered":"2024-09-21T10:25:18.592143+0000","last_clean":"2024-09-21T10:25:18.592143+0000","last_became_active":"2024-09-21T10:23:39.361929+0000","last_became_peered":"2024-09-21T10:23:39.361929+0000","last_unstale":"2024-09-21T10:25:18.592143+0000","last_undegraded":"2024-09-21T10:25:18.592143+0000","last_fullsized":"2024-09-21T10:25:18.592143+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:37.863408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1937,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592268+0000","last_change":"2024-09-21T10:23:39.363032+0000","last_active":"2024-09-21T10:25:18.592268+0000","last_peered":"2024-09-21T10:25:18.592268+0000","last_clean":"2024-09-21T10:25:18.592268+0000","last_became_active":"2024-09-21T10:23:39.362187+0000","last_became_peered":"2024-09-21T10:23:39.362187+0000","last_unstale":"2024-09-21T10:25:18.592268+0000","last_undegraded":"2024-09-21T10:25:18.592268+0000","last_fullsized":"2024-09-21T10:25:18.592268+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:52.282910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1137,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591896+0000","last_change":"2024-09-21T10:23:39.303739+0000","last_active":"2024-09-21T10:25:18.591896+0000","last_peered":"2024-09-21T10:25:18.591896+0000","last_clean":"2024-09-21T10:25:18.591896+0000","last_became_active":"2024-09-21T10:23:39.303418+0000","last_became_peered":"2024-09-21T10:23:39.303418+0000","last_unstale":"2024-09-21T10:25:18.591896+0000","last_undegraded":"2024-09-21T10:25:18.591896+0000","last_fullsized":"2024-09-21T10:25:18.591896+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:22.180697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":614,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592097+0000","last_change":"2024-09-21T10:23:39.364482+0000","last_active":"2024-09-21T10:25:18.592097+0000","last_peered":"2024-09-21T10:25:18.592097+0000","last_clean":"2024-09-21T10:25:18.592097+0000","last_became_active":"2024-09-21T10:23:39.363557+0000","last_became_peered":"2024-09-21T10:23:39.363557+0000","last_unstale":"2024-09-21T10:25:18.592097+0000","last_undegraded":"2024-09-21T10:25:18.592097+0000","last_fullsized":"2024-09-21T10:25:18.592097+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:10.008534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":307,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579460+0000","last_change":"2024-09-21T10:23:39.396151+0000","last_active":"2024-09-21T10:25:07.579460+0000","last_peered":"2024-09-21T10:25:07.579460+0000","last_clean":"2024-09-21T10:25:07.579460+0000","last_became_active":"2024-09-21T10:23:39.395820+0000","last_became_peered":"2024-09-21T10:23:39.395820+0000","last_unstale":"2024-09-21T10:25:07.579460+0000","last_undegraded":"2024-09-21T10:25:07.579460+0000","last_fullsized":"2024-09-21T10:25:07.579460+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:55:32.433084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":532,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916830+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:25:17.916830+0000","last_peered":"2024-09-21T10:25:17.916830+0000","last_clean":"2024-09-21T10:25:17.916830+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:25:17.916830+0000","last_undegraded":"2024-09-21T10:25:17.916830+0000","last_fullsized":"2024-09-21T10:25:17.916830+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":133,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.580460+0000","last_change":"2024-09-21T10:23:39.315690+0000","last_active":"2024-09-21T10:25:07.580460+0000","last_peered":"2024-09-21T10:25:07.580460+0000","last_clean":"2024-09-21T10:25:07.580460+0000","last_became_active":"2024-09-21T10:23:39.315283+0000","last_became_peered":"2024-09-21T10:23:39.315283+0000","last_unstale":"2024-09-21T10:25:07.580460+0000","last_undegraded":"2024-09-21T10:25:07.580460+0000","last_fullsized":"2024-09-21T10:25:07.580460+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:23:44.384937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":103,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916860+0000","last_change":"2024-09-21T10:22:17.009006+0000","last_active":"2024-09-21T10:25:17.916860+0000","last_peered":"2024-09-21T10:25:17.916860+0000","last_clean":"2024-09-21T10:25:17.916860+0000","last_became_active":"2024-09-21T10:22:16.993708+0000","last_became_peered":"2024-09-21T10:22:16.993708+0000","last_unstale":"2024-09-21T10:25:17.916860+0000","last_undegraded":"2024-09-21T10:25:17.916860+0000","last_fullsized":"2024-09-21T10:25:17.916860+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:46.476840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1617,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916289+0000","last_change":"2024-09-21T10:23:39.371348+0000","last_active":"2024-09-21T10:25:17.916289+0000","last_peered":"2024-09-21T10:25:17.916289+0000","last_clean":"2024-09-21T10:25:17.916289+0000","last_became_active":"2024-09-21T10:23:39.370757+0000","last_became_peered":"2024-09-21T10:23:39.370757+0000","last_unstale":"2024-09-21T10:25:17.916289+0000","last_undegraded":"2024-09-21T10:25:17.916289+0000","last_fullsized":"2024-09-21T10:25:17.916289+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:46.292157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":141,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579569+0000","last_change":"2024-09-21T10:23:39.319362+0000","last_active":"2024-09-21T10:25:07.579569+0000","last_peered":"2024-09-21T10:25:07.579569+0000","last_clean":"2024-09-21T10:25:07.579569+0000","last_became_active":"2024-09-21T10:23:39.318981+0000","last_became_peered":"2024-09-21T10:23:39.318981+0000","last_unstale":"2024-09-21T10:25:07.579569+0000","last_undegraded":"2024-09-21T10:25:07.579569+0000","last_fullsized":"2024-09-21T10:25:07.579569+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:54.616448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":876,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916388+0000","last_change":"2024-09-21T10:23:39.412337+0000","last_active":"2024-09-21T10:25:17.916388+0000","last_peered":"2024-09-21T10:25:17.916388+0000","last_clean":"2024-09-21T10:25:17.916388+0000","last_became_active":"2024-09-21T10:23:39.411958+0000","last_became_peered":"2024-09-21T10:23:39.411958+0000","last_unstale":"2024-09-21T10:25:17.916388+0000","last_undegraded":"2024-09-21T10:25:17.916388+0000","last_fullsized":"2024-09-21T10:25:17.916388+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:13.265451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":133,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579368+0000","last_change":"2024-09-21T10:23:39.372274+0000","last_active":"2024-09-21T10:25:07.579368+0000","last_peered":"2024-09-21T10:25:07.579368+0000","last_clean":"2024-09-21T10:25:07.579368+0000","last_became_active":"2024-09-21T10:23:39.371979+0000","last_became_peered":"2024-09-21T10:23:39.371979+0000","last_unstale":"2024-09-21T10:25:07.579368+0000","last_undegraded":"2024-09-21T10:25:07.579368+0000","last_fullsized":"2024-09-21T10:25:07.579368+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:03.528411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2003,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916741+0000","last_change":"2024-09-21T10:23:39.373141+0000","last_active":"2024-09-21T10:25:17.916741+0000","last_peered":"2024-09-21T10:25:17.916741+0000","last_clean":"2024-09-21T10:25:17.916741+0000","last_became_active":"2024-09-21T10:23:39.372532+0000","last_became_peered":"2024-09-21T10:23:39.372532+0000","last_unstale":"2024-09-21T10:25:17.916741+0000","last_undegraded":"2024-09-21T10:25:17.916741+0000","last_fullsized":"2024-09-21T10:25:17.916741+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:21.619250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"361'149","reported_seq":1506,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591828+0000","last_change":"2024-09-21T10:23:39.307102+0000","last_active":"2024-09-21T10:25:18.591828+0000","last_peered":"2024-09-21T10:25:18.591828+0000","last_clean":"2024-09-21T10:25:18.591828+0000","last_became_active":"2024-09-21T10:23:39.306777+0000","last_became_peered":"2024-09-21T10:23:39.306777+0000","last_unstale":"2024-09-21T10:25:18.591828+0000","last_undegraded":"2024-09-21T10:25:18.591828+0000","last_fullsized":"2024-09-21T10:25:18.591828+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:33.910694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":522,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591877+0000","last_change":"2024-09-21T10:23:39.398349+0000","last_active":"2024-09-21T10:25:18.591877+0000","last_peered":"2024-09-21T10:25:18.591877+0000","last_clean":"2024-09-21T10:25:18.591877+0000","last_became_active":"2024-09-21T10:23:39.397605+0000","last_became_peered":"2024-09-21T10:23:39.397605+0000","last_unstale":"2024-09-21T10:25:18.591877+0000","last_undegraded":"2024-09-21T10:25:18.591877+0000","last_fullsized":"2024-09-21T10:25:18.591877+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:37.735612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1611,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592100+0000","last_change":"2024-09-21T10:23:39.391183+0000","last_active":"2024-09-21T10:25:18.592100+0000","last_peered":"2024-09-21T10:25:18.592100+0000","last_clean":"2024-09-21T10:25:18.592100+0000","last_became_active":"2024-09-21T10:23:39.390886+0000","last_became_peered":"2024-09-21T10:23:39.390886+0000","last_unstale":"2024-09-21T10:25:18.592100+0000","last_undegraded":"2024-09-21T10:25:18.592100+0000","last_fullsized":"2024-09-21T10:25:18.592100+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:16:17.625754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"361'149","reported_seq":1506,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579688+0000","last_change":"2024-09-21T10:23:39.315754+0000","last_active":"2024-09-21T10:25:07.579688+0000","last_peered":"2024-09-21T10:25:07.579688+0000","last_clean":"2024-09-21T10:25:07.579688+0000","last_became_active":"2024-09-21T10:23:39.315278+0000","last_became_peered":"2024-09-21T10:23:39.315278+0000","last_unstale":"2024-09-21T10:25:07.579688+0000","last_undegraded":"2024-09-21T10:25:07.579688+0000","last_fullsized":"2024-09-21T10:25:07.579688+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:14.982755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:14.982755+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:08.540111+0000","scrub_duration":0.0028608990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":835,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916483+0000","last_change":"2024-09-21T10:23:39.411248+0000","last_active":"2024-09-21T10:25:17.916483+0000","last_peered":"2024-09-21T10:25:17.916483+0000","last_clean":"2024-09-21T10:25:17.916483+0000","last_became_active":"2024-09-21T10:23:39.410865+0000","last_became_peered":"2024-09-21T10:23:39.410865+0000","last_unstale":"2024-09-21T10:25:17.916483+0000","last_undegraded":"2024-09-21T10:25:17.916483+0000","last_fullsized":"2024-09-21T10:25:17.916483+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:47.337510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":211,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592607+0000","last_change":"2024-09-21T10:23:39.322783+0000","last_active":"2024-09-21T10:25:18.592607+0000","last_peered":"2024-09-21T10:25:18.592607+0000","last_clean":"2024-09-21T10:25:18.592607+0000","last_became_active":"2024-09-21T10:23:39.321617+0000","last_became_peered":"2024-09-21T10:23:39.321617+0000","last_unstale":"2024-09-21T10:25:18.592607+0000","last_undegraded":"2024-09-21T10:25:18.592607+0000","last_fullsized":"2024-09-21T10:25:18.592607+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:38:11.432422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1651,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916180+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:25:17.916180+0000","last_peered":"2024-09-21T10:25:17.916180+0000","last_clean":"2024-09-21T10:25:17.916180+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:25:17.916180+0000","last_undegraded":"2024-09-21T10:25:17.916180+0000","last_fullsized":"2024-09-21T10:25:17.916180+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":130,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916688+0000","last_change":"2024-09-21T10:23:39.349126+0000","last_active":"2024-09-21T10:25:17.916688+0000","last_peered":"2024-09-21T10:25:17.916688+0000","last_clean":"2024-09-21T10:25:17.916688+0000","last_became_active":"2024-09-21T10:23:39.348799+0000","last_became_peered":"2024-09-21T10:23:39.348799+0000","last_unstale":"2024-09-21T10:25:17.916688+0000","last_undegraded":"2024-09-21T10:25:17.916688+0000","last_fullsized":"2024-09-21T10:25:17.916688+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:04.424772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"361'149","reported_seq":1487,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579895+0000","last_change":"2024-09-21T10:24:55.184035+0000","last_active":"2024-09-21T10:25:07.579895+0000","last_peered":"2024-09-21T10:25:07.579895+0000","last_clean":"2024-09-21T10:25:07.579895+0000","last_became_active":"2024-09-21T10:22:20.013780+0000","last_became_peered":"2024-09-21T10:22:20.013780+0000","last_unstale":"2024-09-21T10:25:07.579895+0000","last_undegraded":"2024-09-21T10:25:07.579895+0000","last_fullsized":"2024-09-21T10:25:07.579895+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:55.183929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:55.183929+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:02.561714+0000","scrub_duration":0.001781761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":547,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591742+0000","last_change":"2024-09-21T10:23:39.416450+0000","last_active":"2024-09-21T10:25:18.591742+0000","last_peered":"2024-09-21T10:25:18.591742+0000","last_clean":"2024-09-21T10:25:18.591742+0000","last_became_active":"2024-09-21T10:23:39.414340+0000","last_became_peered":"2024-09-21T10:23:39.414340+0000","last_unstale":"2024-09-21T10:25:18.591742+0000","last_undegraded":"2024-09-21T10:25:18.591742+0000","last_fullsized":"2024-09-21T10:25:18.591742+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:48.376073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2007,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.917003+0000","last_change":"2024-09-21T10:23:39.350772+0000","last_active":"2024-09-21T10:25:17.917003+0000","last_peered":"2024-09-21T10:25:17.917003+0000","last_clean":"2024-09-21T10:25:17.917003+0000","last_became_active":"2024-09-21T10:23:39.345255+0000","last_became_peered":"2024-09-21T10:23:39.345255+0000","last_unstale":"2024-09-21T10:25:17.917003+0000","last_undegraded":"2024-09-21T10:25:17.917003+0000","last_fullsized":"2024-09-21T10:25:17.917003+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:22.523103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1511,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916288+0000","last_change":"2024-09-21T10:23:39.412844+0000","last_active":"2024-09-21T10:25:17.916288+0000","last_peered":"2024-09-21T10:25:17.916288+0000","last_clean":"2024-09-21T10:25:17.916288+0000","last_became_active":"2024-09-21T10:23:39.412591+0000","last_became_peered":"2024-09-21T10:23:39.412591+0000","last_unstale":"2024-09-21T10:25:17.916288+0000","last_undegraded":"2024-09-21T10:25:17.916288+0000","last_fullsized":"2024-09-21T10:25:17.916288+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:15.395609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:15.395609+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:33.836084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":144,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579354+0000","last_change":"2024-09-21T10:23:39.375299+0000","last_active":"2024-09-21T10:25:07.579354+0000","last_peered":"2024-09-21T10:25:07.579354+0000","last_clean":"2024-09-21T10:25:07.579354+0000","last_became_active":"2024-09-21T10:23:39.374244+0000","last_became_peered":"2024-09-21T10:23:39.374244+0000","last_unstale":"2024-09-21T10:25:07.579354+0000","last_undegraded":"2024-09-21T10:25:07.579354+0000","last_fullsized":"2024-09-21T10:25:07.579354+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:25:53.897017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":581,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.580320+0000","last_change":"2024-09-21T10:23:39.305168+0000","last_active":"2024-09-21T10:25:07.580320+0000","last_peered":"2024-09-21T10:25:07.580320+0000","last_clean":"2024-09-21T10:25:07.580320+0000","last_became_active":"2024-09-21T10:23:39.304815+0000","last_became_peered":"2024-09-21T10:23:39.304815+0000","last_unstale":"2024-09-21T10:25:07.580320+0000","last_undegraded":"2024-09-21T10:25:07.580320+0000","last_fullsized":"2024-09-21T10:25:07.580320+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:11.593431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":257,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592359+0000","last_change":"2024-09-21T10:23:39.322438+0000","last_active":"2024-09-21T10:25:18.592359+0000","last_peered":"2024-09-21T10:25:18.592359+0000","last_clean":"2024-09-21T10:25:18.592359+0000","last_became_active":"2024-09-21T10:23:39.321191+0000","last_became_peered":"2024-09-21T10:23:39.321191+0000","last_unstale":"2024-09-21T10:25:18.592359+0000","last_undegraded":"2024-09-21T10:25:18.592359+0000","last_fullsized":"2024-09-21T10:25:18.592359+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.957289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"361'149","reported_seq":1519,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916865+0000","last_change":"2024-09-21T10:22:20.017945+0000","last_active":"2024-09-21T10:25:17.916865+0000","last_peered":"2024-09-21T10:25:17.916865+0000","last_clean":"2024-09-21T10:25:17.916865+0000","last_became_active":"2024-09-21T10:22:20.017453+0000","last_became_peered":"2024-09-21T10:22:20.017453+0000","last_unstale":"2024-09-21T10:25:17.916865+0000","last_undegraded":"2024-09-21T10:25:17.916865+0000","last_fullsized":"2024-09-21T10:25:17.916865+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:32.811754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1299,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916199+0000","last_change":"2024-09-21T10:23:39.413518+0000","last_active":"2024-09-21T10:25:17.916199+0000","last_peered":"2024-09-21T10:25:17.916199+0000","last_clean":"2024-09-21T10:25:17.916199+0000","last_became_active":"2024-09-21T10:23:39.413270+0000","last_became_peered":"2024-09-21T10:23:39.413270+0000","last_unstale":"2024-09-21T10:25:17.916199+0000","last_undegraded":"2024-09-21T10:25:17.916199+0000","last_fullsized":"2024-09-21T10:25:17.916199+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:35.809985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1739,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591548+0000","last_change":"2024-09-21T10:23:39.316171+0000","last_active":"2024-09-21T10:25:18.591548+0000","last_peered":"2024-09-21T10:25:18.591548+0000","last_clean":"2024-09-21T10:25:18.591548+0000","last_became_active":"2024-09-21T10:23:39.315540+0000","last_became_peered":"2024-09-21T10:23:39.315540+0000","last_unstale":"2024-09-21T10:25:18.591548+0000","last_undegraded":"2024-09-21T10:25:18.591548+0000","last_fullsized":"2024-09-21T10:25:18.591548+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.638899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1282,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916709+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:25:17.916709+0000","last_peered":"2024-09-21T10:25:17.916709+0000","last_clean":"2024-09-21T10:25:17.916709+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:25:17.916709+0000","last_undegraded":"2024-09-21T10:25:17.916709+0000","last_fullsized":"2024-09-21T10:25:17.916709+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"361'149","reported_seq":1470,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.580032+0000","last_change":"2024-09-21T10:22:15.938120+0000","last_active":"2024-09-21T10:25:07.580032+0000","last_peered":"2024-09-21T10:25:07.580032+0000","last_clean":"2024-09-21T10:25:07.580032+0000","last_became_active":"2024-09-21T10:21:56.928301+0000","last_became_peered":"2024-09-21T10:21:56.928301+0000","last_unstale":"2024-09-21T10:25:07.580032+0000","last_undegraded":"2024-09-21T10:25:07.580032+0000","last_fullsized":"2024-09-21T10:25:07.580032+0000","mapping_epoch":1358,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1359,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:15.937862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:15.937862+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:01.614339+0000","scrub_duration":0.00329533,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1859,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591580+0000","last_change":"2024-09-21T10:23:39.450173+0000","last_active":"2024-09-21T10:25:18.591580+0000","last_peered":"2024-09-21T10:25:18.591580+0000","last_clean":"2024-09-21T10:25:18.591580+0000","last_became_active":"2024-09-21T10:23:39.449505+0000","last_became_peered":"2024-09-21T10:23:39.449505+0000","last_unstale":"2024-09-21T10:25:18.591580+0000","last_undegraded":"2024-09-21T10:25:18.591580+0000","last_fullsized":"2024-09-21T10:25:18.591580+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:04.778009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2030,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916416+0000","last_change":"2024-09-21T10:23:39.414249+0000","last_active":"2024-09-21T10:25:17.916416+0000","last_peered":"2024-09-21T10:25:17.916416+0000","last_clean":"2024-09-21T10:25:17.916416+0000","last_became_active":"2024-09-21T10:23:39.413975+0000","last_became_peered":"2024-09-21T10:23:39.413975+0000","last_unstale":"2024-09-21T10:25:17.916416+0000","last_undegraded":"2024-09-21T10:25:17.916416+0000","last_fullsized":"2024-09-21T10:25:17.916416+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:20.488016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2023,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916840+0000","last_change":"2024-09-21T10:23:39.412694+0000","last_active":"2024-09-21T10:25:17.916840+0000","last_peered":"2024-09-21T10:25:17.916840+0000","last_clean":"2024-09-21T10:25:17.916840+0000","last_became_active":"2024-09-21T10:23:39.412389+0000","last_became_peered":"2024-09-21T10:23:39.412389+0000","last_unstale":"2024-09-21T10:25:17.916840+0000","last_undegraded":"2024-09-21T10:25:17.916840+0000","last_fullsized":"2024-09-21T10:25:17.916840+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:30.102172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"361'149","reported_seq":1487,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.580223+0000","last_change":"2024-09-21T10:22:20.014154+0000","last_active":"2024-09-21T10:25:07.580223+0000","last_peered":"2024-09-21T10:25:07.580223+0000","last_clean":"2024-09-21T10:25:07.580223+0000","last_became_active":"2024-09-21T10:22:20.013516+0000","last_became_peered":"2024-09-21T10:22:20.013516+0000","last_unstale":"2024-09-21T10:25:07.580223+0000","last_undegraded":"2024-09-21T10:25:07.580223+0000","last_fullsized":"2024-09-21T10:25:07.580223+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:11.089486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:11.089486+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:11:55.464199+0000","scrub_duration":0.010125244,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":141,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579403+0000","last_change":"2024-09-21T10:23:39.346213+0000","last_active":"2024-09-21T10:25:07.579403+0000","last_peered":"2024-09-21T10:25:07.579403+0000","last_clean":"2024-09-21T10:25:07.579403+0000","last_became_active":"2024-09-21T10:23:39.346009+0000","last_became_peered":"2024-09-21T10:23:39.346009+0000","last_unstale":"2024-09-21T10:25:07.579403+0000","last_undegraded":"2024-09-21T10:25:07.579403+0000","last_fullsized":"2024-09-21T10:25:07.579403+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:34.154751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1220,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916406+0000","last_change":"2024-09-21T10:22:16.994613+0000","last_active":"2024-09-21T10:25:17.916406+0000","last_peered":"2024-09-21T10:25:17.916406+0000","last_clean":"2024-09-21T10:25:17.916406+0000","last_became_active":"2024-09-21T10:22:16.994182+0000","last_became_peered":"2024-09-21T10:22:16.994182+0000","last_unstale":"2024-09-21T10:25:17.916406+0000","last_undegraded":"2024-09-21T10:25:17.916406+0000","last_fullsized":"2024-09-21T10:25:17.916406+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:44.627749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1945,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592185+0000","last_change":"2024-09-21T10:23:39.315999+0000","last_active":"2024-09-21T10:25:18.592185+0000","last_peered":"2024-09-21T10:25:18.592185+0000","last_clean":"2024-09-21T10:25:18.592185+0000","last_became_active":"2024-09-21T10:23:39.315576+0000","last_became_peered":"2024-09-21T10:23:39.315576+0000","last_unstale":"2024-09-21T10:25:18.592185+0000","last_undegraded":"2024-09-21T10:25:18.592185+0000","last_fullsized":"2024-09-21T10:25:18.592185+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:21:28.524095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"361'149","reported_seq":1015,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592141+0000","last_change":"2024-09-21T10:23:39.364222+0000","last_active":"2024-09-21T10:25:18.592141+0000","last_peered":"2024-09-21T10:25:18.592141+0000","last_clean":"2024-09-21T10:25:18.592141+0000","last_became_active":"2024-09-21T10:23:39.363177+0000","last_became_peered":"2024-09-21T10:23:39.363177+0000","last_unstale":"2024-09-21T10:25:18.592141+0000","last_undegraded":"2024-09-21T10:25:18.592141+0000","last_fullsized":"2024-09-21T10:25:18.592141+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:36.892949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":322019,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":133,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579552+0000","last_change":"2024-09-21T10:23:39.376204+0000","last_active":"2024-09-21T10:25:07.579552+0000","last_peered":"2024-09-21T10:25:07.579552+0000","last_clean":"2024-09-21T10:25:07.579552+0000","last_became_active":"2024-09-21T10:23:39.376009+0000","last_became_peered":"2024-09-21T10:23:39.376009+0000","last_unstale":"2024-09-21T10:25:07.579552+0000","last_undegraded":"2024-09-21T10:25:07.579552+0000","last_fullsized":"2024-09-21T10:25:07.579552+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:43.583701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2038,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916434+0000","last_change":"2024-09-21T10:23:39.370875+0000","last_active":"2024-09-21T10:25:17.916434+0000","last_peered":"2024-09-21T10:25:17.916434+0000","last_clean":"2024-09-21T10:25:17.916434+0000","last_became_active":"2024-09-21T10:23:39.370134+0000","last_became_peered":"2024-09-21T10:23:39.370134+0000","last_unstale":"2024-09-21T10:25:17.916434+0000","last_undegraded":"2024-09-21T10:25:17.916434+0000","last_fullsized":"2024-09-21T10:25:17.916434+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:24.513775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":553,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916245+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:25:17.916245+0000","last_peered":"2024-09-21T10:25:17.916245+0000","last_clean":"2024-09-21T10:25:17.916245+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:25:17.916245+0000","last_undegraded":"2024-09-21T10:25:17.916245+0000","last_fullsized":"2024-09-21T10:25:17.916245+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"361'149","reported_seq":1015,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592308+0000","last_change":"2024-09-21T10:23:39.302896+0000","last_active":"2024-09-21T10:25:18.592308+0000","last_peered":"2024-09-21T10:25:18.592308+0000","last_clean":"2024-09-21T10:25:18.592308+0000","last_became_active":"2024-09-21T10:23:39.302269+0000","last_became_peered":"2024-09-21T10:23:39.302269+0000","last_unstale":"2024-09-21T10:25:18.592308+0000","last_undegraded":"2024-09-21T10:25:18.592308+0000","last_fullsized":"2024-09-21T10:25:18.592308+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:32.864718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":218,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591895+0000","last_change":"2024-09-21T10:23:39.303934+0000","last_active":"2024-09-21T10:25:18.591895+0000","last_peered":"2024-09-21T10:25:18.591895+0000","last_clean":"2024-09-21T10:25:18.591895+0000","last_became_active":"2024-09-21T10:23:39.303741+0000","last_became_peered":"2024-09-21T10:23:39.303741+0000","last_unstale":"2024-09-21T10:25:18.591895+0000","last_undegraded":"2024-09-21T10:25:18.591895+0000","last_fullsized":"2024-09-21T10:25:18.591895+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:12.138362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1184,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916685+0000","last_change":"2024-09-21T10:23:39.375346+0000","last_active":"2024-09-21T10:25:17.916685+0000","last_peered":"2024-09-21T10:25:17.916685+0000","last_clean":"2024-09-21T10:25:17.916685+0000","last_became_active":"2024-09-21T10:23:39.374849+0000","last_became_peered":"2024-09-21T10:23:39.374849+0000","last_unstale":"2024-09-21T10:25:17.916685+0000","last_undegraded":"2024-09-21T10:25:17.916685+0000","last_fullsized":"2024-09-21T10:25:17.916685+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:10.241167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":109,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.580792+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:25:07.580792+0000","last_peered":"2024-09-21T10:25:07.580792+0000","last_clean":"2024-09-21T10:25:07.580792+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:25:07.580792+0000","last_undegraded":"2024-09-21T10:25:07.580792+0000","last_fullsized":"2024-09-21T10:25:07.580792+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"361'149","reported_seq":1012,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.580847+0000","last_change":"2024-09-21T10:24:57.148783+0000","last_active":"2024-09-21T10:25:07.580847+0000","last_peered":"2024-09-21T10:25:07.580847+0000","last_clean":"2024-09-21T10:25:07.580847+0000","last_became_active":"2024-09-21T10:23:39.318900+0000","last_became_peered":"2024-09-21T10:23:39.318900+0000","last_unstale":"2024-09-21T10:25:07.580847+0000","last_undegraded":"2024-09-21T10:25:07.580847+0000","last_fullsized":"2024-09-21T10:25:07.580847+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:57.148677+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:57.148677+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:19.862692+0000","scrub_duration":0.001855659,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":520,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.582342+0000","last_change":"2024-09-21T10:23:39.305219+0000","last_active":"2024-09-21T10:25:07.582342+0000","last_peered":"2024-09-21T10:25:07.582342+0000","last_clean":"2024-09-21T10:25:07.582342+0000","last_became_active":"2024-09-21T10:23:39.304787+0000","last_became_peered":"2024-09-21T10:23:39.304787+0000","last_unstale":"2024-09-21T10:25:07.582342+0000","last_undegraded":"2024-09-21T10:25:07.582342+0000","last_fullsized":"2024-09-21T10:25:07.582342+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:14.699319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":110,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916615+0000","last_change":"2024-09-21T10:23:39.347172+0000","last_active":"2024-09-21T10:25:17.916615+0000","last_peered":"2024-09-21T10:25:17.916615+0000","last_clean":"2024-09-21T10:25:17.916615+0000","last_became_active":"2024-09-21T10:23:39.345326+0000","last_became_peered":"2024-09-21T10:23:39.345326+0000","last_unstale":"2024-09-21T10:25:17.916615+0000","last_undegraded":"2024-09-21T10:25:17.916615+0000","last_fullsized":"2024-09-21T10:25:17.916615+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:32.454312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":543,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591812+0000","last_change":"2024-09-21T10:23:39.415241+0000","last_active":"2024-09-21T10:25:18.591812+0000","last_peered":"2024-09-21T10:25:18.591812+0000","last_clean":"2024-09-21T10:25:18.591812+0000","last_became_active":"2024-09-21T10:23:39.412720+0000","last_became_peered":"2024-09-21T10:23:39.412720+0000","last_unstale":"2024-09-21T10:25:18.591812+0000","last_undegraded":"2024-09-21T10:25:18.591812+0000","last_fullsized":"2024-09-21T10:25:18.591812+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:09.855474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"361'149","reported_seq":1510,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592190+0000","last_change":"2024-09-21T10:23:39.316659+0000","last_active":"2024-09-21T10:25:18.592190+0000","last_peered":"2024-09-21T10:25:18.592190+0000","last_clean":"2024-09-21T10:25:18.592190+0000","last_became_active":"2024-09-21T10:23:39.316129+0000","last_became_peered":"2024-09-21T10:23:39.316129+0000","last_unstale":"2024-09-21T10:25:18.592190+0000","last_undegraded":"2024-09-21T10:25:18.592190+0000","last_fullsized":"2024-09-21T10:25:18.592190+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:35.853740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":538,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916822+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:25:17.916822+0000","last_peered":"2024-09-21T10:25:17.916822+0000","last_clean":"2024-09-21T10:25:17.916822+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:25:17.916822+0000","last_undegraded":"2024-09-21T10:25:17.916822+0000","last_fullsized":"2024-09-21T10:25:17.916822+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":865,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592207+0000","last_change":"2024-09-21T10:23:39.363897+0000","last_active":"2024-09-21T10:25:18.592207+0000","last_peered":"2024-09-21T10:25:18.592207+0000","last_clean":"2024-09-21T10:25:18.592207+0000","last_became_active":"2024-09-21T10:23:39.362798+0000","last_became_peered":"2024-09-21T10:23:39.362798+0000","last_unstale":"2024-09-21T10:25:18.592207+0000","last_undegraded":"2024-09-21T10:25:18.592207+0000","last_fullsized":"2024-09-21T10:25:18.592207+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:04.258680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":133,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.581543+0000","last_change":"2024-09-21T10:23:39.306440+0000","last_active":"2024-09-21T10:25:07.581543+0000","last_peered":"2024-09-21T10:25:07.581543+0000","last_clean":"2024-09-21T10:25:07.581543+0000","last_became_active":"2024-09-21T10:23:39.306047+0000","last_became_peered":"2024-09-21T10:23:39.306047+0000","last_unstale":"2024-09-21T10:25:07.581543+0000","last_undegraded":"2024-09-21T10:25:07.581543+0000","last_fullsized":"2024-09-21T10:25:07.581543+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:29.308333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1221,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916767+0000","last_change":"2024-09-21T10:22:16.993047+0000","last_active":"2024-09-21T10:25:17.916767+0000","last_peered":"2024-09-21T10:25:17.916767+0000","last_clean":"2024-09-21T10:25:17.916767+0000","last_became_active":"2024-09-21T10:22:16.992606+0000","last_became_peered":"2024-09-21T10:22:16.992606+0000","last_unstale":"2024-09-21T10:25:17.916767+0000","last_undegraded":"2024-09-21T10:25:17.916767+0000","last_fullsized":"2024-09-21T10:25:17.916767+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:35.971920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":288,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.581812+0000","last_change":"2024-09-21T10:23:39.308252+0000","last_active":"2024-09-21T10:25:07.581812+0000","last_peered":"2024-09-21T10:25:07.581812+0000","last_clean":"2024-09-21T10:25:07.581812+0000","last_became_active":"2024-09-21T10:23:39.306643+0000","last_became_peered":"2024-09-21T10:23:39.306643+0000","last_unstale":"2024-09-21T10:25:07.581812+0000","last_undegraded":"2024-09-21T10:25:07.581812+0000","last_fullsized":"2024-09-21T10:25:07.581812+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:07.074129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2006,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916461+0000","last_change":"2024-09-21T10:22:16.994417+0000","last_active":"2024-09-21T10:25:17.916461+0000","last_peered":"2024-09-21T10:25:17.916461+0000","last_clean":"2024-09-21T10:25:17.916461+0000","last_became_active":"2024-09-21T10:22:16.994109+0000","last_became_peered":"2024-09-21T10:22:16.994109+0000","last_unstale":"2024-09-21T10:25:17.916461+0000","last_undegraded":"2024-09-21T10:25:17.916461+0000","last_fullsized":"2024-09-21T10:25:17.916461+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:55.406178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1507,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592582+0000","last_change":"2024-09-21T10:23:39.323087+0000","last_active":"2024-09-21T10:25:18.592582+0000","last_peered":"2024-09-21T10:25:18.592582+0000","last_clean":"2024-09-21T10:25:18.592582+0000","last_became_active":"2024-09-21T10:23:39.322041+0000","last_became_peered":"2024-09-21T10:23:39.322041+0000","last_unstale":"2024-09-21T10:25:18.592582+0000","last_undegraded":"2024-09-21T10:25:18.592582+0000","last_fullsized":"2024-09-21T10:25:18.592582+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:29.126970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":124,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579549+0000","last_change":"2024-09-21T10:23:39.334300+0000","last_active":"2024-09-21T10:25:07.579549+0000","last_peered":"2024-09-21T10:25:07.579549+0000","last_clean":"2024-09-21T10:25:07.579549+0000","last_became_active":"2024-09-21T10:23:39.334044+0000","last_became_peered":"2024-09-21T10:23:39.334044+0000","last_unstale":"2024-09-21T10:25:07.579549+0000","last_undegraded":"2024-09-21T10:25:07.579549+0000","last_fullsized":"2024-09-21T10:25:07.579549+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:26:36.147498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":649,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591779+0000","last_change":"2024-09-21T10:23:39.306893+0000","last_active":"2024-09-21T10:25:18.591779+0000","last_peered":"2024-09-21T10:25:18.591779+0000","last_clean":"2024-09-21T10:25:18.591779+0000","last_became_active":"2024-09-21T10:23:39.306560+0000","last_became_peered":"2024-09-21T10:23:39.306560+0000","last_unstale":"2024-09-21T10:25:18.591779+0000","last_undegraded":"2024-09-21T10:25:18.591779+0000","last_fullsized":"2024-09-21T10:25:18.591779+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:16.170412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":133,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579614+0000","last_change":"2024-09-21T10:23:39.337324+0000","last_active":"2024-09-21T10:25:07.579614+0000","last_peered":"2024-09-21T10:25:07.579614+0000","last_clean":"2024-09-21T10:25:07.579614+0000","last_became_active":"2024-09-21T10:23:39.337017+0000","last_became_peered":"2024-09-21T10:23:39.337017+0000","last_unstale":"2024-09-21T10:25:07.579614+0000","last_undegraded":"2024-09-21T10:25:07.579614+0000","last_fullsized":"2024-09-21T10:25:07.579614+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:53.271860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":633,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591908+0000","last_change":"2024-09-21T10:23:39.414802+0000","last_active":"2024-09-21T10:25:18.591908+0000","last_peered":"2024-09-21T10:25:18.591908+0000","last_clean":"2024-09-21T10:25:18.591908+0000","last_became_active":"2024-09-21T10:23:39.411110+0000","last_became_peered":"2024-09-21T10:23:39.411110+0000","last_unstale":"2024-09-21T10:25:18.591908+0000","last_undegraded":"2024-09-21T10:25:18.591908+0000","last_fullsized":"2024-09-21T10:25:18.591908+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:27.837285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":268,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.579592+0000","last_change":"2024-09-21T10:23:39.337324+0000","last_active":"2024-09-21T10:25:07.579592+0000","last_peered":"2024-09-21T10:25:07.579592+0000","last_clean":"2024-09-21T10:25:07.579592+0000","last_became_active":"2024-09-21T10:23:39.337017+0000","last_became_peered":"2024-09-21T10:23:39.337017+0000","last_unstale":"2024-09-21T10:25:07.579592+0000","last_undegraded":"2024-09-21T10:25:07.579592+0000","last_fullsized":"2024-09-21T10:25:07.579592+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:43.984727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":591,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591639+0000","last_change":"2024-09-21T10:23:39.415574+0000","last_active":"2024-09-21T10:25:18.591639+0000","last_peered":"2024-09-21T10:25:18.591639+0000","last_clean":"2024-09-21T10:25:18.591639+0000","last_became_active":"2024-09-21T10:23:39.413699+0000","last_became_peered":"2024-09-21T10:23:39.413699+0000","last_unstale":"2024-09-21T10:25:18.591639+0000","last_undegraded":"2024-09-21T10:25:18.591639+0000","last_fullsized":"2024-09-21T10:25:18.591639+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:11.990863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1854,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592363+0000","last_change":"2024-09-21T10:23:39.363264+0000","last_active":"2024-09-21T10:25:18.592363+0000","last_peered":"2024-09-21T10:25:18.592363+0000","last_clean":"2024-09-21T10:25:18.592363+0000","last_became_active":"2024-09-21T10:23:39.362615+0000","last_became_peered":"2024-09-21T10:23:39.362615+0000","last_unstale":"2024-09-21T10:25:18.592363+0000","last_undegraded":"2024-09-21T10:25:18.592363+0000","last_fullsized":"2024-09-21T10:25:18.592363+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:55.561333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":296,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916289+0000","last_change":"2024-09-21T10:23:39.371060+0000","last_active":"2024-09-21T10:25:17.916289+0000","last_peered":"2024-09-21T10:25:17.916289+0000","last_clean":"2024-09-21T10:25:17.916289+0000","last_became_active":"2024-09-21T10:23:39.370494+0000","last_became_peered":"2024-09-21T10:23:39.370494+0000","last_unstale":"2024-09-21T10:25:17.916289+0000","last_undegraded":"2024-09-21T10:25:17.916289+0000","last_fullsized":"2024-09-21T10:25:17.916289+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:09.975626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":516,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592694+0000","last_change":"2024-09-21T10:23:39.510198+0000","last_active":"2024-09-21T10:25:18.592694+0000","last_peered":"2024-09-21T10:25:18.592694+0000","last_clean":"2024-09-21T10:25:18.592694+0000","last_became_active":"2024-09-21T10:23:39.509474+0000","last_became_peered":"2024-09-21T10:23:39.509474+0000","last_unstale":"2024-09-21T10:25:18.592694+0000","last_undegraded":"2024-09-21T10:25:18.592694+0000","last_fullsized":"2024-09-21T10:25:18.592694+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.611788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":556,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916546+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:25:17.916546+0000","last_peered":"2024-09-21T10:25:17.916546+0000","last_clean":"2024-09-21T10:25:17.916546+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:25:17.916546+0000","last_undegraded":"2024-09-21T10:25:17.916546+0000","last_fullsized":"2024-09-21T10:25:17.916546+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1896,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916530+0000","last_change":"2024-09-21T10:23:39.370592+0000","last_active":"2024-09-21T10:25:17.916530+0000","last_peered":"2024-09-21T10:25:17.916530+0000","last_clean":"2024-09-21T10:25:17.916530+0000","last_became_active":"2024-09-21T10:23:39.369832+0000","last_became_peered":"2024-09-21T10:23:39.369832+0000","last_unstale":"2024-09-21T10:25:17.916530+0000","last_undegraded":"2024-09-21T10:25:17.916530+0000","last_fullsized":"2024-09-21T10:25:17.916530+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:23.320589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1970,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916625+0000","last_change":"2024-09-21T10:23:39.373474+0000","last_active":"2024-09-21T10:25:17.916625+0000","last_peered":"2024-09-21T10:25:17.916625+0000","last_clean":"2024-09-21T10:25:17.916625+0000","last_became_active":"2024-09-21T10:23:39.373158+0000","last_became_peered":"2024-09-21T10:23:39.373158+0000","last_unstale":"2024-09-21T10:25:17.916625+0000","last_undegraded":"2024-09-21T10:25:17.916625+0000","last_fullsized":"2024-09-21T10:25:17.916625+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:53.362317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1498,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591705+0000","last_change":"2024-09-21T10:23:39.398601+0000","last_active":"2024-09-21T10:25:18.591705+0000","last_peered":"2024-09-21T10:25:18.591705+0000","last_clean":"2024-09-21T10:25:18.591705+0000","last_became_active":"2024-09-21T10:23:39.398064+0000","last_became_peered":"2024-09-21T10:23:39.398064+0000","last_unstale":"2024-09-21T10:25:18.591705+0000","last_undegraded":"2024-09-21T10:25:18.591705+0000","last_fullsized":"2024-09-21T10:25:18.591705+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:05.118881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":113,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410765+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:25:18.410765+0000","last_peered":"2024-09-21T10:25:18.410765+0000","last_clean":"2024-09-21T10:25:18.410765+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:25:18.410765+0000","last_undegraded":"2024-09-21T10:25:18.410765+0000","last_fullsized":"2024-09-21T10:25:18.410765+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1560,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592375+0000","last_change":"2024-09-21T10:23:39.302265+0000","last_active":"2024-09-21T10:25:18.592375+0000","last_peered":"2024-09-21T10:25:18.592375+0000","last_clean":"2024-09-21T10:25:18.592375+0000","last_became_active":"2024-09-21T10:23:39.300210+0000","last_became_peered":"2024-09-21T10:23:39.300210+0000","last_unstale":"2024-09-21T10:25:18.592375+0000","last_undegraded":"2024-09-21T10:25:18.592375+0000","last_fullsized":"2024-09-21T10:25:18.592375+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:23.615984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":141,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410740+0000","last_change":"2024-09-21T10:23:39.375969+0000","last_active":"2024-09-21T10:25:18.410740+0000","last_peered":"2024-09-21T10:25:18.410740+0000","last_clean":"2024-09-21T10:25:18.410740+0000","last_became_active":"2024-09-21T10:23:39.375686+0000","last_became_peered":"2024-09-21T10:23:39.375686+0000","last_unstale":"2024-09-21T10:25:18.410740+0000","last_undegraded":"2024-09-21T10:25:18.410740+0000","last_fullsized":"2024-09-21T10:25:18.410740+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:18:38.252378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":591,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.591663+0000","last_change":"2024-09-21T10:23:39.450465+0000","last_active":"2024-09-21T10:25:18.591663+0000","last_peered":"2024-09-21T10:25:18.591663+0000","last_clean":"2024-09-21T10:25:18.591663+0000","last_became_active":"2024-09-21T10:23:39.449879+0000","last_became_peered":"2024-09-21T10:23:39.449879+0000","last_unstale":"2024-09-21T10:25:18.591663+0000","last_undegraded":"2024-09-21T10:25:18.591663+0000","last_fullsized":"2024-09-21T10:25:18.591663+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:58.113689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":623,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.592056+0000","last_change":"2024-09-21T10:23:39.361838+0000","last_active":"2024-09-21T10:25:18.592056+0000","last_peered":"2024-09-21T10:25:18.592056+0000","last_clean":"2024-09-21T10:25:18.592056+0000","last_became_active":"2024-09-21T10:23:39.361539+0000","last_became_peered":"2024-09-21T10:23:39.361539+0000","last_unstale":"2024-09-21T10:25:18.592056+0000","last_undegraded":"2024-09-21T10:25:18.592056+0000","last_fullsized":"2024-09-21T10:25:18.592056+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:15:39.263560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":241,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:17.916170+0000","last_change":"2024-09-21T10:23:39.378606+0000","last_active":"2024-09-21T10:25:17.916170+0000","last_peered":"2024-09-21T10:25:17.916170+0000","last_clean":"2024-09-21T10:25:17.916170+0000","last_became_active":"2024-09-21T10:23:39.378107+0000","last_became_peered":"2024-09-21T10:23:39.378107+0000","last_unstale":"2024-09-21T10:25:17.916170+0000","last_undegraded":"2024-09-21T10:25:17.916170+0000","last_fullsized":"2024-09-21T10:25:17.916170+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:11.536250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":145,"reported_epoch":1404,"state":"active+clean","last_fresh":"2024-09-21T10:25:07.583127+0000","last_change":"2024-09-21T10:23:39.305452+0000","last_active":"2024-09-21T10:25:07.583127+0000","last_peered":"2024-09-21T10:25:07.583127+0000","last_clean":"2024-09-21T10:25:07.583127+0000","last_became_active":"2024-09-21T10:23:39.305054+0000","last_became_peered":"2024-09-21T10:23:39.305054+0000","last_unstale":"2024-09-21T10:25:07.583127+0000","last_undegraded":"2024-09-21T10:25:07.583127+0000","last_fullsized":"2024-09-21T10:25:07.583127+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:30.307977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":21744,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19584,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":584707,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":117,"num_write_kb":1318,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1824774,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234594,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496230,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757869,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":710920,"kb_used_data":7996,"kb_used_omap":1,"kb_used_meta":702910,"kb_avail":93034232,"statfs":{"total":95995035648,"available":95267053568,"internally_reserved":0,"allocated":8187904,"data_stored":5453929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":719780294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019505,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333508,"kb_used_data":7992,"kb_used_omap":1,"kb_used_meta":1325502,"kb_avail":92411644,"statfs":{"total":95995035648,"available":94629523456,"internally_reserved":0,"allocated":8183808,"data_stored":5453910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1357314502},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1375,"seq":5905580032022,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":189824,"kb_used_data":7988,"kb_used_omap":52,"kb_used_meta":181771,"kb_avail":93555328,"statfs":{"total":95995035648,"available":95800655872,"internally_reserved":0,"allocated":8179712,"data_stored":5453885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54095,"internal_metadata":186133681},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793968040,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047279,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516396,"kb_used_data":7392,"kb_used_omap":41,"kb_used_meta":1508950,"kb_avail":92228756,"statfs":{"total":95995035648,"available":94442246144,"internally_reserved":0,"allocated":7569408,"data_stored":4858075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1545165737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928913,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10112,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:25:25.634 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:25:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[106482]: from='client.46310 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1993177495' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:25.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[110963]: from='client.46310 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1993177495' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:25.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:25 smithi191 ceph-mon[102206]: from='client.46310 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:25 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1993177495' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:25 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-21T10:25:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:26 smithi076 ceph-mon[106482]: pgmap v2334: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:26 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:26 smithi076 ceph-mon[106482]: osdmap e1406: 8 total, 8 up, 4 in 2024-09-21T10:25:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:26 smithi076 ceph-mon[110963]: pgmap v2334: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:26.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:26 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:26 smithi076 ceph-mon[110963]: osdmap e1406: 8 total, 8 up, 4 in 2024-09-21T10:25:27.061 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:26 smithi191 ceph-mon[102206]: pgmap v2334: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:26 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-21T10:25:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:26 smithi191 ceph-mon[102206]: osdmap e1406: 8 total, 8 up, 4 in 2024-09-21T10:25:28.434 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:28.434 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:28.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:28 smithi076 ceph-mon[110963]: pgmap v2336: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:28.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:28 smithi076 ceph-mon[106482]: pgmap v2336: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:28 smithi191 ceph-mon[102206]: pgmap v2336: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:29.729 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:29 smithi076 ceph-mon[106482]: pgmap v2337: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:29.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:29 smithi076 ceph-mon[110963]: pgmap v2337: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:29.993 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph daemon osd.3 dump_historic_ops 2024-09-21T10:25:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:29 smithi191 ceph-mon[102206]: pgmap v2337: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:30.570 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:30.729 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:25:30.730 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:25:32.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:31 smithi076 ceph-mon[106482]: from='client.36423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:32.049 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:31 smithi076 ceph-mon[106482]: pgmap v2338: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:32.049 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:31 smithi076 ceph-mon[110963]: from='client.36423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:32.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:31 smithi076 ceph-mon[110963]: pgmap v2338: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:32.071 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2337,"stamp":"2024-09-21T10:25:29.576362+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590280,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":181,"num_write_kb":1351,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":1846594,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3750684,"kb_used_data":31404,"kb_used_omap":97,"kb_used_meta":3719134,"kb_avail":371229924,"statfs":{"total":383980142592,"available":380139442176,"internally_reserved":0,"allocated":32157696,"data_stored":21243742,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":99354,"internal_metadata":3808394214},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002323"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":593,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815306+0000","last_change":"2024-09-21T10:23:39.397937+0000","last_active":"2024-09-21T10:25:25.815306+0000","last_peered":"2024-09-21T10:25:25.815306+0000","last_clean":"2024-09-21T10:25:25.815306+0000","last_became_active":"2024-09-21T10:23:39.397625+0000","last_became_peered":"2024-09-21T10:23:39.397625+0000","last_unstale":"2024-09-21T10:25:25.815306+0000","last_undegraded":"2024-09-21T10:25:25.815306+0000","last_fullsized":"2024-09-21T10:25:25.815306+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:27.988977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":244,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816369+0000","last_change":"2024-09-21T10:22:17.003755+0000","last_active":"2024-09-21T10:25:25.816369+0000","last_peered":"2024-09-21T10:25:25.816369+0000","last_clean":"2024-09-21T10:25:25.816369+0000","last_became_active":"2024-09-21T10:22:16.991551+0000","last_became_peered":"2024-09-21T10:22:16.991551+0000","last_unstale":"2024-09-21T10:25:25.816369+0000","last_undegraded":"2024-09-21T10:25:25.816369+0000","last_fullsized":"2024-09-21T10:25:25.816369+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:54.135786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410728+0000","last_change":"2024-09-21T10:23:39.376137+0000","last_active":"2024-09-21T10:25:18.410728+0000","last_peered":"2024-09-21T10:25:18.410728+0000","last_clean":"2024-09-21T10:25:18.410728+0000","last_became_active":"2024-09-21T10:23:39.375735+0000","last_became_peered":"2024-09-21T10:23:39.375735+0000","last_unstale":"2024-09-21T10:25:18.410728+0000","last_undegraded":"2024-09-21T10:25:18.410728+0000","last_fullsized":"2024-09-21T10:25:18.410728+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:14.995856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1910,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814729+0000","last_change":"2024-09-21T10:23:39.362192+0000","last_active":"2024-09-21T10:25:25.814729+0000","last_peered":"2024-09-21T10:25:25.814729+0000","last_clean":"2024-09-21T10:25:25.814729+0000","last_became_active":"2024-09-21T10:23:39.361929+0000","last_became_peered":"2024-09-21T10:23:39.361929+0000","last_unstale":"2024-09-21T10:25:25.814729+0000","last_undegraded":"2024-09-21T10:25:25.814729+0000","last_fullsized":"2024-09-21T10:25:25.814729+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:37.863408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1939,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815718+0000","last_change":"2024-09-21T10:23:39.363032+0000","last_active":"2024-09-21T10:25:25.815718+0000","last_peered":"2024-09-21T10:25:25.815718+0000","last_clean":"2024-09-21T10:25:25.815718+0000","last_became_active":"2024-09-21T10:23:39.362187+0000","last_became_peered":"2024-09-21T10:23:39.362187+0000","last_unstale":"2024-09-21T10:25:25.815718+0000","last_undegraded":"2024-09-21T10:25:25.815718+0000","last_fullsized":"2024-09-21T10:25:25.815718+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:52.282910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1139,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815040+0000","last_change":"2024-09-21T10:23:39.303739+0000","last_active":"2024-09-21T10:25:25.815040+0000","last_peered":"2024-09-21T10:25:25.815040+0000","last_clean":"2024-09-21T10:25:25.815040+0000","last_became_active":"2024-09-21T10:23:39.303418+0000","last_became_peered":"2024-09-21T10:23:39.303418+0000","last_unstale":"2024-09-21T10:25:25.815040+0000","last_undegraded":"2024-09-21T10:25:25.815040+0000","last_fullsized":"2024-09-21T10:25:25.815040+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:22.180697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":616,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815349+0000","last_change":"2024-09-21T10:23:39.364482+0000","last_active":"2024-09-21T10:25:25.815349+0000","last_peered":"2024-09-21T10:25:25.815349+0000","last_clean":"2024-09-21T10:25:25.815349+0000","last_became_active":"2024-09-21T10:23:39.363557+0000","last_became_peered":"2024-09-21T10:23:39.363557+0000","last_unstale":"2024-09-21T10:25:25.815349+0000","last_undegraded":"2024-09-21T10:25:25.815349+0000","last_fullsized":"2024-09-21T10:25:25.815349+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:10.008534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":308,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.409980+0000","last_change":"2024-09-21T10:23:39.396151+0000","last_active":"2024-09-21T10:25:18.409980+0000","last_peered":"2024-09-21T10:25:18.409980+0000","last_clean":"2024-09-21T10:25:18.409980+0000","last_became_active":"2024-09-21T10:23:39.395820+0000","last_became_peered":"2024-09-21T10:23:39.395820+0000","last_unstale":"2024-09-21T10:25:18.409980+0000","last_undegraded":"2024-09-21T10:25:18.409980+0000","last_fullsized":"2024-09-21T10:25:18.409980+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:55:32.433084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":534,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816321+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:25:25.816321+0000","last_peered":"2024-09-21T10:25:25.816321+0000","last_clean":"2024-09-21T10:25:25.816321+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:25:25.816321+0000","last_undegraded":"2024-09-21T10:25:25.816321+0000","last_fullsized":"2024-09-21T10:25:25.816321+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410210+0000","last_change":"2024-09-21T10:23:39.315690+0000","last_active":"2024-09-21T10:25:18.410210+0000","last_peered":"2024-09-21T10:25:18.410210+0000","last_clean":"2024-09-21T10:25:18.410210+0000","last_became_active":"2024-09-21T10:23:39.315283+0000","last_became_peered":"2024-09-21T10:23:39.315283+0000","last_unstale":"2024-09-21T10:25:18.410210+0000","last_undegraded":"2024-09-21T10:25:18.410210+0000","last_fullsized":"2024-09-21T10:25:18.410210+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:23:44.384937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":105,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816313+0000","last_change":"2024-09-21T10:22:17.009006+0000","last_active":"2024-09-21T10:25:25.816313+0000","last_peered":"2024-09-21T10:25:25.816313+0000","last_clean":"2024-09-21T10:25:25.816313+0000","last_became_active":"2024-09-21T10:22:16.993708+0000","last_became_peered":"2024-09-21T10:22:16.993708+0000","last_unstale":"2024-09-21T10:25:25.816313+0000","last_undegraded":"2024-09-21T10:25:25.816313+0000","last_fullsized":"2024-09-21T10:25:25.816313+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:46.476840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1619,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815745+0000","last_change":"2024-09-21T10:23:39.371348+0000","last_active":"2024-09-21T10:25:25.815745+0000","last_peered":"2024-09-21T10:25:25.815745+0000","last_clean":"2024-09-21T10:25:25.815745+0000","last_became_active":"2024-09-21T10:23:39.370757+0000","last_became_peered":"2024-09-21T10:23:39.370757+0000","last_unstale":"2024-09-21T10:25:25.815745+0000","last_undegraded":"2024-09-21T10:25:25.815745+0000","last_fullsized":"2024-09-21T10:25:25.815745+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:46.292157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":142,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410888+0000","last_change":"2024-09-21T10:23:39.319362+0000","last_active":"2024-09-21T10:25:18.410888+0000","last_peered":"2024-09-21T10:25:18.410888+0000","last_clean":"2024-09-21T10:25:18.410888+0000","last_became_active":"2024-09-21T10:23:39.318981+0000","last_became_peered":"2024-09-21T10:23:39.318981+0000","last_unstale":"2024-09-21T10:25:18.410888+0000","last_undegraded":"2024-09-21T10:25:18.410888+0000","last_fullsized":"2024-09-21T10:25:18.410888+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:54.616448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":878,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815312+0000","last_change":"2024-09-21T10:23:39.412337+0000","last_active":"2024-09-21T10:25:25.815312+0000","last_peered":"2024-09-21T10:25:25.815312+0000","last_clean":"2024-09-21T10:25:25.815312+0000","last_became_active":"2024-09-21T10:23:39.411958+0000","last_became_peered":"2024-09-21T10:23:39.411958+0000","last_unstale":"2024-09-21T10:25:25.815312+0000","last_undegraded":"2024-09-21T10:25:25.815312+0000","last_fullsized":"2024-09-21T10:25:25.815312+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:13.265451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410795+0000","last_change":"2024-09-21T10:23:39.372274+0000","last_active":"2024-09-21T10:25:18.410795+0000","last_peered":"2024-09-21T10:25:18.410795+0000","last_clean":"2024-09-21T10:25:18.410795+0000","last_became_active":"2024-09-21T10:23:39.371979+0000","last_became_peered":"2024-09-21T10:23:39.371979+0000","last_unstale":"2024-09-21T10:25:18.410795+0000","last_undegraded":"2024-09-21T10:25:18.410795+0000","last_fullsized":"2024-09-21T10:25:18.410795+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:03.528411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2005,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816096+0000","last_change":"2024-09-21T10:23:39.373141+0000","last_active":"2024-09-21T10:25:25.816096+0000","last_peered":"2024-09-21T10:25:25.816096+0000","last_clean":"2024-09-21T10:25:25.816096+0000","last_became_active":"2024-09-21T10:23:39.372532+0000","last_became_peered":"2024-09-21T10:23:39.372532+0000","last_unstale":"2024-09-21T10:25:25.816096+0000","last_undegraded":"2024-09-21T10:25:25.816096+0000","last_fullsized":"2024-09-21T10:25:25.816096+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:21.619250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"361'149","reported_seq":1508,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814935+0000","last_change":"2024-09-21T10:23:39.307102+0000","last_active":"2024-09-21T10:25:25.814935+0000","last_peered":"2024-09-21T10:25:25.814935+0000","last_clean":"2024-09-21T10:25:25.814935+0000","last_became_active":"2024-09-21T10:23:39.306777+0000","last_became_peered":"2024-09-21T10:23:39.306777+0000","last_unstale":"2024-09-21T10:25:25.814935+0000","last_undegraded":"2024-09-21T10:25:25.814935+0000","last_fullsized":"2024-09-21T10:25:25.814935+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:33.910694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":524,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815198+0000","last_change":"2024-09-21T10:23:39.398349+0000","last_active":"2024-09-21T10:25:25.815198+0000","last_peered":"2024-09-21T10:25:25.815198+0000","last_clean":"2024-09-21T10:25:25.815198+0000","last_became_active":"2024-09-21T10:23:39.397605+0000","last_became_peered":"2024-09-21T10:23:39.397605+0000","last_unstale":"2024-09-21T10:25:25.815198+0000","last_undegraded":"2024-09-21T10:25:25.815198+0000","last_fullsized":"2024-09-21T10:25:25.815198+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:37.735612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1613,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815200+0000","last_change":"2024-09-21T10:23:39.391183+0000","last_active":"2024-09-21T10:25:25.815200+0000","last_peered":"2024-09-21T10:25:25.815200+0000","last_clean":"2024-09-21T10:25:25.815200+0000","last_became_active":"2024-09-21T10:23:39.390886+0000","last_became_peered":"2024-09-21T10:23:39.390886+0000","last_unstale":"2024-09-21T10:25:25.815200+0000","last_undegraded":"2024-09-21T10:25:25.815200+0000","last_fullsized":"2024-09-21T10:25:25.815200+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:16:17.625754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"361'149","reported_seq":1507,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.409906+0000","last_change":"2024-09-21T10:23:39.315754+0000","last_active":"2024-09-21T10:25:18.409906+0000","last_peered":"2024-09-21T10:25:18.409906+0000","last_clean":"2024-09-21T10:25:18.409906+0000","last_became_active":"2024-09-21T10:23:39.315278+0000","last_became_peered":"2024-09-21T10:23:39.315278+0000","last_unstale":"2024-09-21T10:25:18.409906+0000","last_undegraded":"2024-09-21T10:25:18.409906+0000","last_fullsized":"2024-09-21T10:25:18.409906+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:14.982755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:14.982755+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:08.540111+0000","scrub_duration":0.0028608990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":837,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815527+0000","last_change":"2024-09-21T10:23:39.411248+0000","last_active":"2024-09-21T10:25:25.815527+0000","last_peered":"2024-09-21T10:25:25.815527+0000","last_clean":"2024-09-21T10:25:25.815527+0000","last_became_active":"2024-09-21T10:23:39.410865+0000","last_became_peered":"2024-09-21T10:23:39.410865+0000","last_unstale":"2024-09-21T10:25:25.815527+0000","last_undegraded":"2024-09-21T10:25:25.815527+0000","last_fullsized":"2024-09-21T10:25:25.815527+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:47.337510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815784+0000","last_change":"2024-09-21T10:23:39.322783+0000","last_active":"2024-09-21T10:25:25.815784+0000","last_peered":"2024-09-21T10:25:25.815784+0000","last_clean":"2024-09-21T10:25:25.815784+0000","last_became_active":"2024-09-21T10:23:39.321617+0000","last_became_peered":"2024-09-21T10:23:39.321617+0000","last_unstale":"2024-09-21T10:25:25.815784+0000","last_undegraded":"2024-09-21T10:25:25.815784+0000","last_fullsized":"2024-09-21T10:25:25.815784+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:38:11.432422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1653,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815606+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:25:25.815606+0000","last_peered":"2024-09-21T10:25:25.815606+0000","last_clean":"2024-09-21T10:25:25.815606+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:25:25.815606+0000","last_undegraded":"2024-09-21T10:25:25.815606+0000","last_fullsized":"2024-09-21T10:25:25.815606+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":132,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816133+0000","last_change":"2024-09-21T10:23:39.349126+0000","last_active":"2024-09-21T10:25:25.816133+0000","last_peered":"2024-09-21T10:25:25.816133+0000","last_clean":"2024-09-21T10:25:25.816133+0000","last_became_active":"2024-09-21T10:23:39.348799+0000","last_became_peered":"2024-09-21T10:23:39.348799+0000","last_unstale":"2024-09-21T10:25:25.816133+0000","last_undegraded":"2024-09-21T10:25:25.816133+0000","last_fullsized":"2024-09-21T10:25:25.816133+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:04.424772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"361'149","reported_seq":1488,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410093+0000","last_change":"2024-09-21T10:24:55.184035+0000","last_active":"2024-09-21T10:25:18.410093+0000","last_peered":"2024-09-21T10:25:18.410093+0000","last_clean":"2024-09-21T10:25:18.410093+0000","last_became_active":"2024-09-21T10:22:20.013780+0000","last_became_peered":"2024-09-21T10:22:20.013780+0000","last_unstale":"2024-09-21T10:25:18.410093+0000","last_undegraded":"2024-09-21T10:25:18.410093+0000","last_fullsized":"2024-09-21T10:25:18.410093+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:55.183929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:55.183929+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:02.561714+0000","scrub_duration":0.001781761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":549,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814906+0000","last_change":"2024-09-21T10:23:39.416450+0000","last_active":"2024-09-21T10:25:25.814906+0000","last_peered":"2024-09-21T10:25:25.814906+0000","last_clean":"2024-09-21T10:25:25.814906+0000","last_became_active":"2024-09-21T10:23:39.414340+0000","last_became_peered":"2024-09-21T10:23:39.414340+0000","last_unstale":"2024-09-21T10:25:25.814906+0000","last_undegraded":"2024-09-21T10:25:25.814906+0000","last_fullsized":"2024-09-21T10:25:25.814906+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:48.376073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2009,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816494+0000","last_change":"2024-09-21T10:23:39.350772+0000","last_active":"2024-09-21T10:25:25.816494+0000","last_peered":"2024-09-21T10:25:25.816494+0000","last_clean":"2024-09-21T10:25:25.816494+0000","last_became_active":"2024-09-21T10:23:39.345255+0000","last_became_peered":"2024-09-21T10:23:39.345255+0000","last_unstale":"2024-09-21T10:25:25.816494+0000","last_undegraded":"2024-09-21T10:25:25.816494+0000","last_fullsized":"2024-09-21T10:25:25.816494+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:22.523103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1513,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815188+0000","last_change":"2024-09-21T10:23:39.412844+0000","last_active":"2024-09-21T10:25:25.815188+0000","last_peered":"2024-09-21T10:25:25.815188+0000","last_clean":"2024-09-21T10:25:25.815188+0000","last_became_active":"2024-09-21T10:23:39.412591+0000","last_became_peered":"2024-09-21T10:23:39.412591+0000","last_unstale":"2024-09-21T10:25:25.815188+0000","last_undegraded":"2024-09-21T10:25:25.815188+0000","last_fullsized":"2024-09-21T10:25:25.815188+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:15.395609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:15.395609+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:33.836084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":145,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410642+0000","last_change":"2024-09-21T10:23:39.375299+0000","last_active":"2024-09-21T10:25:18.410642+0000","last_peered":"2024-09-21T10:25:18.410642+0000","last_clean":"2024-09-21T10:25:18.410642+0000","last_became_active":"2024-09-21T10:23:39.374244+0000","last_became_peered":"2024-09-21T10:23:39.374244+0000","last_unstale":"2024-09-21T10:25:18.410642+0000","last_undegraded":"2024-09-21T10:25:18.410642+0000","last_fullsized":"2024-09-21T10:25:18.410642+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:25:53.897017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":582,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410432+0000","last_change":"2024-09-21T10:23:39.305168+0000","last_active":"2024-09-21T10:25:18.410432+0000","last_peered":"2024-09-21T10:25:18.410432+0000","last_clean":"2024-09-21T10:25:18.410432+0000","last_became_active":"2024-09-21T10:23:39.304815+0000","last_became_peered":"2024-09-21T10:23:39.304815+0000","last_unstale":"2024-09-21T10:25:18.410432+0000","last_undegraded":"2024-09-21T10:25:18.410432+0000","last_fullsized":"2024-09-21T10:25:18.410432+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:11.593431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815660+0000","last_change":"2024-09-21T10:23:39.322438+0000","last_active":"2024-09-21T10:25:25.815660+0000","last_peered":"2024-09-21T10:25:25.815660+0000","last_clean":"2024-09-21T10:25:25.815660+0000","last_became_active":"2024-09-21T10:23:39.321191+0000","last_became_peered":"2024-09-21T10:23:39.321191+0000","last_unstale":"2024-09-21T10:25:25.815660+0000","last_undegraded":"2024-09-21T10:25:25.815660+0000","last_fullsized":"2024-09-21T10:25:25.815660+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.957289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"361'149","reported_seq":1521,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816202+0000","last_change":"2024-09-21T10:22:20.017945+0000","last_active":"2024-09-21T10:25:25.816202+0000","last_peered":"2024-09-21T10:25:25.816202+0000","last_clean":"2024-09-21T10:25:25.816202+0000","last_became_active":"2024-09-21T10:22:20.017453+0000","last_became_peered":"2024-09-21T10:22:20.017453+0000","last_unstale":"2024-09-21T10:25:25.816202+0000","last_undegraded":"2024-09-21T10:25:25.816202+0000","last_fullsized":"2024-09-21T10:25:25.816202+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:32.811754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1301,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815102+0000","last_change":"2024-09-21T10:23:39.413518+0000","last_active":"2024-09-21T10:25:25.815102+0000","last_peered":"2024-09-21T10:25:25.815102+0000","last_clean":"2024-09-21T10:25:25.815102+0000","last_became_active":"2024-09-21T10:23:39.413270+0000","last_became_peered":"2024-09-21T10:23:39.413270+0000","last_unstale":"2024-09-21T10:25:25.815102+0000","last_undegraded":"2024-09-21T10:25:25.815102+0000","last_fullsized":"2024-09-21T10:25:25.815102+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:35.809985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1741,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814529+0000","last_change":"2024-09-21T10:23:39.316171+0000","last_active":"2024-09-21T10:25:25.814529+0000","last_peered":"2024-09-21T10:25:25.814529+0000","last_clean":"2024-09-21T10:25:25.814529+0000","last_became_active":"2024-09-21T10:23:39.315540+0000","last_became_peered":"2024-09-21T10:23:39.315540+0000","last_unstale":"2024-09-21T10:25:25.814529+0000","last_undegraded":"2024-09-21T10:25:25.814529+0000","last_fullsized":"2024-09-21T10:25:25.814529+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.638899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1284,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816189+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:25:25.816189+0000","last_peered":"2024-09-21T10:25:25.816189+0000","last_clean":"2024-09-21T10:25:25.816189+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:25:25.816189+0000","last_undegraded":"2024-09-21T10:25:25.816189+0000","last_fullsized":"2024-09-21T10:25:25.816189+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"361'149","reported_seq":1471,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410119+0000","last_change":"2024-09-21T10:22:15.938120+0000","last_active":"2024-09-21T10:25:18.410119+0000","last_peered":"2024-09-21T10:25:18.410119+0000","last_clean":"2024-09-21T10:25:18.410119+0000","last_became_active":"2024-09-21T10:21:56.928301+0000","last_became_peered":"2024-09-21T10:21:56.928301+0000","last_unstale":"2024-09-21T10:25:18.410119+0000","last_undegraded":"2024-09-21T10:25:18.410119+0000","last_fullsized":"2024-09-21T10:25:18.410119+0000","mapping_epoch":1358,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1359,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:15.937862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:15.937862+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:01.614339+0000","scrub_duration":0.00329533,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1861,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814562+0000","last_change":"2024-09-21T10:23:39.450173+0000","last_active":"2024-09-21T10:25:25.814562+0000","last_peered":"2024-09-21T10:25:25.814562+0000","last_clean":"2024-09-21T10:25:25.814562+0000","last_became_active":"2024-09-21T10:23:39.449505+0000","last_became_peered":"2024-09-21T10:23:39.449505+0000","last_unstale":"2024-09-21T10:25:25.814562+0000","last_undegraded":"2024-09-21T10:25:25.814562+0000","last_fullsized":"2024-09-21T10:25:25.814562+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:04.778009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2032,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815375+0000","last_change":"2024-09-21T10:23:39.414249+0000","last_active":"2024-09-21T10:25:25.815375+0000","last_peered":"2024-09-21T10:25:25.815375+0000","last_clean":"2024-09-21T10:25:25.815375+0000","last_became_active":"2024-09-21T10:23:39.413975+0000","last_became_peered":"2024-09-21T10:23:39.413975+0000","last_unstale":"2024-09-21T10:25:25.815375+0000","last_undegraded":"2024-09-21T10:25:25.815375+0000","last_fullsized":"2024-09-21T10:25:25.815375+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:20.488016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2025,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816401+0000","last_change":"2024-09-21T10:23:39.412694+0000","last_active":"2024-09-21T10:25:25.816401+0000","last_peered":"2024-09-21T10:25:25.816401+0000","last_clean":"2024-09-21T10:25:25.816401+0000","last_became_active":"2024-09-21T10:23:39.412389+0000","last_became_peered":"2024-09-21T10:23:39.412389+0000","last_unstale":"2024-09-21T10:25:25.816401+0000","last_undegraded":"2024-09-21T10:25:25.816401+0000","last_fullsized":"2024-09-21T10:25:25.816401+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:30.102172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"361'149","reported_seq":1488,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410011+0000","last_change":"2024-09-21T10:22:20.014154+0000","last_active":"2024-09-21T10:25:18.410011+0000","last_peered":"2024-09-21T10:25:18.410011+0000","last_clean":"2024-09-21T10:25:18.410011+0000","last_became_active":"2024-09-21T10:22:20.013516+0000","last_became_peered":"2024-09-21T10:22:20.013516+0000","last_unstale":"2024-09-21T10:25:18.410011+0000","last_undegraded":"2024-09-21T10:25:18.410011+0000","last_fullsized":"2024-09-21T10:25:18.410011+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:11.089486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:11.089486+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:11:55.464199+0000","scrub_duration":0.010125244,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":142,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410686+0000","last_change":"2024-09-21T10:23:39.346213+0000","last_active":"2024-09-21T10:25:18.410686+0000","last_peered":"2024-09-21T10:25:18.410686+0000","last_clean":"2024-09-21T10:25:18.410686+0000","last_became_active":"2024-09-21T10:23:39.346009+0000","last_became_peered":"2024-09-21T10:23:39.346009+0000","last_unstale":"2024-09-21T10:25:18.410686+0000","last_undegraded":"2024-09-21T10:25:18.410686+0000","last_fullsized":"2024-09-21T10:25:18.410686+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:34.154751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1222,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815893+0000","last_change":"2024-09-21T10:22:16.994613+0000","last_active":"2024-09-21T10:25:25.815893+0000","last_peered":"2024-09-21T10:25:25.815893+0000","last_clean":"2024-09-21T10:25:25.815893+0000","last_became_active":"2024-09-21T10:22:16.994182+0000","last_became_peered":"2024-09-21T10:22:16.994182+0000","last_unstale":"2024-09-21T10:25:25.815893+0000","last_undegraded":"2024-09-21T10:25:25.815893+0000","last_fullsized":"2024-09-21T10:25:25.815893+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:44.627749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1947,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815038+0000","last_change":"2024-09-21T10:23:39.315999+0000","last_active":"2024-09-21T10:25:25.815038+0000","last_peered":"2024-09-21T10:25:25.815038+0000","last_clean":"2024-09-21T10:25:25.815038+0000","last_became_active":"2024-09-21T10:23:39.315576+0000","last_became_peered":"2024-09-21T10:23:39.315576+0000","last_unstale":"2024-09-21T10:25:25.815038+0000","last_undegraded":"2024-09-21T10:25:25.815038+0000","last_fullsized":"2024-09-21T10:25:25.815038+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:21:28.524095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"361'149","reported_seq":1017,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815384+0000","last_change":"2024-09-21T10:23:39.364222+0000","last_active":"2024-09-21T10:25:25.815384+0000","last_peered":"2024-09-21T10:25:25.815384+0000","last_clean":"2024-09-21T10:25:25.815384+0000","last_became_active":"2024-09-21T10:23:39.363177+0000","last_became_peered":"2024-09-21T10:23:39.363177+0000","last_unstale":"2024-09-21T10:25:25.815384+0000","last_undegraded":"2024-09-21T10:25:25.815384+0000","last_fullsized":"2024-09-21T10:25:25.815384+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:36.892949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":322019,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410955+0000","last_change":"2024-09-21T10:23:39.376204+0000","last_active":"2024-09-21T10:25:18.410955+0000","last_peered":"2024-09-21T10:25:18.410955+0000","last_clean":"2024-09-21T10:25:18.410955+0000","last_became_active":"2024-09-21T10:23:39.376009+0000","last_became_peered":"2024-09-21T10:23:39.376009+0000","last_unstale":"2024-09-21T10:25:18.410955+0000","last_undegraded":"2024-09-21T10:25:18.410955+0000","last_fullsized":"2024-09-21T10:25:18.410955+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:43.583701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2040,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815808+0000","last_change":"2024-09-21T10:23:39.370875+0000","last_active":"2024-09-21T10:25:25.815808+0000","last_peered":"2024-09-21T10:25:25.815808+0000","last_clean":"2024-09-21T10:25:25.815808+0000","last_became_active":"2024-09-21T10:23:39.370134+0000","last_became_peered":"2024-09-21T10:23:39.370134+0000","last_unstale":"2024-09-21T10:25:25.815808+0000","last_undegraded":"2024-09-21T10:25:25.815808+0000","last_fullsized":"2024-09-21T10:25:25.815808+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:24.513775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":555,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815172+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:25:25.815172+0000","last_peered":"2024-09-21T10:25:25.815172+0000","last_clean":"2024-09-21T10:25:25.815172+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:25:25.815172+0000","last_undegraded":"2024-09-21T10:25:25.815172+0000","last_fullsized":"2024-09-21T10:25:25.815172+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"361'149","reported_seq":1017,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815809+0000","last_change":"2024-09-21T10:23:39.302896+0000","last_active":"2024-09-21T10:25:25.815809+0000","last_peered":"2024-09-21T10:25:25.815809+0000","last_clean":"2024-09-21T10:25:25.815809+0000","last_became_active":"2024-09-21T10:23:39.302269+0000","last_became_peered":"2024-09-21T10:23:39.302269+0000","last_unstale":"2024-09-21T10:25:25.815809+0000","last_undegraded":"2024-09-21T10:25:25.815809+0000","last_fullsized":"2024-09-21T10:25:25.815809+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:32.864718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":220,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814708+0000","last_change":"2024-09-21T10:23:39.303934+0000","last_active":"2024-09-21T10:25:25.814708+0000","last_peered":"2024-09-21T10:25:25.814708+0000","last_clean":"2024-09-21T10:25:25.814708+0000","last_became_active":"2024-09-21T10:23:39.303741+0000","last_became_peered":"2024-09-21T10:23:39.303741+0000","last_unstale":"2024-09-21T10:25:25.814708+0000","last_undegraded":"2024-09-21T10:25:25.814708+0000","last_fullsized":"2024-09-21T10:25:25.814708+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:12.138362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1186,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816316+0000","last_change":"2024-09-21T10:23:39.375346+0000","last_active":"2024-09-21T10:25:25.816316+0000","last_peered":"2024-09-21T10:25:25.816316+0000","last_clean":"2024-09-21T10:25:25.816316+0000","last_became_active":"2024-09-21T10:23:39.374849+0000","last_became_peered":"2024-09-21T10:23:39.374849+0000","last_unstale":"2024-09-21T10:25:25.816316+0000","last_undegraded":"2024-09-21T10:25:25.816316+0000","last_fullsized":"2024-09-21T10:25:25.816316+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:10.241167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":110,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410628+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:25:18.410628+0000","last_peered":"2024-09-21T10:25:18.410628+0000","last_clean":"2024-09-21T10:25:18.410628+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:25:18.410628+0000","last_undegraded":"2024-09-21T10:25:18.410628+0000","last_fullsized":"2024-09-21T10:25:18.410628+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"361'149","reported_seq":1013,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410611+0000","last_change":"2024-09-21T10:24:57.148783+0000","last_active":"2024-09-21T10:25:18.410611+0000","last_peered":"2024-09-21T10:25:18.410611+0000","last_clean":"2024-09-21T10:25:18.410611+0000","last_became_active":"2024-09-21T10:23:39.318900+0000","last_became_peered":"2024-09-21T10:23:39.318900+0000","last_unstale":"2024-09-21T10:25:18.410611+0000","last_undegraded":"2024-09-21T10:25:18.410611+0000","last_fullsized":"2024-09-21T10:25:18.410611+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:57.148677+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:57.148677+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:19.862692+0000","scrub_duration":0.001855659,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":521,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411007+0000","last_change":"2024-09-21T10:23:39.305219+0000","last_active":"2024-09-21T10:25:18.411007+0000","last_peered":"2024-09-21T10:25:18.411007+0000","last_clean":"2024-09-21T10:25:18.411007+0000","last_became_active":"2024-09-21T10:23:39.304787+0000","last_became_peered":"2024-09-21T10:23:39.304787+0000","last_unstale":"2024-09-21T10:25:18.411007+0000","last_undegraded":"2024-09-21T10:25:18.411007+0000","last_fullsized":"2024-09-21T10:25:18.411007+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:14.699319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816032+0000","last_change":"2024-09-21T10:23:39.347172+0000","last_active":"2024-09-21T10:25:25.816032+0000","last_peered":"2024-09-21T10:25:25.816032+0000","last_clean":"2024-09-21T10:25:25.816032+0000","last_became_active":"2024-09-21T10:23:39.345326+0000","last_became_peered":"2024-09-21T10:23:39.345326+0000","last_unstale":"2024-09-21T10:25:25.816032+0000","last_undegraded":"2024-09-21T10:25:25.816032+0000","last_fullsized":"2024-09-21T10:25:25.816032+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:32.454312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":545,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814883+0000","last_change":"2024-09-21T10:23:39.415241+0000","last_active":"2024-09-21T10:25:25.814883+0000","last_peered":"2024-09-21T10:25:25.814883+0000","last_clean":"2024-09-21T10:25:25.814883+0000","last_became_active":"2024-09-21T10:23:39.412720+0000","last_became_peered":"2024-09-21T10:23:39.412720+0000","last_unstale":"2024-09-21T10:25:25.814883+0000","last_undegraded":"2024-09-21T10:25:25.814883+0000","last_fullsized":"2024-09-21T10:25:25.814883+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:09.855474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"361'149","reported_seq":1512,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815321+0000","last_change":"2024-09-21T10:23:39.316659+0000","last_active":"2024-09-21T10:25:25.815321+0000","last_peered":"2024-09-21T10:25:25.815321+0000","last_clean":"2024-09-21T10:25:25.815321+0000","last_became_active":"2024-09-21T10:23:39.316129+0000","last_became_peered":"2024-09-21T10:23:39.316129+0000","last_unstale":"2024-09-21T10:25:25.815321+0000","last_undegraded":"2024-09-21T10:25:25.815321+0000","last_fullsized":"2024-09-21T10:25:25.815321+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:35.853740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":540,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816321+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:25:25.816321+0000","last_peered":"2024-09-21T10:25:25.816321+0000","last_clean":"2024-09-21T10:25:25.816321+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:25:25.816321+0000","last_undegraded":"2024-09-21T10:25:25.816321+0000","last_fullsized":"2024-09-21T10:25:25.816321+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":867,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815442+0000","last_change":"2024-09-21T10:23:39.363897+0000","last_active":"2024-09-21T10:25:25.815442+0000","last_peered":"2024-09-21T10:25:25.815442+0000","last_clean":"2024-09-21T10:25:25.815442+0000","last_became_active":"2024-09-21T10:23:39.362798+0000","last_became_peered":"2024-09-21T10:23:39.362798+0000","last_unstale":"2024-09-21T10:25:25.815442+0000","last_undegraded":"2024-09-21T10:25:25.815442+0000","last_fullsized":"2024-09-21T10:25:25.815442+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:04.258680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410229+0000","last_change":"2024-09-21T10:23:39.306440+0000","last_active":"2024-09-21T10:25:18.410229+0000","last_peered":"2024-09-21T10:25:18.410229+0000","last_clean":"2024-09-21T10:25:18.410229+0000","last_became_active":"2024-09-21T10:23:39.306047+0000","last_became_peered":"2024-09-21T10:23:39.306047+0000","last_unstale":"2024-09-21T10:25:18.410229+0000","last_undegraded":"2024-09-21T10:25:18.410229+0000","last_fullsized":"2024-09-21T10:25:18.410229+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:29.308333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1223,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815801+0000","last_change":"2024-09-21T10:22:16.993047+0000","last_active":"2024-09-21T10:25:25.815801+0000","last_peered":"2024-09-21T10:25:25.815801+0000","last_clean":"2024-09-21T10:25:25.815801+0000","last_became_active":"2024-09-21T10:22:16.992606+0000","last_became_peered":"2024-09-21T10:22:16.992606+0000","last_unstale":"2024-09-21T10:25:25.815801+0000","last_undegraded":"2024-09-21T10:25:25.815801+0000","last_fullsized":"2024-09-21T10:25:25.815801+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:35.971920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":289,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410458+0000","last_change":"2024-09-21T10:23:39.308252+0000","last_active":"2024-09-21T10:25:18.410458+0000","last_peered":"2024-09-21T10:25:18.410458+0000","last_clean":"2024-09-21T10:25:18.410458+0000","last_became_active":"2024-09-21T10:23:39.306643+0000","last_became_peered":"2024-09-21T10:23:39.306643+0000","last_unstale":"2024-09-21T10:25:18.410458+0000","last_undegraded":"2024-09-21T10:25:18.410458+0000","last_fullsized":"2024-09-21T10:25:18.410458+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:07.074129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2008,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815449+0000","last_change":"2024-09-21T10:22:16.994417+0000","last_active":"2024-09-21T10:25:25.815449+0000","last_peered":"2024-09-21T10:25:25.815449+0000","last_clean":"2024-09-21T10:25:25.815449+0000","last_became_active":"2024-09-21T10:22:16.994109+0000","last_became_peered":"2024-09-21T10:22:16.994109+0000","last_unstale":"2024-09-21T10:25:25.815449+0000","last_undegraded":"2024-09-21T10:25:25.815449+0000","last_fullsized":"2024-09-21T10:25:25.815449+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:55.406178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1509,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815745+0000","last_change":"2024-09-21T10:23:39.323087+0000","last_active":"2024-09-21T10:25:25.815745+0000","last_peered":"2024-09-21T10:25:25.815745+0000","last_clean":"2024-09-21T10:25:25.815745+0000","last_became_active":"2024-09-21T10:23:39.322041+0000","last_became_peered":"2024-09-21T10:23:39.322041+0000","last_unstale":"2024-09-21T10:25:25.815745+0000","last_undegraded":"2024-09-21T10:25:25.815745+0000","last_fullsized":"2024-09-21T10:25:25.815745+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:29.126970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":125,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411060+0000","last_change":"2024-09-21T10:23:39.334300+0000","last_active":"2024-09-21T10:25:18.411060+0000","last_peered":"2024-09-21T10:25:18.411060+0000","last_clean":"2024-09-21T10:25:18.411060+0000","last_became_active":"2024-09-21T10:23:39.334044+0000","last_became_peered":"2024-09-21T10:23:39.334044+0000","last_unstale":"2024-09-21T10:25:18.411060+0000","last_undegraded":"2024-09-21T10:25:18.411060+0000","last_fullsized":"2024-09-21T10:25:18.411060+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:26:36.147498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":651,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814812+0000","last_change":"2024-09-21T10:23:39.306893+0000","last_active":"2024-09-21T10:25:25.814812+0000","last_peered":"2024-09-21T10:25:25.814812+0000","last_clean":"2024-09-21T10:25:25.814812+0000","last_became_active":"2024-09-21T10:23:39.306560+0000","last_became_peered":"2024-09-21T10:23:39.306560+0000","last_unstale":"2024-09-21T10:25:25.814812+0000","last_undegraded":"2024-09-21T10:25:25.814812+0000","last_fullsized":"2024-09-21T10:25:25.814812+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:16.170412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411145+0000","last_change":"2024-09-21T10:23:39.337324+0000","last_active":"2024-09-21T10:25:18.411145+0000","last_peered":"2024-09-21T10:25:18.411145+0000","last_clean":"2024-09-21T10:25:18.411145+0000","last_became_active":"2024-09-21T10:23:39.337017+0000","last_became_peered":"2024-09-21T10:23:39.337017+0000","last_unstale":"2024-09-21T10:25:18.411145+0000","last_undegraded":"2024-09-21T10:25:18.411145+0000","last_fullsized":"2024-09-21T10:25:18.411145+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:53.271860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":635,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814977+0000","last_change":"2024-09-21T10:23:39.414802+0000","last_active":"2024-09-21T10:25:25.814977+0000","last_peered":"2024-09-21T10:25:25.814977+0000","last_clean":"2024-09-21T10:25:25.814977+0000","last_became_active":"2024-09-21T10:23:39.411110+0000","last_became_peered":"2024-09-21T10:23:39.411110+0000","last_unstale":"2024-09-21T10:25:25.814977+0000","last_undegraded":"2024-09-21T10:25:25.814977+0000","last_fullsized":"2024-09-21T10:25:25.814977+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:27.837285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":269,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410980+0000","last_change":"2024-09-21T10:23:39.337324+0000","last_active":"2024-09-21T10:25:18.410980+0000","last_peered":"2024-09-21T10:25:18.410980+0000","last_clean":"2024-09-21T10:25:18.410980+0000","last_became_active":"2024-09-21T10:23:39.337017+0000","last_became_peered":"2024-09-21T10:23:39.337017+0000","last_unstale":"2024-09-21T10:25:18.410980+0000","last_undegraded":"2024-09-21T10:25:18.410980+0000","last_fullsized":"2024-09-21T10:25:18.410980+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:43.984727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":593,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814716+0000","last_change":"2024-09-21T10:23:39.415574+0000","last_active":"2024-09-21T10:25:25.814716+0000","last_peered":"2024-09-21T10:25:25.814716+0000","last_clean":"2024-09-21T10:25:25.814716+0000","last_became_active":"2024-09-21T10:23:39.413699+0000","last_became_peered":"2024-09-21T10:23:39.413699+0000","last_unstale":"2024-09-21T10:25:25.814716+0000","last_undegraded":"2024-09-21T10:25:25.814716+0000","last_fullsized":"2024-09-21T10:25:25.814716+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:11.990863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1856,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815906+0000","last_change":"2024-09-21T10:23:39.363264+0000","last_active":"2024-09-21T10:25:25.815906+0000","last_peered":"2024-09-21T10:25:25.815906+0000","last_clean":"2024-09-21T10:25:25.815906+0000","last_became_active":"2024-09-21T10:23:39.362615+0000","last_became_peered":"2024-09-21T10:23:39.362615+0000","last_unstale":"2024-09-21T10:25:25.815906+0000","last_undegraded":"2024-09-21T10:25:25.815906+0000","last_fullsized":"2024-09-21T10:25:25.815906+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:55.561333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":298,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815692+0000","last_change":"2024-09-21T10:23:39.371060+0000","last_active":"2024-09-21T10:25:25.815692+0000","last_peered":"2024-09-21T10:25:25.815692+0000","last_clean":"2024-09-21T10:25:25.815692+0000","last_became_active":"2024-09-21T10:23:39.370494+0000","last_became_peered":"2024-09-21T10:23:39.370494+0000","last_unstale":"2024-09-21T10:25:25.815692+0000","last_undegraded":"2024-09-21T10:25:25.815692+0000","last_fullsized":"2024-09-21T10:25:25.815692+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:09.975626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":518,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815864+0000","last_change":"2024-09-21T10:23:39.510198+0000","last_active":"2024-09-21T10:25:25.815864+0000","last_peered":"2024-09-21T10:25:25.815864+0000","last_clean":"2024-09-21T10:25:25.815864+0000","last_became_active":"2024-09-21T10:23:39.509474+0000","last_became_peered":"2024-09-21T10:23:39.509474+0000","last_unstale":"2024-09-21T10:25:25.815864+0000","last_undegraded":"2024-09-21T10:25:25.815864+0000","last_fullsized":"2024-09-21T10:25:25.815864+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.611788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":558,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815609+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:25:25.815609+0000","last_peered":"2024-09-21T10:25:25.815609+0000","last_clean":"2024-09-21T10:25:25.815609+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:25:25.815609+0000","last_undegraded":"2024-09-21T10:25:25.815609+0000","last_fullsized":"2024-09-21T10:25:25.815609+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1898,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815938+0000","last_change":"2024-09-21T10:23:39.370592+0000","last_active":"2024-09-21T10:25:25.815938+0000","last_peered":"2024-09-21T10:25:25.815938+0000","last_clean":"2024-09-21T10:25:25.815938+0000","last_became_active":"2024-09-21T10:23:39.369832+0000","last_became_peered":"2024-09-21T10:23:39.369832+0000","last_unstale":"2024-09-21T10:25:25.815938+0000","last_undegraded":"2024-09-21T10:25:25.815938+0000","last_fullsized":"2024-09-21T10:25:25.815938+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:23.320589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1972,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815998+0000","last_change":"2024-09-21T10:23:39.373474+0000","last_active":"2024-09-21T10:25:25.815998+0000","last_peered":"2024-09-21T10:25:25.815998+0000","last_clean":"2024-09-21T10:25:25.815998+0000","last_became_active":"2024-09-21T10:23:39.373158+0000","last_became_peered":"2024-09-21T10:23:39.373158+0000","last_unstale":"2024-09-21T10:25:25.815998+0000","last_undegraded":"2024-09-21T10:25:25.815998+0000","last_fullsized":"2024-09-21T10:25:25.815998+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:53.362317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1500,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815111+0000","last_change":"2024-09-21T10:23:39.398601+0000","last_active":"2024-09-21T10:25:25.815111+0000","last_peered":"2024-09-21T10:25:25.815111+0000","last_clean":"2024-09-21T10:25:25.815111+0000","last_became_active":"2024-09-21T10:23:39.398064+0000","last_became_peered":"2024-09-21T10:23:39.398064+0000","last_unstale":"2024-09-21T10:25:25.815111+0000","last_undegraded":"2024-09-21T10:25:25.815111+0000","last_fullsized":"2024-09-21T10:25:25.815111+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:05.118881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":114,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:21.781263+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:25:21.781263+0000","last_peered":"2024-09-21T10:25:21.781263+0000","last_clean":"2024-09-21T10:25:21.781263+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:25:21.781263+0000","last_undegraded":"2024-09-21T10:25:21.781263+0000","last_fullsized":"2024-09-21T10:25:21.781263+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1562,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814812+0000","last_change":"2024-09-21T10:23:39.302265+0000","last_active":"2024-09-21T10:25:25.814812+0000","last_peered":"2024-09-21T10:25:25.814812+0000","last_clean":"2024-09-21T10:25:25.814812+0000","last_became_active":"2024-09-21T10:23:39.300210+0000","last_became_peered":"2024-09-21T10:23:39.300210+0000","last_unstale":"2024-09-21T10:25:25.814812+0000","last_undegraded":"2024-09-21T10:25:25.814812+0000","last_fullsized":"2024-09-21T10:25:25.814812+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:23.615984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":142,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:21.781306+0000","last_change":"2024-09-21T10:23:39.375969+0000","last_active":"2024-09-21T10:25:21.781306+0000","last_peered":"2024-09-21T10:25:21.781306+0000","last_clean":"2024-09-21T10:25:21.781306+0000","last_became_active":"2024-09-21T10:23:39.375686+0000","last_became_peered":"2024-09-21T10:23:39.375686+0000","last_unstale":"2024-09-21T10:25:21.781306+0000","last_undegraded":"2024-09-21T10:25:21.781306+0000","last_fullsized":"2024-09-21T10:25:21.781306+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:18:38.252378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":593,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814687+0000","last_change":"2024-09-21T10:23:39.450465+0000","last_active":"2024-09-21T10:25:25.814687+0000","last_peered":"2024-09-21T10:25:25.814687+0000","last_clean":"2024-09-21T10:25:25.814687+0000","last_became_active":"2024-09-21T10:23:39.449879+0000","last_became_peered":"2024-09-21T10:23:39.449879+0000","last_unstale":"2024-09-21T10:25:25.814687+0000","last_undegraded":"2024-09-21T10:25:25.814687+0000","last_fullsized":"2024-09-21T10:25:25.814687+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:58.113689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":625,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815525+0000","last_change":"2024-09-21T10:23:39.361838+0000","last_active":"2024-09-21T10:25:25.815525+0000","last_peered":"2024-09-21T10:25:25.815525+0000","last_clean":"2024-09-21T10:25:25.815525+0000","last_became_active":"2024-09-21T10:23:39.361539+0000","last_became_peered":"2024-09-21T10:23:39.361539+0000","last_unstale":"2024-09-21T10:25:25.815525+0000","last_undegraded":"2024-09-21T10:25:25.815525+0000","last_fullsized":"2024-09-21T10:25:25.815525+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:15:39.263560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":243,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815014+0000","last_change":"2024-09-21T10:23:39.378606+0000","last_active":"2024-09-21T10:25:25.815014+0000","last_peered":"2024-09-21T10:25:25.815014+0000","last_clean":"2024-09-21T10:25:25.815014+0000","last_became_active":"2024-09-21T10:23:39.378107+0000","last_became_peered":"2024-09-21T10:23:39.378107+0000","last_unstale":"2024-09-21T10:25:25.815014+0000","last_undegraded":"2024-09-21T10:25:25.815014+0000","last_fullsized":"2024-09-21T10:25:25.815014+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:11.536250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":146,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411136+0000","last_change":"2024-09-21T10:23:39.305452+0000","last_active":"2024-09-21T10:25:18.411136+0000","last_peered":"2024-09-21T10:25:18.411136+0000","last_clean":"2024-09-21T10:25:18.411136+0000","last_became_active":"2024-09-21T10:23:39.305054+0000","last_became_peered":"2024-09-21T10:23:39.305054+0000","last_unstale":"2024-09-21T10:25:18.411136+0000","last_undegraded":"2024-09-21T10:25:18.411136+0000","last_fullsized":"2024-09-21T10:25:18.411136+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:30.307977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":21744,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19584,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":584707,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":117,"num_write_kb":1318,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1824774,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234595,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496231,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757870,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":710920,"kb_used_data":7996,"kb_used_omap":1,"kb_used_meta":702910,"kb_avail":93034232,"statfs":{"total":95995035648,"available":95267053568,"internally_reserved":0,"allocated":8187904,"data_stored":5453929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":719780294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019506,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333520,"kb_used_data":8004,"kb_used_omap":1,"kb_used_meta":1325502,"kb_avail":92411632,"statfs":{"total":95995035648,"available":94629511168,"internally_reserved":0,"allocated":8196096,"data_stored":5461891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1357314502},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1375,"seq":5905580032023,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":189836,"kb_used_data":8000,"kb_used_omap":52,"kb_used_meta":181771,"kb_avail":93555316,"statfs":{"total":95995035648,"available":95800643584,"internally_reserved":0,"allocated":8192000,"data_stored":5461866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54095,"internal_metadata":186133681},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793968041,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047280,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516408,"kb_used_data":7404,"kb_used_omap":41,"kb_used_meta":1508950,"kb_avail":92228744,"statfs":{"total":95995035648,"available":94442233856,"internally_reserved":0,"allocated":7581696,"data_stored":4866056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1545165737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928914,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10112,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:25:32.075 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:25:32.075 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-21T10:25:32.075 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-21T10:25:32.075 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:25:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:31 smithi191 ceph-mon[102206]: from='client.36423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:31 smithi191 ceph-mon[102206]: pgmap v2338: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:33.084 INFO:teuthology.orchestra.run.smithi076.stdout:ERROR: (22) Invalid argument 2024-09-21T10:25:33.084 INFO:teuthology.orchestra.run.smithi076.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-21T10:25:33.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:25:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:25:33] "GET /metrics HTTP/1.1" 200 34199 "" "Prometheus/2.51.0" 2024-09-21T10:25:33.775 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:34.034 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:33 smithi076 ceph-mon[106482]: pgmap v2339: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:34.035 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:33 smithi076 ceph-mon[110963]: pgmap v2339: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:33 smithi191 ceph-mon[102206]: pgmap v2339: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:35.083 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:34 smithi076 ceph-mon[106482]: 1.3 scrub starts 2024-09-21T10:25:35.084 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:34 smithi076 ceph-mon[106482]: 1.3 scrub ok 2024-09-21T10:25:35.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:34 smithi076 ceph-mon[110963]: 1.3 scrub starts 2024-09-21T10:25:35.084 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:34 smithi076 ceph-mon[110963]: 1.3 scrub ok 2024-09-21T10:25:35.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:34 smithi191 ceph-mon[102206]: 1.3 scrub starts 2024-09-21T10:25:35.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:34 smithi191 ceph-mon[102206]: 1.3 scrub ok 2024-09-21T10:25:35.772 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:35 smithi076 ceph-mon[106482]: 1.0 scrub starts 2024-09-21T10:25:35.772 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:35 smithi076 ceph-mon[106482]: 1.0 scrub ok 2024-09-21T10:25:35.772 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:35 smithi076 ceph-mon[106482]: pgmap v2340: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:35.772 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:35 smithi076 ceph-mon[110963]: 1.0 scrub starts 2024-09-21T10:25:35.772 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:35 smithi076 ceph-mon[110963]: 1.0 scrub ok 2024-09-21T10:25:35.772 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:35 smithi076 ceph-mon[110963]: pgmap v2340: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:36.045 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:25:36.046 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:25:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:35 smithi191 ceph-mon[102206]: 1.0 scrub starts 2024-09-21T10:25:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:35 smithi191 ceph-mon[102206]: 1.0 scrub ok 2024-09-21T10:25:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:35 smithi191 ceph-mon[102206]: pgmap v2340: 83 pgs: 83 active+clean; 576 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:36.886 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2340,"stamp":"2024-09-21T10:25:35.577944+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590280,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":181,"num_write_kb":1351,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":1846594,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3750696,"kb_used_data":31416,"kb_used_omap":97,"kb_used_meta":3719134,"kb_avail":371229912,"statfs":{"total":383980142592,"available":380139429888,"internally_reserved":0,"allocated":32169984,"data_stored":21251723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":99354,"internal_metadata":3808394214},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002437"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":593,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815306+0000","last_change":"2024-09-21T10:23:39.397937+0000","last_active":"2024-09-21T10:25:25.815306+0000","last_peered":"2024-09-21T10:25:25.815306+0000","last_clean":"2024-09-21T10:25:25.815306+0000","last_became_active":"2024-09-21T10:23:39.397625+0000","last_became_peered":"2024-09-21T10:23:39.397625+0000","last_unstale":"2024-09-21T10:25:25.815306+0000","last_undegraded":"2024-09-21T10:25:25.815306+0000","last_fullsized":"2024-09-21T10:25:25.815306+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:27.988977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":244,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816369+0000","last_change":"2024-09-21T10:22:17.003755+0000","last_active":"2024-09-21T10:25:25.816369+0000","last_peered":"2024-09-21T10:25:25.816369+0000","last_clean":"2024-09-21T10:25:25.816369+0000","last_became_active":"2024-09-21T10:22:16.991551+0000","last_became_peered":"2024-09-21T10:22:16.991551+0000","last_unstale":"2024-09-21T10:25:25.816369+0000","last_undegraded":"2024-09-21T10:25:25.816369+0000","last_fullsized":"2024-09-21T10:25:25.816369+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:40:54.135786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410728+0000","last_change":"2024-09-21T10:23:39.376137+0000","last_active":"2024-09-21T10:25:18.410728+0000","last_peered":"2024-09-21T10:25:18.410728+0000","last_clean":"2024-09-21T10:25:18.410728+0000","last_became_active":"2024-09-21T10:23:39.375735+0000","last_became_peered":"2024-09-21T10:23:39.375735+0000","last_unstale":"2024-09-21T10:25:18.410728+0000","last_undegraded":"2024-09-21T10:25:18.410728+0000","last_fullsized":"2024-09-21T10:25:18.410728+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:48:14.995856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1910,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814729+0000","last_change":"2024-09-21T10:23:39.362192+0000","last_active":"2024-09-21T10:25:25.814729+0000","last_peered":"2024-09-21T10:25:25.814729+0000","last_clean":"2024-09-21T10:25:25.814729+0000","last_became_active":"2024-09-21T10:23:39.361929+0000","last_became_peered":"2024-09-21T10:23:39.361929+0000","last_unstale":"2024-09-21T10:25:25.814729+0000","last_undegraded":"2024-09-21T10:25:25.814729+0000","last_fullsized":"2024-09-21T10:25:25.814729+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:42:37.863408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1939,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815718+0000","last_change":"2024-09-21T10:23:39.363032+0000","last_active":"2024-09-21T10:25:25.815718+0000","last_peered":"2024-09-21T10:25:25.815718+0000","last_clean":"2024-09-21T10:25:25.815718+0000","last_became_active":"2024-09-21T10:23:39.362187+0000","last_became_peered":"2024-09-21T10:23:39.362187+0000","last_unstale":"2024-09-21T10:25:25.815718+0000","last_undegraded":"2024-09-21T10:25:25.815718+0000","last_fullsized":"2024-09-21T10:25:25.815718+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:11:52.282910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1139,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815040+0000","last_change":"2024-09-21T10:23:39.303739+0000","last_active":"2024-09-21T10:25:25.815040+0000","last_peered":"2024-09-21T10:25:25.815040+0000","last_clean":"2024-09-21T10:25:25.815040+0000","last_became_active":"2024-09-21T10:23:39.303418+0000","last_became_peered":"2024-09-21T10:23:39.303418+0000","last_unstale":"2024-09-21T10:25:25.815040+0000","last_undegraded":"2024-09-21T10:25:25.815040+0000","last_fullsized":"2024-09-21T10:25:25.815040+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:39:22.180697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":616,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815349+0000","last_change":"2024-09-21T10:23:39.364482+0000","last_active":"2024-09-21T10:25:25.815349+0000","last_peered":"2024-09-21T10:25:25.815349+0000","last_clean":"2024-09-21T10:25:25.815349+0000","last_became_active":"2024-09-21T10:23:39.363557+0000","last_became_peered":"2024-09-21T10:23:39.363557+0000","last_unstale":"2024-09-21T10:25:25.815349+0000","last_undegraded":"2024-09-21T10:25:25.815349+0000","last_fullsized":"2024-09-21T10:25:25.815349+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:26:10.008534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":308,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.409980+0000","last_change":"2024-09-21T10:23:39.396151+0000","last_active":"2024-09-21T10:25:18.409980+0000","last_peered":"2024-09-21T10:25:18.409980+0000","last_clean":"2024-09-21T10:25:18.409980+0000","last_became_active":"2024-09-21T10:23:39.395820+0000","last_became_peered":"2024-09-21T10:23:39.395820+0000","last_unstale":"2024-09-21T10:25:18.409980+0000","last_undegraded":"2024-09-21T10:25:18.409980+0000","last_fullsized":"2024-09-21T10:25:18.409980+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:55:32.433084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":534,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816321+0000","last_change":"2024-09-21T10:17:51.265835+0000","last_active":"2024-09-21T10:25:25.816321+0000","last_peered":"2024-09-21T10:25:25.816321+0000","last_clean":"2024-09-21T10:25:25.816321+0000","last_became_active":"2024-09-21T10:17:51.248291+0000","last_became_peered":"2024-09-21T10:17:51.248291+0000","last_unstale":"2024-09-21T10:25:25.816321+0000","last_undegraded":"2024-09-21T10:25:25.816321+0000","last_fullsized":"2024-09-21T10:25:25.816321+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:09.301300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410210+0000","last_change":"2024-09-21T10:23:39.315690+0000","last_active":"2024-09-21T10:25:18.410210+0000","last_peered":"2024-09-21T10:25:18.410210+0000","last_clean":"2024-09-21T10:25:18.410210+0000","last_became_active":"2024-09-21T10:23:39.315283+0000","last_became_peered":"2024-09-21T10:23:39.315283+0000","last_unstale":"2024-09-21T10:25:18.410210+0000","last_undegraded":"2024-09-21T10:25:18.410210+0000","last_fullsized":"2024-09-21T10:25:18.410210+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:23:44.384937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":105,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816313+0000","last_change":"2024-09-21T10:22:17.009006+0000","last_active":"2024-09-21T10:25:25.816313+0000","last_peered":"2024-09-21T10:25:25.816313+0000","last_clean":"2024-09-21T10:25:25.816313+0000","last_became_active":"2024-09-21T10:22:16.993708+0000","last_became_peered":"2024-09-21T10:22:16.993708+0000","last_unstale":"2024-09-21T10:25:25.816313+0000","last_undegraded":"2024-09-21T10:25:25.816313+0000","last_fullsized":"2024-09-21T10:25:25.816313+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:23:46.476840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1619,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815745+0000","last_change":"2024-09-21T10:23:39.371348+0000","last_active":"2024-09-21T10:25:25.815745+0000","last_peered":"2024-09-21T10:25:25.815745+0000","last_clean":"2024-09-21T10:25:25.815745+0000","last_became_active":"2024-09-21T10:23:39.370757+0000","last_became_peered":"2024-09-21T10:23:39.370757+0000","last_unstale":"2024-09-21T10:25:25.815745+0000","last_undegraded":"2024-09-21T10:25:25.815745+0000","last_fullsized":"2024-09-21T10:25:25.815745+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:08:46.292157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":142,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410888+0000","last_change":"2024-09-21T10:23:39.319362+0000","last_active":"2024-09-21T10:25:18.410888+0000","last_peered":"2024-09-21T10:25:18.410888+0000","last_clean":"2024-09-21T10:25:18.410888+0000","last_became_active":"2024-09-21T10:23:39.318981+0000","last_became_peered":"2024-09-21T10:23:39.318981+0000","last_unstale":"2024-09-21T10:25:18.410888+0000","last_undegraded":"2024-09-21T10:25:18.410888+0000","last_fullsized":"2024-09-21T10:25:18.410888+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:27:54.616448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":878,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815312+0000","last_change":"2024-09-21T10:23:39.412337+0000","last_active":"2024-09-21T10:25:25.815312+0000","last_peered":"2024-09-21T10:25:25.815312+0000","last_clean":"2024-09-21T10:25:25.815312+0000","last_became_active":"2024-09-21T10:23:39.411958+0000","last_became_peered":"2024-09-21T10:23:39.411958+0000","last_unstale":"2024-09-21T10:25:25.815312+0000","last_undegraded":"2024-09-21T10:25:25.815312+0000","last_fullsized":"2024-09-21T10:25:25.815312+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:02:13.265451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410795+0000","last_change":"2024-09-21T10:23:39.372274+0000","last_active":"2024-09-21T10:25:18.410795+0000","last_peered":"2024-09-21T10:25:18.410795+0000","last_clean":"2024-09-21T10:25:18.410795+0000","last_became_active":"2024-09-21T10:23:39.371979+0000","last_became_peered":"2024-09-21T10:23:39.371979+0000","last_unstale":"2024-09-21T10:25:18.410795+0000","last_undegraded":"2024-09-21T10:25:18.410795+0000","last_fullsized":"2024-09-21T10:25:18.410795+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:03.528411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2005,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816096+0000","last_change":"2024-09-21T10:23:39.373141+0000","last_active":"2024-09-21T10:25:25.816096+0000","last_peered":"2024-09-21T10:25:25.816096+0000","last_clean":"2024-09-21T10:25:25.816096+0000","last_became_active":"2024-09-21T10:23:39.372532+0000","last_became_peered":"2024-09-21T10:23:39.372532+0000","last_unstale":"2024-09-21T10:25:25.816096+0000","last_undegraded":"2024-09-21T10:25:25.816096+0000","last_fullsized":"2024-09-21T10:25:25.816096+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:34:21.619250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"361'149","reported_seq":1508,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814935+0000","last_change":"2024-09-21T10:23:39.307102+0000","last_active":"2024-09-21T10:25:25.814935+0000","last_peered":"2024-09-21T10:25:25.814935+0000","last_clean":"2024-09-21T10:25:25.814935+0000","last_became_active":"2024-09-21T10:23:39.306777+0000","last_became_peered":"2024-09-21T10:23:39.306777+0000","last_unstale":"2024-09-21T10:25:25.814935+0000","last_undegraded":"2024-09-21T10:25:25.814935+0000","last_fullsized":"2024-09-21T10:25:25.814935+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:33.910694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":524,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815198+0000","last_change":"2024-09-21T10:23:39.398349+0000","last_active":"2024-09-21T10:25:25.815198+0000","last_peered":"2024-09-21T10:25:25.815198+0000","last_clean":"2024-09-21T10:25:25.815198+0000","last_became_active":"2024-09-21T10:23:39.397605+0000","last_became_peered":"2024-09-21T10:23:39.397605+0000","last_unstale":"2024-09-21T10:25:25.815198+0000","last_undegraded":"2024-09-21T10:25:25.815198+0000","last_fullsized":"2024-09-21T10:25:25.815198+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:20:37.735612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1613,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815200+0000","last_change":"2024-09-21T10:23:39.391183+0000","last_active":"2024-09-21T10:25:25.815200+0000","last_peered":"2024-09-21T10:25:25.815200+0000","last_clean":"2024-09-21T10:25:25.815200+0000","last_became_active":"2024-09-21T10:23:39.390886+0000","last_became_peered":"2024-09-21T10:23:39.390886+0000","last_unstale":"2024-09-21T10:25:25.815200+0000","last_undegraded":"2024-09-21T10:25:25.815200+0000","last_fullsized":"2024-09-21T10:25:25.815200+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:16:17.625754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"361'149","reported_seq":1507,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.409906+0000","last_change":"2024-09-21T10:23:39.315754+0000","last_active":"2024-09-21T10:25:18.409906+0000","last_peered":"2024-09-21T10:25:18.409906+0000","last_clean":"2024-09-21T10:25:18.409906+0000","last_became_active":"2024-09-21T10:23:39.315278+0000","last_became_peered":"2024-09-21T10:23:39.315278+0000","last_unstale":"2024-09-21T10:25:18.409906+0000","last_undegraded":"2024-09-21T10:25:18.409906+0000","last_fullsized":"2024-09-21T10:25:18.409906+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:14.982755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:14.982755+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:03:08.540111+0000","scrub_duration":0.0028608990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":837,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815527+0000","last_change":"2024-09-21T10:23:39.411248+0000","last_active":"2024-09-21T10:25:25.815527+0000","last_peered":"2024-09-21T10:25:25.815527+0000","last_clean":"2024-09-21T10:25:25.815527+0000","last_became_active":"2024-09-21T10:23:39.410865+0000","last_became_peered":"2024-09-21T10:23:39.410865+0000","last_unstale":"2024-09-21T10:25:25.815527+0000","last_undegraded":"2024-09-21T10:25:25.815527+0000","last_fullsized":"2024-09-21T10:25:25.815527+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:54:47.337510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815784+0000","last_change":"2024-09-21T10:23:39.322783+0000","last_active":"2024-09-21T10:25:25.815784+0000","last_peered":"2024-09-21T10:25:25.815784+0000","last_clean":"2024-09-21T10:25:25.815784+0000","last_became_active":"2024-09-21T10:23:39.321617+0000","last_became_peered":"2024-09-21T10:23:39.321617+0000","last_unstale":"2024-09-21T10:25:25.815784+0000","last_undegraded":"2024-09-21T10:25:25.815784+0000","last_fullsized":"2024-09-21T10:25:25.815784+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:38:11.432422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1653,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815606+0000","last_change":"2024-09-21T10:17:51.250802+0000","last_active":"2024-09-21T10:25:25.815606+0000","last_peered":"2024-09-21T10:25:25.815606+0000","last_clean":"2024-09-21T10:25:25.815606+0000","last_became_active":"2024-09-21T10:17:51.250385+0000","last_became_peered":"2024-09-21T10:17:51.250385+0000","last_unstale":"2024-09-21T10:25:25.815606+0000","last_undegraded":"2024-09-21T10:25:25.815606+0000","last_fullsized":"2024-09-21T10:25:25.815606+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:53:10.262260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":132,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816133+0000","last_change":"2024-09-21T10:23:39.349126+0000","last_active":"2024-09-21T10:25:25.816133+0000","last_peered":"2024-09-21T10:25:25.816133+0000","last_clean":"2024-09-21T10:25:25.816133+0000","last_became_active":"2024-09-21T10:23:39.348799+0000","last_became_peered":"2024-09-21T10:23:39.348799+0000","last_unstale":"2024-09-21T10:25:25.816133+0000","last_undegraded":"2024-09-21T10:25:25.816133+0000","last_fullsized":"2024-09-21T10:25:25.816133+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:42:04.424772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"361'149","reported_seq":1488,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410093+0000","last_change":"2024-09-21T10:24:55.184035+0000","last_active":"2024-09-21T10:25:18.410093+0000","last_peered":"2024-09-21T10:25:18.410093+0000","last_clean":"2024-09-21T10:25:18.410093+0000","last_became_active":"2024-09-21T10:22:20.013780+0000","last_became_peered":"2024-09-21T10:22:20.013780+0000","last_unstale":"2024-09-21T10:25:18.410093+0000","last_undegraded":"2024-09-21T10:25:18.410093+0000","last_fullsized":"2024-09-21T10:25:18.410093+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:55.183929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:55.183929+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:14:02.561714+0000","scrub_duration":0.001781761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":549,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814906+0000","last_change":"2024-09-21T10:23:39.416450+0000","last_active":"2024-09-21T10:25:25.814906+0000","last_peered":"2024-09-21T10:25:25.814906+0000","last_clean":"2024-09-21T10:25:25.814906+0000","last_became_active":"2024-09-21T10:23:39.414340+0000","last_became_peered":"2024-09-21T10:23:39.414340+0000","last_unstale":"2024-09-21T10:25:25.814906+0000","last_undegraded":"2024-09-21T10:25:25.814906+0000","last_fullsized":"2024-09-21T10:25:25.814906+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:08:48.376073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2009,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816494+0000","last_change":"2024-09-21T10:23:39.350772+0000","last_active":"2024-09-21T10:25:25.816494+0000","last_peered":"2024-09-21T10:25:25.816494+0000","last_clean":"2024-09-21T10:25:25.816494+0000","last_became_active":"2024-09-21T10:23:39.345255+0000","last_became_peered":"2024-09-21T10:23:39.345255+0000","last_unstale":"2024-09-21T10:25:25.816494+0000","last_undegraded":"2024-09-21T10:25:25.816494+0000","last_fullsized":"2024-09-21T10:25:25.816494+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:22.523103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1513,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815188+0000","last_change":"2024-09-21T10:23:39.412844+0000","last_active":"2024-09-21T10:25:25.815188+0000","last_peered":"2024-09-21T10:25:25.815188+0000","last_clean":"2024-09-21T10:25:25.815188+0000","last_became_active":"2024-09-21T10:23:39.412591+0000","last_became_peered":"2024-09-21T10:23:39.412591+0000","last_unstale":"2024-09-21T10:25:25.815188+0000","last_undegraded":"2024-09-21T10:25:25.815188+0000","last_fullsized":"2024-09-21T10:25:25.815188+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:15.395609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:15.395609+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:33.836084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":145,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410642+0000","last_change":"2024-09-21T10:23:39.375299+0000","last_active":"2024-09-21T10:25:18.410642+0000","last_peered":"2024-09-21T10:25:18.410642+0000","last_clean":"2024-09-21T10:25:18.410642+0000","last_became_active":"2024-09-21T10:23:39.374244+0000","last_became_peered":"2024-09-21T10:23:39.374244+0000","last_unstale":"2024-09-21T10:25:18.410642+0000","last_undegraded":"2024-09-21T10:25:18.410642+0000","last_fullsized":"2024-09-21T10:25:18.410642+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:25:53.897017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":582,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410432+0000","last_change":"2024-09-21T10:23:39.305168+0000","last_active":"2024-09-21T10:25:18.410432+0000","last_peered":"2024-09-21T10:25:18.410432+0000","last_clean":"2024-09-21T10:25:18.410432+0000","last_became_active":"2024-09-21T10:23:39.304815+0000","last_became_peered":"2024-09-21T10:23:39.304815+0000","last_unstale":"2024-09-21T10:25:18.410432+0000","last_undegraded":"2024-09-21T10:25:18.410432+0000","last_fullsized":"2024-09-21T10:25:18.410432+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:11.593431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815660+0000","last_change":"2024-09-21T10:23:39.322438+0000","last_active":"2024-09-21T10:25:25.815660+0000","last_peered":"2024-09-21T10:25:25.815660+0000","last_clean":"2024-09-21T10:25:25.815660+0000","last_became_active":"2024-09-21T10:23:39.321191+0000","last_became_peered":"2024-09-21T10:23:39.321191+0000","last_unstale":"2024-09-21T10:25:25.815660+0000","last_undegraded":"2024-09-21T10:25:25.815660+0000","last_fullsized":"2024-09-21T10:25:25.815660+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.957289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"361'149","reported_seq":1521,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816202+0000","last_change":"2024-09-21T10:22:20.017945+0000","last_active":"2024-09-21T10:25:25.816202+0000","last_peered":"2024-09-21T10:25:25.816202+0000","last_clean":"2024-09-21T10:25:25.816202+0000","last_became_active":"2024-09-21T10:22:20.017453+0000","last_became_peered":"2024-09-21T10:22:20.017453+0000","last_unstale":"2024-09-21T10:25:25.816202+0000","last_undegraded":"2024-09-21T10:25:25.816202+0000","last_fullsized":"2024-09-21T10:25:25.816202+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:32.811754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1301,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815102+0000","last_change":"2024-09-21T10:23:39.413518+0000","last_active":"2024-09-21T10:25:25.815102+0000","last_peered":"2024-09-21T10:25:25.815102+0000","last_clean":"2024-09-21T10:25:25.815102+0000","last_became_active":"2024-09-21T10:23:39.413270+0000","last_became_peered":"2024-09-21T10:23:39.413270+0000","last_unstale":"2024-09-21T10:25:25.815102+0000","last_undegraded":"2024-09-21T10:25:25.815102+0000","last_fullsized":"2024-09-21T10:25:25.815102+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:35.809985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1741,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814529+0000","last_change":"2024-09-21T10:23:39.316171+0000","last_active":"2024-09-21T10:25:25.814529+0000","last_peered":"2024-09-21T10:25:25.814529+0000","last_clean":"2024-09-21T10:25:25.814529+0000","last_became_active":"2024-09-21T10:23:39.315540+0000","last_became_peered":"2024-09-21T10:23:39.315540+0000","last_unstale":"2024-09-21T10:25:25.814529+0000","last_undegraded":"2024-09-21T10:25:25.814529+0000","last_fullsized":"2024-09-21T10:25:25.814529+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.638899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1284,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816189+0000","last_change":"2024-09-21T10:17:51.252035+0000","last_active":"2024-09-21T10:25:25.816189+0000","last_peered":"2024-09-21T10:25:25.816189+0000","last_clean":"2024-09-21T10:25:25.816189+0000","last_became_active":"2024-09-21T10:17:51.250816+0000","last_became_peered":"2024-09-21T10:17:51.250816+0000","last_unstale":"2024-09-21T10:25:25.816189+0000","last_undegraded":"2024-09-21T10:25:25.816189+0000","last_fullsized":"2024-09-21T10:25:25.816189+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:41.776582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"361'149","reported_seq":1471,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410119+0000","last_change":"2024-09-21T10:22:15.938120+0000","last_active":"2024-09-21T10:25:18.410119+0000","last_peered":"2024-09-21T10:25:18.410119+0000","last_clean":"2024-09-21T10:25:18.410119+0000","last_became_active":"2024-09-21T10:21:56.928301+0000","last_became_peered":"2024-09-21T10:21:56.928301+0000","last_unstale":"2024-09-21T10:25:18.410119+0000","last_undegraded":"2024-09-21T10:25:18.410119+0000","last_fullsized":"2024-09-21T10:25:18.410119+0000","mapping_epoch":1358,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1359,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:15.937862+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:15.937862+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:01.614339+0000","scrub_duration":0.00329533,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1861,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814562+0000","last_change":"2024-09-21T10:23:39.450173+0000","last_active":"2024-09-21T10:25:25.814562+0000","last_peered":"2024-09-21T10:25:25.814562+0000","last_clean":"2024-09-21T10:25:25.814562+0000","last_became_active":"2024-09-21T10:23:39.449505+0000","last_became_peered":"2024-09-21T10:23:39.449505+0000","last_unstale":"2024-09-21T10:25:25.814562+0000","last_undegraded":"2024-09-21T10:25:25.814562+0000","last_fullsized":"2024-09-21T10:25:25.814562+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:04.778009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2032,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815375+0000","last_change":"2024-09-21T10:23:39.414249+0000","last_active":"2024-09-21T10:25:25.815375+0000","last_peered":"2024-09-21T10:25:25.815375+0000","last_clean":"2024-09-21T10:25:25.815375+0000","last_became_active":"2024-09-21T10:23:39.413975+0000","last_became_peered":"2024-09-21T10:23:39.413975+0000","last_unstale":"2024-09-21T10:25:25.815375+0000","last_undegraded":"2024-09-21T10:25:25.815375+0000","last_fullsized":"2024-09-21T10:25:25.815375+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:20.488016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2025,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816401+0000","last_change":"2024-09-21T10:23:39.412694+0000","last_active":"2024-09-21T10:25:25.816401+0000","last_peered":"2024-09-21T10:25:25.816401+0000","last_clean":"2024-09-21T10:25:25.816401+0000","last_became_active":"2024-09-21T10:23:39.412389+0000","last_became_peered":"2024-09-21T10:23:39.412389+0000","last_unstale":"2024-09-21T10:25:25.816401+0000","last_undegraded":"2024-09-21T10:25:25.816401+0000","last_fullsized":"2024-09-21T10:25:25.816401+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:06:30.102172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"361'149","reported_seq":1488,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410011+0000","last_change":"2024-09-21T10:22:20.014154+0000","last_active":"2024-09-21T10:25:18.410011+0000","last_peered":"2024-09-21T10:25:18.410011+0000","last_clean":"2024-09-21T10:25:18.410011+0000","last_became_active":"2024-09-21T10:22:20.013516+0000","last_became_peered":"2024-09-21T10:22:20.013516+0000","last_unstale":"2024-09-21T10:25:18.410011+0000","last_undegraded":"2024-09-21T10:25:18.410011+0000","last_fullsized":"2024-09-21T10:25:18.410011+0000","mapping_epoch":1364,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:11.089486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:11.089486+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T22:11:55.464199+0000","scrub_duration":0.010125244,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":142,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410686+0000","last_change":"2024-09-21T10:23:39.346213+0000","last_active":"2024-09-21T10:25:18.410686+0000","last_peered":"2024-09-21T10:25:18.410686+0000","last_clean":"2024-09-21T10:25:18.410686+0000","last_became_active":"2024-09-21T10:23:39.346009+0000","last_became_peered":"2024-09-21T10:23:39.346009+0000","last_unstale":"2024-09-21T10:25:18.410686+0000","last_undegraded":"2024-09-21T10:25:18.410686+0000","last_fullsized":"2024-09-21T10:25:18.410686+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:59:34.154751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1222,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815893+0000","last_change":"2024-09-21T10:22:16.994613+0000","last_active":"2024-09-21T10:25:25.815893+0000","last_peered":"2024-09-21T10:25:25.815893+0000","last_clean":"2024-09-21T10:25:25.815893+0000","last_became_active":"2024-09-21T10:22:16.994182+0000","last_became_peered":"2024-09-21T10:22:16.994182+0000","last_unstale":"2024-09-21T10:25:25.815893+0000","last_undegraded":"2024-09-21T10:25:25.815893+0000","last_fullsized":"2024-09-21T10:25:25.815893+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:44.627749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1947,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815038+0000","last_change":"2024-09-21T10:23:39.315999+0000","last_active":"2024-09-21T10:25:25.815038+0000","last_peered":"2024-09-21T10:25:25.815038+0000","last_clean":"2024-09-21T10:25:25.815038+0000","last_became_active":"2024-09-21T10:23:39.315576+0000","last_became_peered":"2024-09-21T10:23:39.315576+0000","last_unstale":"2024-09-21T10:25:25.815038+0000","last_undegraded":"2024-09-21T10:25:25.815038+0000","last_fullsized":"2024-09-21T10:25:25.815038+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:21:28.524095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"361'149","reported_seq":1017,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815384+0000","last_change":"2024-09-21T10:23:39.364222+0000","last_active":"2024-09-21T10:25:25.815384+0000","last_peered":"2024-09-21T10:25:25.815384+0000","last_clean":"2024-09-21T10:25:25.815384+0000","last_became_active":"2024-09-21T10:23:39.363177+0000","last_became_peered":"2024-09-21T10:23:39.363177+0000","last_unstale":"2024-09-21T10:25:25.815384+0000","last_undegraded":"2024-09-21T10:25:25.815384+0000","last_fullsized":"2024-09-21T10:25:25.815384+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:36.892949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":322019,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410955+0000","last_change":"2024-09-21T10:23:39.376204+0000","last_active":"2024-09-21T10:25:18.410955+0000","last_peered":"2024-09-21T10:25:18.410955+0000","last_clean":"2024-09-21T10:25:18.410955+0000","last_became_active":"2024-09-21T10:23:39.376009+0000","last_became_peered":"2024-09-21T10:23:39.376009+0000","last_unstale":"2024-09-21T10:25:18.410955+0000","last_undegraded":"2024-09-21T10:25:18.410955+0000","last_fullsized":"2024-09-21T10:25:18.410955+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:43.583701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2040,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815808+0000","last_change":"2024-09-21T10:23:39.370875+0000","last_active":"2024-09-21T10:25:25.815808+0000","last_peered":"2024-09-21T10:25:25.815808+0000","last_clean":"2024-09-21T10:25:25.815808+0000","last_became_active":"2024-09-21T10:23:39.370134+0000","last_became_peered":"2024-09-21T10:23:39.370134+0000","last_unstale":"2024-09-21T10:25:25.815808+0000","last_undegraded":"2024-09-21T10:25:25.815808+0000","last_fullsized":"2024-09-21T10:25:25.815808+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:57:24.513775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":555,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815172+0000","last_change":"2024-09-21T10:17:51.252259+0000","last_active":"2024-09-21T10:25:25.815172+0000","last_peered":"2024-09-21T10:25:25.815172+0000","last_clean":"2024-09-21T10:25:25.815172+0000","last_became_active":"2024-09-21T10:17:51.251574+0000","last_became_peered":"2024-09-21T10:17:51.251574+0000","last_unstale":"2024-09-21T10:25:25.815172+0000","last_undegraded":"2024-09-21T10:25:25.815172+0000","last_fullsized":"2024-09-21T10:25:25.815172+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:09:26.778140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"361'149","reported_seq":1017,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815809+0000","last_change":"2024-09-21T10:23:39.302896+0000","last_active":"2024-09-21T10:25:25.815809+0000","last_peered":"2024-09-21T10:25:25.815809+0000","last_clean":"2024-09-21T10:25:25.815809+0000","last_became_active":"2024-09-21T10:23:39.302269+0000","last_became_peered":"2024-09-21T10:23:39.302269+0000","last_unstale":"2024-09-21T10:25:25.815809+0000","last_undegraded":"2024-09-21T10:25:25.815809+0000","last_fullsized":"2024-09-21T10:25:25.815809+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:32.864718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":268349,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":220,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814708+0000","last_change":"2024-09-21T10:23:39.303934+0000","last_active":"2024-09-21T10:25:25.814708+0000","last_peered":"2024-09-21T10:25:25.814708+0000","last_clean":"2024-09-21T10:25:25.814708+0000","last_became_active":"2024-09-21T10:23:39.303741+0000","last_became_peered":"2024-09-21T10:23:39.303741+0000","last_unstale":"2024-09-21T10:25:25.814708+0000","last_undegraded":"2024-09-21T10:25:25.814708+0000","last_fullsized":"2024-09-21T10:25:25.814708+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:57:12.138362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1186,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816316+0000","last_change":"2024-09-21T10:23:39.375346+0000","last_active":"2024-09-21T10:25:25.816316+0000","last_peered":"2024-09-21T10:25:25.816316+0000","last_clean":"2024-09-21T10:25:25.816316+0000","last_became_active":"2024-09-21T10:23:39.374849+0000","last_became_peered":"2024-09-21T10:23:39.374849+0000","last_unstale":"2024-09-21T10:25:25.816316+0000","last_undegraded":"2024-09-21T10:25:25.816316+0000","last_fullsized":"2024-09-21T10:25:25.816316+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:45:10.241167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":110,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410628+0000","last_change":"2024-09-21T10:17:51.261674+0000","last_active":"2024-09-21T10:25:18.410628+0000","last_peered":"2024-09-21T10:25:18.410628+0000","last_clean":"2024-09-21T10:25:18.410628+0000","last_became_active":"2024-09-21T10:17:51.261355+0000","last_became_peered":"2024-09-21T10:17:51.261355+0000","last_unstale":"2024-09-21T10:25:18.410628+0000","last_undegraded":"2024-09-21T10:25:18.410628+0000","last_fullsized":"2024-09-21T10:25:18.410628+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:40:48.016826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"361'149","reported_seq":1013,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410611+0000","last_change":"2024-09-21T10:24:57.148783+0000","last_active":"2024-09-21T10:25:18.410611+0000","last_peered":"2024-09-21T10:25:18.410611+0000","last_clean":"2024-09-21T10:25:18.410611+0000","last_became_active":"2024-09-21T10:23:39.318900+0000","last_became_peered":"2024-09-21T10:23:39.318900+0000","last_unstale":"2024-09-21T10:25:18.410611+0000","last_undegraded":"2024-09-21T10:25:18.410611+0000","last_fullsized":"2024-09-21T10:25:18.410611+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:57.148677+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:57.148677+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:25:19.862692+0000","scrub_duration":0.001855659,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":521,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411007+0000","last_change":"2024-09-21T10:23:39.305219+0000","last_active":"2024-09-21T10:25:18.411007+0000","last_peered":"2024-09-21T10:25:18.411007+0000","last_clean":"2024-09-21T10:25:18.411007+0000","last_became_active":"2024-09-21T10:23:39.304787+0000","last_became_peered":"2024-09-21T10:23:39.304787+0000","last_unstale":"2024-09-21T10:25:18.411007+0000","last_undegraded":"2024-09-21T10:25:18.411007+0000","last_fullsized":"2024-09-21T10:25:18.411007+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:54:14.699319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816032+0000","last_change":"2024-09-21T10:23:39.347172+0000","last_active":"2024-09-21T10:25:25.816032+0000","last_peered":"2024-09-21T10:25:25.816032+0000","last_clean":"2024-09-21T10:25:25.816032+0000","last_became_active":"2024-09-21T10:23:39.345326+0000","last_became_peered":"2024-09-21T10:23:39.345326+0000","last_unstale":"2024-09-21T10:25:25.816032+0000","last_undegraded":"2024-09-21T10:25:25.816032+0000","last_fullsized":"2024-09-21T10:25:25.816032+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:33:32.454312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":545,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814883+0000","last_change":"2024-09-21T10:23:39.415241+0000","last_active":"2024-09-21T10:25:25.814883+0000","last_peered":"2024-09-21T10:25:25.814883+0000","last_clean":"2024-09-21T10:25:25.814883+0000","last_became_active":"2024-09-21T10:23:39.412720+0000","last_became_peered":"2024-09-21T10:23:39.412720+0000","last_unstale":"2024-09-21T10:25:25.814883+0000","last_undegraded":"2024-09-21T10:25:25.814883+0000","last_fullsized":"2024-09-21T10:25:25.814883+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:09.855474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"361'149","reported_seq":1512,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815321+0000","last_change":"2024-09-21T10:23:39.316659+0000","last_active":"2024-09-21T10:25:25.815321+0000","last_peered":"2024-09-21T10:25:25.815321+0000","last_clean":"2024-09-21T10:25:25.815321+0000","last_became_active":"2024-09-21T10:23:39.316129+0000","last_became_peered":"2024-09-21T10:23:39.316129+0000","last_unstale":"2024-09-21T10:25:25.815321+0000","last_undegraded":"2024-09-21T10:25:25.815321+0000","last_fullsized":"2024-09-21T10:25:25.815321+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T09:32:50.664564+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-21T10:25:35.853740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":540,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.816321+0000","last_change":"2024-09-21T10:17:51.274066+0000","last_active":"2024-09-21T10:25:25.816321+0000","last_peered":"2024-09-21T10:25:25.816321+0000","last_clean":"2024-09-21T10:25:25.816321+0000","last_became_active":"2024-09-21T10:17:51.249233+0000","last_became_peered":"2024-09-21T10:17:51.249233+0000","last_unstale":"2024-09-21T10:25:25.816321+0000","last_undegraded":"2024-09-21T10:25:25.816321+0000","last_fullsized":"2024-09-21T10:25:25.816321+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:17.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":867,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815442+0000","last_change":"2024-09-21T10:23:39.363897+0000","last_active":"2024-09-21T10:25:25.815442+0000","last_peered":"2024-09-21T10:25:25.815442+0000","last_clean":"2024-09-21T10:25:25.815442+0000","last_became_active":"2024-09-21T10:23:39.362798+0000","last_became_peered":"2024-09-21T10:23:39.362798+0000","last_unstale":"2024-09-21T10:25:25.815442+0000","last_undegraded":"2024-09-21T10:25:25.815442+0000","last_fullsized":"2024-09-21T10:25:25.815442+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:46:04.258680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410229+0000","last_change":"2024-09-21T10:23:39.306440+0000","last_active":"2024-09-21T10:25:18.410229+0000","last_peered":"2024-09-21T10:25:18.410229+0000","last_clean":"2024-09-21T10:25:18.410229+0000","last_became_active":"2024-09-21T10:23:39.306047+0000","last_became_peered":"2024-09-21T10:23:39.306047+0000","last_unstale":"2024-09-21T10:25:18.410229+0000","last_undegraded":"2024-09-21T10:25:18.410229+0000","last_fullsized":"2024-09-21T10:25:18.410229+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:37:29.308333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1223,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815801+0000","last_change":"2024-09-21T10:22:16.993047+0000","last_active":"2024-09-21T10:25:25.815801+0000","last_peered":"2024-09-21T10:25:25.815801+0000","last_clean":"2024-09-21T10:25:25.815801+0000","last_became_active":"2024-09-21T10:22:16.992606+0000","last_became_peered":"2024-09-21T10:22:16.992606+0000","last_unstale":"2024-09-21T10:25:25.815801+0000","last_undegraded":"2024-09-21T10:25:25.815801+0000","last_fullsized":"2024-09-21T10:25:25.815801+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:41:35.971920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":289,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410458+0000","last_change":"2024-09-21T10:23:39.308252+0000","last_active":"2024-09-21T10:25:18.410458+0000","last_peered":"2024-09-21T10:25:18.410458+0000","last_clean":"2024-09-21T10:25:18.410458+0000","last_became_active":"2024-09-21T10:23:39.306643+0000","last_became_peered":"2024-09-21T10:23:39.306643+0000","last_unstale":"2024-09-21T10:25:18.410458+0000","last_undegraded":"2024-09-21T10:25:18.410458+0000","last_fullsized":"2024-09-21T10:25:18.410458+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:47:07.074129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19024,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2008,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815449+0000","last_change":"2024-09-21T10:22:16.994417+0000","last_active":"2024-09-21T10:25:25.815449+0000","last_peered":"2024-09-21T10:25:25.815449+0000","last_clean":"2024-09-21T10:25:25.815449+0000","last_became_active":"2024-09-21T10:22:16.994109+0000","last_became_peered":"2024-09-21T10:22:16.994109+0000","last_unstale":"2024-09-21T10:25:25.815449+0000","last_undegraded":"2024-09-21T10:25:25.815449+0000","last_fullsized":"2024-09-21T10:25:25.815449+0000","mapping_epoch":1361,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:55.406178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1509,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815745+0000","last_change":"2024-09-21T10:23:39.323087+0000","last_active":"2024-09-21T10:25:25.815745+0000","last_peered":"2024-09-21T10:25:25.815745+0000","last_clean":"2024-09-21T10:25:25.815745+0000","last_became_active":"2024-09-21T10:23:39.322041+0000","last_became_peered":"2024-09-21T10:23:39.322041+0000","last_unstale":"2024-09-21T10:25:25.815745+0000","last_undegraded":"2024-09-21T10:25:25.815745+0000","last_fullsized":"2024-09-21T10:25:25.815745+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:35:29.126970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":125,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411060+0000","last_change":"2024-09-21T10:23:39.334300+0000","last_active":"2024-09-21T10:25:18.411060+0000","last_peered":"2024-09-21T10:25:18.411060+0000","last_clean":"2024-09-21T10:25:18.411060+0000","last_became_active":"2024-09-21T10:23:39.334044+0000","last_became_peered":"2024-09-21T10:23:39.334044+0000","last_unstale":"2024-09-21T10:25:18.411060+0000","last_undegraded":"2024-09-21T10:25:18.411060+0000","last_fullsized":"2024-09-21T10:25:18.411060+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:26:36.147498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":651,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814812+0000","last_change":"2024-09-21T10:23:39.306893+0000","last_active":"2024-09-21T10:25:25.814812+0000","last_peered":"2024-09-21T10:25:25.814812+0000","last_clean":"2024-09-21T10:25:25.814812+0000","last_became_active":"2024-09-21T10:23:39.306560+0000","last_became_peered":"2024-09-21T10:23:39.306560+0000","last_unstale":"2024-09-21T10:25:25.814812+0000","last_undegraded":"2024-09-21T10:25:25.814812+0000","last_fullsized":"2024-09-21T10:25:25.814812+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:16.170412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":134,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411145+0000","last_change":"2024-09-21T10:23:39.337324+0000","last_active":"2024-09-21T10:25:18.411145+0000","last_peered":"2024-09-21T10:25:18.411145+0000","last_clean":"2024-09-21T10:25:18.411145+0000","last_became_active":"2024-09-21T10:23:39.337017+0000","last_became_peered":"2024-09-21T10:23:39.337017+0000","last_unstale":"2024-09-21T10:25:18.411145+0000","last_undegraded":"2024-09-21T10:25:18.411145+0000","last_fullsized":"2024-09-21T10:25:18.411145+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:53.271860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":635,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814977+0000","last_change":"2024-09-21T10:23:39.414802+0000","last_active":"2024-09-21T10:25:25.814977+0000","last_peered":"2024-09-21T10:25:25.814977+0000","last_clean":"2024-09-21T10:25:25.814977+0000","last_became_active":"2024-09-21T10:23:39.411110+0000","last_became_peered":"2024-09-21T10:23:39.411110+0000","last_unstale":"2024-09-21T10:25:25.814977+0000","last_undegraded":"2024-09-21T10:25:25.814977+0000","last_fullsized":"2024-09-21T10:25:25.814977+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:27.837285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":269,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.410980+0000","last_change":"2024-09-21T10:23:39.337324+0000","last_active":"2024-09-21T10:25:18.410980+0000","last_peered":"2024-09-21T10:25:18.410980+0000","last_clean":"2024-09-21T10:25:18.410980+0000","last_became_active":"2024-09-21T10:23:39.337017+0000","last_became_peered":"2024-09-21T10:23:39.337017+0000","last_unstale":"2024-09-21T10:25:18.410980+0000","last_undegraded":"2024-09-21T10:25:18.410980+0000","last_fullsized":"2024-09-21T10:25:18.410980+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:03:43.984727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":593,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814716+0000","last_change":"2024-09-21T10:23:39.415574+0000","last_active":"2024-09-21T10:25:25.814716+0000","last_peered":"2024-09-21T10:25:25.814716+0000","last_clean":"2024-09-21T10:25:25.814716+0000","last_became_active":"2024-09-21T10:23:39.413699+0000","last_became_peered":"2024-09-21T10:23:39.413699+0000","last_unstale":"2024-09-21T10:25:25.814716+0000","last_undegraded":"2024-09-21T10:25:25.814716+0000","last_fullsized":"2024-09-21T10:25:25.814716+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:10:11.990863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1856,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815906+0000","last_change":"2024-09-21T10:23:39.363264+0000","last_active":"2024-09-21T10:25:25.815906+0000","last_peered":"2024-09-21T10:25:25.815906+0000","last_clean":"2024-09-21T10:25:25.815906+0000","last_became_active":"2024-09-21T10:23:39.362615+0000","last_became_peered":"2024-09-21T10:23:39.362615+0000","last_unstale":"2024-09-21T10:25:25.815906+0000","last_undegraded":"2024-09-21T10:25:25.815906+0000","last_fullsized":"2024-09-21T10:25:25.815906+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:06:55.561333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":298,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815692+0000","last_change":"2024-09-21T10:23:39.371060+0000","last_active":"2024-09-21T10:25:25.815692+0000","last_peered":"2024-09-21T10:25:25.815692+0000","last_clean":"2024-09-21T10:25:25.815692+0000","last_became_active":"2024-09-21T10:23:39.370494+0000","last_became_peered":"2024-09-21T10:23:39.370494+0000","last_unstale":"2024-09-21T10:25:25.815692+0000","last_undegraded":"2024-09-21T10:25:25.815692+0000","last_fullsized":"2024-09-21T10:25:25.815692+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:09.975626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":518,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815864+0000","last_change":"2024-09-21T10:23:39.510198+0000","last_active":"2024-09-21T10:25:25.815864+0000","last_peered":"2024-09-21T10:25:25.815864+0000","last_clean":"2024-09-21T10:25:25.815864+0000","last_became_active":"2024-09-21T10:23:39.509474+0000","last_became_peered":"2024-09-21T10:23:39.509474+0000","last_unstale":"2024-09-21T10:25:25.815864+0000","last_undegraded":"2024-09-21T10:25:25.815864+0000","last_fullsized":"2024-09-21T10:25:25.815864+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.611788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":558,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815609+0000","last_change":"2024-09-21T10:17:51.250624+0000","last_active":"2024-09-21T10:25:25.815609+0000","last_peered":"2024-09-21T10:25:25.815609+0000","last_clean":"2024-09-21T10:25:25.815609+0000","last_became_active":"2024-09-21T10:17:51.248657+0000","last_became_peered":"2024-09-21T10:17:51.248657+0000","last_unstale":"2024-09-21T10:25:25.815609+0000","last_undegraded":"2024-09-21T10:25:25.815609+0000","last_fullsized":"2024-09-21T10:25:25.815609+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:57.653267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1898,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815938+0000","last_change":"2024-09-21T10:23:39.370592+0000","last_active":"2024-09-21T10:25:25.815938+0000","last_peered":"2024-09-21T10:25:25.815938+0000","last_clean":"2024-09-21T10:25:25.815938+0000","last_became_active":"2024-09-21T10:23:39.369832+0000","last_became_peered":"2024-09-21T10:23:39.369832+0000","last_unstale":"2024-09-21T10:25:25.815938+0000","last_undegraded":"2024-09-21T10:25:25.815938+0000","last_fullsized":"2024-09-21T10:25:25.815938+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:26:23.320589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1972,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815998+0000","last_change":"2024-09-21T10:23:39.373474+0000","last_active":"2024-09-21T10:25:25.815998+0000","last_peered":"2024-09-21T10:25:25.815998+0000","last_clean":"2024-09-21T10:25:25.815998+0000","last_became_active":"2024-09-21T10:23:39.373158+0000","last_became_peered":"2024-09-21T10:23:39.373158+0000","last_unstale":"2024-09-21T10:25:25.815998+0000","last_undegraded":"2024-09-21T10:25:25.815998+0000","last_fullsized":"2024-09-21T10:25:25.815998+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:41:53.362317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1500,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815111+0000","last_change":"2024-09-21T10:23:39.398601+0000","last_active":"2024-09-21T10:25:25.815111+0000","last_peered":"2024-09-21T10:25:25.815111+0000","last_clean":"2024-09-21T10:25:25.815111+0000","last_became_active":"2024-09-21T10:23:39.398064+0000","last_became_peered":"2024-09-21T10:23:39.398064+0000","last_unstale":"2024-09-21T10:25:25.815111+0000","last_undegraded":"2024-09-21T10:25:25.815111+0000","last_fullsized":"2024-09-21T10:25:25.815111+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:20:05.118881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":114,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:21.781263+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:25:21.781263+0000","last_peered":"2024-09-21T10:25:21.781263+0000","last_clean":"2024-09-21T10:25:21.781263+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:25:21.781263+0000","last_undegraded":"2024-09-21T10:25:21.781263+0000","last_fullsized":"2024-09-21T10:25:21.781263+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1562,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814812+0000","last_change":"2024-09-21T10:23:39.302265+0000","last_active":"2024-09-21T10:25:25.814812+0000","last_peered":"2024-09-21T10:25:25.814812+0000","last_clean":"2024-09-21T10:25:25.814812+0000","last_became_active":"2024-09-21T10:23:39.300210+0000","last_became_peered":"2024-09-21T10:23:39.300210+0000","last_unstale":"2024-09-21T10:25:25.814812+0000","last_undegraded":"2024-09-21T10:25:25.814812+0000","last_fullsized":"2024-09-21T10:25:25.814812+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:56:23.615984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":142,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:21.781306+0000","last_change":"2024-09-21T10:23:39.375969+0000","last_active":"2024-09-21T10:25:21.781306+0000","last_peered":"2024-09-21T10:25:21.781306+0000","last_clean":"2024-09-21T10:25:21.781306+0000","last_became_active":"2024-09-21T10:23:39.375686+0000","last_became_peered":"2024-09-21T10:23:39.375686+0000","last_unstale":"2024-09-21T10:25:21.781306+0000","last_undegraded":"2024-09-21T10:25:21.781306+0000","last_fullsized":"2024-09-21T10:25:21.781306+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:18:38.252378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":593,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.814687+0000","last_change":"2024-09-21T10:23:39.450465+0000","last_active":"2024-09-21T10:25:25.814687+0000","last_peered":"2024-09-21T10:25:25.814687+0000","last_clean":"2024-09-21T10:25:25.814687+0000","last_became_active":"2024-09-21T10:23:39.449879+0000","last_became_peered":"2024-09-21T10:23:39.449879+0000","last_unstale":"2024-09-21T10:25:25.814687+0000","last_undegraded":"2024-09-21T10:25:25.814687+0000","last_fullsized":"2024-09-21T10:25:25.814687+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:58.113689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":625,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815525+0000","last_change":"2024-09-21T10:23:39.361838+0000","last_active":"2024-09-21T10:25:25.815525+0000","last_peered":"2024-09-21T10:25:25.815525+0000","last_clean":"2024-09-21T10:25:25.815525+0000","last_became_active":"2024-09-21T10:23:39.361539+0000","last_became_peered":"2024-09-21T10:23:39.361539+0000","last_unstale":"2024-09-21T10:25:25.815525+0000","last_undegraded":"2024-09-21T10:25:25.815525+0000","last_fullsized":"2024-09-21T10:25:25.815525+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:15:39.263560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":243,"reported_epoch":1406,"state":"active+clean","last_fresh":"2024-09-21T10:25:25.815014+0000","last_change":"2024-09-21T10:23:39.378606+0000","last_active":"2024-09-21T10:25:25.815014+0000","last_peered":"2024-09-21T10:25:25.815014+0000","last_clean":"2024-09-21T10:25:25.815014+0000","last_became_active":"2024-09-21T10:23:39.378107+0000","last_became_peered":"2024-09-21T10:23:39.378107+0000","last_unstale":"2024-09-21T10:25:25.815014+0000","last_undegraded":"2024-09-21T10:25:25.815014+0000","last_fullsized":"2024-09-21T10:25:25.815014+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:11.536250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":146,"reported_epoch":1405,"state":"active+clean","last_fresh":"2024-09-21T10:25:18.411136+0000","last_change":"2024-09-21T10:23:39.305452+0000","last_active":"2024-09-21T10:25:18.411136+0000","last_peered":"2024-09-21T10:25:18.411136+0000","last_clean":"2024-09-21T10:25:18.411136+0000","last_became_active":"2024-09-21T10:23:39.305054+0000","last_became_peered":"2024-09-21T10:23:39.305054+0000","last_unstale":"2024-09-21T10:25:18.411136+0000","last_undegraded":"2024-09-21T10:25:18.411136+0000","last_fullsized":"2024-09-21T10:25:18.411136+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:30.307977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2328,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":21744,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":184320,"data_stored":32630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19584,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":76,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":584707,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":117,"num_write_kb":1318,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1824774,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234596,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496232,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757871,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":710932,"kb_used_data":8008,"kb_used_omap":1,"kb_used_meta":702910,"kb_avail":93034220,"statfs":{"total":95995035648,"available":95267041280,"internally_reserved":0,"allocated":8200192,"data_stored":5461910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":719780294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019507,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1333520,"kb_used_data":8004,"kb_used_omap":1,"kb_used_meta":1325502,"kb_avail":92411632,"statfs":{"total":95995035648,"available":94629511168,"internally_reserved":0,"allocated":8196096,"data_stored":5461891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1357314502},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1375,"seq":5905580032024,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":189836,"kb_used_data":8000,"kb_used_omap":52,"kb_used_meta":181771,"kb_avail":93555316,"statfs":{"total":95995035648,"available":95800643584,"internally_reserved":0,"allocated":8192000,"data_stored":5461866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54095,"internal_metadata":186133681},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":0,"seq":3418793968043,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047281,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516408,"kb_used_data":7404,"kb_used_omap":41,"kb_used_meta":1508950,"kb_avail":92228744,"statfs":{"total":95995035648,"available":94442233856,"internally_reserved":0,"allocated":7581696,"data_stored":4866056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1545165737},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":5656471928916,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4133,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10112,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:25:36.889 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-21T10:25:36.890 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pg_num 21 2024-09-21T10:25:37.092 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:36 smithi191 ceph-mon[102206]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:37.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:36 smithi076 ceph-mon[106482]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:37.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:36 smithi076 ceph-mon[110963]: from='client.46328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:25:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:37 smithi076 ceph-mon[110963]: 1.6 scrub starts 2024-09-21T10:25:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:37 smithi076 ceph-mon[110963]: 1.6 scrub ok 2024-09-21T10:25:37.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:37 smithi076 ceph-mon[110963]: pgmap v2341: 83 pgs: 83 active+clean; 472 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:37 smithi076 ceph-mon[106482]: 1.6 scrub starts 2024-09-21T10:25:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:37 smithi076 ceph-mon[106482]: 1.6 scrub ok 2024-09-21T10:25:37.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:37 smithi076 ceph-mon[106482]: pgmap v2341: 83 pgs: 83 active+clean; 472 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:37 smithi191 ceph-mon[102206]: 1.6 scrub starts 2024-09-21T10:25:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:37 smithi191 ceph-mon[102206]: 1.6 scrub ok 2024-09-21T10:25:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:37 smithi191 ceph-mon[102206]: pgmap v2341: 83 pgs: 83 active+clean; 472 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:38.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:38 smithi076 ceph-mon[106482]: 1.9 scrub starts 2024-09-21T10:25:38.846 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:38 smithi076 ceph-mon[106482]: 1.9 scrub ok 2024-09-21T10:25:38.847 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:38 smithi076 ceph-mon[110963]: 1.9 scrub starts 2024-09-21T10:25:38.847 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:38 smithi076 ceph-mon[110963]: 1.9 scrub ok 2024-09-21T10:25:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:38 smithi191 ceph-mon[102206]: 1.9 scrub starts 2024-09-21T10:25:39.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:38 smithi191 ceph-mon[102206]: 1.9 scrub ok 2024-09-21T10:25:39.691 INFO:teuthology.orchestra.run.smithi076.stderr:set pool 1 pg_num to 21 2024-09-21T10:25:40.054 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/33529714' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:25:40.054 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:39 smithi076 ceph-mon[106482]: pgmap v2342: 83 pgs: 83 active+clean; 367 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:40.055 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/33529714' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:25:40.055 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:39 smithi076 ceph-mon[110963]: pgmap v2342: 83 pgs: 83 active+clean; 367 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/33529714' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-21T10:25:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:39 smithi191 ceph-mon[102206]: pgmap v2342: 83 pgs: 83 active+clean; 367 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:41.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:40 smithi191 ceph-mon[102206]: 1.4 scrub starts 2024-09-21T10:25:41.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:40 smithi191 ceph-mon[102206]: 1.4 scrub ok 2024-09-21T10:25:41.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:40 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/33529714' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:25:41.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:40 smithi191 ceph-mon[102206]: osdmap e1407: 8 total, 8 up, 4 in 2024-09-21T10:25:41.015 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[106482]: 1.4 scrub starts 2024-09-21T10:25:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[106482]: 1.4 scrub ok 2024-09-21T10:25:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/33529714' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:25:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[106482]: osdmap e1407: 8 total, 8 up, 4 in 2024-09-21T10:25:41.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:41.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[110963]: 1.4 scrub starts 2024-09-21T10:25:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[110963]: 1.4 scrub ok 2024-09-21T10:25:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/33529714' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-21T10:25:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[110963]: osdmap e1407: 8 total, 8 up, 4 in 2024-09-21T10:25:41.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:41 smithi191 ceph-mon[102206]: pgmap v2344: 83 pgs: 83 active+clean; 367 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:25:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:41 smithi076 ceph-mon[106482]: pgmap v2344: 83 pgs: 83 active+clean; 367 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:25:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:41 smithi076 ceph-mon[110963]: pgmap v2344: 83 pgs: 83 active+clean; 367 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-21T10:25:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:42 smithi191 ceph-mon[102206]: 1.7 scrub starts 2024-09-21T10:25:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:42 smithi191 ceph-mon[102206]: 1.7 scrub ok 2024-09-21T10:25:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:42 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:25:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:42 smithi191 ceph-mon[102206]: osdmap e1408: 8 total, 8 up, 4 in 2024-09-21T10:25:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[106482]: 1.7 scrub starts 2024-09-21T10:25:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[106482]: 1.7 scrub ok 2024-09-21T10:25:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:25:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[106482]: osdmap e1408: 8 total, 8 up, 4 in 2024-09-21T10:25:43.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[110963]: 1.7 scrub starts 2024-09-21T10:25:43.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[110963]: 1.7 scrub ok 2024-09-21T10:25:43.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-21T10:25:43.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:42 smithi076 ceph-mon[110963]: osdmap e1408: 8 total, 8 up, 4 in 2024-09-21T10:25:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:25:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:25:43] "GET /metrics HTTP/1.1" 200 34198 "" "Prometheus/2.51.0" 2024-09-21T10:25:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:43 smithi191 ceph-mon[102206]: osdmap e1409: 8 total, 8 up, 4 in 2024-09-21T10:25:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:43 smithi191 ceph-mon[102206]: pgmap v2347: 93 pgs: 8 peering, 10 unknown, 75 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:43 smithi076 ceph-mon[106482]: osdmap e1409: 8 total, 8 up, 4 in 2024-09-21T10:25:44.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:43 smithi076 ceph-mon[106482]: pgmap v2347: 93 pgs: 8 peering, 10 unknown, 75 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:44.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:43 smithi076 ceph-mon[110963]: osdmap e1409: 8 total, 8 up, 4 in 2024-09-21T10:25:44.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:43 smithi076 ceph-mon[110963]: pgmap v2347: 93 pgs: 8 peering, 10 unknown, 75 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:44 smithi191 ceph-mon[102206]: 1.1 scrub starts 2024-09-21T10:25:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:44 smithi191 ceph-mon[102206]: 1.1 scrub ok 2024-09-21T10:25:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:44 smithi076 ceph-mon[106482]: 1.1 scrub starts 2024-09-21T10:25:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:44 smithi076 ceph-mon[106482]: 1.1 scrub ok 2024-09-21T10:25:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:44 smithi076 ceph-mon[110963]: 1.1 scrub starts 2024-09-21T10:25:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:44 smithi076 ceph-mon[110963]: 1.1 scrub ok 2024-09-21T10:25:45.434 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-21T10:25:45.434 INFO:tasks.thrashosds.thrasher:Adding osd 2 2024-09-21T10:25:45.435 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 2 2024-09-21T10:25:45.642 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:46.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:45 smithi076 ceph-mon[106482]: 1.2 scrub starts 2024-09-21T10:25:46.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:45 smithi076 ceph-mon[106482]: 1.2 scrub ok 2024-09-21T10:25:46.025 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:45 smithi076 ceph-mon[106482]: pgmap v2348: 93 pgs: 8 peering, 10 unknown, 75 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:46.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:45 smithi076 ceph-mon[110963]: 1.2 scrub starts 2024-09-21T10:25:46.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:45 smithi076 ceph-mon[110963]: 1.2 scrub ok 2024-09-21T10:25:46.026 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:45 smithi076 ceph-mon[110963]: pgmap v2348: 93 pgs: 8 peering, 10 unknown, 75 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:45 smithi191 ceph-mon[102206]: 1.2 scrub starts 2024-09-21T10:25:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:45 smithi191 ceph-mon[102206]: 1.2 scrub ok 2024-09-21T10:25:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:45 smithi191 ceph-mon[102206]: pgmap v2348: 93 pgs: 8 peering, 10 unknown, 75 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:46.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:46 smithi076 ceph-mon[110963]: 1.4 scrub starts 2024-09-21T10:25:46.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:46 smithi076 ceph-mon[110963]: 1.4 scrub ok 2024-09-21T10:25:46.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:46 smithi076 ceph-mon[106482]: 1.4 scrub starts 2024-09-21T10:25:46.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:46 smithi076 ceph-mon[106482]: 1.4 scrub ok 2024-09-21T10:25:47.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:46 smithi191 ceph-mon[102206]: 1.4 scrub starts 2024-09-21T10:25:47.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:46 smithi191 ceph-mon[102206]: 1.4 scrub ok 2024-09-21T10:25:47.728 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.2. 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:25:47.630+0000 7f240edc2640 -1 mon.a@0(leader).osd e1409 definitely_dead 0 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[106482]: 1.5 scrub starts 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[106482]: 1.5 scrub ok 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[106482]: pgmap v2349: 93 pgs: 3 peering, 4 unknown, 86 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1145772073' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[110963]: 1.5 scrub starts 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[110963]: 1.5 scrub ok 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[110963]: pgmap v2349: 93 pgs: 3 peering, 4 unknown, 86 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:48.089 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:47 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1145772073' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-21T10:25:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:47 smithi191 ceph-mon[102206]: 1.5 scrub starts 2024-09-21T10:25:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:47 smithi191 ceph-mon[102206]: 1.5 scrub ok 2024-09-21T10:25:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:47 smithi191 ceph-mon[102206]: pgmap v2349: 93 pgs: 3 peering, 4 unknown, 86 active+clean; 582 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:48.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:47 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1145772073' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-21T10:25:48.560 INFO:tasks.thrashosds.thrasher:Added osd 2 2024-09-21T10:25:48.560 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-21T10:25:48.560 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-09-21T10:25:48.560 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 6 2024-09-21T10:25:48.791 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1145772073' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-21T10:25:48.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:48 smithi076 ceph-mon[106482]: osdmap e1410: 8 total, 8 up, 5 in 2024-09-21T10:25:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1145772073' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-21T10:25:48.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:48 smithi076 ceph-mon[110963]: osdmap e1410: 8 total, 8 up, 5 in 2024-09-21T10:25:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1145772073' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-21T10:25:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:48 smithi191 ceph-mon[102206]: osdmap e1410: 8 total, 8 up, 5 in 2024-09-21T10:25:49.962 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: 1.14 scrub starts 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: 1.14 scrub ok 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: osdmap e1411: 8 total, 8 up, 5 in 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: pgmap v2352: 93 pgs: 93 active+clean; 475 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[106482]: osdmap e1412: 8 total, 8 up, 5 in 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: 1.14 scrub starts 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: 1.14 scrub ok 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: osdmap e1411: 8 total, 8 up, 5 in 2024-09-21T10:25:49.963 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: pgmap v2352: 93 pgs: 93 active+clean; 475 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:49.964 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:25:49.964 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:25:49.964 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:49 smithi076 ceph-mon[110963]: osdmap e1412: 8 total, 8 up, 5 in 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: 1.14 scrub starts 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: 1.14 scrub ok 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: osdmap e1411: 8 total, 8 up, 5 in 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: pgmap v2352: 93 pgs: 93 active+clean; 475 KiB data, 3.6 GiB used, 354 GiB / 358 GiB avail 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-21T10:25:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:49 smithi191 ceph-mon[102206]: osdmap e1412: 8 total, 8 up, 5 in 2024-09-21T10:25:50.745 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.6. 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:25:50.680+0000 7f240edc2640 -1 mon.a@0(leader).osd e1412 definitely_dead 0 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[106482]: 1.c scrub starts 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[106482]: 1.c scrub ok 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1060553472' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:25:51.075 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[106482]: osdmap e1413: 8 total, 8 up, 6 in 2024-09-21T10:25:51.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[110963]: 1.c scrub starts 2024-09-21T10:25:51.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[110963]: 1.c scrub ok 2024-09-21T10:25:51.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1060553472' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:25:51.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:25:51.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:25:51.076 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:50 smithi076 ceph-mon[110963]: osdmap e1413: 8 total, 8 up, 6 in 2024-09-21T10:25:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:50 smithi191 ceph-mon[102206]: 1.c scrub starts 2024-09-21T10:25:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:50 smithi191 ceph-mon[102206]: 1.c scrub ok 2024-09-21T10:25:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:50 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1060553472' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:25:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:50 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-09-21T10:25:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:50 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-09-21T10:25:51.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:50 smithi191 ceph-mon[102206]: osdmap e1413: 8 total, 8 up, 6 in 2024-09-21T10:25:51.509 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-09-21T10:25:51.509 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-21T10:25:51.509 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-09-21T10:25:51.510 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 0 2024-09-21T10:25:51.713 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:52.018 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:51 smithi076 ceph-mon[106482]: pgmap v2355: 93 pgs: 93 active+clean; 475 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:25:52.019 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:51 smithi076 ceph-mon[106482]: osdmap e1414: 8 total, 8 up, 6 in 2024-09-21T10:25:52.019 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:51 smithi076 ceph-mon[110963]: pgmap v2355: 93 pgs: 93 active+clean; 475 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:25:52.019 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:51 smithi076 ceph-mon[110963]: osdmap e1414: 8 total, 8 up, 6 in 2024-09-21T10:25:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:51 smithi191 ceph-mon[102206]: pgmap v2355: 93 pgs: 93 active+clean; 475 KiB data, 4.3 GiB used, 443 GiB / 447 GiB avail 2024-09-21T10:25:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:51 smithi191 ceph-mon[102206]: osdmap e1414: 8 total, 8 up, 6 in 2024-09-21T10:25:53.494 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:25:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:25:53] "GET /metrics HTTP/1.1" 200 34198 "" "Prometheus/2.51.0" 2024-09-21T10:25:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:53 smithi191 ceph-mon[102206]: osdmap e1415: 8 total, 8 up, 6 in 2024-09-21T10:25:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:53 smithi191 ceph-mon[102206]: pgmap v2358: 93 pgs: 1 peering, 6 active+undersized+remapped, 9 remapped+peering, 77 active+clean; 661 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 28 KiB/s rd, 82 KiB/s wr, 20 op/s; 170 KiB/s, 4 keys/s, 6 objects/s recovering 2024-09-21T10:25:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:53 smithi076 ceph-mon[110963]: osdmap e1415: 8 total, 8 up, 6 in 2024-09-21T10:25:54.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:53 smithi076 ceph-mon[110963]: pgmap v2358: 93 pgs: 1 peering, 6 active+undersized+remapped, 9 remapped+peering, 77 active+clean; 661 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 28 KiB/s rd, 82 KiB/s wr, 20 op/s; 170 KiB/s, 4 keys/s, 6 objects/s recovering 2024-09-21T10:25:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:53 smithi076 ceph-mon[106482]: osdmap e1415: 8 total, 8 up, 6 in 2024-09-21T10:25:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:53 smithi076 ceph-mon[106482]: pgmap v2358: 93 pgs: 1 peering, 6 active+undersized+remapped, 9 remapped+peering, 77 active+clean; 661 KiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 28 KiB/s rd, 82 KiB/s wr, 20 op/s; 170 KiB/s, 4 keys/s, 6 objects/s recovering 2024-09-21T10:25:54.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:25:53.777+0000 7f240edc2640 -1 mon.a@0(leader).osd e1416 definitely_dead 0 2024-09-21T10:25:54.745 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.0. 2024-09-21T10:25:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:54 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:25:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:54 smithi191 ceph-mon[102206]: osdmap e1416: 8 total, 8 up, 6 in 2024-09-21T10:25:55.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:54 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2466120183' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:25:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:54 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:25:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:54 smithi076 ceph-mon[106482]: osdmap e1416: 8 total, 8 up, 6 in 2024-09-21T10:25:55.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:54 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2466120183' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:25:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:54 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-21T10:25:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:54 smithi076 ceph-mon[110963]: osdmap e1416: 8 total, 8 up, 6 in 2024-09-21T10:25:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:54 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2466120183' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-21T10:25:55.545 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-09-21T10:25:55.545 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-21T10:25:55.545 INFO:tasks.thrashosds.thrasher:Adding osd 7 2024-09-21T10:25:55.545 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd in 7 2024-09-21T10:25:55.755 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2466120183' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[106482]: osdmap e1417: 8 total, 8 up, 7 in 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[106482]: pgmap v2361: 93 pgs: 16 peering, 5 active+undersized+remapped, 3 unknown, 8 remapped+peering, 61 active+clean; 661 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 28 KiB/s rd, 82 KiB/s wr, 20 op/s; 170 KiB/s, 4 keys/s, 5 objects/s recovering 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[106482]: osdmap e1418: 8 total, 8 up, 7 in 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2466120183' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:25:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[110963]: osdmap e1417: 8 total, 8 up, 7 in 2024-09-21T10:25:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[110963]: pgmap v2361: 93 pgs: 16 peering, 5 active+undersized+remapped, 3 unknown, 8 remapped+peering, 61 active+clean; 661 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 28 KiB/s rd, 82 KiB/s wr, 20 op/s; 170 KiB/s, 4 keys/s, 5 objects/s recovering 2024-09-21T10:25:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:55.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:55 smithi076 ceph-mon[110963]: osdmap e1418: 8 total, 8 up, 7 in 2024-09-21T10:25:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2466120183' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-21T10:25:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:55 smithi191 ceph-mon[102206]: osdmap e1417: 8 total, 8 up, 7 in 2024-09-21T10:25:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:55 smithi191 ceph-mon[102206]: pgmap v2361: 93 pgs: 16 peering, 5 active+undersized+remapped, 3 unknown, 8 remapped+peering, 61 active+clean; 661 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 28 KiB/s rd, 82 KiB/s wr, 20 op/s; 170 KiB/s, 4 keys/s, 5 objects/s recovering 2024-09-21T10:25:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:25:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:25:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:55 smithi191 ceph-mon[102206]: osdmap e1418: 8 total, 8 up, 7 in 2024-09-21T10:25:57.045 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:56 smithi076 ceph-mon[106482]: 1.12 scrub starts 2024-09-21T10:25:57.045 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:56 smithi076 ceph-mon[106482]: 1.12 scrub ok 2024-09-21T10:25:57.045 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:56 smithi076 ceph-mon[106482]: osdmap e1419: 8 total, 8 up, 7 in 2024-09-21T10:25:57.046 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:56 smithi076 ceph-mon[110963]: 1.12 scrub starts 2024-09-21T10:25:57.046 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:56 smithi076 ceph-mon[110963]: 1.12 scrub ok 2024-09-21T10:25:57.046 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:56 smithi076 ceph-mon[110963]: osdmap e1419: 8 total, 8 up, 7 in 2024-09-21T10:25:57.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:56 smithi191 ceph-mon[102206]: 1.12 scrub starts 2024-09-21T10:25:57.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:56 smithi191 ceph-mon[102206]: 1.12 scrub ok 2024-09-21T10:25:57.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:56 smithi191 ceph-mon[102206]: osdmap e1419: 8 total, 8 up, 7 in 2024-09-21T10:25:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:57 smithi191 ceph-mon[102206]: 1.e scrub starts 2024-09-21T10:25:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:57 smithi191 ceph-mon[102206]: 1.e scrub ok 2024-09-21T10:25:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:57 smithi191 ceph-mon[102206]: 1.7 scrub starts 2024-09-21T10:25:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:57 smithi191 ceph-mon[102206]: 1.7 scrub ok 2024-09-21T10:25:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:57 smithi191 ceph-mon[102206]: pgmap v2364: 93 pgs: 1 active+remapped, 15 peering, 4 active+undersized+remapped, 1 unknown, 2 remapped+peering, 70 active+clean; 634 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:25:58.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:57 smithi191 ceph-mon[102206]: osdmap e1420: 8 total, 8 up, 7 in 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[110963]: 1.e scrub starts 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[110963]: 1.e scrub ok 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[110963]: 1.7 scrub starts 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[110963]: 1.7 scrub ok 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[110963]: pgmap v2364: 93 pgs: 1 active+remapped, 15 peering, 4 active+undersized+remapped, 1 unknown, 2 remapped+peering, 70 active+clean; 634 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[110963]: osdmap e1420: 8 total, 8 up, 7 in 2024-09-21T10:25:58.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[106482]: 1.e scrub starts 2024-09-21T10:25:58.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[106482]: 1.e scrub ok 2024-09-21T10:25:58.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[106482]: 1.7 scrub starts 2024-09-21T10:25:58.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[106482]: 1.7 scrub ok 2024-09-21T10:25:58.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[106482]: pgmap v2364: 93 pgs: 1 active+remapped, 15 peering, 4 active+undersized+remapped, 1 unknown, 2 remapped+peering, 70 active+clean; 634 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:25:58.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-mon[106482]: osdmap e1420: 8 total, 8 up, 7 in 2024-09-21T10:25:58.124 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:57 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:25:57.793+0000 7f240edc2640 -1 mon.a@0(leader).osd e1420 definitely_dead 0 2024-09-21T10:25:58.756 INFO:teuthology.orchestra.run.smithi076.stderr:marked in osd.7. 2024-09-21T10:25:59.052 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1411502395' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:25:59.052 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:25:59.052 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-21T10:25:59.052 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[106482]: osdmap e1421: 8 total, 8 up, 8 in 2024-09-21T10:25:59.052 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1411502395' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:25:59.053 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:25:59.053 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-21T10:25:59.053 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:58 smithi076 ceph-mon[110963]: osdmap e1421: 8 total, 8 up, 8 in 2024-09-21T10:25:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1411502395' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:25:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:58 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["7"]} : dispatch 2024-09-21T10:25:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:58 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2024-09-21T10:25:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:58 smithi191 ceph-mon[102206]: osdmap e1421: 8 total, 8 up, 8 in 2024-09-21T10:25:59.486 INFO:tasks.thrashosds.thrasher:Added osd 7 2024-09-21T10:25:59.486 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-21T10:25:59.486 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd pool set .mgr pgp_num 21 2024-09-21T10:25:59.692 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:00.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:59 smithi076 ceph-mon[106482]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-21T10:26:00.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:59 smithi076 ceph-mon[106482]: osdmap e1422: 8 total, 8 up, 8 in 2024-09-21T10:26:00.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:25:59 smithi076 ceph-mon[106482]: pgmap v2368: 93 pgs: 1 active+remapped, 15 peering, 5 active+undersized+remapped, 1 unknown, 71 active+clean; 634 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 133 KiB/s, 2 objects/s recovering 2024-09-21T10:26:00.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:59 smithi076 ceph-mon[110963]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-21T10:26:00.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:59 smithi076 ceph-mon[110963]: osdmap e1422: 8 total, 8 up, 8 in 2024-09-21T10:26:00.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:25:59 smithi076 ceph-mon[110963]: pgmap v2368: 93 pgs: 1 active+remapped, 15 peering, 5 active+undersized+remapped, 1 unknown, 71 active+clean; 634 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 133 KiB/s, 2 objects/s recovering 2024-09-21T10:26:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:59 smithi191 ceph-mon[102206]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-21T10:26:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:59 smithi191 ceph-mon[102206]: osdmap e1422: 8 total, 8 up, 8 in 2024-09-21T10:26:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:25:59 smithi191 ceph-mon[102206]: pgmap v2368: 93 pgs: 1 active+remapped, 15 peering, 5 active+undersized+remapped, 1 unknown, 71 active+clean; 634 KiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 133 KiB/s, 2 objects/s recovering 2024-09-21T10:26:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[106482]: 1.11 scrub starts 2024-09-21T10:26:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[106482]: 1.11 scrub ok 2024-09-21T10:26:00.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[106482]: 1.13 scrub starts 2024-09-21T10:26:00.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[106482]: 1.13 scrub ok 2024-09-21T10:26:00.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[106482]: osdmap e1423: 8 total, 8 up, 8 in 2024-09-21T10:26:00.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[110963]: 1.11 scrub starts 2024-09-21T10:26:00.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[110963]: 1.11 scrub ok 2024-09-21T10:26:00.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[110963]: 1.13 scrub starts 2024-09-21T10:26:00.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[110963]: 1.13 scrub ok 2024-09-21T10:26:00.875 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:00 smithi076 ceph-mon[110963]: osdmap e1423: 8 total, 8 up, 8 in 2024-09-21T10:26:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:00 smithi191 ceph-mon[102206]: 1.11 scrub starts 2024-09-21T10:26:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:00 smithi191 ceph-mon[102206]: 1.11 scrub ok 2024-09-21T10:26:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:00 smithi191 ceph-mon[102206]: 1.13 scrub starts 2024-09-21T10:26:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:00 smithi191 ceph-mon[102206]: 1.13 scrub ok 2024-09-21T10:26:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:00 smithi191 ceph-mon[102206]: osdmap e1423: 8 total, 8 up, 8 in 2024-09-21T10:26:02.330 INFO:teuthology.orchestra.run.smithi076.stderr:set pool 1 pgp_num to 21 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: osdmap e1424: 8 total, 8 up, 8 in 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: pgmap v2371: 93 pgs: 4 active+undersized+remapped, 4 remapped+peering, 85 active+clean; 634 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 117 B/s, 0 objects/s recovering 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: 1.f scrub starts 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: 1.f scrub ok 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/101834139' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:02 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: osdmap e1424: 8 total, 8 up, 8 in 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: pgmap v2371: 93 pgs: 4 active+undersized+remapped, 4 remapped+peering, 85 active+clean; 634 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 117 B/s, 0 objects/s recovering 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: 1.f scrub starts 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: 1.f scrub ok 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/101834139' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: osdmap e1424: 8 total, 8 up, 8 in 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: pgmap v2371: 93 pgs: 4 active+undersized+remapped, 4 remapped+peering, 85 active+clean; 634 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 117 B/s, 0 objects/s recovering 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: 1.f scrub starts 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: 1.f scrub ok 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/101834139' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"} : dispatch 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:02.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:02 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:03.236 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-21T10:26:03.520 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:26:03] "GET /metrics HTTP/1.1" 200 34216 "" "Prometheus/2.51.0" 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/101834139' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[106482]: osdmap e1425: 8 total, 8 up, 8 in 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[106482]: 1.6 scrub starts 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[106482]: 1.6 scrub ok 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:26:03.521 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/101834139' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-21T10:26:03.522 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[110963]: osdmap e1425: 8 total, 8 up, 8 in 2024-09-21T10:26:03.522 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[110963]: 1.6 scrub starts 2024-09-21T10:26:03.522 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:03 smithi076 ceph-mon[110963]: 1.6 scrub ok 2024-09-21T10:26:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:03 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-21T10:26:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:03 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:26:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:03 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/101834139' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2024-09-21T10:26:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:03 smithi191 ceph-mon[102206]: osdmap e1425: 8 total, 8 up, 8 in 2024-09-21T10:26:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:03 smithi191 ceph-mon[102206]: 1.6 scrub starts 2024-09-21T10:26:03.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:03 smithi191 ceph-mon[102206]: 1.6 scrub ok 2024-09-21T10:26:03.750 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:04.566 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:04 smithi191 ceph-mon[102206]: pgmap v2373: 93 pgs: 2 peering, 4 remapped+peering, 87 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:26:04.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:04 smithi076 ceph-mon[106482]: pgmap v2373: 93 pgs: 2 peering, 4 remapped+peering, 87 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:26:04.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:04 smithi076 ceph-mon[110963]: pgmap v2373: 93 pgs: 2 peering, 4 remapped+peering, 87 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:26:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:05 smithi191 ceph-mon[102206]: 1.b scrub starts 2024-09-21T10:26:05.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:05 smithi191 ceph-mon[102206]: 1.b scrub ok 2024-09-21T10:26:05.596 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:05 smithi076 ceph-mon[106482]: 1.b scrub starts 2024-09-21T10:26:05.596 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:05 smithi076 ceph-mon[106482]: 1.b scrub ok 2024-09-21T10:26:05.596 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:05 smithi076 ceph-mon[110963]: 1.b scrub starts 2024-09-21T10:26:05.596 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:05 smithi076 ceph-mon[110963]: 1.b scrub ok 2024-09-21T10:26:05.601 INFO:teuthology.orchestra.run.smithi076.stdout:mon.a: {} 2024-09-21T10:26:05.602 INFO:teuthology.orchestra.run.smithi076.stdout:mon.b: {} 2024-09-21T10:26:05.602 INFO:teuthology.orchestra.run.smithi076.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:26:05.602 INFO:teuthology.orchestra.run.smithi076.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:26:05.604 INFO:teuthology.orchestra.run.smithi076.stdout:mon.c: {} 2024-09-21T10:26:05.604 INFO:teuthology.orchestra.run.smithi076.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-21T10:26:06.549 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3232046847' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:26:06.549 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[106482]: pgmap v2374: 93 pgs: 2 peering, 4 remapped+peering, 87 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:26:06.550 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:06.550 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:06.550 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3232046847' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:26:06.550 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[110963]: pgmap v2374: 93 pgs: 2 peering, 4 remapped+peering, 87 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:26:06.550 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:06.550 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:06 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:06.572 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 1 1 2024-09-21T10:26:06.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:06 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3232046847' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-21T10:26:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:06 smithi191 ceph-mon[102206]: pgmap v2374: 93 pgs: 2 peering, 4 remapped+peering, 87 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-21T10:26:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:06.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:06 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:07.446 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:07.597 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:07 smithi076 ceph-mon[106482]: 1.10 scrub starts 2024-09-21T10:26:07.597 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:07 smithi076 ceph-mon[106482]: 1.10 scrub ok 2024-09-21T10:26:07.598 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:07 smithi076 ceph-mon[110963]: 1.10 scrub starts 2024-09-21T10:26:07.598 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:07 smithi076 ceph-mon[110963]: 1.10 scrub ok 2024-09-21T10:26:07.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:07 smithi191 ceph-mon[102206]: 1.10 scrub starts 2024-09-21T10:26:07.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:07 smithi191 ceph-mon[102206]: 1.10 scrub ok 2024-09-21T10:26:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:08 smithi191 ceph-mon[102206]: 1.d scrub starts 2024-09-21T10:26:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:08 smithi191 ceph-mon[102206]: 1.d scrub ok 2024-09-21T10:26:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:08 smithi191 ceph-mon[102206]: 1.3 scrub starts 2024-09-21T10:26:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:08 smithi191 ceph-mon[102206]: 1.3 scrub ok 2024-09-21T10:26:08.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:08 smithi191 ceph-mon[102206]: pgmap v2375: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[106482]: 1.d scrub starts 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[106482]: 1.d scrub ok 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[106482]: 1.3 scrub starts 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[106482]: 1.3 scrub ok 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[106482]: pgmap v2375: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[110963]: 1.d scrub starts 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[110963]: 1.d scrub ok 2024-09-21T10:26:08.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[110963]: 1.3 scrub starts 2024-09-21T10:26:08.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[110963]: 1.3 scrub ok 2024-09-21T10:26:08.624 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:08 smithi076 ceph-mon[110963]: pgmap v2375: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:10.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:09 smithi076 ceph-mon[106482]: pgmap v2376: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:10.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:09 smithi076 ceph-mon[110963]: pgmap v2376: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:10.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:09 smithi191 ceph-mon[102206]: pgmap v2376: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:10.679 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.1 to 1 (10000) 2024-09-21T10:26:11.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:10 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2794862849' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:11.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:10 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:11.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:11.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:11.049 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2794862849' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:11.049 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:11.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:11.050 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:11.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2794862849' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:11.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:11.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:11.050 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:11.396 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 1 1 2024-09-21T10:26:11.593 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:12.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: 1.e scrub starts 2024-09-21T10:26:12.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: 1.e scrub ok 2024-09-21T10:26:12.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:26:12.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: osdmap e1426: 8 total, 8 up, 8 in 2024-09-21T10:26:12.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.029 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[106482]: pgmap v2378: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: 1.e scrub starts 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: 1.e scrub ok 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: osdmap e1426: 8 total, 8 up, 8 in 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.030 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:26:12.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:26:12.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.031 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:11 smithi076 ceph-mon[110963]: pgmap v2378: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: 1.e scrub starts 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: 1.e scrub ok 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: osdmap e1426: 8 total, 8 up, 8 in 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:26:12.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:26:12.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:26:12.114 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:11 smithi191 ceph-mon[102206]: pgmap v2378: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:12.742 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:12 smithi076 ceph-mon[106482]: 1.0 scrub starts 2024-09-21T10:26:12.742 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:12 smithi076 ceph-mon[106482]: 1.0 scrub ok 2024-09-21T10:26:12.742 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:12 smithi076 ceph-mon[110963]: 1.0 scrub starts 2024-09-21T10:26:12.743 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:12 smithi076 ceph-mon[110963]: 1.0 scrub ok 2024-09-21T10:26:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:12 smithi191 ceph-mon[102206]: 1.0 scrub starts 2024-09-21T10:26:13.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:12 smithi191 ceph-mon[102206]: 1.0 scrub ok 2024-09-21T10:26:13.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:26:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:26:13] "GET /metrics HTTP/1.1" 200 34197 "" "Prometheus/2.51.0" 2024-09-21T10:26:13.691 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-21T10:26:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:13 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/775887199' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:14.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:13 smithi191 ceph-mon[102206]: pgmap v2379: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:13 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/775887199' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:14.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:13 smithi076 ceph-mon[106482]: pgmap v2379: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:13 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/775887199' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-21T10:26:14.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:13 smithi076 ceph-mon[110963]: pgmap v2379: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:14.501 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 2 1 2024-09-21T10:26:14.705 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:14.825 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:14 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/775887199' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:26:14.825 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:14 smithi076 ceph-mon[106482]: osdmap e1427: 8 total, 8 up, 8 in 2024-09-21T10:26:14.826 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:14 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/775887199' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:26:14.826 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:14 smithi076 ceph-mon[110963]: osdmap e1427: 8 total, 8 up, 8 in 2024-09-21T10:26:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:14 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/775887199' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-21T10:26:15.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:14 smithi191 ceph-mon[102206]: osdmap e1427: 8 total, 8 up, 8 in 2024-09-21T10:26:15.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:15 smithi076 ceph-mon[106482]: pgmap v2381: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:15.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:15 smithi076 ceph-mon[110963]: pgmap v2381: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:16.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:15 smithi191 ceph-mon[102206]: pgmap v2381: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:16.708 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.2 to 1 (10000) 2024-09-21T10:26:17.077 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:16 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1606092124' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:17.077 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:16 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1606092124' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:17.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:16 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1606092124' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:17.533 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 2 1 2024-09-21T10:26:17.734 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:17 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1606092124' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:17 smithi076 ceph-mon[106482]: osdmap e1428: 8 total, 8 up, 8 in 2024-09-21T10:26:17.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:17 smithi076 ceph-mon[106482]: pgmap v2383: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:17.856 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:17 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1606092124' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:17.856 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:17 smithi076 ceph-mon[110963]: osdmap e1428: 8 total, 8 up, 8 in 2024-09-21T10:26:17.856 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:17 smithi076 ceph-mon[110963]: pgmap v2383: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:17 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1606092124' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:17 smithi191 ceph-mon[102206]: osdmap e1428: 8 total, 8 up, 8 in 2024-09-21T10:26:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:17 smithi191 ceph-mon[102206]: pgmap v2383: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:19.691 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-21T10:26:20.071 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:19 smithi076 ceph-mon[106482]: pgmap v2384: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:20.071 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:19 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/830160415' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:20.071 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:19 smithi076 ceph-mon[110963]: pgmap v2384: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:20.071 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:19 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/830160415' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:19 smithi191 ceph-mon[102206]: pgmap v2384: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:19 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/830160415' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-21T10:26:20.461 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 4 1 2024-09-21T10:26:20.658 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:20.700 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:20 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/830160415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:20.700 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:20 smithi076 ceph-mon[110963]: osdmap e1429: 8 total, 8 up, 8 in 2024-09-21T10:26:21.059 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:20 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/830160415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:21.059 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:20 smithi076 ceph-mon[106482]: osdmap e1429: 8 total, 8 up, 8 in 2024-09-21T10:26:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:20 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/830160415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-21T10:26:21.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:20 smithi191 ceph-mon[102206]: osdmap e1429: 8 total, 8 up, 8 in 2024-09-21T10:26:21.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:21 smithi076 ceph-mon[106482]: osdmap e1430: 8 total, 8 up, 8 in 2024-09-21T10:26:21.855 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:21 smithi076 ceph-mon[106482]: pgmap v2387: 93 pgs: 7 peering, 86 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:21.855 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:21 smithi076 ceph-mon[110963]: osdmap e1430: 8 total, 8 up, 8 in 2024-09-21T10:26:21.856 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:21 smithi076 ceph-mon[110963]: pgmap v2387: 93 pgs: 7 peering, 86 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:21 smithi191 ceph-mon[102206]: osdmap e1430: 8 total, 8 up, 8 in 2024-09-21T10:26:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:21 smithi191 ceph-mon[102206]: pgmap v2387: 93 pgs: 7 peering, 86 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:22.717 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.4 to 1 (10000) 2024-09-21T10:26:23.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:22 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/205445430' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:23.072 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:22 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:23.072 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:22 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/205445430' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:23.072 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:22 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:22 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/205445430' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:23.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:22 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:23.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:26:23] "GET /metrics HTTP/1.1" 200 34197 "" "Prometheus/2.51.0" 2024-09-21T10:26:23.530 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 4 1 2024-09-21T10:26:23.729 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:23.850 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:26:23.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-mon[106482]: osdmap e1431: 8 total, 8 up, 8 in 2024-09-21T10:26:23.851 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-mon[106482]: pgmap v2389: 93 pgs: 7 peering, 86 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:23.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:26:23.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-mon[110963]: osdmap e1431: 8 total, 8 up, 8 in 2024-09-21T10:26:23.851 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:23 smithi076 ceph-mon[110963]: pgmap v2389: 93 pgs: 7 peering, 86 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:23 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:26:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:23 smithi191 ceph-mon[102206]: osdmap e1431: 8 total, 8 up, 8 in 2024-09-21T10:26:24.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:23 smithi191 ceph-mon[102206]: pgmap v2389: 93 pgs: 7 peering, 86 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:25 smithi191 ceph-mon[102206]: pgmap v2390: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:26.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:25 smithi076 ceph-mon[110963]: pgmap v2390: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:26.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:25 smithi076 ceph-mon[106482]: pgmap v2390: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:26.656 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-21T10:26:27.048 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/156888071' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:27.049 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/156888071' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/156888071' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-21T10:26:27.350 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 5 1 2024-09-21T10:26:27.552 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:27.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:27 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/156888071' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:26:27.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:27 smithi076 ceph-mon[106482]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-21T10:26:27.997 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:27 smithi076 ceph-mon[106482]: pgmap v2392: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.5 KiB/s rd, 16 KiB/s wr, 3 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:26:27.997 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:27 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/156888071' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:26:27.997 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:27 smithi076 ceph-mon[110963]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-21T10:26:27.997 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:27 smithi076 ceph-mon[110963]: pgmap v2392: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.5 KiB/s rd, 16 KiB/s wr, 3 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:26:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:27 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/156888071' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-21T10:26:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:27 smithi191 ceph-mon[102206]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-21T10:26:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:27 smithi191 ceph-mon[102206]: pgmap v2392: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.5 KiB/s rd, 16 KiB/s wr, 3 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:26:28.957 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[106482]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[106482]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[106482]: 1.e scrub starts 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[106482]: 1.e scrub ok 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[110963]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[110963]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[110963]: 1.e scrub starts 2024-09-21T10:26:28.958 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:28 smithi076 ceph-mon[110963]: 1.e scrub ok 2024-09-21T10:26:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:28 smithi191 ceph-mon[102206]: Health check failed: Reduced data availability: 3 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-21T10:26:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:28 smithi191 ceph-mon[102206]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-21T10:26:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:28 smithi191 ceph-mon[102206]: 1.e scrub starts 2024-09-21T10:26:29.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:28 smithi191 ceph-mon[102206]: 1.e scrub ok 2024-09-21T10:26:29.671 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.5 to 1 (10000) 2024-09-21T10:26:30.001 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:29 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2210496685' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:30.001 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:29 smithi076 ceph-mon[106482]: pgmap v2394: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.5 KiB/s rd, 16 KiB/s wr, 3 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:26:30.001 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:29 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2210496685' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:30.001 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:29 smithi076 ceph-mon[110963]: pgmap v2394: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.5 KiB/s rd, 16 KiB/s wr, 3 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:26:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:29 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2210496685' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:30.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:29 smithi191 ceph-mon[102206]: pgmap v2394: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5.5 KiB/s rd, 16 KiB/s wr, 3 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-21T10:26:30.437 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 5 1 2024-09-21T10:26:30.671 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:30.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:30 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2210496685' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:30.960 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:30 smithi076 ceph-mon[106482]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-21T10:26:30.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:30 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2210496685' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:30.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:30 smithi076 ceph-mon[110963]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-21T10:26:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:30 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2210496685' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:31.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:30 smithi191 ceph-mon[102206]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-21T10:26:31.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:31 smithi076 ceph-mon[106482]: pgmap v2396: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6.3 KiB/s rd, 18 KiB/s wr, 4 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-21T10:26:31.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:31 smithi076 ceph-mon[110963]: pgmap v2396: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6.3 KiB/s rd, 18 KiB/s wr, 4 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-21T10:26:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:31 smithi191 ceph-mon[102206]: pgmap v2396: 93 pgs: 15 peering, 78 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6.3 KiB/s rd, 18 KiB/s wr, 4 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-21T10:26:32.692 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-21T10:26:32.704 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:32 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2253777009' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:32.704 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:32 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:32.704 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:32 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2253777009' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:32.705 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:32 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:33.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:32 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2253777009' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:33.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:32 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-21T10:26:33.366 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:26:33] "GET /metrics HTTP/1.1" 200 34189 "" "Prometheus/2.51.0" 2024-09-21T10:26:33.389 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 6 1 2024-09-21T10:26:33.589 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:34.011 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:34.011 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-mon[106482]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-21T10:26:34.011 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-mon[106482]: pgmap v2398: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:34.011 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:34.011 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-mon[110963]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-21T10:26:34.012 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:33 smithi076 ceph-mon[110963]: pgmap v2398: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:33 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-21T10:26:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:33 smithi191 ceph-mon[102206]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-21T10:26:34.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:33 smithi191 ceph-mon[102206]: pgmap v2398: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:34.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:34 smithi076 ceph-mon[106482]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-21T10:26:34.831 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:34 smithi076 ceph-mon[106482]: Cluster is now healthy 2024-09-21T10:26:34.831 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:34 smithi076 ceph-mon[110963]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-21T10:26:34.831 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:34 smithi076 ceph-mon[110963]: Cluster is now healthy 2024-09-21T10:26:35.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:34 smithi191 ceph-mon[102206]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 5 pgs peering) 2024-09-21T10:26:35.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:34 smithi191 ceph-mon[102206]: Cluster is now healthy 2024-09-21T10:26:35.692 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.6 to 1 (10000) 2024-09-21T10:26:36.064 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[106482]: 1.5 scrub starts 2024-09-21T10:26:36.064 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[106482]: 1.5 scrub ok 2024-09-21T10:26:36.065 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1811877761' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:36.065 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[106482]: pgmap v2399: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:36.065 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[110963]: 1.5 scrub starts 2024-09-21T10:26:36.065 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[110963]: 1.5 scrub ok 2024-09-21T10:26:36.065 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1811877761' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:36.065 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:35 smithi076 ceph-mon[110963]: pgmap v2399: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:35 smithi191 ceph-mon[102206]: 1.5 scrub starts 2024-09-21T10:26:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:35 smithi191 ceph-mon[102206]: 1.5 scrub ok 2024-09-21T10:26:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:35 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1811877761' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:35 smithi191 ceph-mon[102206]: pgmap v2399: 93 pgs: 93 active+clean; 608 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:36.394 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 6 1 2024-09-21T10:26:36.591 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:36.958 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:36 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1811877761' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:36.959 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:36 smithi076 ceph-mon[110963]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-21T10:26:36.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:36 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1811877761' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:36.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:36 smithi076 ceph-mon[106482]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-21T10:26:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:36 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1811877761' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:37.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:36 smithi191 ceph-mon[102206]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-21T10:26:38.055 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:37 smithi076 ceph-mon[106482]: pgmap v2401: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:38.055 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:37 smithi076 ceph-mon[110963]: pgmap v2401: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:37 smithi191 ceph-mon[102206]: pgmap v2401: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:38.800 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-21T10:26:39.103 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:38 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1395894635' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:39.103 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:38 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1395894635' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:39.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:38 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1395894635' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-21T10:26:39.504 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 7 1 2024-09-21T10:26:39.706 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:40.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1395894635' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:39 smithi191 ceph-mon[102206]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-21T10:26:40.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:39 smithi191 ceph-mon[102206]: pgmap v2403: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1395894635' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:39 smithi076 ceph-mon[106482]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-21T10:26:40.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:39 smithi076 ceph-mon[106482]: pgmap v2403: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1395894635' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-21T10:26:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:39 smithi076 ceph-mon[110963]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-21T10:26:40.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:39 smithi076 ceph-mon[110963]: pgmap v2403: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:40.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:40 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:40.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:40 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:41.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:40 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:41.824 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.7 to 1 (10000) 2024-09-21T10:26:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:41 smithi191 ceph-mon[102206]: pgmap v2404: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:42.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:41 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/691815812' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:26:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:41 smithi076 ceph-mon[110963]: pgmap v2404: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:42.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:41 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/691815812' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:26:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:41 smithi076 ceph-mon[106482]: pgmap v2404: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:42.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:41 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/691815812' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:26:42.566 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 7 1 2024-09-21T10:26:42.765 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:42 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/691815812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:26:43.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:42 smithi191 ceph-mon[102206]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-21T10:26:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:42 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/691815812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:26:43.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:42 smithi076 ceph-mon[106482]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-21T10:26:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:42 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/691815812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:26:43.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:42 smithi076 ceph-mon[110963]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-21T10:26:43.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:26:43] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.51.0" 2024-09-21T10:26:43.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-09-21T10:26:43.959 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4, 3, 5]} : dispatch 2024-09-21T10:26:43.960 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-mon[106482]: pgmap v2406: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:43.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-09-21T10:26:43.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4, 3, 5]} : dispatch 2024-09-21T10:26:43.960 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:43 smithi076 ceph-mon[110963]: pgmap v2406: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-09-21T10:26:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:43 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4, 3, 5]} : dispatch 2024-09-21T10:26:44.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:43 smithi191 ceph-mon[102206]: pgmap v2406: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:44.836 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-21T10:26:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T10:26:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:44 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4, 3, 5]}]': finished 2024-09-21T10:26:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:44 smithi191 ceph-mon[102206]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-21T10:26:45.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3169266754' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4, 3, 5]}]': finished 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[110963]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3169266754' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 4, 3, 5]}]': finished 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[106482]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-21T10:26:45.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3169266754' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-21T10:26:45.648 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 0 1 2024-09-21T10:26:45.849 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:45.970 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:45 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3169266754' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:26:45.970 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:45 smithi076 ceph-mon[106482]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-21T10:26:45.970 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:45 smithi076 ceph-mon[106482]: pgmap v2409: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:45.970 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:45 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3169266754' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:26:45.970 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:45 smithi076 ceph-mon[110963]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-21T10:26:45.970 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:45 smithi076 ceph-mon[110963]: pgmap v2409: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:45 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3169266754' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-21T10:26:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:45 smithi191 ceph-mon[102206]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-21T10:26:46.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:45 smithi191 ceph-mon[102206]: pgmap v2409: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:47 smithi191 ceph-mon[102206]: pgmap v2410: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:48.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:47 smithi076 ceph-mon[106482]: pgmap v2410: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:48.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:47 smithi076 ceph-mon[110963]: pgmap v2410: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:48.688 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.0 to 1 (10000) 2024-09-21T10:26:49.014 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:48 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1276372062' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:49.015 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:48 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:49.015 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:48 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1276372062' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:49.015 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:48 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:48 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1276372062' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:49.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:48 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:49.458 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 0 1 2024-09-21T10:26:49.673 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:49.946 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:49 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:26:49.946 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:49 smithi076 ceph-mon[106482]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-21T10:26:49.946 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:49 smithi076 ceph-mon[106482]: pgmap v2412: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:49.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:49 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:26:49.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:49 smithi076 ceph-mon[110963]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-21T10:26:49.946 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:49 smithi076 ceph-mon[110963]: pgmap v2412: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:49 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:26:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:49 smithi191 ceph-mon[102206]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-21T10:26:50.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:49 smithi191 ceph-mon[102206]: pgmap v2412: 93 pgs: 1 peering, 92 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:51 smithi191 ceph-mon[102206]: pgmap v2413: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:51 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4113779370' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:51 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:51 smithi076 ceph-mon[106482]: pgmap v2413: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:51 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4113779370' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:52.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:51 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:51 smithi076 ceph-mon[110963]: pgmap v2413: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:52.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:51 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4113779370' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:52.124 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:51 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-21T10:26:52.660 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-21T10:26:53.431 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd reweight 3 1 2024-09-21T10:26:53.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:26:53] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.51.0" 2024-09-21T10:26:53.635 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:53.914 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-mon[106482]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:26:53.915 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-mon[106482]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-21T10:26:53.915 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-mon[106482]: pgmap v2415: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:53.915 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-mon[110963]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:26:53.915 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-mon[110963]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-21T10:26:53.915 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:53 smithi076 ceph-mon[110963]: pgmap v2415: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:53 smithi191 ceph-mon[102206]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-21T10:26:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:53 smithi191 ceph-mon[102206]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-21T10:26:54.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:53 smithi191 ceph-mon[102206]: pgmap v2415: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:55.674 INFO:teuthology.orchestra.run.smithi076.stderr:reweighted osd.3 to 1 (10000) 2024-09-21T10:26:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:55 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:55 smithi076 ceph-mon[110963]: pgmap v2416: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:55.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:55 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2896417120' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:55.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:55 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:55.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:55 smithi076 ceph-mon[106482]: pgmap v2416: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:55.874 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:55 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2896417120' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:55 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:26:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:55 smithi191 ceph-mon[102206]: pgmap v2416: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:56.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:55 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2896417120' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:56.494 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd primary-affinity 3 1 2024-09-21T10:26:56.693 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:26:56.813 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:56 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/2896417120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:26:56.813 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:56 smithi076 ceph-mon[106482]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-21T10:26:56.813 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:56 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/2896417120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:26:56.813 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:56 smithi076 ceph-mon[110963]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-21T10:26:57.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:56 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/2896417120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:26:57.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:56 smithi191 ceph-mon[102206]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-21T10:26:57.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:57 smithi076 ceph-mon[106482]: pgmap v2418: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:57.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:57 smithi076 ceph-mon[110963]: pgmap v2418: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:58.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:57 smithi191 ceph-mon[102206]: pgmap v2418: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:26:58.687 INFO:teuthology.orchestra.run.smithi076.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-21T10:26:58.700 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:58 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/753591491' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:58.700 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:58 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/753591491' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:59.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:58 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/753591491' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-21T10:26:59.406 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-21T10:26:59.406 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-21T10:26:59.406 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-21T10:26:59.406 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-21T10:26:59.406 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T10:26:59.406 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:26:59.624 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:00.046 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:59 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/753591491' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:27:00.046 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:59 smithi076 ceph-mon[106482]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-21T10:27:00.046 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:26:59 smithi076 ceph-mon[106482]: pgmap v2420: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:00.046 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:59 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/753591491' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:27:00.046 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:59 smithi076 ceph-mon[110963]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-21T10:27:00.047 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:26:59 smithi076 ceph-mon[110963]: pgmap v2420: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:59 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/753591491' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-21T10:27:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:59 smithi191 ceph-mon[102206]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-21T10:27:00.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:26:59 smithi191 ceph-mon[102206]: pgmap v2420: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:00.833 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:00 smithi076 ceph-mon[106482]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-21T10:27:00.833 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:00 smithi076 ceph-mon[110963]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-21T10:27:01.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:00 smithi191 ceph-mon[102206]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-21T10:27:01.492 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:27:01.492 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":1445,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T10:26:59.684809+0000","last_up_change":"2024-09-21T10:23:38.639657+0000","last_in_change":"2024-09-21T10:25:58.749000+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":21,"pg_placement_num":21,"pg_placement_num_target":21,"pg_num_target":21,"pg_num_pending":21,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1425","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1408","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8999999761581421,"score_stable":1.8999999761581421,"optimal_score":1,"raw_score_acting":1.8999999761581421,"raw_score_stable":1.8999999761581421,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":730,"last_clean_end":1292,"up_from":1317,"up_thru":1423,"down_at":1293,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6827","nonce":399575421}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6829","nonce":399575421}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6833","nonce":399575421}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6831","nonce":399575421}]},"public_addr":"172.21.15.76:6827/399575421","cluster_addr":"172.21.15.76:6829/399575421","heartbeat_back_addr":"172.21.15.76:6833/399575421","heartbeat_front_addr":"172.21.15.76:6831/399575421","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":750,"up_from":763,"up_thru":1423,"down_at":751,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6811","nonce":46966950}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6813","nonce":46966950}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6817","nonce":46966950}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6815","nonce":46966950}]},"public_addr":"172.21.15.76:6811/46966950","cluster_addr":"172.21.15.76:6813/46966950","heartbeat_back_addr":"172.21.15.76:6817/46966950","heartbeat_front_addr":"172.21.15.76:6815/46966950","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":783,"up_from":796,"up_thru":1439,"down_at":784,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6819","nonce":3488608517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6821","nonce":3488608517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6825","nonce":3488608517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6823","nonce":3488608517}]},"public_addr":"172.21.15.76:6819/3488608517","cluster_addr":"172.21.15.76:6821/3488608517","heartbeat_back_addr":"172.21.15.76:6825/3488608517","heartbeat_front_addr":"172.21.15.76:6823/3488608517","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":1301,"last_clean_end":1365,"up_from":1375,"up_thru":1444,"down_at":1366,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6803","nonce":1408876515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6805","nonce":1408876515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6809","nonce":1408876515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6807","nonce":1408876515}]},"public_addr":"172.21.15.76:6803/1408876515","cluster_addr":"172.21.15.76:6805/1408876515","heartbeat_back_addr":"172.21.15.76:6809/1408876515","heartbeat_front_addr":"172.21.15.76:6807/1408876515","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":1439,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":1424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":1292},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":750},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":783},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":1365},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":1,"to":3}]},{"pgid":"3.4","mappings":[{"from":1,"to":3}]},{"pgid":"3.1d","mappings":[{"from":4,"to":3}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":4},{"from":3,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.191:6835/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/1830630483":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6801/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.76:0/3937727837":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/1931405138":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/4144106685":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6834/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1616224256":"2024-09-22T09:39:37.220289+0000","172.21.15.191:0/1868415942":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/3270182435":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/3629581092":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1280485976":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6801/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/4175697025":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.191:0/739062149":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/3392089630":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6800/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/3117602468":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/1469488034":"2024-09-22T09:42:40.197673+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:27:01.798 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:01 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3090749274' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:27:01.799 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:01 smithi076 ceph-mon[106482]: pgmap v2422: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:01.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:01 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3090749274' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:27:01.799 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:01 smithi076 ceph-mon[110963]: pgmap v2422: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:02.104 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T10:27:02.104 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:27:02.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:01 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3090749274' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:27:02.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:01 smithi191 ceph-mon[102206]: pgmap v2422: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:02.303 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:03.373 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:03 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:27:03] "GET /metrics HTTP/1.1" 200 34183 "" "Prometheus/2.51.0" 2024-09-21T10:27:03.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:03 smithi076 ceph-mon[106482]: pgmap v2423: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:03.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:03 smithi076 ceph-mon[110963]: pgmap v2423: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:04.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:03 smithi191 ceph-mon[102206]: pgmap v2423: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:04.297 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:27:04.297 INFO:teuthology.orchestra.run.smithi076.stdout:{"epoch":1445,"fsid":"e97bd93e-77fb-11ef-baf6-efdc52797490","created":"2024-09-21T09:29:53.043145+0000","modified":"2024-09-21T10:26:59.684809+0000","last_up_change":"2024-09-21T10:23:38.639657+0000","last_in_change":"2024-09-21T10:25:58.749000+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:32:50.664150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":21,"pg_placement_num":21,"pg_placement_num_target":21,"pg_num_target":21,"pg_num_pending":21,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1425","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1408","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8999999761581421,"score_stable":1.8999999761581421,"optimal_score":1,"raw_score_acting":1.8999999761581421,"raw_score_stable":1.8999999761581421,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:35:23.012145+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-21T09:36:38.810335+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-21T09:36:39.117800+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e79ed056-c2aa-44c5-915a-5680ca7c081e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":730,"last_clean_end":1292,"up_from":1317,"up_thru":1423,"down_at":1293,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6826","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6827","nonce":399575421}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6828","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6829","nonce":399575421}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6832","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6833","nonce":399575421}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6830","nonce":399575421},{"type":"v1","addr":"172.21.15.76:6831","nonce":399575421}]},"public_addr":"172.21.15.76:6827/399575421","cluster_addr":"172.21.15.76:6829/399575421","heartbeat_back_addr":"172.21.15.76:6833/399575421","heartbeat_front_addr":"172.21.15.76:6831/399575421","state":["exists","up"]},{"osd":1,"uuid":"1de58204-a1cd-4ff3-96f7-e03b83aa0041","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":750,"up_from":763,"up_thru":1423,"down_at":751,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6810","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6811","nonce":46966950}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6812","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6813","nonce":46966950}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6816","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6817","nonce":46966950}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6814","nonce":46966950},{"type":"v1","addr":"172.21.15.76:6815","nonce":46966950}]},"public_addr":"172.21.15.76:6811/46966950","cluster_addr":"172.21.15.76:6813/46966950","heartbeat_back_addr":"172.21.15.76:6817/46966950","heartbeat_front_addr":"172.21.15.76:6815/46966950","state":["exists","up"]},{"osd":2,"uuid":"f5de7797-a6a1-4301-8efc-5fe9a50da8cb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":783,"up_from":796,"up_thru":1439,"down_at":784,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6818","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6819","nonce":3488608517}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6820","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6821","nonce":3488608517}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6824","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6825","nonce":3488608517}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6822","nonce":3488608517},{"type":"v1","addr":"172.21.15.76:6823","nonce":3488608517}]},"public_addr":"172.21.15.76:6819/3488608517","cluster_addr":"172.21.15.76:6821/3488608517","heartbeat_back_addr":"172.21.15.76:6825/3488608517","heartbeat_front_addr":"172.21.15.76:6823/3488608517","state":["exists","up"]},{"osd":3,"uuid":"357ff6e9-4d95-42ac-8a3f-a9f3c4a48835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":1301,"last_clean_end":1365,"up_from":1375,"up_thru":1444,"down_at":1366,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6802","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6803","nonce":1408876515}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6804","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6805","nonce":1408876515}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6808","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6809","nonce":1408876515}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.76:6806","nonce":1408876515},{"type":"v1","addr":"172.21.15.76:6807","nonce":1408876515}]},"public_addr":"172.21.15.76:6803/1408876515","cluster_addr":"172.21.15.76:6805/1408876515","heartbeat_back_addr":"172.21.15.76:6809/1408876515","heartbeat_front_addr":"172.21.15.76:6807/1408876515","state":["exists","up"]},{"osd":4,"uuid":"c25d7b87-1abd-4472-af56-e090a83c5835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":1439,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6800","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6801","nonce":2797067433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6802","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6803","nonce":2797067433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6806","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6807","nonce":2797067433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6804","nonce":2797067433},{"type":"v1","addr":"172.21.15.191:6805","nonce":2797067433}]},"public_addr":"172.21.15.191:6801/2797067433","cluster_addr":"172.21.15.191:6803/2797067433","heartbeat_back_addr":"172.21.15.191:6807/2797067433","heartbeat_front_addr":"172.21.15.191:6805/2797067433","state":["exists","up"]},{"osd":5,"uuid":"68bcb809-42fe-4390-ae82-4ab0f2f9c54e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":1424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6808","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6809","nonce":4066883840}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6810","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6811","nonce":4066883840}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6814","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6815","nonce":4066883840}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6812","nonce":4066883840},{"type":"v1","addr":"172.21.15.191:6813","nonce":4066883840}]},"public_addr":"172.21.15.191:6809/4066883840","cluster_addr":"172.21.15.191:6811/4066883840","heartbeat_back_addr":"172.21.15.191:6815/4066883840","heartbeat_front_addr":"172.21.15.191:6813/4066883840","state":["exists","up"]},{"osd":6,"uuid":"0592317d-31bb-4c86-81c9-5f134a98cba0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6816","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6817","nonce":2601453952}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6818","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6819","nonce":2601453952}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6822","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6823","nonce":2601453952}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6820","nonce":2601453952},{"type":"v1","addr":"172.21.15.191:6821","nonce":2601453952}]},"public_addr":"172.21.15.191:6817/2601453952","cluster_addr":"172.21.15.191:6819/2601453952","heartbeat_back_addr":"172.21.15.191:6823/2601453952","heartbeat_front_addr":"172.21.15.191:6821/2601453952","state":["exists","up"]},{"osd":7,"uuid":"08ee88fa-c070-487e-96c8-d85878cc1249","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1424,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6824","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6825","nonce":3899643071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6826","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6827","nonce":3899643071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6830","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6831","nonce":3899643071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.191:6828","nonce":3899643071},{"type":"v1","addr":"172.21.15.191:6829","nonce":3899643071}]},"public_addr":"172.21.15.191:6825/3899643071","cluster_addr":"172.21.15.191:6827/3899643071","heartbeat_back_addr":"172.21.15.191:6831/3899643071","heartbeat_front_addr":"172.21.15.191:6829/3899643071","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:31:58.910804+0000","dead_epoch":1292},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:23.163390+0000","dead_epoch":750},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:32:47.855923+0000","dead_epoch":783},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:12.495143+0000","dead_epoch":1365},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:34.489498+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:33:58.876115+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:23.318536+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:34:47.347079+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.6","mappings":[{"from":1,"to":3}]},{"pgid":"3.4","mappings":[{"from":1,"to":3}]},{"pgid":"3.1d","mappings":[{"from":4,"to":3}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":4},{"from":3,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.76:0/3937727837":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6801/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.76:6800/2928923484":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6835/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1469488034":"2024-09-22T09:42:40.197673+0000","172.21.15.76:0/3117602468":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6801/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6800/771074100":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/3280887082":"2024-09-22T09:35:18.636475+0000","172.21.15.76:0/4005379925":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/1830630483":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1931405138":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6801/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/4144106685":"2024-09-22T09:42:40.197673+0000","172.21.15.191:6834/2769276191":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/1946495829":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/720143062":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/1616224256":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/1280485976":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/3129788088":"2024-09-22T09:30:15.293339+0000","172.21.15.76:0/3625371489":"2024-09-22T09:35:18.636475+0000","172.21.15.76:6801/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/4089488545":"2024-09-22T09:35:18.636475+0000","172.21.15.191:0/4175697025":"2024-09-22T09:40:06.660926+0000","172.21.15.76:6800/606566558":"2024-09-22T09:30:15.293339+0000","172.21.15.76:6801/2955791868":"2024-09-22T09:30:38.364115+0000","172.21.15.191:0/739062149":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/3392089630":"2024-09-22T09:39:37.220289+0000","172.21.15.76:6800/23865699":"2024-09-22T09:39:37.220289+0000","172.21.15.76:0/200756880":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/2990785964":"2024-09-22T09:30:38.364115+0000","172.21.15.76:0/3270182435":"2024-09-22T09:42:40.197673+0000","172.21.15.191:0/3629581092":"2024-09-22T09:40:06.660926+0000","172.21.15.191:0/1868415942":"2024-09-22T09:40:06.660926+0000","172.21.15.76:0/659358277":"2024-09-22T09:30:38.364115+0000","172.21.15.76:6800/2955791868":"2024-09-22T09:30:38.364115+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:27:04.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:04 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3714674347' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:27:04.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:04 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3714674347' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:27:04.939 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T10:27:04.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T10:27:04.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T10:27:04.940 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T10:27:04.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T10:27:04.941 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T10:27:04.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T10:27:04.942 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T10:27:05.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:04 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3714674347' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:27:05.349 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.351 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.352 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.353 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.355 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.355 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.360 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.390 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:05.767 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:05 smithi076 ceph-mon[106482]: pgmap v2424: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:05.767 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:05 smithi076 ceph-mon[110963]: pgmap v2424: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:06.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:05 smithi191 ceph-mon[102206]: pgmap v2424: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:07.864 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:07 smithi076 ceph-mon[106482]: pgmap v2425: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:07.865 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:07 smithi076 ceph-mon[110963]: pgmap v2425: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:08.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:07 smithi191 ceph-mon[102206]: pgmap v2425: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:09.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:09 smithi076 ceph-mon[110963]: pgmap v2426: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:09.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:09 smithi076 ceph-mon[106482]: pgmap v2426: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:10.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:09 smithi191 ceph-mon[102206]: pgmap v2426: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:11.014 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:10 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:11.053 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:10 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:11.054 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:10 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:11.994 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:11 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:27:11.995 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:11 smithi191 ceph-mon[102206]: pgmap v2427: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:12.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:11 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:27:12.058 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:11 smithi076 ceph-mon[106482]: pgmap v2427: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:12.058 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:11 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:27:12.058 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:11 smithi076 ceph-mon[110963]: pgmap v2427: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:13.567 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:13 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:27:13] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.51.0" 2024-09-21T10:27:13.735 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:13 smithi191 ceph-mon[102206]: pgmap v2428: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:13.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:13 smithi076 ceph-mon[106482]: pgmap v2428: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:13.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:13 smithi076 ceph-mon[110963]: pgmap v2428: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:16.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:16.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:16 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:16.808 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:16 smithi076 ceph-mon[106482]: pgmap v2429: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:16.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:16.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:16 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:16.809 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:16 smithi076 ceph-mon[110963]: pgmap v2429: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:16 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:16.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:16 smithi191 ceph-mon[102206]: pgmap v2429: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:17.749 INFO:teuthology.orchestra.run.smithi076.stdout:163208757892 2024-09-21T10:27:17.749 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T10:27:18.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:17 smithi191 ceph-mon[102206]: pgmap v2430: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:18.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:17 smithi076 ceph-mon[106482]: pgmap v2430: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:18.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:17 smithi076 ceph-mon[110963]: pgmap v2430: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:18.958 INFO:teuthology.orchestra.run.smithi076.stdout:5656471928936 2024-09-21T10:27:18.958 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T10:27:19.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:19 smithi076 ceph-mon[106482]: pgmap v2431: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:19.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:19 smithi076 ceph-mon[110963]: pgmap v2431: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:20.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:19 smithi191 ceph-mon[102206]: pgmap v2431: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:20.939 INFO:teuthology.orchestra.run.smithi076.stdout:3418793968064 2024-09-21T10:27:20.939 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T10:27:21.457 INFO:teuthology.orchestra.run.smithi076.stdout:5905580032045 2024-09-21T10:27:21.457 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T10:27:21.730 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:21 smithi076 ceph-mon[106482]: pgmap v2432: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:21.730 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:21 smithi076 ceph-mon[110963]: pgmap v2432: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:22.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:21 smithi191 ceph-mon[102206]: pgmap v2432: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:22.364 INFO:teuthology.orchestra.run.smithi076.stdout:231928234619 2024-09-21T10:27:22.364 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T10:27:23.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:23 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:27:23] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.51.0" 2024-09-21T10:27:23.666 INFO:teuthology.orchestra.run.smithi076.stdout:197568496254 2024-09-21T10:27:23.666 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T10:27:24.079 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:23 smithi076 ceph-mon[106482]: pgmap v2433: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:24.080 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:23 smithi076 ceph-mon[110963]: pgmap v2433: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:24.099 INFO:teuthology.orchestra.run.smithi076.stdout:128849019530 2024-09-21T10:27:24.100 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T10:27:24.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:23 smithi191 ceph-mon[102206]: pgmap v2433: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:24.116 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:24.561 INFO:teuthology.orchestra.run.smithi076.stdout:3277060047302 2024-09-21T10:27:24.561 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T10:27:26.059 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:25 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:26.059 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:25 smithi076 ceph-mon[106482]: pgmap v2434: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:26.059 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:25 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:26.060 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:25 smithi076 ceph-mon[110963]: pgmap v2434: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:26.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:25 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:26.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:25 smithi191 ceph-mon[102206]: pgmap v2434: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:26.786 INFO:teuthology.orchestra.run.smithi076.stdout:163208757894 2024-09-21T10:27:26.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:26 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/572606051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:27:26.874 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:26 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/572606051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:27:26.878 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:27.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:26 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/572606051' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-21T10:27:27.310 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:27.311 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:27.928 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:27.929 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:28.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:27 smithi191 ceph-mon[102206]: pgmap v2435: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:28.123 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:27 smithi076 ceph-mon[106482]: pgmap v2435: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:28.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:27 smithi076 ceph-mon[110963]: pgmap v2435: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:28.463 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757892 got 163208757894 for osd.5 2024-09-21T10:27:28.463 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:29.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:29 smithi076 ceph-mon[106482]: pgmap v2436: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:29.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:29 smithi076 ceph-mon[110963]: pgmap v2436: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:30.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:29 smithi191 ceph-mon[102206]: pgmap v2436: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:32.048 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:31 smithi076 ceph-mon[106482]: pgmap v2437: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:32.048 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:31 smithi076 ceph-mon[110963]: pgmap v2437: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:32.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:31 smithi191 ceph-mon[102206]: pgmap v2437: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:33.415 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:33 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:27:33] "GET /metrics HTTP/1.1" 200 34187 "" "Prometheus/2.51.0" 2024-09-21T10:27:33.986 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:33 smithi076 ceph-mon[106482]: pgmap v2438: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:33.987 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:33 smithi076 ceph-mon[110963]: pgmap v2438: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:34.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:33 smithi191 ceph-mon[102206]: pgmap v2438: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:34.169 INFO:teuthology.orchestra.run.smithi076.stdout:5656471928940 2024-09-21T10:27:34.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:34 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1675088939' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:27:34.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:34 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1675088939' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:27:34.891 INFO:teuthology.orchestra.run.smithi076.stdout:5905580032049 2024-09-21T10:27:35.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:34 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1675088939' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-21T10:27:35.620 INFO:teuthology.orchestra.run.smithi076.stdout:3418793968067 2024-09-21T10:27:35.855 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:35 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3385071337' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:27:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:35 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3566628792' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:27:35.873 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:35 smithi076 ceph-mon[106482]: pgmap v2439: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:35 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3385071337' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:27:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:35 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3566628792' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:27:35.873 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:35 smithi076 ceph-mon[110963]: pgmap v2439: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:35 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3385071337' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-21T10:27:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:35 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3566628792' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-21T10:27:36.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:35 smithi191 ceph-mon[102206]: pgmap v2439: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:36.354 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3418793968064 got 3418793968067 for osd.2 2024-09-21T10:27:36.354 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:37.097 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5656471928936 got 5656471928940 for osd.0 2024-09-21T10:27:37.097 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:37.499 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5905580032045 got 5905580032049 for osd.3 2024-09-21T10:27:37.500 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:37.752 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:37 smithi076 ceph-mon[106482]: pgmap v2440: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:37.753 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:37 smithi076 ceph-mon[110963]: pgmap v2440: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:38.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:37 smithi191 ceph-mon[102206]: pgmap v2440: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:38.287 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:39.099 INFO:teuthology.orchestra.run.smithi076.stdout:3277060047307 2024-09-21T10:27:39.345 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:39 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/1953651572' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:27:39.346 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:39 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/1953651572' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:27:39.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:39 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/1953651572' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-21T10:27:40.422 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:40 smithi076 ceph-mon[106482]: pgmap v2441: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:40.423 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:40 smithi076 ceph-mon[110963]: pgmap v2441: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:40.612 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:40 smithi191 ceph-mon[102206]: pgmap v2441: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:40.698 INFO:teuthology.orchestra.run.smithi076.stdout:231928234622 2024-09-21T10:27:41.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:41 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:41.351 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:41 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/36203742' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:27:41.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:41 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:41.351 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:41 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/36203742' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:27:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:41 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:27:41.514 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:41 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/36203742' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-21T10:27:42.341 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:42 smithi076 ceph-mon[106482]: pgmap v2442: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:42.341 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:42 smithi076 ceph-mon[110963]: pgmap v2442: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:42.495 INFO:teuthology.orchestra.run.smithi076.stdout:197568496259 2024-09-21T10:27:42.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:42 smithi191 ceph-mon[102206]: pgmap v2442: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:43.525 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:43 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/4277653254' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:27:43.526 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:43 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:27:43] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.51.0" 2024-09-21T10:27:43.526 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:43 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/4277653254' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:27:43.545 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234619 got 231928234622 for osd.7 2024-09-21T10:27:43.546 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:43.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:43 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/4277653254' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-21T10:27:44.410 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3277060047302 got 3277060047307 for osd.1 2024-09-21T10:27:44.410 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:44.533 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:44 smithi076 ceph-mon[106482]: pgmap v2443: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:44.533 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:44 smithi076 ceph-mon[106482]: from='client.? 172.21.15.76:0/3631129826' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:27:44.533 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:44 smithi076 ceph-mon[110963]: pgmap v2443: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:44.533 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:44 smithi076 ceph-mon[110963]: from='client.? 172.21.15.76:0/3631129826' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:27:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:44 smithi191 ceph-mon[102206]: pgmap v2443: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:44.613 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:44 smithi191 ceph-mon[102206]: from='client.? 172.21.15.76:0/3631129826' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-21T10:27:44.840 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496254 got 197568496259 for osd.6 2024-09-21T10:27:44.840 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:44.958 INFO:teuthology.orchestra.run.smithi076.stdout:128849019534 2024-09-21T10:27:45.766 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019530 got 128849019534 for osd.4 2024-09-21T10:27:45.766 DEBUG:teuthology.parallel:result is None 2024-09-21T10:27:45.766 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-21T10:27:45.766 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:27:46.099 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:45 smithi076 ceph-mon[106482]: pgmap v2444: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:46.099 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:45 smithi076 ceph-mon[110963]: pgmap v2444: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:46.112 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:45 smithi191 ceph-mon[102206]: pgmap v2444: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:46.561 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:48.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:47 smithi191 ceph-mon[102206]: pgmap v2445: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:48.121 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:47 smithi076 ceph-mon[110963]: pgmap v2445: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:48.121 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:47 smithi076 ceph-mon[106482]: pgmap v2445: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:49.139 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:27:49.140 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:27:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:27:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:27:49.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:49 smithi191 ceph-mon[102206]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[106482]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.373 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:27:49.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:27:49.374 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:49 smithi076 ceph-mon[110963]: from='mgr.25058 172.21.15.76:0/415390362' entity='mgr.y' 2024-09-21T10:27:49.961 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2445,"stamp":"2024-09-21T10:27:47.610506+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":1193923,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3188,"ondisk_log_size":3188,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6213964,"kb_used_data":64728,"kb_used_omap":260,"kb_used_meta":6148859,"kb_avail":743747252,"statfs":{"total":767960285184,"available":761597186048,"internally_reserved":0,"allocated":66281472,"data_stored":43294079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":266323,"internal_metadata":6296432557},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002838"},"pg_stats":[{"pgid":"1.11","version":"361'149","reported_seq":1501,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694709+0000","last_change":"2024-09-21T10:25:59.370742+0000","last_active":"2024-09-21T10:26:59.694709+0000","last_peered":"2024-09-21T10:26:59.694709+0000","last_clean":"2024-09-21T10:26:59.694709+0000","last_became_active":"2024-09-21T10:25:58.778558+0000","last_became_peered":"2024-09-21T10:25:58.778558+0000","last_unstale":"2024-09-21T10:26:59.694709+0000","last_undegraded":"2024-09-21T10:26:59.694709+0000","last_fullsized":"2024-09-21T10:26:59.694709+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:59.370683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:59.370683+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:28.814303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":686,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692950+0000","last_change":"2024-09-21T10:25:59.331647+0000","last_active":"2024-09-21T10:26:59.692950+0000","last_peered":"2024-09-21T10:26:59.692950+0000","last_clean":"2024-09-21T10:26:59.692950+0000","last_became_active":"2024-09-21T10:25:59.330669+0000","last_became_peered":"2024-09-21T10:25:59.330669+0000","last_unstale":"2024-09-21T10:26:59.692950+0000","last_undegraded":"2024-09-21T10:26:59.692950+0000","last_fullsized":"2024-09-21T10:26:59.692950+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:51.037327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":40,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686713+0000","last_change":"2024-09-21T10:25:59.344700+0000","last_active":"2024-09-21T10:26:58.686713+0000","last_peered":"2024-09-21T10:26:58.686713+0000","last_clean":"2024-09-21T10:26:58.686713+0000","last_became_active":"2024-09-21T10:25:59.333045+0000","last_became_peered":"2024-09-21T10:25:59.333045+0000","last_unstale":"2024-09-21T10:26:58.686713+0000","last_undegraded":"2024-09-21T10:26:58.686713+0000","last_fullsized":"2024-09-21T10:26:58.686713+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:38.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"361'149","reported_seq":1610,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.688970+0000","last_change":"2024-09-21T10:26:06.076093+0000","last_active":"2024-09-21T10:26:58.688970+0000","last_peered":"2024-09-21T10:26:58.688970+0000","last_clean":"2024-09-21T10:26:58.688970+0000","last_became_active":"2024-09-21T10:26:02.338696+0000","last_became_peered":"2024-09-21T10:26:02.338696+0000","last_unstale":"2024-09-21T10:26:58.688970+0000","last_undegraded":"2024-09-21T10:26:58.688970+0000","last_fullsized":"2024-09-21T10:26:58.688970+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:06.075988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:06.075988+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:55.599577+0000","scrub_duration":0.001354278,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836344+0000","last_change":"2024-09-21T10:25:59.330891+0000","last_active":"2024-09-21T10:26:59.836344+0000","last_peered":"2024-09-21T10:26:59.836344+0000","last_clean":"2024-09-21T10:26:59.836344+0000","last_became_active":"2024-09-21T10:25:59.330561+0000","last_became_peered":"2024-09-21T10:25:59.330561+0000","last_unstale":"2024-09-21T10:26:59.836344+0000","last_undegraded":"2024-09-21T10:26:59.836344+0000","last_fullsized":"2024-09-21T10:26:59.836344+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:31.076727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694658+0000","last_change":"2024-09-21T10:25:59.328066+0000","last_active":"2024-09-21T10:26:59.694658+0000","last_peered":"2024-09-21T10:26:59.694658+0000","last_clean":"2024-09-21T10:26:59.694658+0000","last_became_active":"2024-09-21T10:25:59.327651+0000","last_became_peered":"2024-09-21T10:25:59.327651+0000","last_unstale":"2024-09-21T10:26:59.694658+0000","last_undegraded":"2024-09-21T10:26:59.694658+0000","last_fullsized":"2024-09-21T10:26:59.694658+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:28.029256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"361'149","reported_seq":1634,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836753+0000","last_change":"2024-09-21T10:26:00.001350+0000","last_active":"2024-09-21T10:26:59.836753+0000","last_peered":"2024-09-21T10:26:59.836753+0000","last_clean":"2024-09-21T10:26:59.836753+0000","last_became_active":"2024-09-21T10:25:53.751867+0000","last_became_peered":"2024-09-21T10:25:53.751867+0000","last_unstale":"2024-09-21T10:26:59.836753+0000","last_undegraded":"2024-09-21T10:26:59.836753+0000","last_fullsized":"2024-09-21T10:26:59.836753+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:00.001298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:00.001298+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:44.983255+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694837+0000","last_change":"2024-09-21T10:25:59.328382+0000","last_active":"2024-09-21T10:26:59.694837+0000","last_peered":"2024-09-21T10:26:59.694837+0000","last_clean":"2024-09-21T10:26:59.694837+0000","last_became_active":"2024-09-21T10:25:59.328072+0000","last_became_peered":"2024-09-21T10:25:59.328072+0000","last_unstale":"2024-09-21T10:26:59.694837+0000","last_undegraded":"2024-09-21T10:26:59.694837+0000","last_fullsized":"2024-09-21T10:26:59.694837+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:37:32.814312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686395+0000","last_change":"2024-09-21T10:25:59.333563+0000","last_active":"2024-09-21T10:26:58.686395+0000","last_peered":"2024-09-21T10:26:58.686395+0000","last_clean":"2024-09-21T10:26:58.686395+0000","last_became_active":"2024-09-21T10:25:59.333207+0000","last_became_peered":"2024-09-21T10:25:59.333207+0000","last_unstale":"2024-09-21T10:26:58.686395+0000","last_undegraded":"2024-09-21T10:26:58.686395+0000","last_fullsized":"2024-09-21T10:26:58.686395+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:47.763170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"361'149","reported_seq":1550,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689702+0000","last_change":"2024-09-21T10:25:55.905397+0000","last_active":"2024-09-21T10:26:58.689702+0000","last_peered":"2024-09-21T10:26:58.689702+0000","last_clean":"2024-09-21T10:26:58.689702+0000","last_became_active":"2024-09-21T10:25:53.756592+0000","last_became_peered":"2024-09-21T10:25:53.756592+0000","last_unstale":"2024-09-21T10:26:58.689702+0000","last_undegraded":"2024-09-21T10:26:58.689702+0000","last_fullsized":"2024-09-21T10:26:58.689702+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:55.905292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:55.905292+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:44.493209+0000","scrub_duration":0.0015904389999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":709,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693455+0000","last_change":"2024-09-21T10:25:59.330640+0000","last_active":"2024-09-21T10:26:59.693455+0000","last_peered":"2024-09-21T10:26:59.693455+0000","last_clean":"2024-09-21T10:26:59.693455+0000","last_became_active":"2024-09-21T10:25:59.330330+0000","last_became_peered":"2024-09-21T10:25:59.330330+0000","last_unstale":"2024-09-21T10:26:59.693455+0000","last_undegraded":"2024-09-21T10:26:59.693455+0000","last_fullsized":"2024-09-21T10:26:59.693455+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:00.831233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":71,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690155+0000","last_change":"2024-09-21T10:25:55.780663+0000","last_active":"2024-09-21T10:26:58.690155+0000","last_peered":"2024-09-21T10:26:58.690155+0000","last_clean":"2024-09-21T10:26:58.690155+0000","last_became_active":"2024-09-21T10:25:55.759508+0000","last_became_peered":"2024-09-21T10:25:55.759508+0000","last_unstale":"2024-09-21T10:26:58.690155+0000","last_undegraded":"2024-09-21T10:26:58.690155+0000","last_fullsized":"2024-09-21T10:26:58.690155+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:37:51.356439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"361'149","reported_seq":1101,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686702+0000","last_change":"2024-09-21T10:26:06.996191+0000","last_active":"2024-09-21T10:26:58.686702+0000","last_peered":"2024-09-21T10:26:58.686702+0000","last_clean":"2024-09-21T10:26:58.686702+0000","last_became_active":"2024-09-21T10:26:02.341215+0000","last_became_peered":"2024-09-21T10:26:02.341215+0000","last_unstale":"2024-09-21T10:26:58.686702+0000","last_undegraded":"2024-09-21T10:26:58.686702+0000","last_fullsized":"2024-09-21T10:26:58.686702+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:06.996089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:06.996089+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:45.116333+0000","scrub_duration":0.0015455410000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":38,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.687095+0000","last_change":"2024-09-21T10:25:59.349222+0000","last_active":"2024-09-21T10:26:58.687095+0000","last_peered":"2024-09-21T10:26:58.687095+0000","last_clean":"2024-09-21T10:26:58.687095+0000","last_became_active":"2024-09-21T10:25:59.333321+0000","last_became_peered":"2024-09-21T10:25:59.333321+0000","last_unstale":"2024-09-21T10:26:58.687095+0000","last_undegraded":"2024-09-21T10:26:58.687095+0000","last_fullsized":"2024-09-21T10:26:58.687095+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:36.232685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":192,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.688959+0000","last_change":"2024-09-21T10:25:59.334339+0000","last_active":"2024-09-21T10:26:58.688959+0000","last_peered":"2024-09-21T10:26:58.688959+0000","last_clean":"2024-09-21T10:26:58.688959+0000","last_became_active":"2024-09-21T10:25:59.334012+0000","last_became_peered":"2024-09-21T10:25:59.334012+0000","last_unstale":"2024-09-21T10:26:58.688959+0000","last_undegraded":"2024-09-21T10:26:58.688959+0000","last_fullsized":"2024-09-21T10:26:58.688959+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:01.812552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"361'149","reported_seq":1635,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693145+0000","last_change":"2024-09-21T10:25:58.779214+0000","last_active":"2024-09-21T10:26:59.693145+0000","last_peered":"2024-09-21T10:26:59.693145+0000","last_clean":"2024-09-21T10:26:59.693145+0000","last_became_active":"2024-09-21T10:25:58.778718+0000","last_became_peered":"2024-09-21T10:25:58.778718+0000","last_unstale":"2024-09-21T10:26:59.693145+0000","last_undegraded":"2024-09-21T10:26:59.693145+0000","last_fullsized":"2024-09-21T10:26:59.693145+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:49.156681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:49.156681+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:10.356625+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":38,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686962+0000","last_change":"2024-09-21T10:25:59.342009+0000","last_active":"2024-09-21T10:26:58.686962+0000","last_peered":"2024-09-21T10:26:58.686962+0000","last_clean":"2024-09-21T10:26:58.686962+0000","last_became_active":"2024-09-21T10:25:59.331715+0000","last_became_peered":"2024-09-21T10:25:59.331715+0000","last_unstale":"2024-09-21T10:26:58.686962+0000","last_undegraded":"2024-09-21T10:26:58.686962+0000","last_fullsized":"2024-09-21T10:26:58.686962+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:21.984045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1573,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807513+0000","last_change":"2024-09-21T10:26:27.668856+0000","last_active":"2024-09-21T10:26:58.807513+0000","last_peered":"2024-09-21T10:26:58.807513+0000","last_clean":"2024-09-21T10:26:58.807513+0000","last_became_active":"2024-09-21T10:26:27.668553+0000","last_became_peered":"2024-09-21T10:26:27.668553+0000","last_unstale":"2024-09-21T10:26:58.807513+0000","last_undegraded":"2024-09-21T10:26:58.807513+0000","last_fullsized":"2024-09-21T10:26:58.807513+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:05.350182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"361'149","reported_seq":995,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689900+0000","last_change":"2024-09-21T10:26:01.591818+0000","last_active":"2024-09-21T10:26:58.689900+0000","last_peered":"2024-09-21T10:26:58.689900+0000","last_clean":"2024-09-21T10:26:58.689900+0000","last_became_active":"2024-09-21T10:25:54.758256+0000","last_became_peered":"2024-09-21T10:25:54.758256+0000","last_unstale":"2024-09-21T10:26:58.689900+0000","last_undegraded":"2024-09-21T10:26:58.689900+0000","last_fullsized":"2024-09-21T10:26:58.689900+0000","mapping_epoch":1416,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1417,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:01.591718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:01.591718+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:33.331048+0000","scrub_duration":0.001673234,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":38,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686974+0000","last_change":"2024-09-21T10:25:59.350950+0000","last_active":"2024-09-21T10:26:58.686974+0000","last_peered":"2024-09-21T10:26:58.686974+0000","last_clean":"2024-09-21T10:26:58.686974+0000","last_became_active":"2024-09-21T10:25:59.335076+0000","last_became_peered":"2024-09-21T10:25:59.335076+0000","last_unstale":"2024-09-21T10:26:58.686974+0000","last_undegraded":"2024-09-21T10:26:58.686974+0000","last_fullsized":"2024-09-21T10:26:58.686974+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:34.161654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":51,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689896+0000","last_change":"2024-09-21T10:25:59.336918+0000","last_active":"2024-09-21T10:26:58.689896+0000","last_peered":"2024-09-21T10:26:58.689896+0000","last_clean":"2024-09-21T10:26:58.689896+0000","last_became_active":"2024-09-21T10:25:59.336042+0000","last_became_peered":"2024-09-21T10:25:59.336042+0000","last_unstale":"2024-09-21T10:26:58.689896+0000","last_undegraded":"2024-09-21T10:26:58.689896+0000","last_fullsized":"2024-09-21T10:26:58.689896+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:46.593265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"361'149","reported_seq":841,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808154+0000","last_change":"2024-09-21T10:26:28.112787+0000","last_active":"2024-09-21T10:26:58.808154+0000","last_peered":"2024-09-21T10:26:58.808154+0000","last_clean":"2024-09-21T10:26:58.808154+0000","last_became_active":"2024-09-21T10:26:27.670564+0000","last_became_peered":"2024-09-21T10:26:27.670564+0000","last_unstale":"2024-09-21T10:26:58.808154+0000","last_undegraded":"2024-09-21T10:26:58.808154+0000","last_fullsized":"2024-09-21T10:26:58.808154+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:28.112686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:28.112686+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:48.621317+0000","scrub_duration":0.002132002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":183,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689917+0000","last_change":"2024-09-21T10:25:51.761914+0000","last_active":"2024-09-21T10:26:58.689917+0000","last_peered":"2024-09-21T10:26:58.689917+0000","last_clean":"2024-09-21T10:26:58.689917+0000","last_became_active":"2024-09-21T10:25:51.761153+0000","last_became_peered":"2024-09-21T10:25:51.761153+0000","last_unstale":"2024-09-21T10:26:58.689917+0000","last_undegraded":"2024-09-21T10:26:58.689917+0000","last_fullsized":"2024-09-21T10:26:58.689917+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:29.366348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694842+0000","last_change":"2024-09-21T10:25:55.770244+0000","last_active":"2024-09-21T10:26:59.694842+0000","last_peered":"2024-09-21T10:26:59.694842+0000","last_clean":"2024-09-21T10:26:59.694842+0000","last_became_active":"2024-09-21T10:25:55.769692+0000","last_became_peered":"2024-09-21T10:25:55.769692+0000","last_unstale":"2024-09-21T10:26:59.694842+0000","last_undegraded":"2024-09-21T10:26:59.694842+0000","last_fullsized":"2024-09-21T10:26:59.694842+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:29.852715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"361'149","reported_seq":1563,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686512+0000","last_change":"2024-09-21T10:26:02.343335+0000","last_active":"2024-09-21T10:26:58.686512+0000","last_peered":"2024-09-21T10:26:58.686512+0000","last_clean":"2024-09-21T10:26:58.686512+0000","last_became_active":"2024-09-21T10:26:02.342969+0000","last_became_peered":"2024-09-21T10:26:02.342969+0000","last_unstale":"2024-09-21T10:26:58.686512+0000","last_undegraded":"2024-09-21T10:26:58.686512+0000","last_fullsized":"2024-09-21T10:26:58.686512+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:37.252419+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:37.252419+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:01.154484+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":614,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692997+0000","last_change":"2024-09-21T10:25:55.787837+0000","last_active":"2024-09-21T10:26:59.692997+0000","last_peered":"2024-09-21T10:26:59.692997+0000","last_clean":"2024-09-21T10:26:59.692997+0000","last_became_active":"2024-09-21T10:25:55.768035+0000","last_became_peered":"2024-09-21T10:25:55.768035+0000","last_unstale":"2024-09-21T10:26:59.692997+0000","last_undegraded":"2024-09-21T10:26:59.692997+0000","last_fullsized":"2024-09-21T10:26:59.692997+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:52.239344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1650,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808037+0000","last_change":"2024-09-21T10:26:27.669551+0000","last_active":"2024-09-21T10:26:58.808037+0000","last_peered":"2024-09-21T10:26:58.808037+0000","last_clean":"2024-09-21T10:26:58.808037+0000","last_became_active":"2024-09-21T10:26:27.669230+0000","last_became_peered":"2024-09-21T10:26:27.669230+0000","last_unstale":"2024-09-21T10:26:58.808037+0000","last_undegraded":"2024-09-21T10:26:58.808037+0000","last_fullsized":"2024-09-21T10:26:58.808037+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:27.533026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"361'149","reported_seq":1568,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.688792+0000","last_change":"2024-09-21T10:25:51.751718+0000","last_active":"2024-09-21T10:26:58.688792+0000","last_peered":"2024-09-21T10:26:58.688792+0000","last_clean":"2024-09-21T10:26:58.688792+0000","last_became_active":"2024-09-21T10:25:51.751250+0000","last_became_peered":"2024-09-21T10:25:51.751250+0000","last_unstale":"2024-09-21T10:26:58.688792+0000","last_undegraded":"2024-09-21T10:26:58.688792+0000","last_fullsized":"2024-09-21T10:26:58.688792+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:14.982755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:14.982755+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:38.779659+0000","scrub_duration":0.0028608990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":42,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689986+0000","last_change":"2024-09-21T10:25:51.751943+0000","last_active":"2024-09-21T10:26:58.689986+0000","last_peered":"2024-09-21T10:26:58.689986+0000","last_clean":"2024-09-21T10:26:58.689986+0000","last_became_active":"2024-09-21T10:25:51.751607+0000","last_became_peered":"2024-09-21T10:25:51.751607+0000","last_unstale":"2024-09-21T10:26:58.689986+0000","last_undegraded":"2024-09-21T10:26:58.689986+0000","last_fullsized":"2024-09-21T10:26:58.689986+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:12:22.046064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1136,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808213+0000","last_change":"2024-09-21T10:26:27.668640+0000","last_active":"2024-09-21T10:26:58.808213+0000","last_peered":"2024-09-21T10:26:58.808213+0000","last_clean":"2024-09-21T10:26:58.808213+0000","last_became_active":"2024-09-21T10:26:27.668326+0000","last_became_peered":"2024-09-21T10:26:27.668326+0000","last_unstale":"2024-09-21T10:26:58.808213+0000","last_undegraded":"2024-09-21T10:26:58.808213+0000","last_fullsized":"2024-09-21T10:26:58.808213+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:20:43.801509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"361'149","reported_seq":1644,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837098+0000","last_change":"2024-09-21T10:26:04.019563+0000","last_active":"2024-09-21T10:26:59.837098+0000","last_peered":"2024-09-21T10:26:59.837098+0000","last_clean":"2024-09-21T10:26:59.837098+0000","last_became_active":"2024-09-21T10:25:58.768756+0000","last_became_peered":"2024-09-21T10:25:58.768756+0000","last_unstale":"2024-09-21T10:26:59.837098+0000","last_undegraded":"2024-09-21T10:26:59.837098+0000","last_fullsized":"2024-09-21T10:26:59.837098+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:04.019508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:04.019508+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:06.722178+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1688,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807739+0000","last_change":"2024-09-21T10:26:27.668358+0000","last_active":"2024-09-21T10:26:58.807739+0000","last_peered":"2024-09-21T10:26:58.807739+0000","last_clean":"2024-09-21T10:26:58.807739+0000","last_became_active":"2024-09-21T10:26:27.667963+0000","last_became_peered":"2024-09-21T10:26:27.667963+0000","last_unstale":"2024-09-21T10:26:58.807739+0000","last_undegraded":"2024-09-21T10:26:58.807739+0000","last_fullsized":"2024-09-21T10:26:58.807739+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:03:04.974419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1139,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808284+0000","last_change":"2024-09-21T10:26:27.670533+0000","last_active":"2024-09-21T10:26:58.808284+0000","last_peered":"2024-09-21T10:26:58.808284+0000","last_clean":"2024-09-21T10:26:58.808284+0000","last_became_active":"2024-09-21T10:26:27.670236+0000","last_became_peered":"2024-09-21T10:26:27.670236+0000","last_unstale":"2024-09-21T10:26:58.808284+0000","last_undegraded":"2024-09-21T10:26:58.808284+0000","last_fullsized":"2024-09-21T10:26:58.808284+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:48.498697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"361'149","reported_seq":1553,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689488+0000","last_change":"2024-09-21T10:25:54.767294+0000","last_active":"2024-09-21T10:26:58.689488+0000","last_peered":"2024-09-21T10:26:58.689488+0000","last_clean":"2024-09-21T10:26:58.689488+0000","last_became_active":"2024-09-21T10:25:54.766557+0000","last_became_peered":"2024-09-21T10:25:54.766557+0000","last_unstale":"2024-09-21T10:26:58.689488+0000","last_undegraded":"2024-09-21T10:26:58.689488+0000","last_fullsized":"2024-09-21T10:26:58.689488+0000","mapping_epoch":1416,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1417,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:55.183929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:55.183929+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:06.295861+0000","scrub_duration":0.001781761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":635,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692641+0000","last_change":"2024-09-21T10:25:59.332733+0000","last_active":"2024-09-21T10:26:59.692641+0000","last_peered":"2024-09-21T10:26:59.692641+0000","last_clean":"2024-09-21T10:26:59.692641+0000","last_became_active":"2024-09-21T10:25:59.332246+0000","last_became_peered":"2024-09-21T10:25:59.332246+0000","last_unstale":"2024-09-21T10:26:59.692641+0000","last_undegraded":"2024-09-21T10:26:59.692641+0000","last_fullsized":"2024-09-21T10:26:59.692641+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:28.477050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2088,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837293+0000","last_change":"2024-09-21T10:23:39.350772+0000","last_active":"2024-09-21T10:26:59.837293+0000","last_peered":"2024-09-21T10:26:59.837293+0000","last_clean":"2024-09-21T10:26:59.837293+0000","last_became_active":"2024-09-21T10:23:39.345255+0000","last_became_peered":"2024-09-21T10:23:39.345255+0000","last_unstale":"2024-09-21T10:26:59.837293+0000","last_undegraded":"2024-09-21T10:26:59.837293+0000","last_fullsized":"2024-09-21T10:26:59.837293+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:22.523103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1602,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686688+0000","last_change":"2024-09-21T10:26:11.015613+0000","last_active":"2024-09-21T10:26:58.686688+0000","last_peered":"2024-09-21T10:26:58.686688+0000","last_clean":"2024-09-21T10:26:58.686688+0000","last_became_active":"2024-09-21T10:26:02.338686+0000","last_became_peered":"2024-09-21T10:26:02.338686+0000","last_unstale":"2024-09-21T10:26:58.686688+0000","last_undegraded":"2024-09-21T10:26:58.686688+0000","last_fullsized":"2024-09-21T10:26:58.686688+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":0,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:11.015392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:11.015392+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:59.399953+0000","scrub_duration":0.0022519319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":211,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689521+0000","last_change":"2024-09-21T10:25:59.338345+0000","last_active":"2024-09-21T10:26:58.689521+0000","last_peered":"2024-09-21T10:26:58.689521+0000","last_clean":"2024-09-21T10:26:58.689521+0000","last_became_active":"2024-09-21T10:25:59.330952+0000","last_became_peered":"2024-09-21T10:25:59.330952+0000","last_unstale":"2024-09-21T10:26:58.689521+0000","last_undegraded":"2024-09-21T10:26:58.689521+0000","last_fullsized":"2024-09-21T10:26:58.689521+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:44.580358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":603,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837482+0000","last_change":"2024-09-21T10:26:59.837150+0000","last_active":"2024-09-21T10:26:59.837482+0000","last_peered":"2024-09-21T10:26:59.837482+0000","last_clean":"2024-09-21T10:26:59.837482+0000","last_became_active":"2024-09-21T10:26:59.836806+0000","last_became_peered":"2024-09-21T10:26:59.836806+0000","last_unstale":"2024-09-21T10:26:59.837482+0000","last_undegraded":"2024-09-21T10:26:59.837482+0000","last_fullsized":"2024-09-21T10:26:59.837482+0000","mapping_epoch":1444,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:30.255001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690149+0000","last_change":"2024-09-21T10:25:59.336761+0000","last_active":"2024-09-21T10:26:58.690149+0000","last_peered":"2024-09-21T10:26:58.690149+0000","last_clean":"2024-09-21T10:26:58.690149+0000","last_became_active":"2024-09-21T10:25:59.336441+0000","last_became_peered":"2024-09-21T10:25:59.336441+0000","last_unstale":"2024-09-21T10:26:58.690149+0000","last_undegraded":"2024-09-21T10:26:58.690149+0000","last_fullsized":"2024-09-21T10:26:58.690149+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:36.725859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"361'149","reported_seq":1636,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837073+0000","last_change":"2024-09-21T10:26:07.019202+0000","last_active":"2024-09-21T10:26:59.837073+0000","last_peered":"2024-09-21T10:26:59.837073+0000","last_clean":"2024-09-21T10:26:59.837073+0000","last_became_active":"2024-09-21T10:25:54.763354+0000","last_became_peered":"2024-09-21T10:25:54.763354+0000","last_unstale":"2024-09-21T10:26:59.837073+0000","last_undegraded":"2024-09-21T10:26:59.837073+0000","last_fullsized":"2024-09-21T10:26:59.837073+0000","mapping_epoch":1416,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1417,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:07.019156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:07.019156+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:57.651395+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":34,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686516+0000","last_change":"2024-09-21T10:25:59.331309+0000","last_active":"2024-09-21T10:26:58.686516+0000","last_peered":"2024-09-21T10:26:58.686516+0000","last_clean":"2024-09-21T10:26:58.686516+0000","last_became_active":"2024-09-21T10:25:59.330812+0000","last_became_peered":"2024-09-21T10:25:59.330812+0000","last_unstale":"2024-09-21T10:26:58.686516+0000","last_undegraded":"2024-09-21T10:26:58.686516+0000","last_fullsized":"2024-09-21T10:26:58.686516+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:33.640990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694514+0000","last_change":"2024-09-21T10:25:55.773832+0000","last_active":"2024-09-21T10:26:59.694514+0000","last_peered":"2024-09-21T10:26:59.694514+0000","last_clean":"2024-09-21T10:26:59.694514+0000","last_became_active":"2024-09-21T10:25:55.773515+0000","last_became_peered":"2024-09-21T10:25:55.773515+0000","last_unstale":"2024-09-21T10:26:59.694514+0000","last_undegraded":"2024-09-21T10:26:59.694514+0000","last_fullsized":"2024-09-21T10:26:59.694514+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:34.195830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694985+0000","last_change":"2024-09-21T10:25:55.764647+0000","last_active":"2024-09-21T10:26:59.694985+0000","last_peered":"2024-09-21T10:26:59.694985+0000","last_clean":"2024-09-21T10:26:59.694985+0000","last_became_active":"2024-09-21T10:25:55.764349+0000","last_became_peered":"2024-09-21T10:25:55.764349+0000","last_unstale":"2024-09-21T10:26:59.694985+0000","last_undegraded":"2024-09-21T10:26:59.694985+0000","last_fullsized":"2024-09-21T10:26:59.694985+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:10.832495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"361'149","reported_seq":1538,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686819+0000","last_change":"2024-09-21T10:26:02.340359+0000","last_active":"2024-09-21T10:26:58.686819+0000","last_peered":"2024-09-21T10:26:58.686819+0000","last_clean":"2024-09-21T10:26:58.686819+0000","last_became_active":"2024-09-21T10:26:02.339971+0000","last_became_peered":"2024-09-21T10:26:02.339971+0000","last_unstale":"2024-09-21T10:26:58.686819+0000","last_undegraded":"2024-09-21T10:26:58.686819+0000","last_fullsized":"2024-09-21T10:26:58.686819+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:44.849304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:44.849304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:27.019549+0000","scrub_duration":0.002061387,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1933,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835700+0000","last_change":"2024-09-21T10:26:20.732018+0000","last_active":"2024-09-21T10:26:59.835700+0000","last_peered":"2024-09-21T10:26:59.835700+0000","last_clean":"2024-09-21T10:26:59.835700+0000","last_became_active":"2024-09-21T10:26:20.731669+0000","last_became_peered":"2024-09-21T10:26:20.731669+0000","last_unstale":"2024-09-21T10:26:59.835700+0000","last_undegraded":"2024-09-21T10:26:59.835700+0000","last_fullsized":"2024-09-21T10:26:59.835700+0000","mapping_epoch":1429,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:09:59.605809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2126,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836106+0000","last_change":"2024-09-21T10:25:51.749756+0000","last_active":"2024-09-21T10:26:59.836106+0000","last_peered":"2024-09-21T10:26:59.836106+0000","last_clean":"2024-09-21T10:26:59.836106+0000","last_became_active":"2024-09-21T10:25:51.749404+0000","last_became_peered":"2024-09-21T10:25:51.749404+0000","last_unstale":"2024-09-21T10:26:59.836106+0000","last_undegraded":"2024-09-21T10:26:59.836106+0000","last_fullsized":"2024-09-21T10:26:59.836106+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:58.620370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2111,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835744+0000","last_change":"2024-09-21T10:25:55.761083+0000","last_active":"2024-09-21T10:26:59.835744+0000","last_peered":"2024-09-21T10:26:59.835744+0000","last_clean":"2024-09-21T10:26:59.835744+0000","last_became_active":"2024-09-21T10:25:55.760784+0000","last_became_peered":"2024-09-21T10:25:55.760784+0000","last_unstale":"2024-09-21T10:26:59.835744+0000","last_undegraded":"2024-09-21T10:26:59.835744+0000","last_fullsized":"2024-09-21T10:26:59.835744+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:56.095026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"361'149","reported_seq":1583,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689028+0000","last_change":"2024-09-21T10:25:58.767837+0000","last_active":"2024-09-21T10:26:58.689028+0000","last_peered":"2024-09-21T10:26:58.689028+0000","last_clean":"2024-09-21T10:26:58.689028+0000","last_became_active":"2024-09-21T10:25:58.767256+0000","last_became_peered":"2024-09-21T10:25:58.767256+0000","last_unstale":"2024-09-21T10:26:58.689028+0000","last_undegraded":"2024-09-21T10:26:58.689028+0000","last_fullsized":"2024-09-21T10:26:58.689028+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:43.836750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:43.836750+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:50:20.192943+0000","scrub_duration":0.0019745000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":195,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689950+0000","last_change":"2024-09-21T10:25:51.778636+0000","last_active":"2024-09-21T10:26:58.689950+0000","last_peered":"2024-09-21T10:26:58.689950+0000","last_clean":"2024-09-21T10:26:58.689950+0000","last_became_active":"2024-09-21T10:25:51.762888+0000","last_became_peered":"2024-09-21T10:25:51.762888+0000","last_unstale":"2024-09-21T10:26:58.689950+0000","last_undegraded":"2024-09-21T10:26:58.689950+0000","last_fullsized":"2024-09-21T10:26:58.689950+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:53.548491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1147,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808015+0000","last_change":"2024-09-21T10:26:27.928815+0000","last_active":"2024-09-21T10:26:58.808015+0000","last_peered":"2024-09-21T10:26:58.808015+0000","last_clean":"2024-09-21T10:26:58.808015+0000","last_became_active":"2024-09-21T10:26:27.928530+0000","last_became_peered":"2024-09-21T10:26:27.928530+0000","last_unstale":"2024-09-21T10:26:58.808015+0000","last_undegraded":"2024-09-21T10:26:58.808015+0000","last_fullsized":"2024-09-21T10:26:58.808015+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:36.669221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1991,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807869+0000","last_change":"2024-09-21T10:26:45.227401+0000","last_active":"2024-09-21T10:26:58.807869+0000","last_peered":"2024-09-21T10:26:58.807869+0000","last_clean":"2024-09-21T10:26:58.807869+0000","last_became_active":"2024-09-21T10:26:45.227024+0000","last_became_peered":"2024-09-21T10:26:45.227024+0000","last_unstale":"2024-09-21T10:26:58.807869+0000","last_undegraded":"2024-09-21T10:26:58.807869+0000","last_fullsized":"2024-09-21T10:26:58.807869+0000","mapping_epoch":1439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:09.279771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"361'149","reported_seq":1120,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689890+0000","last_change":"2024-09-21T10:26:02.341872+0000","last_active":"2024-09-21T10:26:58.689890+0000","last_peered":"2024-09-21T10:26:58.689890+0000","last_clean":"2024-09-21T10:26:58.689890+0000","last_became_active":"2024-09-21T10:26:02.341507+0000","last_became_peered":"2024-09-21T10:26:02.341507+0000","last_unstale":"2024-09-21T10:26:58.689890+0000","last_undegraded":"2024-09-21T10:26:58.689890+0000","last_fullsized":"2024-09-21T10:26:58.689890+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:56.612468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:56.612468+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:20.739592+0000","scrub_duration":0.0018069690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690155+0000","last_change":"2024-09-21T10:25:48.754301+0000","last_active":"2024-09-21T10:26:58.690155+0000","last_peered":"2024-09-21T10:26:58.690155+0000","last_clean":"2024-09-21T10:26:58.690155+0000","last_became_active":"2024-09-21T10:25:48.753829+0000","last_became_peered":"2024-09-21T10:25:48.753829+0000","last_unstale":"2024-09-21T10:26:58.690155+0000","last_undegraded":"2024-09-21T10:26:58.690155+0000","last_fullsized":"2024-09-21T10:26:58.690155+0000","mapping_epoch":1410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:21.386439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2140,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836807+0000","last_change":"2024-09-21T10:25:59.331850+0000","last_active":"2024-09-21T10:26:59.836807+0000","last_peered":"2024-09-21T10:26:59.836807+0000","last_clean":"2024-09-21T10:26:59.836807+0000","last_became_active":"2024-09-21T10:25:59.331517+0000","last_became_peered":"2024-09-21T10:25:59.331517+0000","last_unstale":"2024-09-21T10:26:59.836807+0000","last_undegraded":"2024-09-21T10:26:59.836807+0000","last_fullsized":"2024-09-21T10:26:59.836807+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:00.625058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":22,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835614+0000","last_change":"2024-09-21T10:26:45.227342+0000","last_active":"2024-09-21T10:26:59.835614+0000","last_peered":"2024-09-21T10:26:59.835614+0000","last_clean":"2024-09-21T10:26:59.835614+0000","last_became_active":"2024-09-21T10:26:45.227047+0000","last_became_peered":"2024-09-21T10:26:45.227047+0000","last_unstale":"2024-09-21T10:26:59.835614+0000","last_undegraded":"2024-09-21T10:26:59.835614+0000","last_fullsized":"2024-09-21T10:26:59.835614+0000","mapping_epoch":1439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:32.930354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"361'149","reported_seq":1112,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690002+0000","last_change":"2024-09-21T10:26:02.579300+0000","last_active":"2024-09-21T10:26:58.690002+0000","last_peered":"2024-09-21T10:26:58.690002+0000","last_clean":"2024-09-21T10:26:58.690002+0000","last_became_active":"2024-09-21T10:25:58.775781+0000","last_became_peered":"2024-09-21T10:25:58.775781+0000","last_unstale":"2024-09-21T10:26:58.690002+0000","last_undegraded":"2024-09-21T10:26:58.690002+0000","last_fullsized":"2024-09-21T10:26:58.690002+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:02.579205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:02.579205+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:49.183837+0000","scrub_duration":0.0014992790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":306,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693046+0000","last_change":"2024-09-21T10:25:48.749057+0000","last_active":"2024-09-21T10:26:59.693046+0000","last_peered":"2024-09-21T10:26:59.693046+0000","last_clean":"2024-09-21T10:26:59.693046+0000","last_became_active":"2024-09-21T10:25:48.748471+0000","last_became_peered":"2024-09-21T10:25:48.748471+0000","last_unstale":"2024-09-21T10:26:59.693046+0000","last_undegraded":"2024-09-21T10:26:59.693046+0000","last_fullsized":"2024-09-21T10:26:59.693046+0000","mapping_epoch":1410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:39.024192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686742+0000","last_change":"2024-09-21T10:25:59.332732+0000","last_active":"2024-09-21T10:26:58.686742+0000","last_peered":"2024-09-21T10:26:58.686742+0000","last_clean":"2024-09-21T10:26:58.686742+0000","last_became_active":"2024-09-21T10:25:59.332420+0000","last_became_peered":"2024-09-21T10:25:59.332420+0000","last_unstale":"2024-09-21T10:26:58.686742+0000","last_undegraded":"2024-09-21T10:26:58.686742+0000","last_fullsized":"2024-09-21T10:26:58.686742+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:56.981194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694748+0000","last_change":"2024-09-21T10:25:59.328950+0000","last_active":"2024-09-21T10:26:59.694748+0000","last_peered":"2024-09-21T10:26:59.694748+0000","last_clean":"2024-09-21T10:26:59.694748+0000","last_became_active":"2024-09-21T10:25:59.328619+0000","last_became_peered":"2024-09-21T10:25:59.328619+0000","last_unstale":"2024-09-21T10:26:59.694748+0000","last_undegraded":"2024-09-21T10:26:59.694748+0000","last_fullsized":"2024-09-21T10:26:59.694748+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:06.020852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"361'149","reported_seq":849,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808038+0000","last_change":"2024-09-21T10:26:35.102300+0000","last_active":"2024-09-21T10:26:58.808038+0000","last_peered":"2024-09-21T10:26:58.808038+0000","last_clean":"2024-09-21T10:26:58.808038+0000","last_became_active":"2024-09-21T10:26:27.669120+0000","last_became_peered":"2024-09-21T10:26:27.669120+0000","last_unstale":"2024-09-21T10:26:58.808038+0000","last_undegraded":"2024-09-21T10:26:58.808038+0000","last_fullsized":"2024-09-21T10:26:58.808038+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:35.102188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:35.102188+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:29:17.236873+0000","scrub_duration":0.0020000119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":429,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.696395+0000","last_change":"2024-09-21T10:26:59.696125+0000","last_active":"2024-09-21T10:26:59.696395+0000","last_peered":"2024-09-21T10:26:59.696395+0000","last_clean":"2024-09-21T10:26:59.696395+0000","last_became_active":"2024-09-21T10:26:59.695782+0000","last_became_peered":"2024-09-21T10:26:59.695782+0000","last_unstale":"2024-09-21T10:26:59.696395+0000","last_undegraded":"2024-09-21T10:26:59.696395+0000","last_fullsized":"2024-09-21T10:26:59.696395+0000","mapping_epoch":1444,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:12:18.288410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":202,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693380+0000","last_change":"2024-09-21T10:25:51.755726+0000","last_active":"2024-09-21T10:26:59.693380+0000","last_peered":"2024-09-21T10:26:59.693380+0000","last_clean":"2024-09-21T10:26:59.693380+0000","last_became_active":"2024-09-21T10:25:51.755373+0000","last_became_peered":"2024-09-21T10:25:51.755373+0000","last_unstale":"2024-09-21T10:26:59.693380+0000","last_undegraded":"2024-09-21T10:26:59.693380+0000","last_fullsized":"2024-09-21T10:26:59.693380+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:02:52.571798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":639,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692704+0000","last_change":"2024-09-21T10:25:59.331761+0000","last_active":"2024-09-21T10:26:59.692704+0000","last_peered":"2024-09-21T10:26:59.692704+0000","last_clean":"2024-09-21T10:26:59.692704+0000","last_became_active":"2024-09-21T10:25:59.330633+0000","last_became_peered":"2024-09-21T10:25:59.330633+0000","last_unstale":"2024-09-21T10:26:59.692704+0000","last_undegraded":"2024-09-21T10:26:59.692704+0000","last_fullsized":"2024-09-21T10:26:59.692704+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:03:37.270144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"361'149","reported_seq":1636,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693257+0000","last_change":"2024-09-21T10:26:02.341199+0000","last_active":"2024-09-21T10:26:59.693257+0000","last_peered":"2024-09-21T10:26:59.693257+0000","last_clean":"2024-09-21T10:26:59.693257+0000","last_became_active":"2024-09-21T10:26:02.340534+0000","last_became_peered":"2024-09-21T10:26:02.340534+0000","last_unstale":"2024-09-21T10:26:59.693257+0000","last_undegraded":"2024-09-21T10:26:59.693257+0000","last_fullsized":"2024-09-21T10:26:59.693257+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:45.334710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:45.334710+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:32.459672+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694872+0000","last_change":"2024-09-21T10:25:55.780088+0000","last_active":"2024-09-21T10:26:59.694872+0000","last_peered":"2024-09-21T10:26:59.694872+0000","last_clean":"2024-09-21T10:26:59.694872+0000","last_became_active":"2024-09-21T10:25:55.765081+0000","last_became_peered":"2024-09-21T10:25:55.765081+0000","last_unstale":"2024-09-21T10:26:59.694872+0000","last_undegraded":"2024-09-21T10:26:59.694872+0000","last_fullsized":"2024-09-21T10:26:59.694872+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:43.407232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":51,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689990+0000","last_change":"2024-09-21T10:25:59.337330+0000","last_active":"2024-09-21T10:26:58.689990+0000","last_peered":"2024-09-21T10:26:58.689990+0000","last_clean":"2024-09-21T10:26:58.689990+0000","last_became_active":"2024-09-21T10:25:59.337011+0000","last_became_peered":"2024-09-21T10:25:59.337011+0000","last_unstale":"2024-09-21T10:26:58.689990+0000","last_undegraded":"2024-09-21T10:26:58.689990+0000","last_fullsized":"2024-09-21T10:26:58.689990+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:07.800833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1116,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807619+0000","last_change":"2024-09-21T10:26:27.929354+0000","last_active":"2024-09-21T10:26:58.807619+0000","last_peered":"2024-09-21T10:26:58.807619+0000","last_clean":"2024-09-21T10:26:58.807619+0000","last_became_active":"2024-09-21T10:26:27.929042+0000","last_became_peered":"2024-09-21T10:26:27.929042+0000","last_unstale":"2024-09-21T10:26:58.807619+0000","last_undegraded":"2024-09-21T10:26:58.807619+0000","last_fullsized":"2024-09-21T10:26:58.807619+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:40.088951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1148,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807971+0000","last_change":"2024-09-21T10:26:27.670767+0000","last_active":"2024-09-21T10:26:58.807971+0000","last_peered":"2024-09-21T10:26:58.807971+0000","last_clean":"2024-09-21T10:26:58.807971+0000","last_became_active":"2024-09-21T10:26:27.670458+0000","last_became_peered":"2024-09-21T10:26:27.670458+0000","last_unstale":"2024-09-21T10:26:58.807971+0000","last_undegraded":"2024-09-21T10:26:58.807971+0000","last_fullsized":"2024-09-21T10:26:58.807971+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:11.886413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1167,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808186+0000","last_change":"2024-09-21T10:26:27.669387+0000","last_active":"2024-09-21T10:26:58.808186+0000","last_peered":"2024-09-21T10:26:58.808186+0000","last_clean":"2024-09-21T10:26:58.808186+0000","last_became_active":"2024-09-21T10:26:27.668952+0000","last_became_peered":"2024-09-21T10:26:27.668952+0000","last_unstale":"2024-09-21T10:26:58.808186+0000","last_undegraded":"2024-09-21T10:26:58.808186+0000","last_fullsized":"2024-09-21T10:26:58.808186+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:03.375664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2101,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836325+0000","last_change":"2024-09-21T10:25:51.750944+0000","last_active":"2024-09-21T10:26:59.836325+0000","last_peered":"2024-09-21T10:26:59.836325+0000","last_clean":"2024-09-21T10:26:59.836325+0000","last_became_active":"2024-09-21T10:25:51.750521+0000","last_became_peered":"2024-09-21T10:25:51.750521+0000","last_unstale":"2024-09-21T10:26:59.836325+0000","last_undegraded":"2024-09-21T10:26:59.836325+0000","last_fullsized":"2024-09-21T10:26:59.836325+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:00.271350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":40,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686799+0000","last_change":"2024-09-21T10:25:59.347575+0000","last_active":"2024-09-21T10:26:58.686799+0000","last_peered":"2024-09-21T10:26:58.686799+0000","last_clean":"2024-09-21T10:26:58.686799+0000","last_became_active":"2024-09-21T10:25:59.333758+0000","last_became_peered":"2024-09-21T10:25:59.333758+0000","last_unstale":"2024-09-21T10:26:58.686799+0000","last_undegraded":"2024-09-21T10:26:58.686799+0000","last_fullsized":"2024-09-21T10:26:58.686799+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:20.061779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"361'149","reported_seq":1590,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835993+0000","last_change":"2024-09-21T10:26:20.698780+0000","last_active":"2024-09-21T10:26:59.835993+0000","last_peered":"2024-09-21T10:26:59.835993+0000","last_clean":"2024-09-21T10:26:59.835993+0000","last_became_active":"2024-09-21T10:26:20.698460+0000","last_became_peered":"2024-09-21T10:26:20.698460+0000","last_unstale":"2024-09-21T10:26:59.835993+0000","last_undegraded":"2024-09-21T10:26:59.835993+0000","last_fullsized":"2024-09-21T10:26:59.835993+0000","mapping_epoch":1429,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1430,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:48.196311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:48.196311+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:26.100373+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":174,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836423+0000","last_change":"2024-09-21T10:25:59.331970+0000","last_active":"2024-09-21T10:26:59.836423+0000","last_peered":"2024-09-21T10:26:59.836423+0000","last_clean":"2024-09-21T10:26:59.836423+0000","last_became_active":"2024-09-21T10:25:59.331636+0000","last_became_peered":"2024-09-21T10:25:59.331636+0000","last_unstale":"2024-09-21T10:26:59.836423+0000","last_undegraded":"2024-09-21T10:26:59.836423+0000","last_fullsized":"2024-09-21T10:26:59.836423+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:14.908930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":751,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693239+0000","last_change":"2024-09-21T10:25:59.330908+0000","last_active":"2024-09-21T10:26:59.693239+0000","last_peered":"2024-09-21T10:26:59.693239+0000","last_clean":"2024-09-21T10:26:59.693239+0000","last_became_active":"2024-09-21T10:25:59.330587+0000","last_became_peered":"2024-09-21T10:25:59.330587+0000","last_unstale":"2024-09-21T10:26:59.693239+0000","last_undegraded":"2024-09-21T10:26:59.693239+0000","last_fullsized":"2024-09-21T10:26:59.693239+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:31.829657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694885+0000","last_change":"2024-09-21T10:25:55.771157+0000","last_active":"2024-09-21T10:26:59.694885+0000","last_peered":"2024-09-21T10:26:59.694885+0000","last_clean":"2024-09-21T10:26:59.694885+0000","last_became_active":"2024-09-21T10:25:55.770841+0000","last_became_peered":"2024-09-21T10:25:55.770841+0000","last_unstale":"2024-09-21T10:26:59.694885+0000","last_undegraded":"2024-09-21T10:26:59.694885+0000","last_fullsized":"2024-09-21T10:26:59.694885+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:00:39.889923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":721,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692742+0000","last_change":"2024-09-21T10:25:51.748938+0000","last_active":"2024-09-21T10:26:59.692742+0000","last_peered":"2024-09-21T10:26:59.692742+0000","last_clean":"2024-09-21T10:26:59.692742+0000","last_became_active":"2024-09-21T10:25:51.748472+0000","last_became_peered":"2024-09-21T10:25:51.748472+0000","last_unstale":"2024-09-21T10:26:59.692742+0000","last_undegraded":"2024-09-21T10:26:59.692742+0000","last_fullsized":"2024-09-21T10:26:59.692742+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:13.956736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":42,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690077+0000","last_change":"2024-09-21T10:25:51.761452+0000","last_active":"2024-09-21T10:26:58.690077+0000","last_peered":"2024-09-21T10:26:58.690077+0000","last_clean":"2024-09-21T10:26:58.690077+0000","last_became_active":"2024-09-21T10:25:51.760427+0000","last_became_peered":"2024-09-21T10:25:51.760427+0000","last_unstale":"2024-09-21T10:26:58.690077+0000","last_undegraded":"2024-09-21T10:26:58.690077+0000","last_fullsized":"2024-09-21T10:26:58.690077+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:08.783792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":686,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692512+0000","last_change":"2024-09-21T10:25:55.771479+0000","last_active":"2024-09-21T10:26:59.692512+0000","last_peered":"2024-09-21T10:26:59.692512+0000","last_clean":"2024-09-21T10:26:59.692512+0000","last_became_active":"2024-09-21T10:25:55.770758+0000","last_became_peered":"2024-09-21T10:25:55.770758+0000","last_unstale":"2024-09-21T10:26:59.692512+0000","last_undegraded":"2024-09-21T10:26:59.692512+0000","last_fullsized":"2024-09-21T10:26:59.692512+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:13.903497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694709+0000","last_change":"2024-09-21T10:25:59.328440+0000","last_active":"2024-09-21T10:26:59.694709+0000","last_peered":"2024-09-21T10:26:59.694709+0000","last_clean":"2024-09-21T10:26:59.694709+0000","last_became_active":"2024-09-21T10:25:59.328124+0000","last_became_peered":"2024-09-21T10:25:59.328124+0000","last_unstale":"2024-09-21T10:26:59.694709+0000","last_undegraded":"2024-09-21T10:26:59.694709+0000","last_fullsized":"2024-09-21T10:26:59.694709+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:24:21.543744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":377,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836050+0000","last_change":"2024-09-21T10:23:39.371060+0000","last_active":"2024-09-21T10:26:59.836050+0000","last_peered":"2024-09-21T10:26:59.836050+0000","last_clean":"2024-09-21T10:26:59.836050+0000","last_became_active":"2024-09-21T10:23:39.370494+0000","last_became_peered":"2024-09-21T10:23:39.370494+0000","last_unstale":"2024-09-21T10:26:59.836050+0000","last_undegraded":"2024-09-21T10:26:59.836050+0000","last_fullsized":"2024-09-21T10:26:59.836050+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:09.975626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":597,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693222+0000","last_change":"2024-09-21T10:23:39.510198+0000","last_active":"2024-09-21T10:26:59.693222+0000","last_peered":"2024-09-21T10:26:59.693222+0000","last_clean":"2024-09-21T10:26:59.693222+0000","last_became_active":"2024-09-21T10:23:39.509474+0000","last_became_peered":"2024-09-21T10:23:39.509474+0000","last_unstale":"2024-09-21T10:26:59.693222+0000","last_undegraded":"2024-09-21T10:26:59.693222+0000","last_fullsized":"2024-09-21T10:26:59.693222+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.611788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1137,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807777+0000","last_change":"2024-09-21T10:26:27.669976+0000","last_active":"2024-09-21T10:26:58.807777+0000","last_peered":"2024-09-21T10:26:58.807777+0000","last_clean":"2024-09-21T10:26:58.807777+0000","last_became_active":"2024-09-21T10:26:27.669656+0000","last_became_peered":"2024-09-21T10:26:27.669656+0000","last_unstale":"2024-09-21T10:26:58.807777+0000","last_undegraded":"2024-09-21T10:26:58.807777+0000","last_fullsized":"2024-09-21T10:26:58.807777+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:51.213576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1948,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836141+0000","last_change":"2024-09-21T10:26:20.732541+0000","last_active":"2024-09-21T10:26:59.836141+0000","last_peered":"2024-09-21T10:26:59.836141+0000","last_clean":"2024-09-21T10:26:59.836141+0000","last_became_active":"2024-09-21T10:26:20.732244+0000","last_became_peered":"2024-09-21T10:26:20.732244+0000","last_unstale":"2024-09-21T10:26:59.836141+0000","last_undegraded":"2024-09-21T10:26:59.836141+0000","last_fullsized":"2024-09-21T10:26:59.836141+0000","mapping_epoch":1429,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:09.876339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694778+0000","last_change":"2024-09-21T10:25:59.328227+0000","last_active":"2024-09-21T10:26:59.694778+0000","last_peered":"2024-09-21T10:26:59.694778+0000","last_clean":"2024-09-21T10:26:59.694778+0000","last_became_active":"2024-09-21T10:25:59.327831+0000","last_became_peered":"2024-09-21T10:25:59.327831+0000","last_unstale":"2024-09-21T10:26:59.694778+0000","last_undegraded":"2024-09-21T10:26:59.694778+0000","last_fullsized":"2024-09-21T10:26:59.694778+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:42:49.483388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1454,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807910+0000","last_change":"2024-09-21T10:26:27.671125+0000","last_active":"2024-09-21T10:26:58.807910+0000","last_peered":"2024-09-21T10:26:58.807910+0000","last_clean":"2024-09-21T10:26:58.807910+0000","last_became_active":"2024-09-21T10:26:27.670843+0000","last_became_peered":"2024-09-21T10:26:27.670843+0000","last_unstale":"2024-09-21T10:26:58.807910+0000","last_undegraded":"2024-09-21T10:26:58.807910+0000","last_fullsized":"2024-09-21T10:26:58.807910+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:31.387998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":153,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689822+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:26:58.689822+0000","last_peered":"2024-09-21T10:26:58.689822+0000","last_clean":"2024-09-21T10:26:58.689822+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:26:58.689822+0000","last_undegraded":"2024-09-21T10:26:58.689822+0000","last_fullsized":"2024-09-21T10:26:58.689822+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1614,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807660+0000","last_change":"2024-09-21T10:26:27.928592+0000","last_active":"2024-09-21T10:26:58.807660+0000","last_peered":"2024-09-21T10:26:58.807660+0000","last_clean":"2024-09-21T10:26:58.807660+0000","last_became_active":"2024-09-21T10:26:27.928169+0000","last_became_peered":"2024-09-21T10:26:27.928169+0000","last_unstale":"2024-09-21T10:26:58.807660+0000","last_undegraded":"2024-09-21T10:26:58.807660+0000","last_fullsized":"2024-09-21T10:26:58.807660+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:06.164201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":194,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690010+0000","last_change":"2024-09-21T10:25:51.774977+0000","last_active":"2024-09-21T10:26:58.690010+0000","last_peered":"2024-09-21T10:26:58.690010+0000","last_clean":"2024-09-21T10:26:58.690010+0000","last_became_active":"2024-09-21T10:25:51.760806+0000","last_became_peered":"2024-09-21T10:25:51.760806+0000","last_unstale":"2024-09-21T10:26:58.690010+0000","last_undegraded":"2024-09-21T10:26:58.690010+0000","last_fullsized":"2024-09-21T10:26:58.690010+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:06.374137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":686,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692868+0000","last_change":"2024-09-21T10:25:55.769834+0000","last_active":"2024-09-21T10:26:59.692868+0000","last_peered":"2024-09-21T10:26:59.692868+0000","last_clean":"2024-09-21T10:26:59.692868+0000","last_became_active":"2024-09-21T10:25:55.769394+0000","last_became_peered":"2024-09-21T10:25:55.769394+0000","last_unstale":"2024-09-21T10:26:59.692868+0000","last_undegraded":"2024-09-21T10:26:59.692868+0000","last_fullsized":"2024-09-21T10:26:59.692868+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:13.871492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":719,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693393+0000","last_change":"2024-09-21T10:25:51.756534+0000","last_active":"2024-09-21T10:26:59.693393+0000","last_peered":"2024-09-21T10:26:59.693393+0000","last_clean":"2024-09-21T10:26:59.693393+0000","last_became_active":"2024-09-21T10:25:51.755414+0000","last_became_peered":"2024-09-21T10:25:51.755414+0000","last_unstale":"2024-09-21T10:26:59.693393+0000","last_undegraded":"2024-09-21T10:26:59.693393+0000","last_fullsized":"2024-09-21T10:26:59.693393+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:29.592895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1138,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807448+0000","last_change":"2024-09-21T10:26:27.670145+0000","last_active":"2024-09-21T10:26:58.807448+0000","last_peered":"2024-09-21T10:26:58.807448+0000","last_clean":"2024-09-21T10:26:58.807448+0000","last_became_active":"2024-09-21T10:26:27.669855+0000","last_became_peered":"2024-09-21T10:26:27.669855+0000","last_unstale":"2024-09-21T10:26:58.807448+0000","last_undegraded":"2024-09-21T10:26:58.807448+0000","last_fullsized":"2024-09-21T10:26:58.807448+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:41.688953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":72,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.695006+0000","last_change":"2024-09-21T10:25:55.776057+0000","last_active":"2024-09-21T10:26:59.695006+0000","last_peered":"2024-09-21T10:26:59.695006+0000","last_clean":"2024-09-21T10:26:59.695006+0000","last_became_active":"2024-09-21T10:25:55.764033+0000","last_became_peered":"2024-09-21T10:25:55.764033+0000","last_unstale":"2024-09-21T10:26:59.695006+0000","last_undegraded":"2024-09-21T10:26:59.695006+0000","last_fullsized":"2024-09-21T10:26:59.695006+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:11:05.749188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":13092,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25311,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234624,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":223584,"kb_used_data":7888,"kb_used_omap":1,"kb_used_meta":215678,"kb_avail":93521568,"statfs":{"total":95995035648,"available":95766085632,"internally_reserved":0,"allocated":8077312,"data_stored":5193248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":220854731},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496260,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1439380,"kb_used_data":8200,"kb_used_omap":1,"kb_used_meta":1431166,"kb_avail":92305772,"statfs":{"total":95995035648,"available":94521110528,"internally_reserved":0,"allocated":8396800,"data_stored":5520801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1465514439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757899,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":711060,"kb_used_data":8132,"kb_used_omap":1,"kb_used_meta":702910,"kb_avail":93034092,"statfs":{"total":95995035648,"available":95266910208,"internally_reserved":0,"allocated":8327168,"data_stored":5451635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":719780294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019535,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1337496,"kb_used_data":7884,"kb_used_omap":1,"kb_used_meta":1329598,"kb_avail":92407656,"statfs":{"total":95995035648,"available":94625439744,"internally_reserved":0,"allocated":8073216,"data_stored":5193178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1361508806},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1375,"seq":5905580032051,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190284,"kb_used_data":8444,"kb_used_omap":52,"kb_used_meta":181771,"kb_avail":93554868,"statfs":{"total":95995035648,"available":95800184832,"internally_reserved":0,"allocated":8646656,"data_stored":5778746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54095,"internal_metadata":186133681},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793968070,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":752400,"kb_used_data":8128,"kb_used_omap":41,"kb_used_meta":744214,"kb_avail":92992752,"statfs":{"total":95995035648,"available":95224578048,"internally_reserved":0,"allocated":8323072,"data_stored":5451592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":762075551},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047309,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521032,"kb_used_data":7864,"kb_used_omap":41,"kb_used_meta":1513110,"kb_avail":92224120,"statfs":{"total":95995035648,"available":94437498880,"internally_reserved":0,"allocated":8052736,"data_stored":5188307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1549425577},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1317,"seq":5656471928943,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":38728,"kb_used_data":8188,"kb_used_omap":118,"kb_used_meta":30409,"kb_avail":93706424,"statfs":{"total":95995035648,"available":95955378176,"internally_reserved":0,"allocated":8384512,"data_stored":5516572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121194,"internal_metadata":31139478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9860,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10112,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:27:49.965 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:27:50.167 INFO:teuthology.orchestra.run.smithi076.stderr:Inferring config /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:27:50.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:50 smithi076 ceph-mon[106482]: from='client.36687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:50.288 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:50 smithi076 ceph-mon[106482]: pgmap v2446: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:50.288 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:50 smithi076 ceph-mon[110963]: from='client.36687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:50.288 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:50 smithi076 ceph-mon[110963]: pgmap v2446: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:50 smithi191 ceph-mon[102206]: from='client.36687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:50.363 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:50 smithi191 ceph-mon[102206]: pgmap v2446: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:52.008 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:51 smithi076 ceph-mon[106482]: pgmap v2447: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:52.009 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:51 smithi076 ceph-mon[110963]: pgmap v2447: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:52.113 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:51 smithi191 ceph-mon[102206]: pgmap v2447: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-21T10:27:52.153 INFO:teuthology.orchestra.run.smithi076.stdout: 2024-09-21T10:27:52.154 INFO:teuthology.orchestra.run.smithi076.stderr:dumped all 2024-09-21T10:27:52.943 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:52 smithi076 ceph-mon[106482]: from='client.46592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:52.944 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:52 smithi076 ceph-mon[110963]: from='client.46592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:52.944 INFO:teuthology.orchestra.run.smithi076.stdout:{"pg_ready":true,"pg_map":{"version":2447,"stamp":"2024-09-21T10:27:51.611644+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":1193923,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3188,"ondisk_log_size":3188,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6213964,"kb_used_data":64728,"kb_used_omap":260,"kb_used_meta":6148859,"kb_avail":743747252,"statfs":{"total":767960285184,"available":761597186048,"internally_reserved":0,"allocated":66281472,"data_stored":43294079,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":266323,"internal_metadata":6296432557},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003090"},"pg_stats":[{"pgid":"1.11","version":"361'149","reported_seq":1501,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694709+0000","last_change":"2024-09-21T10:25:59.370742+0000","last_active":"2024-09-21T10:26:59.694709+0000","last_peered":"2024-09-21T10:26:59.694709+0000","last_clean":"2024-09-21T10:26:59.694709+0000","last_became_active":"2024-09-21T10:25:58.778558+0000","last_became_peered":"2024-09-21T10:25:58.778558+0000","last_unstale":"2024-09-21T10:26:59.694709+0000","last_undegraded":"2024-09-21T10:26:59.694709+0000","last_fullsized":"2024-09-21T10:26:59.694709+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:59.370683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:59.370683+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:28.814303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":686,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692950+0000","last_change":"2024-09-21T10:25:59.331647+0000","last_active":"2024-09-21T10:26:59.692950+0000","last_peered":"2024-09-21T10:26:59.692950+0000","last_clean":"2024-09-21T10:26:59.692950+0000","last_became_active":"2024-09-21T10:25:59.330669+0000","last_became_peered":"2024-09-21T10:25:59.330669+0000","last_unstale":"2024-09-21T10:26:59.692950+0000","last_undegraded":"2024-09-21T10:26:59.692950+0000","last_fullsized":"2024-09-21T10:26:59.692950+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:41:51.037327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":40,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686713+0000","last_change":"2024-09-21T10:25:59.344700+0000","last_active":"2024-09-21T10:26:58.686713+0000","last_peered":"2024-09-21T10:26:58.686713+0000","last_clean":"2024-09-21T10:26:58.686713+0000","last_became_active":"2024-09-21T10:25:59.333045+0000","last_became_peered":"2024-09-21T10:25:59.333045+0000","last_unstale":"2024-09-21T10:26:58.686713+0000","last_undegraded":"2024-09-21T10:26:58.686713+0000","last_fullsized":"2024-09-21T10:26:58.686713+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:38.037693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"361'149","reported_seq":1610,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.688970+0000","last_change":"2024-09-21T10:26:06.076093+0000","last_active":"2024-09-21T10:26:58.688970+0000","last_peered":"2024-09-21T10:26:58.688970+0000","last_clean":"2024-09-21T10:26:58.688970+0000","last_became_active":"2024-09-21T10:26:02.338696+0000","last_became_peered":"2024-09-21T10:26:02.338696+0000","last_unstale":"2024-09-21T10:26:58.688970+0000","last_undegraded":"2024-09-21T10:26:58.688970+0000","last_fullsized":"2024-09-21T10:26:58.688970+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:06.075988+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:06.075988+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:55.599577+0000","scrub_duration":0.001354278,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836344+0000","last_change":"2024-09-21T10:25:59.330891+0000","last_active":"2024-09-21T10:26:59.836344+0000","last_peered":"2024-09-21T10:26:59.836344+0000","last_clean":"2024-09-21T10:26:59.836344+0000","last_became_active":"2024-09-21T10:25:59.330561+0000","last_became_peered":"2024-09-21T10:25:59.330561+0000","last_unstale":"2024-09-21T10:26:59.836344+0000","last_undegraded":"2024-09-21T10:26:59.836344+0000","last_fullsized":"2024-09-21T10:26:59.836344+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:31.076727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694658+0000","last_change":"2024-09-21T10:25:59.328066+0000","last_active":"2024-09-21T10:26:59.694658+0000","last_peered":"2024-09-21T10:26:59.694658+0000","last_clean":"2024-09-21T10:26:59.694658+0000","last_became_active":"2024-09-21T10:25:59.327651+0000","last_became_peered":"2024-09-21T10:25:59.327651+0000","last_unstale":"2024-09-21T10:26:59.694658+0000","last_undegraded":"2024-09-21T10:26:59.694658+0000","last_fullsized":"2024-09-21T10:26:59.694658+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:28:28.029256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"361'149","reported_seq":1634,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836753+0000","last_change":"2024-09-21T10:26:00.001350+0000","last_active":"2024-09-21T10:26:59.836753+0000","last_peered":"2024-09-21T10:26:59.836753+0000","last_clean":"2024-09-21T10:26:59.836753+0000","last_became_active":"2024-09-21T10:25:53.751867+0000","last_became_peered":"2024-09-21T10:25:53.751867+0000","last_unstale":"2024-09-21T10:26:59.836753+0000","last_undegraded":"2024-09-21T10:26:59.836753+0000","last_fullsized":"2024-09-21T10:26:59.836753+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:00.001298+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:00.001298+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:57:44.983255+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694837+0000","last_change":"2024-09-21T10:25:59.328382+0000","last_active":"2024-09-21T10:26:59.694837+0000","last_peered":"2024-09-21T10:26:59.694837+0000","last_clean":"2024-09-21T10:26:59.694837+0000","last_became_active":"2024-09-21T10:25:59.328072+0000","last_became_peered":"2024-09-21T10:25:59.328072+0000","last_unstale":"2024-09-21T10:26:59.694837+0000","last_undegraded":"2024-09-21T10:26:59.694837+0000","last_fullsized":"2024-09-21T10:26:59.694837+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:37:32.814312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686395+0000","last_change":"2024-09-21T10:25:59.333563+0000","last_active":"2024-09-21T10:26:58.686395+0000","last_peered":"2024-09-21T10:26:58.686395+0000","last_clean":"2024-09-21T10:26:58.686395+0000","last_became_active":"2024-09-21T10:25:59.333207+0000","last_became_peered":"2024-09-21T10:25:59.333207+0000","last_unstale":"2024-09-21T10:26:58.686395+0000","last_undegraded":"2024-09-21T10:26:58.686395+0000","last_fullsized":"2024-09-21T10:26:58.686395+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:47.763170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"361'149","reported_seq":1550,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689702+0000","last_change":"2024-09-21T10:25:55.905397+0000","last_active":"2024-09-21T10:26:58.689702+0000","last_peered":"2024-09-21T10:26:58.689702+0000","last_clean":"2024-09-21T10:26:58.689702+0000","last_became_active":"2024-09-21T10:25:53.756592+0000","last_became_peered":"2024-09-21T10:25:53.756592+0000","last_unstale":"2024-09-21T10:26:58.689702+0000","last_undegraded":"2024-09-21T10:26:58.689702+0000","last_fullsized":"2024-09-21T10:26:58.689702+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:55.905292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:55.905292+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:05:44.493209+0000","scrub_duration":0.0015904389999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":709,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693455+0000","last_change":"2024-09-21T10:25:59.330640+0000","last_active":"2024-09-21T10:26:59.693455+0000","last_peered":"2024-09-21T10:26:59.693455+0000","last_clean":"2024-09-21T10:26:59.693455+0000","last_became_active":"2024-09-21T10:25:59.330330+0000","last_became_peered":"2024-09-21T10:25:59.330330+0000","last_unstale":"2024-09-21T10:26:59.693455+0000","last_undegraded":"2024-09-21T10:26:59.693455+0000","last_fullsized":"2024-09-21T10:26:59.693455+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:00.831233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":71,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690155+0000","last_change":"2024-09-21T10:25:55.780663+0000","last_active":"2024-09-21T10:26:58.690155+0000","last_peered":"2024-09-21T10:26:58.690155+0000","last_clean":"2024-09-21T10:26:58.690155+0000","last_became_active":"2024-09-21T10:25:55.759508+0000","last_became_peered":"2024-09-21T10:25:55.759508+0000","last_unstale":"2024-09-21T10:26:58.690155+0000","last_undegraded":"2024-09-21T10:26:58.690155+0000","last_fullsized":"2024-09-21T10:26:58.690155+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:37:51.356439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"361'149","reported_seq":1101,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686702+0000","last_change":"2024-09-21T10:26:06.996191+0000","last_active":"2024-09-21T10:26:58.686702+0000","last_peered":"2024-09-21T10:26:58.686702+0000","last_clean":"2024-09-21T10:26:58.686702+0000","last_became_active":"2024-09-21T10:26:02.341215+0000","last_became_peered":"2024-09-21T10:26:02.341215+0000","last_unstale":"2024-09-21T10:26:58.686702+0000","last_undegraded":"2024-09-21T10:26:58.686702+0000","last_fullsized":"2024-09-21T10:26:58.686702+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:06.996089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:06.996089+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:04:45.116333+0000","scrub_duration":0.0015455410000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":38,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.687095+0000","last_change":"2024-09-21T10:25:59.349222+0000","last_active":"2024-09-21T10:26:58.687095+0000","last_peered":"2024-09-21T10:26:58.687095+0000","last_clean":"2024-09-21T10:26:58.687095+0000","last_became_active":"2024-09-21T10:25:59.333321+0000","last_became_peered":"2024-09-21T10:25:59.333321+0000","last_unstale":"2024-09-21T10:26:58.687095+0000","last_undegraded":"2024-09-21T10:26:58.687095+0000","last_fullsized":"2024-09-21T10:26:58.687095+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:01:36.232685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":192,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.688959+0000","last_change":"2024-09-21T10:25:59.334339+0000","last_active":"2024-09-21T10:26:58.688959+0000","last_peered":"2024-09-21T10:26:58.688959+0000","last_clean":"2024-09-21T10:26:58.688959+0000","last_became_active":"2024-09-21T10:25:59.334012+0000","last_became_peered":"2024-09-21T10:25:59.334012+0000","last_unstale":"2024-09-21T10:26:58.688959+0000","last_undegraded":"2024-09-21T10:26:58.688959+0000","last_fullsized":"2024-09-21T10:26:58.688959+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:28:01.812552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"361'149","reported_seq":1635,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693145+0000","last_change":"2024-09-21T10:25:58.779214+0000","last_active":"2024-09-21T10:26:59.693145+0000","last_peered":"2024-09-21T10:26:59.693145+0000","last_clean":"2024-09-21T10:26:59.693145+0000","last_became_active":"2024-09-21T10:25:58.778718+0000","last_became_peered":"2024-09-21T10:25:58.778718+0000","last_unstale":"2024-09-21T10:26:59.693145+0000","last_undegraded":"2024-09-21T10:26:59.693145+0000","last_fullsized":"2024-09-21T10:26:59.693145+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:49.156681+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:49.156681+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:39:10.356625+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":38,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686962+0000","last_change":"2024-09-21T10:25:59.342009+0000","last_active":"2024-09-21T10:26:58.686962+0000","last_peered":"2024-09-21T10:26:58.686962+0000","last_clean":"2024-09-21T10:26:58.686962+0000","last_became_active":"2024-09-21T10:25:59.331715+0000","last_became_peered":"2024-09-21T10:25:59.331715+0000","last_unstale":"2024-09-21T10:26:58.686962+0000","last_undegraded":"2024-09-21T10:26:58.686962+0000","last_fullsized":"2024-09-21T10:26:58.686962+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:24:21.984045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1573,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807513+0000","last_change":"2024-09-21T10:26:27.668856+0000","last_active":"2024-09-21T10:26:58.807513+0000","last_peered":"2024-09-21T10:26:58.807513+0000","last_clean":"2024-09-21T10:26:58.807513+0000","last_became_active":"2024-09-21T10:26:27.668553+0000","last_became_peered":"2024-09-21T10:26:27.668553+0000","last_unstale":"2024-09-21T10:26:58.807513+0000","last_undegraded":"2024-09-21T10:26:58.807513+0000","last_fullsized":"2024-09-21T10:26:58.807513+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:44:05.350182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"361'149","reported_seq":995,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689900+0000","last_change":"2024-09-21T10:26:01.591818+0000","last_active":"2024-09-21T10:26:58.689900+0000","last_peered":"2024-09-21T10:26:58.689900+0000","last_clean":"2024-09-21T10:26:58.689900+0000","last_became_active":"2024-09-21T10:25:54.758256+0000","last_became_peered":"2024-09-21T10:25:54.758256+0000","last_unstale":"2024-09-21T10:26:58.689900+0000","last_undegraded":"2024-09-21T10:26:58.689900+0000","last_fullsized":"2024-09-21T10:26:58.689900+0000","mapping_epoch":1416,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1417,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:01.591718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:01.591718+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:33.331048+0000","scrub_duration":0.001673234,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":38,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686974+0000","last_change":"2024-09-21T10:25:59.350950+0000","last_active":"2024-09-21T10:26:58.686974+0000","last_peered":"2024-09-21T10:26:58.686974+0000","last_clean":"2024-09-21T10:26:58.686974+0000","last_became_active":"2024-09-21T10:25:59.335076+0000","last_became_peered":"2024-09-21T10:25:59.335076+0000","last_unstale":"2024-09-21T10:26:58.686974+0000","last_undegraded":"2024-09-21T10:26:58.686974+0000","last_fullsized":"2024-09-21T10:26:58.686974+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:34.161654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":51,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689896+0000","last_change":"2024-09-21T10:25:59.336918+0000","last_active":"2024-09-21T10:26:58.689896+0000","last_peered":"2024-09-21T10:26:58.689896+0000","last_clean":"2024-09-21T10:26:58.689896+0000","last_became_active":"2024-09-21T10:25:59.336042+0000","last_became_peered":"2024-09-21T10:25:59.336042+0000","last_unstale":"2024-09-21T10:26:58.689896+0000","last_undegraded":"2024-09-21T10:26:58.689896+0000","last_fullsized":"2024-09-21T10:26:58.689896+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:07:46.593265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"361'149","reported_seq":841,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808154+0000","last_change":"2024-09-21T10:26:28.112787+0000","last_active":"2024-09-21T10:26:58.808154+0000","last_peered":"2024-09-21T10:26:58.808154+0000","last_clean":"2024-09-21T10:26:58.808154+0000","last_became_active":"2024-09-21T10:26:27.670564+0000","last_became_peered":"2024-09-21T10:26:27.670564+0000","last_unstale":"2024-09-21T10:26:58.808154+0000","last_undegraded":"2024-09-21T10:26:58.808154+0000","last_fullsized":"2024-09-21T10:26:58.808154+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:28.112686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:28.112686+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:18:48.621317+0000","scrub_duration":0.002132002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":183,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689917+0000","last_change":"2024-09-21T10:25:51.761914+0000","last_active":"2024-09-21T10:26:58.689917+0000","last_peered":"2024-09-21T10:26:58.689917+0000","last_clean":"2024-09-21T10:26:58.689917+0000","last_became_active":"2024-09-21T10:25:51.761153+0000","last_became_peered":"2024-09-21T10:25:51.761153+0000","last_unstale":"2024-09-21T10:26:58.689917+0000","last_undegraded":"2024-09-21T10:26:58.689917+0000","last_fullsized":"2024-09-21T10:26:58.689917+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:17:29.366348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694842+0000","last_change":"2024-09-21T10:25:55.770244+0000","last_active":"2024-09-21T10:26:59.694842+0000","last_peered":"2024-09-21T10:26:59.694842+0000","last_clean":"2024-09-21T10:26:59.694842+0000","last_became_active":"2024-09-21T10:25:55.769692+0000","last_became_peered":"2024-09-21T10:25:55.769692+0000","last_unstale":"2024-09-21T10:26:59.694842+0000","last_undegraded":"2024-09-21T10:26:59.694842+0000","last_fullsized":"2024-09-21T10:26:59.694842+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:29.852715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"361'149","reported_seq":1563,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686512+0000","last_change":"2024-09-21T10:26:02.343335+0000","last_active":"2024-09-21T10:26:58.686512+0000","last_peered":"2024-09-21T10:26:58.686512+0000","last_clean":"2024-09-21T10:26:58.686512+0000","last_became_active":"2024-09-21T10:26:02.342969+0000","last_became_peered":"2024-09-21T10:26:02.342969+0000","last_unstale":"2024-09-21T10:26:58.686512+0000","last_undegraded":"2024-09-21T10:26:58.686512+0000","last_fullsized":"2024-09-21T10:26:58.686512+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:37.252419+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:37.252419+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:33:01.154484+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":614,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692997+0000","last_change":"2024-09-21T10:25:55.787837+0000","last_active":"2024-09-21T10:26:59.692997+0000","last_peered":"2024-09-21T10:26:59.692997+0000","last_clean":"2024-09-21T10:26:59.692997+0000","last_became_active":"2024-09-21T10:25:55.768035+0000","last_became_peered":"2024-09-21T10:25:55.768035+0000","last_unstale":"2024-09-21T10:26:59.692997+0000","last_undegraded":"2024-09-21T10:26:59.692997+0000","last_fullsized":"2024-09-21T10:26:59.692997+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:44:52.239344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1650,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808037+0000","last_change":"2024-09-21T10:26:27.669551+0000","last_active":"2024-09-21T10:26:58.808037+0000","last_peered":"2024-09-21T10:26:58.808037+0000","last_clean":"2024-09-21T10:26:58.808037+0000","last_became_active":"2024-09-21T10:26:27.669230+0000","last_became_peered":"2024-09-21T10:26:27.669230+0000","last_unstale":"2024-09-21T10:26:58.808037+0000","last_undegraded":"2024-09-21T10:26:58.808037+0000","last_fullsized":"2024-09-21T10:26:58.808037+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:29:27.533026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"361'149","reported_seq":1568,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.688792+0000","last_change":"2024-09-21T10:25:51.751718+0000","last_active":"2024-09-21T10:26:58.688792+0000","last_peered":"2024-09-21T10:26:58.688792+0000","last_clean":"2024-09-21T10:26:58.688792+0000","last_became_active":"2024-09-21T10:25:51.751250+0000","last_became_peered":"2024-09-21T10:25:51.751250+0000","last_unstale":"2024-09-21T10:26:58.688792+0000","last_undegraded":"2024-09-21T10:26:58.688792+0000","last_fullsized":"2024-09-21T10:26:58.688792+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:22:14.982755+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:22:14.982755+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:25:38.779659+0000","scrub_duration":0.0028608990000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":42,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689986+0000","last_change":"2024-09-21T10:25:51.751943+0000","last_active":"2024-09-21T10:26:58.689986+0000","last_peered":"2024-09-21T10:26:58.689986+0000","last_clean":"2024-09-21T10:26:58.689986+0000","last_became_active":"2024-09-21T10:25:51.751607+0000","last_became_peered":"2024-09-21T10:25:51.751607+0000","last_unstale":"2024-09-21T10:26:58.689986+0000","last_undegraded":"2024-09-21T10:26:58.689986+0000","last_fullsized":"2024-09-21T10:26:58.689986+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:12:22.046064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1136,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808213+0000","last_change":"2024-09-21T10:26:27.668640+0000","last_active":"2024-09-21T10:26:58.808213+0000","last_peered":"2024-09-21T10:26:58.808213+0000","last_clean":"2024-09-21T10:26:58.808213+0000","last_became_active":"2024-09-21T10:26:27.668326+0000","last_became_peered":"2024-09-21T10:26:27.668326+0000","last_unstale":"2024-09-21T10:26:58.808213+0000","last_undegraded":"2024-09-21T10:26:58.808213+0000","last_fullsized":"2024-09-21T10:26:58.808213+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:20:43.801509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"361'149","reported_seq":1644,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837098+0000","last_change":"2024-09-21T10:26:04.019563+0000","last_active":"2024-09-21T10:26:59.837098+0000","last_peered":"2024-09-21T10:26:59.837098+0000","last_clean":"2024-09-21T10:26:59.837098+0000","last_became_active":"2024-09-21T10:25:58.768756+0000","last_became_peered":"2024-09-21T10:25:58.768756+0000","last_unstale":"2024-09-21T10:26:59.837098+0000","last_undegraded":"2024-09-21T10:26:59.837098+0000","last_fullsized":"2024-09-21T10:26:59.837098+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:04.019508+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:04.019508+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:08:06.722178+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1688,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807739+0000","last_change":"2024-09-21T10:26:27.668358+0000","last_active":"2024-09-21T10:26:58.807739+0000","last_peered":"2024-09-21T10:26:58.807739+0000","last_clean":"2024-09-21T10:26:58.807739+0000","last_became_active":"2024-09-21T10:26:27.667963+0000","last_became_peered":"2024-09-21T10:26:27.667963+0000","last_unstale":"2024-09-21T10:26:58.807739+0000","last_undegraded":"2024-09-21T10:26:58.807739+0000","last_fullsized":"2024-09-21T10:26:58.807739+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:03:04.974419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1139,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808284+0000","last_change":"2024-09-21T10:26:27.670533+0000","last_active":"2024-09-21T10:26:58.808284+0000","last_peered":"2024-09-21T10:26:58.808284+0000","last_clean":"2024-09-21T10:26:58.808284+0000","last_became_active":"2024-09-21T10:26:27.670236+0000","last_became_peered":"2024-09-21T10:26:27.670236+0000","last_unstale":"2024-09-21T10:26:58.808284+0000","last_undegraded":"2024-09-21T10:26:58.808284+0000","last_fullsized":"2024-09-21T10:26:58.808284+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:48.498697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"361'149","reported_seq":1553,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689488+0000","last_change":"2024-09-21T10:25:54.767294+0000","last_active":"2024-09-21T10:26:58.689488+0000","last_peered":"2024-09-21T10:26:58.689488+0000","last_clean":"2024-09-21T10:26:58.689488+0000","last_became_active":"2024-09-21T10:25:54.766557+0000","last_became_peered":"2024-09-21T10:25:54.766557+0000","last_unstale":"2024-09-21T10:26:58.689488+0000","last_undegraded":"2024-09-21T10:26:58.689488+0000","last_fullsized":"2024-09-21T10:26:58.689488+0000","mapping_epoch":1416,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1417,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:24:55.183929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:24:55.183929+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:06.295861+0000","scrub_duration":0.001781761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":635,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692641+0000","last_change":"2024-09-21T10:25:59.332733+0000","last_active":"2024-09-21T10:26:59.692641+0000","last_peered":"2024-09-21T10:26:59.692641+0000","last_clean":"2024-09-21T10:26:59.692641+0000","last_became_active":"2024-09-21T10:25:59.332246+0000","last_became_peered":"2024-09-21T10:25:59.332246+0000","last_unstale":"2024-09-21T10:26:59.692641+0000","last_undegraded":"2024-09-21T10:26:59.692641+0000","last_fullsized":"2024-09-21T10:26:59.692641+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:16:28.477050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2088,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837293+0000","last_change":"2024-09-21T10:23:39.350772+0000","last_active":"2024-09-21T10:26:59.837293+0000","last_peered":"2024-09-21T10:26:59.837293+0000","last_clean":"2024-09-21T10:26:59.837293+0000","last_became_active":"2024-09-21T10:23:39.345255+0000","last_became_peered":"2024-09-21T10:23:39.345255+0000","last_unstale":"2024-09-21T10:26:59.837293+0000","last_undegraded":"2024-09-21T10:26:59.837293+0000","last_fullsized":"2024-09-21T10:26:59.837293+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:22.523103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"361'149","reported_seq":1602,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686688+0000","last_change":"2024-09-21T10:26:11.015613+0000","last_active":"2024-09-21T10:26:58.686688+0000","last_peered":"2024-09-21T10:26:58.686688+0000","last_clean":"2024-09-21T10:26:58.686688+0000","last_became_active":"2024-09-21T10:26:02.338686+0000","last_became_peered":"2024-09-21T10:26:02.338686+0000","last_unstale":"2024-09-21T10:26:58.686688+0000","last_undegraded":"2024-09-21T10:26:58.686688+0000","last_fullsized":"2024-09-21T10:26:58.686688+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":0,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:11.015392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:11.015392+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:59.399953+0000","scrub_duration":0.0022519319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":211,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689521+0000","last_change":"2024-09-21T10:25:59.338345+0000","last_active":"2024-09-21T10:26:58.689521+0000","last_peered":"2024-09-21T10:26:58.689521+0000","last_clean":"2024-09-21T10:26:58.689521+0000","last_became_active":"2024-09-21T10:25:59.330952+0000","last_became_peered":"2024-09-21T10:25:59.330952+0000","last_unstale":"2024-09-21T10:26:58.689521+0000","last_undegraded":"2024-09-21T10:26:58.689521+0000","last_fullsized":"2024-09-21T10:26:58.689521+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:44.580358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039961100000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":603,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837482+0000","last_change":"2024-09-21T10:26:59.837150+0000","last_active":"2024-09-21T10:26:59.837482+0000","last_peered":"2024-09-21T10:26:59.837482+0000","last_clean":"2024-09-21T10:26:59.837482+0000","last_became_active":"2024-09-21T10:26:59.836806+0000","last_became_peered":"2024-09-21T10:26:59.836806+0000","last_unstale":"2024-09-21T10:26:59.837482+0000","last_undegraded":"2024-09-21T10:26:59.837482+0000","last_fullsized":"2024-09-21T10:26:59.837482+0000","mapping_epoch":1444,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:31:30.255001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":60,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690149+0000","last_change":"2024-09-21T10:25:59.336761+0000","last_active":"2024-09-21T10:26:58.690149+0000","last_peered":"2024-09-21T10:26:58.690149+0000","last_clean":"2024-09-21T10:26:58.690149+0000","last_became_active":"2024-09-21T10:25:59.336441+0000","last_became_peered":"2024-09-21T10:25:59.336441+0000","last_unstale":"2024-09-21T10:26:58.690149+0000","last_undegraded":"2024-09-21T10:26:58.690149+0000","last_fullsized":"2024-09-21T10:26:58.690149+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:47:36.725859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"361'149","reported_seq":1636,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.837073+0000","last_change":"2024-09-21T10:26:07.019202+0000","last_active":"2024-09-21T10:26:59.837073+0000","last_peered":"2024-09-21T10:26:59.837073+0000","last_clean":"2024-09-21T10:26:59.837073+0000","last_became_active":"2024-09-21T10:25:54.763354+0000","last_became_peered":"2024-09-21T10:25:54.763354+0000","last_unstale":"2024-09-21T10:26:59.837073+0000","last_undegraded":"2024-09-21T10:26:59.837073+0000","last_fullsized":"2024-09-21T10:26:59.837073+0000","mapping_epoch":1416,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1417,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:07.019156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:07.019156+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:57.651395+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":34,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686516+0000","last_change":"2024-09-21T10:25:59.331309+0000","last_active":"2024-09-21T10:26:58.686516+0000","last_peered":"2024-09-21T10:26:58.686516+0000","last_clean":"2024-09-21T10:26:58.686516+0000","last_became_active":"2024-09-21T10:25:59.330812+0000","last_became_peered":"2024-09-21T10:25:59.330812+0000","last_unstale":"2024-09-21T10:26:58.686516+0000","last_undegraded":"2024-09-21T10:26:58.686516+0000","last_fullsized":"2024-09-21T10:26:58.686516+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:33.640990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032247500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694514+0000","last_change":"2024-09-21T10:25:55.773832+0000","last_active":"2024-09-21T10:26:59.694514+0000","last_peered":"2024-09-21T10:26:59.694514+0000","last_clean":"2024-09-21T10:26:59.694514+0000","last_became_active":"2024-09-21T10:25:55.773515+0000","last_became_peered":"2024-09-21T10:25:55.773515+0000","last_unstale":"2024-09-21T10:26:59.694514+0000","last_undegraded":"2024-09-21T10:26:59.694514+0000","last_fullsized":"2024-09-21T10:26:59.694514+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:34.195830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694985+0000","last_change":"2024-09-21T10:25:55.764647+0000","last_active":"2024-09-21T10:26:59.694985+0000","last_peered":"2024-09-21T10:26:59.694985+0000","last_clean":"2024-09-21T10:26:59.694985+0000","last_became_active":"2024-09-21T10:25:55.764349+0000","last_became_peered":"2024-09-21T10:25:55.764349+0000","last_unstale":"2024-09-21T10:26:59.694985+0000","last_undegraded":"2024-09-21T10:26:59.694985+0000","last_fullsized":"2024-09-21T10:26:59.694985+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:36:10.832495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"361'149","reported_seq":1538,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686819+0000","last_change":"2024-09-21T10:26:02.340359+0000","last_active":"2024-09-21T10:26:58.686819+0000","last_peered":"2024-09-21T10:26:58.686819+0000","last_clean":"2024-09-21T10:26:58.686819+0000","last_became_active":"2024-09-21T10:26:02.339971+0000","last_became_peered":"2024-09-21T10:26:02.339971+0000","last_unstale":"2024-09-21T10:26:58.686819+0000","last_undegraded":"2024-09-21T10:26:58.686819+0000","last_fullsized":"2024-09-21T10:26:58.686819+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:44.849304+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:44.849304+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:43:27.019549+0000","scrub_duration":0.002061387,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1933,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835700+0000","last_change":"2024-09-21T10:26:20.732018+0000","last_active":"2024-09-21T10:26:59.835700+0000","last_peered":"2024-09-21T10:26:59.835700+0000","last_clean":"2024-09-21T10:26:59.835700+0000","last_became_active":"2024-09-21T10:26:20.731669+0000","last_became_peered":"2024-09-21T10:26:20.731669+0000","last_unstale":"2024-09-21T10:26:59.835700+0000","last_undegraded":"2024-09-21T10:26:59.835700+0000","last_fullsized":"2024-09-21T10:26:59.835700+0000","mapping_epoch":1429,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:09:59.605809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039901299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2126,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836106+0000","last_change":"2024-09-21T10:25:51.749756+0000","last_active":"2024-09-21T10:26:59.836106+0000","last_peered":"2024-09-21T10:26:59.836106+0000","last_clean":"2024-09-21T10:26:59.836106+0000","last_became_active":"2024-09-21T10:25:51.749404+0000","last_became_peered":"2024-09-21T10:25:51.749404+0000","last_unstale":"2024-09-21T10:26:59.836106+0000","last_undegraded":"2024-09-21T10:26:59.836106+0000","last_fullsized":"2024-09-21T10:26:59.836106+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:42:58.620370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2111,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835744+0000","last_change":"2024-09-21T10:25:55.761083+0000","last_active":"2024-09-21T10:26:59.835744+0000","last_peered":"2024-09-21T10:26:59.835744+0000","last_clean":"2024-09-21T10:26:59.835744+0000","last_became_active":"2024-09-21T10:25:55.760784+0000","last_became_peered":"2024-09-21T10:25:55.760784+0000","last_unstale":"2024-09-21T10:26:59.835744+0000","last_undegraded":"2024-09-21T10:26:59.835744+0000","last_fullsized":"2024-09-21T10:26:59.835744+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:48:56.095026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"361'149","reported_seq":1583,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689028+0000","last_change":"2024-09-21T10:25:58.767837+0000","last_active":"2024-09-21T10:26:58.689028+0000","last_peered":"2024-09-21T10:26:58.689028+0000","last_clean":"2024-09-21T10:26:58.689028+0000","last_became_active":"2024-09-21T10:25:58.767256+0000","last_became_peered":"2024-09-21T10:25:58.767256+0000","last_unstale":"2024-09-21T10:26:58.689028+0000","last_undegraded":"2024-09-21T10:26:58.689028+0000","last_fullsized":"2024-09-21T10:26:58.689028+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":4,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:43.836750+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:43.836750+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:50:20.192943+0000","scrub_duration":0.0019745000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":195,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689950+0000","last_change":"2024-09-21T10:25:51.778636+0000","last_active":"2024-09-21T10:26:58.689950+0000","last_peered":"2024-09-21T10:26:58.689950+0000","last_clean":"2024-09-21T10:26:58.689950+0000","last_became_active":"2024-09-21T10:25:51.762888+0000","last_became_peered":"2024-09-21T10:25:51.762888+0000","last_unstale":"2024-09-21T10:26:58.689950+0000","last_undegraded":"2024-09-21T10:26:58.689950+0000","last_fullsized":"2024-09-21T10:26:58.689950+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:43:53.548491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1147,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808015+0000","last_change":"2024-09-21T10:26:27.928815+0000","last_active":"2024-09-21T10:26:58.808015+0000","last_peered":"2024-09-21T10:26:58.808015+0000","last_clean":"2024-09-21T10:26:58.808015+0000","last_became_active":"2024-09-21T10:26:27.928530+0000","last_became_peered":"2024-09-21T10:26:27.928530+0000","last_unstale":"2024-09-21T10:26:58.808015+0000","last_undegraded":"2024-09-21T10:26:58.808015+0000","last_fullsized":"2024-09-21T10:26:58.808015+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:56:36.669221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1991,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807869+0000","last_change":"2024-09-21T10:26:45.227401+0000","last_active":"2024-09-21T10:26:58.807869+0000","last_peered":"2024-09-21T10:26:58.807869+0000","last_clean":"2024-09-21T10:26:58.807869+0000","last_became_active":"2024-09-21T10:26:45.227024+0000","last_became_peered":"2024-09-21T10:26:45.227024+0000","last_unstale":"2024-09-21T10:26:58.807869+0000","last_undegraded":"2024-09-21T10:26:58.807869+0000","last_fullsized":"2024-09-21T10:26:58.807869+0000","mapping_epoch":1439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:09:09.279771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"361'149","reported_seq":1120,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689890+0000","last_change":"2024-09-21T10:26:02.341872+0000","last_active":"2024-09-21T10:26:58.689890+0000","last_peered":"2024-09-21T10:26:58.689890+0000","last_clean":"2024-09-21T10:26:58.689890+0000","last_became_active":"2024-09-21T10:26:02.341507+0000","last_became_peered":"2024-09-21T10:26:02.341507+0000","last_unstale":"2024-09-21T10:26:58.689890+0000","last_undegraded":"2024-09-21T10:26:58.689890+0000","last_fullsized":"2024-09-21T10:26:58.689890+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:56.612468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:56.612468+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:24:20.739592+0000","scrub_duration":0.0018069690000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690155+0000","last_change":"2024-09-21T10:25:48.754301+0000","last_active":"2024-09-21T10:26:58.690155+0000","last_peered":"2024-09-21T10:26:58.690155+0000","last_clean":"2024-09-21T10:26:58.690155+0000","last_became_active":"2024-09-21T10:25:48.753829+0000","last_became_peered":"2024-09-21T10:25:48.753829+0000","last_unstale":"2024-09-21T10:26:58.690155+0000","last_undegraded":"2024-09-21T10:26:58.690155+0000","last_fullsized":"2024-09-21T10:26:58.690155+0000","mapping_epoch":1410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:40:21.386439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2140,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836807+0000","last_change":"2024-09-21T10:25:59.331850+0000","last_active":"2024-09-21T10:26:59.836807+0000","last_peered":"2024-09-21T10:26:59.836807+0000","last_clean":"2024-09-21T10:26:59.836807+0000","last_became_active":"2024-09-21T10:25:59.331517+0000","last_became_peered":"2024-09-21T10:25:59.331517+0000","last_unstale":"2024-09-21T10:26:59.836807+0000","last_undegraded":"2024-09-21T10:26:59.836807+0000","last_fullsized":"2024-09-21T10:26:59.836807+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:55:00.625058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032114900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":22,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835614+0000","last_change":"2024-09-21T10:26:45.227342+0000","last_active":"2024-09-21T10:26:59.835614+0000","last_peered":"2024-09-21T10:26:59.835614+0000","last_clean":"2024-09-21T10:26:59.835614+0000","last_became_active":"2024-09-21T10:26:45.227047+0000","last_became_peered":"2024-09-21T10:26:45.227047+0000","last_unstale":"2024-09-21T10:26:59.835614+0000","last_undegraded":"2024-09-21T10:26:59.835614+0000","last_fullsized":"2024-09-21T10:26:59.835614+0000","mapping_epoch":1439,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1440,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:49:32.930354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"361'149","reported_seq":1112,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690002+0000","last_change":"2024-09-21T10:26:02.579300+0000","last_active":"2024-09-21T10:26:58.690002+0000","last_peered":"2024-09-21T10:26:58.690002+0000","last_clean":"2024-09-21T10:26:58.690002+0000","last_became_active":"2024-09-21T10:25:58.775781+0000","last_became_peered":"2024-09-21T10:25:58.775781+0000","last_unstale":"2024-09-21T10:26:58.690002+0000","last_undegraded":"2024-09-21T10:26:58.690002+0000","last_fullsized":"2024-09-21T10:26:58.690002+0000","mapping_epoch":1420,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1421,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:02.579205+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:02.579205+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:45:49.183837+0000","scrub_duration":0.0014992790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":307,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693091+0000","last_change":"2024-09-21T10:25:48.749057+0000","last_active":"2024-09-21T10:26:59.693091+0000","last_peered":"2024-09-21T10:26:59.693091+0000","last_clean":"2024-09-21T10:26:59.693091+0000","last_became_active":"2024-09-21T10:25:48.748471+0000","last_became_peered":"2024-09-21T10:25:48.748471+0000","last_unstale":"2024-09-21T10:26:59.693091+0000","last_undegraded":"2024-09-21T10:26:59.693091+0000","last_fullsized":"2024-09-21T10:26:59.693091+0000","mapping_epoch":1410,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:11:39.024192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":34,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686742+0000","last_change":"2024-09-21T10:25:59.332732+0000","last_active":"2024-09-21T10:26:58.686742+0000","last_peered":"2024-09-21T10:26:58.686742+0000","last_clean":"2024-09-21T10:26:58.686742+0000","last_became_active":"2024-09-21T10:25:59.332420+0000","last_became_peered":"2024-09-21T10:25:59.332420+0000","last_unstale":"2024-09-21T10:26:58.686742+0000","last_undegraded":"2024-09-21T10:26:58.686742+0000","last_fullsized":"2024-09-21T10:26:58.686742+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:24:56.981194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029397800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694748+0000","last_change":"2024-09-21T10:25:59.328950+0000","last_active":"2024-09-21T10:26:59.694748+0000","last_peered":"2024-09-21T10:26:59.694748+0000","last_clean":"2024-09-21T10:26:59.694748+0000","last_became_active":"2024-09-21T10:25:59.328619+0000","last_became_peered":"2024-09-21T10:25:59.328619+0000","last_unstale":"2024-09-21T10:26:59.694748+0000","last_undegraded":"2024-09-21T10:26:59.694748+0000","last_fullsized":"2024-09-21T10:26:59.694748+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:06.020852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"361'149","reported_seq":849,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808038+0000","last_change":"2024-09-21T10:26:35.102300+0000","last_active":"2024-09-21T10:26:58.808038+0000","last_peered":"2024-09-21T10:26:58.808038+0000","last_clean":"2024-09-21T10:26:58.808038+0000","last_became_active":"2024-09-21T10:26:27.669120+0000","last_became_peered":"2024-09-21T10:26:27.669120+0000","last_unstale":"2024-09-21T10:26:58.808038+0000","last_undegraded":"2024-09-21T10:26:58.808038+0000","last_fullsized":"2024-09-21T10:26:58.808038+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:26:35.102188+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:26:35.102188+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:29:17.236873+0000","scrub_duration":0.0020000119999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":429,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.696395+0000","last_change":"2024-09-21T10:26:59.696125+0000","last_active":"2024-09-21T10:26:59.696395+0000","last_peered":"2024-09-21T10:26:59.696395+0000","last_clean":"2024-09-21T10:26:59.696395+0000","last_became_active":"2024-09-21T10:26:59.695782+0000","last_became_peered":"2024-09-21T10:26:59.695782+0000","last_unstale":"2024-09-21T10:26:59.696395+0000","last_undegraded":"2024-09-21T10:26:59.696395+0000","last_fullsized":"2024-09-21T10:26:59.696395+0000","mapping_epoch":1444,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1445,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:12:18.288410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":202,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693380+0000","last_change":"2024-09-21T10:25:51.755726+0000","last_active":"2024-09-21T10:26:59.693380+0000","last_peered":"2024-09-21T10:26:59.693380+0000","last_clean":"2024-09-21T10:26:59.693380+0000","last_became_active":"2024-09-21T10:25:51.755373+0000","last_became_peered":"2024-09-21T10:25:51.755373+0000","last_unstale":"2024-09-21T10:26:59.693380+0000","last_undegraded":"2024-09-21T10:26:59.693380+0000","last_fullsized":"2024-09-21T10:26:59.693380+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:02:52.571798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":639,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692704+0000","last_change":"2024-09-21T10:25:59.331761+0000","last_active":"2024-09-21T10:26:59.692704+0000","last_peered":"2024-09-21T10:26:59.692704+0000","last_clean":"2024-09-21T10:26:59.692704+0000","last_became_active":"2024-09-21T10:25:59.330633+0000","last_became_peered":"2024-09-21T10:25:59.330633+0000","last_unstale":"2024-09-21T10:26:59.692704+0000","last_undegraded":"2024-09-21T10:26:59.692704+0000","last_fullsized":"2024-09-21T10:26:59.692704+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:03:37.270144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"361'149","reported_seq":1637,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693574+0000","last_change":"2024-09-21T10:26:02.341199+0000","last_active":"2024-09-21T10:26:59.693574+0000","last_peered":"2024-09-21T10:26:59.693574+0000","last_clean":"2024-09-21T10:26:59.693574+0000","last_became_active":"2024-09-21T10:26:02.340534+0000","last_became_peered":"2024-09-21T10:26:02.340534+0000","last_unstale":"2024-09-21T10:26:59.693574+0000","last_undegraded":"2024-09-21T10:26:59.693574+0000","last_fullsized":"2024-09-21T10:26:59.693574+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":1352,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":3,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:45.334710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:45.334710+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:32.459672+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":70,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694872+0000","last_change":"2024-09-21T10:25:55.780088+0000","last_active":"2024-09-21T10:26:59.694872+0000","last_peered":"2024-09-21T10:26:59.694872+0000","last_clean":"2024-09-21T10:26:59.694872+0000","last_became_active":"2024-09-21T10:25:55.765081+0000","last_became_peered":"2024-09-21T10:25:55.765081+0000","last_unstale":"2024-09-21T10:26:59.694872+0000","last_undegraded":"2024-09-21T10:26:59.694872+0000","last_fullsized":"2024-09-21T10:26:59.694872+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:43.407232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":51,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689990+0000","last_change":"2024-09-21T10:25:59.337330+0000","last_active":"2024-09-21T10:26:58.689990+0000","last_peered":"2024-09-21T10:26:58.689990+0000","last_clean":"2024-09-21T10:26:58.689990+0000","last_became_active":"2024-09-21T10:25:59.337011+0000","last_became_peered":"2024-09-21T10:25:59.337011+0000","last_unstale":"2024-09-21T10:26:58.689990+0000","last_undegraded":"2024-09-21T10:26:58.689990+0000","last_fullsized":"2024-09-21T10:26:58.689990+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:35:23.726156+0000","last_clean_scrub_stamp":"2024-09-21T09:35:23.726156+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:07.800833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000235226,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1116,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807619+0000","last_change":"2024-09-21T10:26:27.929354+0000","last_active":"2024-09-21T10:26:58.807619+0000","last_peered":"2024-09-21T10:26:58.807619+0000","last_clean":"2024-09-21T10:26:58.807619+0000","last_became_active":"2024-09-21T10:26:27.929042+0000","last_became_peered":"2024-09-21T10:26:27.929042+0000","last_unstale":"2024-09-21T10:26:58.807619+0000","last_undegraded":"2024-09-21T10:26:58.807619+0000","last_fullsized":"2024-09-21T10:26:58.807619+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:45:40.088951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1148,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807971+0000","last_change":"2024-09-21T10:26:27.670767+0000","last_active":"2024-09-21T10:26:58.807971+0000","last_peered":"2024-09-21T10:26:58.807971+0000","last_clean":"2024-09-21T10:26:58.807971+0000","last_became_active":"2024-09-21T10:26:27.670458+0000","last_became_peered":"2024-09-21T10:26:27.670458+0000","last_unstale":"2024-09-21T10:26:58.807971+0000","last_undegraded":"2024-09-21T10:26:58.807971+0000","last_fullsized":"2024-09-21T10:26:58.807971+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:04:11.886413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"443'23","reported_seq":1167,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.808186+0000","last_change":"2024-09-21T10:26:27.669387+0000","last_active":"2024-09-21T10:26:58.808186+0000","last_peered":"2024-09-21T10:26:58.808186+0000","last_clean":"2024-09-21T10:26:58.808186+0000","last_became_active":"2024-09-21T10:26:27.668952+0000","last_became_peered":"2024-09-21T10:26:27.668952+0000","last_unstale":"2024-09-21T10:26:58.808186+0000","last_undegraded":"2024-09-21T10:26:58.808186+0000","last_fullsized":"2024-09-21T10:26:58.808186+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:42:03.375664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2101,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836325+0000","last_change":"2024-09-21T10:25:51.750944+0000","last_active":"2024-09-21T10:26:59.836325+0000","last_peered":"2024-09-21T10:26:59.836325+0000","last_clean":"2024-09-21T10:26:59.836325+0000","last_became_active":"2024-09-21T10:25:51.750521+0000","last_became_peered":"2024-09-21T10:25:51.750521+0000","last_unstale":"2024-09-21T10:26:59.836325+0000","last_undegraded":"2024-09-21T10:26:59.836325+0000","last_fullsized":"2024-09-21T10:26:59.836325+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:35:00.271350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":40,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.686799+0000","last_change":"2024-09-21T10:25:59.347575+0000","last_active":"2024-09-21T10:26:58.686799+0000","last_peered":"2024-09-21T10:26:58.686799+0000","last_clean":"2024-09-21T10:26:58.686799+0000","last_became_active":"2024-09-21T10:25:59.333758+0000","last_became_peered":"2024-09-21T10:25:59.333758+0000","last_unstale":"2024-09-21T10:26:58.686799+0000","last_undegraded":"2024-09-21T10:26:58.686799+0000","last_fullsized":"2024-09-21T10:26:58.686799+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:20.061779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"361'149","reported_seq":1590,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.835993+0000","last_change":"2024-09-21T10:26:20.698780+0000","last_active":"2024-09-21T10:26:59.835993+0000","last_peered":"2024-09-21T10:26:59.835993+0000","last_clean":"2024-09-21T10:26:59.835993+0000","last_became_active":"2024-09-21T10:26:20.698460+0000","last_became_peered":"2024-09-21T10:26:20.698460+0000","last_unstale":"2024-09-21T10:26:59.835993+0000","last_undegraded":"2024-09-21T10:26:59.835993+0000","last_fullsized":"2024-09-21T10:26:59.835993+0000","mapping_epoch":1429,"log_start":"0'0","ondisk_log_start":"0'0","created":1408,"last_epoch_clean":1430,"parent":"0.0","parent_split_bits":5,"last_scrub":"361'149","last_scrub_stamp":"2024-09-21T10:25:48.196311+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:32:50.664564+0000","last_clean_scrub_stamp":"2024-09-21T10:25:48.196311+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:26.100373+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":174,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836423+0000","last_change":"2024-09-21T10:25:59.331970+0000","last_active":"2024-09-21T10:26:59.836423+0000","last_peered":"2024-09-21T10:26:59.836423+0000","last_clean":"2024-09-21T10:26:59.836423+0000","last_became_active":"2024-09-21T10:25:59.331636+0000","last_became_peered":"2024-09-21T10:25:59.331636+0000","last_unstale":"2024-09-21T10:26:59.836423+0000","last_undegraded":"2024-09-21T10:26:59.836423+0000","last_fullsized":"2024-09-21T10:26:59.836423+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:47:14.908930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":751,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693239+0000","last_change":"2024-09-21T10:25:59.330908+0000","last_active":"2024-09-21T10:26:59.693239+0000","last_peered":"2024-09-21T10:26:59.693239+0000","last_clean":"2024-09-21T10:26:59.693239+0000","last_became_active":"2024-09-21T10:25:59.330587+0000","last_became_peered":"2024-09-21T10:25:59.330587+0000","last_unstale":"2024-09-21T10:26:59.693239+0000","last_undegraded":"2024-09-21T10:26:59.693239+0000","last_fullsized":"2024-09-21T10:26:59.693239+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:01:31.829657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":66,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694885+0000","last_change":"2024-09-21T10:25:55.771157+0000","last_active":"2024-09-21T10:26:59.694885+0000","last_peered":"2024-09-21T10:26:59.694885+0000","last_clean":"2024-09-21T10:26:59.694885+0000","last_became_active":"2024-09-21T10:25:55.770841+0000","last_became_peered":"2024-09-21T10:25:55.770841+0000","last_unstale":"2024-09-21T10:26:59.694885+0000","last_undegraded":"2024-09-21T10:26:59.694885+0000","last_fullsized":"2024-09-21T10:26:59.694885+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:00:39.889923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":721,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692742+0000","last_change":"2024-09-21T10:25:51.748938+0000","last_active":"2024-09-21T10:26:59.692742+0000","last_peered":"2024-09-21T10:26:59.692742+0000","last_clean":"2024-09-21T10:26:59.692742+0000","last_became_active":"2024-09-21T10:25:51.748472+0000","last_became_peered":"2024-09-21T10:25:51.748472+0000","last_unstale":"2024-09-21T10:26:59.692742+0000","last_undegraded":"2024-09-21T10:26:59.692742+0000","last_fullsized":"2024-09-21T10:26:59.692742+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:32:13.956736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":42,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690077+0000","last_change":"2024-09-21T10:25:51.761452+0000","last_active":"2024-09-21T10:26:58.690077+0000","last_peered":"2024-09-21T10:26:58.690077+0000","last_clean":"2024-09-21T10:26:58.690077+0000","last_became_active":"2024-09-21T10:25:51.760427+0000","last_became_peered":"2024-09-21T10:25:51.760427+0000","last_unstale":"2024-09-21T10:26:58.690077+0000","last_undegraded":"2024-09-21T10:26:58.690077+0000","last_fullsized":"2024-09-21T10:26:58.690077+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:08.783792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":686,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692512+0000","last_change":"2024-09-21T10:25:55.771479+0000","last_active":"2024-09-21T10:26:59.692512+0000","last_peered":"2024-09-21T10:26:59.692512+0000","last_clean":"2024-09-21T10:26:59.692512+0000","last_became_active":"2024-09-21T10:25:55.770758+0000","last_became_peered":"2024-09-21T10:25:55.770758+0000","last_unstale":"2024-09-21T10:26:59.692512+0000","last_undegraded":"2024-09-21T10:26:59.692512+0000","last_fullsized":"2024-09-21T10:26:59.692512+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:13.903497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694709+0000","last_change":"2024-09-21T10:25:59.328440+0000","last_active":"2024-09-21T10:26:59.694709+0000","last_peered":"2024-09-21T10:26:59.694709+0000","last_clean":"2024-09-21T10:26:59.694709+0000","last_became_active":"2024-09-21T10:25:59.328124+0000","last_became_peered":"2024-09-21T10:25:59.328124+0000","last_unstale":"2024-09-21T10:26:59.694709+0000","last_undegraded":"2024-09-21T10:26:59.694709+0000","last_fullsized":"2024-09-21T10:26:59.694709+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:24:21.543744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":377,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836050+0000","last_change":"2024-09-21T10:23:39.371060+0000","last_active":"2024-09-21T10:26:59.836050+0000","last_peered":"2024-09-21T10:26:59.836050+0000","last_clean":"2024-09-21T10:26:59.836050+0000","last_became_active":"2024-09-21T10:23:39.370494+0000","last_became_peered":"2024-09-21T10:23:39.370494+0000","last_unstale":"2024-09-21T10:26:59.836050+0000","last_undegraded":"2024-09-21T10:26:59.836050+0000","last_fullsized":"2024-09-21T10:26:59.836050+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:00:09.975626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":597,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693222+0000","last_change":"2024-09-21T10:23:39.510198+0000","last_active":"2024-09-21T10:26:59.693222+0000","last_peered":"2024-09-21T10:26:59.693222+0000","last_clean":"2024-09-21T10:26:59.693222+0000","last_became_active":"2024-09-21T10:23:39.509474+0000","last_became_peered":"2024-09-21T10:23:39.509474+0000","last_unstale":"2024-09-21T10:26:59.693222+0000","last_undegraded":"2024-09-21T10:26:59.693222+0000","last_fullsized":"2024-09-21T10:26:59.693222+0000","mapping_epoch":1375,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:34:00.611788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1137,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807777+0000","last_change":"2024-09-21T10:26:27.669976+0000","last_active":"2024-09-21T10:26:58.807777+0000","last_peered":"2024-09-21T10:26:58.807777+0000","last_clean":"2024-09-21T10:26:58.807777+0000","last_became_active":"2024-09-21T10:26:27.669656+0000","last_became_peered":"2024-09-21T10:26:27.669656+0000","last_unstale":"2024-09-21T10:26:58.807777+0000","last_undegraded":"2024-09-21T10:26:58.807777+0000","last_fullsized":"2024-09-21T10:26:58.807777+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:51.213576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1948,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.836141+0000","last_change":"2024-09-21T10:26:20.732541+0000","last_active":"2024-09-21T10:26:59.836141+0000","last_peered":"2024-09-21T10:26:59.836141+0000","last_clean":"2024-09-21T10:26:59.836141+0000","last_became_active":"2024-09-21T10:26:20.732244+0000","last_became_peered":"2024-09-21T10:26:20.732244+0000","last_unstale":"2024-09-21T10:26:59.836141+0000","last_undegraded":"2024-09-21T10:26:59.836141+0000","last_fullsized":"2024-09-21T10:26:59.836141+0000","mapping_epoch":1429,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1430,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:45:09.876339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":73,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.694778+0000","last_change":"2024-09-21T10:25:59.328227+0000","last_active":"2024-09-21T10:26:59.694778+0000","last_peered":"2024-09-21T10:26:59.694778+0000","last_clean":"2024-09-21T10:26:59.694778+0000","last_became_active":"2024-09-21T10:25:59.327831+0000","last_became_peered":"2024-09-21T10:25:59.327831+0000","last_unstale":"2024-09-21T10:26:59.694778+0000","last_undegraded":"2024-09-21T10:26:59.694778+0000","last_fullsized":"2024-09-21T10:26:59.694778+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:42:49.483388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1454,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807910+0000","last_change":"2024-09-21T10:26:27.671125+0000","last_active":"2024-09-21T10:26:58.807910+0000","last_peered":"2024-09-21T10:26:58.807910+0000","last_clean":"2024-09-21T10:26:58.807910+0000","last_became_active":"2024-09-21T10:26:27.670843+0000","last_became_peered":"2024-09-21T10:26:27.670843+0000","last_unstale":"2024-09-21T10:26:58.807910+0000","last_undegraded":"2024-09-21T10:26:58.807910+0000","last_fullsized":"2024-09-21T10:26:58.807910+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:40:31.387998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":153,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.689822+0000","last_change":"2024-09-21T10:17:51.272416+0000","last_active":"2024-09-21T10:26:58.689822+0000","last_peered":"2024-09-21T10:26:58.689822+0000","last_clean":"2024-09-21T10:26:58.689822+0000","last_became_active":"2024-09-21T10:17:51.246503+0000","last_became_peered":"2024-09-21T10:17:51.246503+0000","last_unstale":"2024-09-21T10:26:58.689822+0000","last_undegraded":"2024-09-21T10:26:58.689822+0000","last_fullsized":"2024-09-21T10:26:58.689822+0000","mapping_epoch":1308,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:08:54.193461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1614,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807660+0000","last_change":"2024-09-21T10:26:27.928592+0000","last_active":"2024-09-21T10:26:58.807660+0000","last_peered":"2024-09-21T10:26:58.807660+0000","last_clean":"2024-09-21T10:26:58.807660+0000","last_became_active":"2024-09-21T10:26:27.928169+0000","last_became_peered":"2024-09-21T10:26:27.928169+0000","last_unstale":"2024-09-21T10:26:58.807660+0000","last_undegraded":"2024-09-21T10:26:58.807660+0000","last_fullsized":"2024-09-21T10:26:58.807660+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:34:06.164201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":194,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.690010+0000","last_change":"2024-09-21T10:25:51.774977+0000","last_active":"2024-09-21T10:26:58.690010+0000","last_peered":"2024-09-21T10:26:58.690010+0000","last_clean":"2024-09-21T10:26:58.690010+0000","last_became_active":"2024-09-21T10:25:51.760806+0000","last_became_peered":"2024-09-21T10:25:51.760806+0000","last_unstale":"2024-09-21T10:26:58.690010+0000","last_undegraded":"2024-09-21T10:26:58.690010+0000","last_fullsized":"2024-09-21T10:26:58.690010+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:06.374137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":686,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.692868+0000","last_change":"2024-09-21T10:25:55.769834+0000","last_active":"2024-09-21T10:26:59.692868+0000","last_peered":"2024-09-21T10:26:59.692868+0000","last_clean":"2024-09-21T10:26:59.692868+0000","last_became_active":"2024-09-21T10:25:55.769394+0000","last_became_peered":"2024-09-21T10:25:55.769394+0000","last_unstale":"2024-09-21T10:26:59.692868+0000","last_undegraded":"2024-09-21T10:26:59.692868+0000","last_fullsized":"2024-09-21T10:26:59.692868+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:13.871492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":719,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.693393+0000","last_change":"2024-09-21T10:25:51.756534+0000","last_active":"2024-09-21T10:26:59.693393+0000","last_peered":"2024-09-21T10:26:59.693393+0000","last_clean":"2024-09-21T10:26:59.693393+0000","last_became_active":"2024-09-21T10:25:51.755414+0000","last_became_peered":"2024-09-21T10:25:51.755414+0000","last_unstale":"2024-09-21T10:26:59.693393+0000","last_undegraded":"2024-09-21T10:26:59.693393+0000","last_fullsized":"2024-09-21T10:26:59.693393+0000","mapping_epoch":1413,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:36:29.592895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1138,"reported_epoch":1444,"state":"active+clean","last_fresh":"2024-09-21T10:26:58.807448+0000","last_change":"2024-09-21T10:26:27.670145+0000","last_active":"2024-09-21T10:26:58.807448+0000","last_peered":"2024-09-21T10:26:58.807448+0000","last_clean":"2024-09-21T10:26:58.807448+0000","last_became_active":"2024-09-21T10:26:27.669855+0000","last_became_peered":"2024-09-21T10:26:27.669855+0000","last_unstale":"2024-09-21T10:26:58.807448+0000","last_undegraded":"2024-09-21T10:26:58.807448+0000","last_fullsized":"2024-09-21T10:26:58.807448+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:39.862093+0000","last_clean_scrub_stamp":"2024-09-21T09:36:39.862093+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:43:41.688953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":72,"reported_epoch":1445,"state":"active+clean","last_fresh":"2024-09-21T10:26:59.695006+0000","last_change":"2024-09-21T10:25:55.776057+0000","last_active":"2024-09-21T10:26:59.695006+0000","last_peered":"2024-09-21T10:26:59.695006+0000","last_clean":"2024-09-21T10:26:59.695006+0000","last_became_active":"2024-09-21T10:25:55.764033+0000","last_became_peered":"2024-09-21T10:25:55.764033+0000","last_unstale":"2024-09-21T10:26:59.695006+0000","last_undegraded":"2024-09-21T10:26:59.695006+0000","last_fullsized":"2024-09-21T10:26:59.695006+0000","mapping_epoch":1417,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:36:38.860454+0000","last_clean_scrub_stamp":"2024-09-21T09:36:38.860454+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:11:05.749188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2910,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":89,"num_bytes_recovered":13092,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25311,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234625,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":223584,"kb_used_data":7888,"kb_used_omap":1,"kb_used_meta":215678,"kb_avail":93521568,"statfs":{"total":95995035648,"available":95766085632,"internally_reserved":0,"allocated":8077312,"data_stored":5193248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":220854731},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496261,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1439380,"kb_used_data":8200,"kb_used_omap":1,"kb_used_meta":1431166,"kb_avail":92305772,"statfs":{"total":95995035648,"available":94521110528,"internally_reserved":0,"allocated":8396800,"data_stored":5520801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1465514439},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757899,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":711060,"kb_used_data":8132,"kb_used_omap":1,"kb_used_meta":702910,"kb_avail":93034092,"statfs":{"total":95995035648,"available":95266910208,"internally_reserved":0,"allocated":8327168,"data_stored":5451635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":719780294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019536,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1337496,"kb_used_data":7884,"kb_used_omap":1,"kb_used_meta":1329598,"kb_avail":92407656,"statfs":{"total":95995035648,"available":94625439744,"internally_reserved":0,"allocated":8073216,"data_stored":5193178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1361508806},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1375,"seq":5905580032052,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":190284,"kb_used_data":8444,"kb_used_omap":52,"kb_used_meta":181771,"kb_avail":93554868,"statfs":{"total":95995035648,"available":95800184832,"internally_reserved":0,"allocated":8646656,"data_stored":5778746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54095,"internal_metadata":186133681},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":796,"seq":3418793968071,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":752400,"kb_used_data":8128,"kb_used_omap":41,"kb_used_meta":744214,"kb_avail":92992752,"statfs":{"total":95995035648,"available":95224578048,"internally_reserved":0,"allocated":8323072,"data_stored":5451592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42593,"internal_metadata":762075551},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":763,"seq":3277060047309,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521032,"kb_used_data":7864,"kb_used_omap":41,"kb_used_meta":1513110,"kb_avail":92224120,"statfs":{"total":95995035648,"available":94437498880,"internally_reserved":0,"allocated":8052736,"data_stored":5188307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42071,"internal_metadata":1549425577},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1317,"seq":5656471928944,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":38728,"kb_used_data":8188,"kb_used_omap":118,"kb_used_meta":30409,"kb_avail":93706424,"statfs":{"total":95995035648,"available":95955378176,"internally_reserved":0,"allocated":8384512,"data_stored":5516572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":121194,"internal_metadata":31139478},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9860,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3821,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1518,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10112,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:27:52.948 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-21T10:27:52.948 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi076 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:27:53.110 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=0f8b84bb8a9c42d2bc542cfe2114c311 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi076 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:27:53.113 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T10:27:53.123 INFO:tasks.cephadm:Teardown begin 2024-09-21T10:27:53.123 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:27:53.129 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:52 smithi191 ceph-mon[102206]: from='client.46592 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:27:53.180 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:27:53.213 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T10:27:53.213 DEBUG:teuthology.orchestra.run.smithi076:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:27:53.237 DEBUG:teuthology.orchestra.run.smithi191:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:27:53.271 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T10:27:53.271 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-21T10:27:53.271 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a 2024-09-21T10:27:53.578 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:53 smithi076 systemd[1]: Stopping Ceph mon.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:27:53.578 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: ::ffff:172.21.15.191 - - [21/Sep/2024:10:27:53] "GET /metrics HTTP/1.1" 200 34185 "" "Prometheus/2.51.0" 2024-09-21T10:27:53.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:27:53.575+0000 7f24145cd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:27:53.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a[106459]: 2024-09-21T10:27:53.575+0000 7f24145cd640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-21T10:27:53.858 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:53 smithi076 podman[372106]: 2024-09-21 10:27:53.627004401 +0000 UTC m=+0.152917879 container died 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:27:53.858 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:10:27:53] ENGINE Bus STOPPING 2024-09-21T10:27:53.859 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:10:27:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:27:53.859 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:10:27:53] ENGINE Bus STOPPED 2024-09-21T10:27:53.859 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:10:27:53] ENGINE Bus STARTING 2024-09-21T10:27:53.859 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:10:27:53] ENGINE Serving on http://:::9283 2024-09-21T10:27:53.859 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:53 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: [21/Sep/2024:10:27:53] ENGINE Bus STARTED 2024-09-21T10:27:54.467 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:54 smithi076 podman[372106]: 2024-09-21 10:27:54.132995114 +0000 UTC m=+0.658908591 container cleanup 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T10:27:54.467 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:54 smithi076 bash[372106]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a 2024-09-21T10:27:54.467 INFO:journalctl@ceph.mon.a.smithi076.stdout:Sep 21 10:27:54 smithi076 podman[372123]: 2024-09-21 10:27:54.359289787 +0000 UTC m=+0.730004702 container remove 034b7218a76b479b441826cfa054315c0cfdfaec79136f2cf4933ac3482c3a87 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T10:27:54.481 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.a.service' 2024-09-21T10:27:54.519 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:27:54.519 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-21T10:27:54.519 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-21T10:27:54.519 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.c 2024-09-21T10:27:54.867 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:54 smithi076 systemd[1]: Stopping Ceph mon.c for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:27:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:54 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c[110940]: 2024-09-21T10:27:54.864+0000 7fe90d798640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:27:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:54 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c[110940]: 2024-09-21T10:27:54.864+0000 7fe90d798640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:27:55.123 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:55 smithi076 podman[372249]: 2024-09-21 10:27:55.02872605 +0000 UTC m=+0.286357747 container died a49012ff5abf941d6ef55e86a2ff344112002060a04a0b2a1b5d0321fc2bba5b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-62da54b) 2024-09-21T10:27:55.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:55 smithi076 podman[372249]: 2024-09-21 10:27:55.387886491 +0000 UTC m=+0.645518175 container cleanup a49012ff5abf941d6ef55e86a2ff344112002060a04a0b2a1b5d0321fc2bba5b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:27:55.623 INFO:journalctl@ceph.mon.c.smithi076.stdout:Sep 21 10:27:55 smithi076 bash[372249]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-c 2024-09-21T10:27:55.826 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.c.service' 2024-09-21T10:27:55.861 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:27:55.862 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-21T10:27:55.862 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-21T10:27:55.862 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.b 2024-09-21T10:27:56.188 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:55 smithi191 systemd[1]: Stopping Ceph mon.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:27:56.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:56 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b[102183]: 2024-09-21T10:27:56.186+0000 7f8c216bc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:27:56.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:56 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b[102183]: 2024-09-21T10:27:56.186+0000 7f8c216bc640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:27:56.565 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:56 smithi191 podman[261801]: 2024-09-21 10:27:56.257669341 +0000 UTC m=+0.187854275 container died bb6de71151ea53bd9376ee82f3878ff4d138086270d3ca1fd65f4cd56a765428 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731) 2024-09-21T10:27:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:56 smithi191 podman[261801]: 2024-09-21 10:27:56.776200555 +0000 UTC m=+0.706385473 container cleanup bb6de71151ea53bd9376ee82f3878ff4d138086270d3ca1fd65f4cd56a765428 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T10:27:56.863 INFO:journalctl@ceph.mon.b.smithi191.stdout:Sep 21 10:27:56 smithi191 bash[261801]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mon-b 2024-09-21T10:27:57.287 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mon.b.service' 2024-09-21T10:27:57.331 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:27:57.332 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-21T10:27:57.332 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-21T10:27:57.332 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y 2024-09-21T10:27:57.623 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:57 smithi076 systemd[1]: Stopping Ceph mgr.y for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:27:58.020 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:57 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T10:27:57.644+0000 7f871a4ca640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:27:58.020 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:57 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y[89204]: 2024-09-21T10:27:57.644+0000 7f871a4ca640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-21T10:27:58.020 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:57 smithi076 podman[372381]: 2024-09-21 10:27:57.676452745 +0000 UTC m=+0.158907802 container died 151afa33d0eeb7e634a87f1659f7562e17d0cc4fbb37a4fd13e75dc33c14dea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T10:27:58.337 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:58 smithi076 podman[372381]: 2024-09-21 10:27:58.14291934 +0000 UTC m=+0.625374398 container cleanup 151afa33d0eeb7e634a87f1659f7562e17d0cc4fbb37a4fd13e75dc33c14dea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:27:58.338 INFO:journalctl@ceph.mgr.y.smithi076.stdout:Sep 21 10:27:58 smithi076 bash[372381]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-y 2024-09-21T10:27:58.562 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.y.service' 2024-09-21T10:27:58.596 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:27:58.596 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-21T10:27:58.597 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-21T10:27:58.597 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x 2024-09-21T10:27:58.924 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 10:27:58 smithi191 systemd[1]: Stopping Ceph mgr.x for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:27:59.242 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 10:27:58 smithi191 podman[261933]: 2024-09-21 10:27:58.922344799 +0000 UTC m=+0.133848848 container died 141a29e6e0055f07498cb1ef4e7c1be24573454ddd3742bd40886a11d7abb384 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T10:27:59.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 10:27:59 smithi191 podman[261933]: 2024-09-21 10:27:59.34755068 +0000 UTC m=+0.559054739 container cleanup 141a29e6e0055f07498cb1ef4e7c1be24573454ddd3742bd40886a11d7abb384 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:27:59.613 INFO:journalctl@ceph.mgr.x.smithi191.stdout:Sep 21 10:27:59 smithi191 bash[261933]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-mgr-x 2024-09-21T10:27:59.784 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@mgr.x.service' 2024-09-21T10:27:59.818 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:27:59.819 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-21T10:27:59.819 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:27:59.819 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:27:59.909 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:27:59 smithi076 systemd[1]: Stopping Ceph osd.0 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:00.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[311601]: 2024-09-21T10:28:00.132+0000 7f00af80b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:00.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[311601]: 2024-09-21T10:28:00.132+0000 7f00af80b640 -1 osd.0 1445 *** Got signal Terminated *** 2024-09-21T10:28:00.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:00 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0[311601]: 2024-09-21T10:28:00.132+0000 7f00af80b640 -1 osd.0 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:05.785 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:05 smithi076 podman[372512]: 2024-09-21 10:28:05.472201652 +0000 UTC m=+5.451365240 container died 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T10:28:06.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 podman[372512]: 2024-09-21 10:28:06.015326301 +0000 UTC m=+5.994489941 container cleanup 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T10:28:06.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 bash[372512]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:28:06.623 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 podman[372546]: 2024-09-21 10:28:06.225244191 +0000 UTC m=+0.744482277 container remove 3a9c62c344aec699f8caaa7da5fe84150264797369f39201e5422ef1e5ddff67 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:06.914 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 podman[372638]: 2024-09-21 10:28:06.531981117 +0000 UTC m=+0.016258752 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:28:06.914 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 podman[372638]: 2024-09-21 10:28:06.70130564 +0000 UTC m=+0.185583266 container create 2ddd84ef64e4d2f4a33878cdedf161f119fc4835db005fb1080146d02aab09f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.schema-version=1.0) 2024-09-21T10:28:06.914 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 podman[372638]: 2024-09-21 10:28:06.912280211 +0000 UTC m=+0.396557847 container init 2ddd84ef64e4d2f4a33878cdedf161f119fc4835db005fb1080146d02aab09f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True) 2024-09-21T10:28:07.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:06 smithi076 podman[372638]: 2024-09-21 10:28:06.917158712 +0000 UTC m=+0.401436339 container start 2ddd84ef64e4d2f4a33878cdedf161f119fc4835db005fb1080146d02aab09f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, RELEASE=main-62da54b, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:28:07.373 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:07 smithi076 podman[372638]: 2024-09-21 10:28:07.087139774 +0000 UTC m=+0.571417400 container attach 2ddd84ef64e4d2f4a33878cdedf161f119fc4835db005fb1080146d02aab09f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:08.123 INFO:journalctl@ceph.osd.0.smithi076.stdout:Sep 21 10:28:07 smithi076 podman[372638]: 2024-09-21 10:28:07.822301755 +0000 UTC m=+1.306579398 container died 2ddd84ef64e4d2f4a33878cdedf161f119fc4835db005fb1080146d02aab09f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T10:28:08.472 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:28:08.512 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:08.512 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:28:08.512 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T10:28:08.512 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:28:08.832 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:08 smithi076 systemd[1]: Stopping Ceph osd.1 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:09.123 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[127682]: 2024-09-21T10:28:08.829+0000 7f8448ee0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:09.123 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[127682]: 2024-09-21T10:28:08.829+0000 7f8448ee0640 -1 osd.1 1445 *** Got signal Terminated *** 2024-09-21T10:28:09.123 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1[127682]: 2024-09-21T10:28:08.829+0000 7f8448ee0640 -1 osd.1 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.019Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:10.363 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.023Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.023Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.025Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.025Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.026Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:10.364 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:28:10 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:28:10.026Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.76:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.76:8765: connect: connection refused" 2024-09-21T10:28:14.757 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:14 smithi076 podman[372769]: 2024-09-21 10:28:14.483144614 +0000 UTC m=+5.751296535 container died c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, maintainer=Guillaume Abrioux ) 2024-09-21T10:28:15.123 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:14 smithi076 podman[372769]: 2024-09-21 10:28:14.969956319 +0000 UTC m=+6.238108227 container cleanup c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T10:28:15.123 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:14 smithi076 bash[372769]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:28:15.557 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372791]: 2024-09-21 10:28:15.238217504 +0000 UTC m=+0.753156027 container remove c1218f57993c90ad31760016648f2f01f89b9e7b372674882f166aa70db42586 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T10:28:15.870 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372882]: 2024-09-21 10:28:15.458447319 +0000 UTC m=+0.016324093 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:28:15.870 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372882]: 2024-09-21 10:28:15.566047123 +0000 UTC m=+0.123923882 container create 451feb4f7f407b7da10e0ef5336839968d935b1e1a18e9270695f9c47d3b6dea (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:28:15.870 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372882]: 2024-09-21 10:28:15.776903024 +0000 UTC m=+0.334779792 container init 451feb4f7f407b7da10e0ef5336839968d935b1e1a18e9270695f9c47d3b6dea (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:28:15.870 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372882]: 2024-09-21 10:28:15.78180471 +0000 UTC m=+0.339681475 container start 451feb4f7f407b7da10e0ef5336839968d935b1e1a18e9270695f9c47d3b6dea (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T10:28:15.870 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372882]: 2024-09-21 10:28:15.86840254 +0000 UTC m=+0.426279308 container attach 451feb4f7f407b7da10e0ef5336839968d935b1e1a18e9270695f9c47d3b6dea (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:28:16.123 INFO:journalctl@ceph.osd.1.smithi076.stdout:Sep 21 10:28:15 smithi076 podman[372882]: 2024-09-21 10:28:15.971789736 +0000 UTC m=+0.529666511 container died 451feb4f7f407b7da10e0ef5336839968d935b1e1a18e9270695f9c47d3b6dea (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True) 2024-09-21T10:28:16.562 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T10:28:16.600 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:16.601 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T10:28:16.601 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T10:28:16.601 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.2 2024-09-21T10:28:16.873 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:16 smithi076 systemd[1]: Stopping Ceph osd.2 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:17.373 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T10:28:16.968+0000 7f5c7e747640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:17.373 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T10:28:16.968+0000 7f5c7e747640 -1 osd.2 1445 *** Got signal Terminated *** 2024-09-21T10:28:17.374 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:16 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2[132757]: 2024-09-21T10:28:16.968+0000 7f5c7e747640 -1 osd.2 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:22.711 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:22 smithi076 podman[373012]: 2024-09-21 10:28:22.451825696 +0000 UTC m=+5.626709307 container died d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T10:28:23.123 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:22 smithi076 podman[373012]: 2024-09-21 10:28:22.891996767 +0000 UTC m=+6.066880372 container cleanup d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T10:28:23.123 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:22 smithi076 bash[373012]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:28:23.534 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:23 smithi076 podman[373032]: 2024-09-21 10:28:23.210154783 +0000 UTC m=+0.762258948 container remove d1afb0dbaa23f6d519e27d94f542ff1cd0d37ff26a2e80fca5e86dd61189182d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:28:23.790 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:23 smithi076 podman[373121]: 2024-09-21 10:28:23.435903708 +0000 UTC m=+0.016014414 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:28:23.790 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:23 smithi076 podman[373121]: 2024-09-21 10:28:23.561097967 +0000 UTC m=+0.141208667 container create 7d4c72104dcd3ade1b6bcb3946b5dac64aeb7a2711bdb142bfb4add2e1d69996 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:28:23.790 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:23 smithi076 podman[373121]: 2024-09-21 10:28:23.788777902 +0000 UTC m=+0.368888602 container init 7d4c72104dcd3ade1b6bcb3946b5dac64aeb7a2711bdb142bfb4add2e1d69996 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b) 2024-09-21T10:28:24.123 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:23 smithi076 podman[373121]: 2024-09-21 10:28:23.793335648 +0000 UTC m=+0.373446349 container start 7d4c72104dcd3ade1b6bcb3946b5dac64aeb7a2711bdb142bfb4add2e1d69996 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T10:28:24.123 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:23 smithi076 podman[373121]: 2024-09-21 10:28:23.88044149 +0000 UTC m=+0.460552191 container attach 7d4c72104dcd3ade1b6bcb3946b5dac64aeb7a2711bdb142bfb4add2e1d69996 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T10:28:24.123 INFO:journalctl@ceph.osd.2.smithi076.stdout:Sep 21 10:28:24 smithi076 podman[373163]: 2024-09-21 10:28:24.009652561 +0000 UTC m=+0.016577672 container died 7d4c72104dcd3ade1b6bcb3946b5dac64aeb7a2711bdb142bfb4add2e1d69996 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T10:28:24.653 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T10:28:24.693 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:24.693 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T10:28:24.693 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:28:24.693 DEBUG:teuthology.orchestra.run.smithi076:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:28:25.032 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:24 smithi076 systemd[1]: Stopping Ceph osd.3 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:25.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:28:25.030+0000 7fcb4005f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:25.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:28:25.030+0000 7fcb4005f640 -1 osd.3 1445 *** Got signal Terminated *** 2024-09-21T10:28:25.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:25 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:28:25.030+0000 7fcb4005f640 -1 osd.3 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:27.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:27 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:27.044+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:27.626 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:27 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:27.573+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:28.362 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:28 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:28.188+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:28.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:28 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:28.037+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:28.623 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:28 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:28:28.352+0000 7fcb3866c640 -1 osd.3 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:03.262776+0000 front 2024-09-21T10:28:03.262864+0000 (oldest deadline 2024-09-21T10:28:27.962990+0000) 2024-09-21T10:28:28.862 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:28 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:28.542+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:28.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:28 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:28.535+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:29.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:29 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:29.166+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:29.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:29 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:29.070+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:29.853 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:29 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3[347317]: 2024-09-21T10:28:29.390+0000 7fcb3866c640 -1 osd.3 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:03.262776+0000 front 2024-09-21T10:28:03.262864+0000 (oldest deadline 2024-09-21T10:28:27.962990+0000) 2024-09-21T10:28:29.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:29 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:29.492+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:29.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:29 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:29.563+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:30.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:30 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:30.125+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:30.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:30 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:30.039+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:30.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:30 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:30.452+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:30.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:30 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:30.566+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:30.873 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:30 smithi076 podman[373248]: 2024-09-21 10:28:30.617391739 +0000 UTC m=+5.696446339 container died 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T10:28:31.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:31.130+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:31.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:31.081+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:31.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 podman[373248]: 2024-09-21 10:28:31.070390019 +0000 UTC m=+6.149444653 container cleanup 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T10:28:31.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 bash[373248]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:28:31.726 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 podman[373280]: 2024-09-21 10:28:31.38827462 +0000 UTC m=+0.766473001 container remove 4bcc684cf1482d39fbb6efde1c962aa0579c83d7a582c391867173110e8ddc4f (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:28:31.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:31.473+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:31.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:31 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:31.555+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:31.993 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 podman[373374]: 2024-09-21 10:28:31.627954644 +0000 UTC m=+0.015836098 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:28:31.993 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 podman[373374]: 2024-09-21 10:28:31.747811633 +0000 UTC m=+0.135693107 container create c4db70795e715b3c964566de9dfa7b30d710efa293ffd1ca973530740bc9f588 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:28:31.993 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 podman[373374]: 2024-09-21 10:28:31.991946711 +0000 UTC m=+0.379828155 container init c4db70795e715b3c964566de9dfa7b30d710efa293ffd1ca973530740bc9f588 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T10:28:32.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:31 smithi076 podman[373374]: 2024-09-21 10:28:31.99686974 +0000 UTC m=+0.384751181 container start c4db70795e715b3c964566de9dfa7b30d710efa293ffd1ca973530740bc9f588 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-62da54b, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:28:32.373 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:32 smithi076 podman[373374]: 2024-09-21 10:28:32.083505267 +0000 UTC m=+0.471386699 container attach c4db70795e715b3c964566de9dfa7b30d710efa293ffd1ca973530740bc9f588 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:28:32.374 INFO:journalctl@ceph.osd.3.smithi076.stdout:Sep 21 10:28:32 smithi076 podman[373374]: 2024-09-21 10:28:32.188144962 +0000 UTC m=+0.576026409 container died c4db70795e715b3c964566de9dfa7b30d710efa293ffd1ca973530740bc9f588 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b) 2024-09-21T10:28:32.483 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:32 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:32.111+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:32.483 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:32 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:32.136+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:32.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:32 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:32.480+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:32.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:32 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:32.510+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:32.909 DEBUG:teuthology.orchestra.run.smithi076:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:28:32.950 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:32.950 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:28:32.950 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T10:28:32.950 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:28:33.277 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:33.067+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:33.278 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:33 smithi191 systemd[1]: Stopping Ceph osd.4 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:33.278 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:33.274+0000 7f554ec20640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:33.278 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:33.179+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:33.531 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:33.274+0000 7f554ec20640 -1 osd.4 1445 *** Got signal Terminated *** 2024-09-21T10:28:33.532 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:33.274+0000 7f554ec20640 -1 osd.4 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:33.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:33.528+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:33.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:33 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:33.535+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:34.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:34 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:34.166+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:34.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:34 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:34.099+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:34.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:34 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:34.532+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:34.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:34 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:34.577+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:35.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:35.090+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:35.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:35.201+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:35.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:35.577+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:35.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:35.577+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:08.538964+0000 front 2024-09-21T10:28:08.538981+0000 (oldest deadline 2024-09-21T10:28:34.939321+0000) 2024-09-21T10:28:35.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:35.610+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:35.864 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:35 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:35.610+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:36.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:36.191+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:36.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:36.092+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:36.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:36.092+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:36.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:36.608+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:36.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:36.608+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:08.538964+0000 front 2024-09-21T10:28:08.538981+0000 (oldest deadline 2024-09-21T10:28:34.939321+0000) 2024-09-21T10:28:36.864 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:36.592+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:36.864 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:36 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:36.592+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:37.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:37.180+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:37.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:37.104+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:37.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:37.104+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:37.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:37.592+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:02.837397+0000 front 2024-09-21T10:28:02.837593+0000 (oldest deadline 2024-09-21T10:28:27.537546+0000) 2024-09-21T10:28:37.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4[38691]: 2024-09-21T10:28:37.592+0000 7f554b234640 -1 osd.4 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:08.538964+0000 front 2024-09-21T10:28:08.538981+0000 (oldest deadline 2024-09-21T10:28:34.939321+0000) 2024-09-21T10:28:37.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:37.601+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:37.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:37 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:37.601+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:38.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:38.203+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:38.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:38.057+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:38.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:38.058+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:38.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:38 smithi191 podman[262078]: 2024-09-21 10:28:38.843514678 +0000 UTC m=+5.691219934 container died 01f1dafb6a2187db795ab4a07352fa3215d9d57fade5264898fd001a66449fca (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T10:28:38.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:38.598+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:38.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:38 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:38.598+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:39.363 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:39 smithi191 podman[262078]: 2024-09-21 10:28:39.232312816 +0000 UTC m=+6.080017997 container cleanup 01f1dafb6a2187db795ab4a07352fa3215d9d57fade5264898fd001a66449fca (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:28:39.363 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:39 smithi191 bash[262078]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:28:39.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:39.233+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:39.364 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:39.088+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:39.364 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:39.088+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:39.863 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:39 smithi191 podman[262099]: 2024-09-21 10:28:39.551869055 +0000 UTC m=+0.697152397 container remove 01f1dafb6a2187db795ab4a07352fa3215d9d57fade5264898fd001a66449fca (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T10:28:39.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:39.590+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:39.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:39 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:39.590+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:40.139 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:39 smithi191 podman[262194]: 2024-09-21 10:28:39.770662976 +0000 UTC m=+0.016576523 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:28:40.139 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:39 smithi191 podman[262194]: 2024-09-21 10:28:39.876592626 +0000 UTC m=+0.122506172 container create c0fbbae4f9861dd5efada87393f973a5e146a960d9f96d16ae950a9334999f07 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:28:40.139 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:40 smithi191 podman[262194]: 2024-09-21 10:28:40.112225159 +0000 UTC m=+0.358138718 container init c0fbbae4f9861dd5efada87393f973a5e146a960d9f96d16ae950a9334999f07 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8) 2024-09-21T10:28:40.140 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:40 smithi191 podman[262194]: 2024-09-21 10:28:40.116941819 +0000 UTC m=+0.362855365 container start c0fbbae4f9861dd5efada87393f973a5e146a960d9f96d16ae950a9334999f07 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:28:40.482 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:40 smithi191 podman[262194]: 2024-09-21 10:28:40.220881349 +0000 UTC m=+0.466794910 container attach c0fbbae4f9861dd5efada87393f973a5e146a960d9f96d16ae950a9334999f07 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T10:28:40.483 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:40.136+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:40.483 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:40.136+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:40.483 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:40.219+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:40.792 INFO:journalctl@ceph.osd.4.smithi191.stdout:Sep 21 10:28:40 smithi191 podman[262194]: 2024-09-21 10:28:40.481805064 +0000 UTC m=+0.727718609 container died c0fbbae4f9861dd5efada87393f973a5e146a960d9f96d16ae950a9334999f07 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, RELEASE=reef-6a1bdf8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:40.792 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:40.631+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:40.793 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:40 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:40.631+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:41.148 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T10:28:41.188 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:41.188 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T10:28:41.188 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T10:28:41.188 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:28:41.362 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:41 smithi191 systemd[1]: Stopping Ceph osd.5 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:41.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:41.226+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:41.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:41.226+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:41.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:41.093+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:41.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:41.093+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:41.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:41.511+0000 7faf306f2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:41.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:41.511+0000 7faf306f2640 -1 osd.5 1445 *** Got signal Terminated *** 2024-09-21T10:28:41.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:41.511+0000 7faf306f2640 -1 osd.5 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:41.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:41.599+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:41.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:41 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:41.599+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:42.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:42 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:42.205+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:42.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:42 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:42.205+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:42.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:42 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:42.052+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:42.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:42 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:42.052+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:43.072 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:42 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:42.613+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:43.072 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:42 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:42.613+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:43.362 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:43 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:43.245+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:43.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:43 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:43.245+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:43.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:43 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:43.070+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:43.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:43 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:43.070+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:43.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:43 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:43.600+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:43.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:43 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:43.600+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:44.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:44.245+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:44.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:44.245+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:44.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:44.036+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:44.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:44.036+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:44.862 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:44.582+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:44.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:44 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:44.582+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:45.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:45.006+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:45.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:45.006+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:45.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:45.198+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:45.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:45.198+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:45.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:45.585+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.933469+0000 front 2024-09-21T10:28:04.933610+0000 (oldest deadline 2024-09-21T10:28:28.433257+0000) 2024-09-21T10:28:45.863 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:45 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5[44799]: 2024-09-21T10:28:45.585+0000 7faf28cfe640 -1 osd.5 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:11.934066+0000 front 2024-09-21T10:28:11.934118+0000 (oldest deadline 2024-09-21T10:28:35.433952+0000) 2024-09-21T10:28:46.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:46.000+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:46.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:46.000+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:46.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:46.000+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:46.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:46.188+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:46.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:46.188+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:46.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:46 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:46.188+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:47.297 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:46 smithi191 podman[262322]: 2024-09-21 10:28:46.998394638 +0000 UTC m=+5.592132406 container died 19b2c345ce827d0338f3579b1fc08721896f16b791f1bb94c5d040f4f6f40d11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:28:47.297 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:47.219+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:47.297 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:47.219+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:47.298 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:47.219+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:47.298 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:47.031+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:47.298 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:47.031+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:47.298 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:47 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:47.031+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:47.612 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:47 smithi191 podman[262322]: 2024-09-21 10:28:47.38550062 +0000 UTC m=+5.979238392 container cleanup 19b2c345ce827d0338f3579b1fc08721896f16b791f1bb94c5d040f4f6f40d11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:28:47.613 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:47 smithi191 bash[262322]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:28:47.999 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:47 smithi191 podman[262344]: 2024-09-21 10:28:47.686433216 +0000 UTC m=+0.680196370 container remove 19b2c345ce827d0338f3579b1fc08721896f16b791f1bb94c5d040f4f6f40d11 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:28:48.350 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:48.227+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:48.350 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:48.227+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:48.350 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:48.227+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:48.005+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:48.005+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:48.005+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:47 smithi191 podman[262437]: 2024-09-21 10:28:47.900312975 +0000 UTC m=+0.016032134 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:48 smithi191 podman[262437]: 2024-09-21 10:28:48.012719138 +0000 UTC m=+0.128438356 container create 2964fa7746666e4c27d6b37271e2e023756183c8b635ac9e948d0141a800cd35 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=reef-6a1bdf8) 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:48 smithi191 podman[262437]: 2024-09-21 10:28:48.240324083 +0000 UTC m=+0.356043242 container init 2964fa7746666e4c27d6b37271e2e023756183c8b635ac9e948d0141a800cd35 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:48.351 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:48 smithi191 podman[262437]: 2024-09-21 10:28:48.245203421 +0000 UTC m=+0.360922581 container start 2964fa7746666e4c27d6b37271e2e023756183c8b635ac9e948d0141a800cd35 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T10:28:48.352 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:48 smithi191 podman[262437]: 2024-09-21 10:28:48.348381882 +0000 UTC m=+0.464101042 container attach 2964fa7746666e4c27d6b37271e2e023756183c8b635ac9e948d0141a800cd35 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T10:28:48.613 INFO:journalctl@ceph.osd.5.smithi191.stdout:Sep 21 10:28:48 smithi191 podman[262437]: 2024-09-21 10:28:48.447298631 +0000 UTC m=+0.563017807 container died 2964fa7746666e4c27d6b37271e2e023756183c8b635ac9e948d0141a800cd35 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:28:49.040 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T10:28:49.081 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:49.081 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T10:28:49.082 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T10:28:49.082 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.6 2024-09-21T10:28:49.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:48.994+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:49.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:48.994+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:49.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:48 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:48.994+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:49.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 systemd[1]: Stopping Ceph osd.6 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:49.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:49.224+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:49.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:49.224+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:49.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:49.224+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:49.863 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:49.423+0000 7f846e12a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:49.863 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:49.423+0000 7f846e12a640 -1 osd.6 1445 *** Got signal Terminated *** 2024-09-21T10:28:49.863 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:49 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:49.423+0000 7f846e12a640 -1 osd.6 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:50.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:50 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:50.044+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:50.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:50 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:50.044+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:50.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:50 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:50.044+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:50.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:50 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:50.189+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:50.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:50 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:50.189+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:50.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:50 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:50.189+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:51.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:51.213+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:51.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:51.213+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:51.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:51.213+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:51.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:51.068+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:51.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:51.068+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:51.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:51 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:51.068+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:52.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:52.040+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:52.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:52.040+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:52.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:52.040+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:52.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:52.263+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:52.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:52.263+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:52.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:52.263+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:52.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:52 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:52.263+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.951551+0000 front 2024-09-21T10:28:27.951488+0000 (oldest deadline 2024-09-21T10:28:52.051500+0000) 2024-09-21T10:28:53.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:53.073+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:53.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:53.073+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:53.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:53.073+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:53.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:53.305+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:53.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:53.305+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:53.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:53.305+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:53.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:53 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:53.305+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.951551+0000 front 2024-09-21T10:28:27.951488+0000 (oldest deadline 2024-09-21T10:28:52.051500+0000) 2024-09-21T10:28:54.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:54.097+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:54.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:54.097+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:54.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:54.097+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:54.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:54.097+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:28:54.363 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:54.292+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.648500+0000 front 2024-09-21T10:28:04.648754+0000 (oldest deadline 2024-09-21T10:28:28.148509+0000) 2024-09-21T10:28:54.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:54.292+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:12.750007+0000 front 2024-09-21T10:28:12.749944+0000 (oldest deadline 2024-09-21T10:28:40.350466+0000) 2024-09-21T10:28:54.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:54.292+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:20.350957+0000 front 2024-09-21T10:28:20.350848+0000 (oldest deadline 2024-09-21T10:28:45.650795+0000) 2024-09-21T10:28:54.364 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:54 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6[51120]: 2024-09-21T10:28:54.292+0000 7f846a73e640 -1 osd.6 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.951551+0000 front 2024-09-21T10:28:27.951488+0000 (oldest deadline 2024-09-21T10:28:52.051500+0000) 2024-09-21T10:28:55.215 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:54 smithi191 podman[262565]: 2024-09-21 10:28:54.929247626 +0000 UTC m=+5.624304693 container died 6a3f1d2d2637ad1fe21d35d32c595d98ebe1b45b9eb255cefae63252110e6200 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:28:55.216 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:55 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:55.084+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:55.216 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:55 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:55.084+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:55.216 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:55 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:55.084+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:55.216 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:55 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:55.084+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:28:55.613 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:55 smithi191 podman[262565]: 2024-09-21 10:28:55.421700843 +0000 UTC m=+6.116757964 container cleanup 6a3f1d2d2637ad1fe21d35d32c595d98ebe1b45b9eb255cefae63252110e6200 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-6a1bdf8, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T10:28:55.613 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:55 smithi191 bash[262565]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6 2024-09-21T10:28:55.982 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:55 smithi191 podman[262588]: 2024-09-21 10:28:55.664690531 +0000 UTC m=+0.733572251 container remove 6a3f1d2d2637ad1fe21d35d32c595d98ebe1b45b9eb255cefae63252110e6200 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:28:56.286 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:56 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:56.051+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:56.286 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:56 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:56.051+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:56.286 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:56 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:56.051+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:56.286 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:56 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:56.051+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:28:56.287 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:55 smithi191 podman[262681]: 2024-09-21 10:28:55.884155352 +0000 UTC m=+0.016012019 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:28:56.287 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:55 smithi191 podman[262681]: 2024-09-21 10:28:55.99087072 +0000 UTC m=+0.122727371 container create 67cc2e3f8636af4788b971dd58f4ab1a124f2a5c52cdaa4f5d8260041ced7c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, RELEASE=reef-6a1bdf8, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T10:28:56.287 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:56 smithi191 podman[262681]: 2024-09-21 10:28:56.193255897 +0000 UTC m=+0.325112560 container init 67cc2e3f8636af4788b971dd58f4ab1a124f2a5c52cdaa4f5d8260041ced7c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:56.287 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:56 smithi191 podman[262681]: 2024-09-21 10:28:56.19819803 +0000 UTC m=+0.330054683 container start 67cc2e3f8636af4788b971dd58f4ab1a124f2a5c52cdaa4f5d8260041ced7c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:56.287 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:56 smithi191 podman[262681]: 2024-09-21 10:28:56.284645359 +0000 UTC m=+0.416502025 container attach 67cc2e3f8636af4788b971dd58f4ab1a124f2a5c52cdaa4f5d8260041ced7c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:28:56.612 INFO:journalctl@ceph.osd.6.smithi191.stdout:Sep 21 10:28:56 smithi191 podman[262681]: 2024-09-21 10:28:56.385700625 +0000 UTC m=+0.517557278 container died 67cc2e3f8636af4788b971dd58f4ab1a124f2a5c52cdaa4f5d8260041ced7c0f (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=reef-6a1bdf8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:28:57.094 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T10:28:57.134 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:28:57.134 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T10:28:57.134 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T10:28:57.135 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.7 2024-09-21T10:28:57.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.029+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:57.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.029+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:57.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.029+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:57.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.029+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:28:57.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 systemd[1]: Stopping Ceph osd.7 for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:28:57.863 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.469+0000 7fd95f126640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:28:57.863 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.469+0000 7fd95f126640 -1 osd.7 1445 *** Got signal Terminated *** 2024-09-21T10:28:57.863 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.469+0000 7fd95f126640 -1 osd.7 1445 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:28:58.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.989+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:58.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.989+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:58.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.989+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:58.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:57 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:57.989+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:28:59.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:58 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:58.940+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:28:59.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:58 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:58.940+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:28:59.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:58 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:58.940+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:28:59.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:58 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:58.940+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:29:00.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:59 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:59.949+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:29:00.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:59 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:59.949+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:29:00.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:59 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:59.949+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:29:00.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:28:59 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:28:59.949+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:29:01.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:00 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:00.981+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:29:01.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:00 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:00.981+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:29:01.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:00 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:00.981+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:29:01.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:00 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:00.981+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:29:01.364 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:00 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:00.981+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.191:6804 osd.4 since back 2024-09-21T10:28:34.308808+0000 front 2024-09-21T10:28:34.308746+0000 (oldest deadline 2024-09-21T10:29:00.208826+0000) 2024-09-21T10:29:02.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:01 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:01.945+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6830 osd.0 since back 2024-09-21T10:28:04.704385+0000 front 2024-09-21T10:28:04.704173+0000 (oldest deadline 2024-09-21T10:28:27.004238+0000) 2024-09-21T10:29:02.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:01 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:01.945+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6814 osd.1 since back 2024-09-21T10:28:09.305069+0000 front 2024-09-21T10:28:09.305107+0000 (oldest deadline 2024-09-21T10:28:35.205020+0000) 2024-09-21T10:29:02.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:01 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:01.945+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6822 osd.2 since back 2024-09-21T10:28:19.806802+0000 front 2024-09-21T10:28:19.806653+0000 (oldest deadline 2024-09-21T10:28:45.106626+0000) 2024-09-21T10:29:02.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:01 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:01.945+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.76:6806 osd.3 since back 2024-09-21T10:28:27.407553+0000 front 2024-09-21T10:28:27.407567+0000 (oldest deadline 2024-09-21T10:28:53.207775+0000) 2024-09-21T10:29:02.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:01 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7[57171]: 2024-09-21T10:29:01.945+0000 7fd95b73a640 -1 osd.7 1445 heartbeat_check: no reply from 172.21.15.191:6804 osd.4 since back 2024-09-21T10:28:34.308808+0000 front 2024-09-21T10:28:34.308746+0000 (oldest deadline 2024-09-21T10:29:00.208826+0000) 2024-09-21T10:29:03.112 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:02 smithi191 podman[262808]: 2024-09-21 10:29:02.832757217 +0000 UTC m=+5.479562794 container died b84d3545a30575e59b43881ccca38a287fa96d7f5d5d18e1016028fa24c7ef5e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8) 2024-09-21T10:29:03.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:03 smithi191 podman[262808]: 2024-09-21 10:29:03.275229517 +0000 UTC m=+5.922035094 container cleanup b84d3545a30575e59b43881ccca38a287fa96d7f5d5d18e1016028fa24c7ef5e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T10:29:03.363 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:03 smithi191 bash[262808]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7 2024-09-21T10:29:03.863 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:03 smithi191 podman[262842]: 2024-09-21 10:29:03.47655792 +0000 UTC m=+0.637688023 container remove b84d3545a30575e59b43881ccca38a287fa96d7f5d5d18e1016028fa24c7ef5e (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-6a1bdf8, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:29:04.231 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:03 smithi191 podman[262935]: 2024-09-21 10:29:03.80019854 +0000 UTC m=+0.016022866 image pull 0adab276c97e6f3143ffbca5eeff37fcd7dc3b5c0f339a9f747381d9993602de quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6 2024-09-21T10:29:04.231 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:03 smithi191 podman[262935]: 2024-09-21 10:29:03.919207312 +0000 UTC m=+0.135031679 container create 0318dfedb9d20fb26a916a70f1f8491f197f1b4ffa2b92c6a83bb363b0233606 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:29:04.231 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:04 smithi191 podman[262935]: 2024-09-21 10:29:04.137992807 +0000 UTC m=+0.353817131 container init 0318dfedb9d20fb26a916a70f1f8491f197f1b4ffa2b92c6a83bb363b0233606 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:29:04.232 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:04 smithi191 podman[262935]: 2024-09-21 10:29:04.143236132 +0000 UTC m=+0.359060451 container start 0318dfedb9d20fb26a916a70f1f8491f197f1b4ffa2b92c6a83bb363b0233606 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-6a1bdf8, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T10:29:04.612 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:04 smithi191 podman[262935]: 2024-09-21 10:29:04.229568008 +0000 UTC m=+0.445392334 container attach 0318dfedb9d20fb26a916a70f1f8491f197f1b4ffa2b92c6a83bb363b0233606 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-6a1bdf8, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T10:29:04.613 INFO:journalctl@ceph.osd.7.smithi191.stdout:Sep 21 10:29:04 smithi191 podman[262935]: 2024-09-21 10:29:04.331350561 +0000 UTC m=+0.547174884 container died 0318dfedb9d20fb26a916a70f1f8491f197f1b4ffa2b92c6a83bb363b0233606 (image=quay.ceph.io/ceph-ci/ceph@sha256:de4fa798d334aaeb6a820ab2dad2fbcb86bcd7f118d94ed3a02ce126be31cec6, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-osd-7-deactivate, RELEASE=reef-6a1bdf8, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T10:29:04.958 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T10:29:04.997 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:29:04.997 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T10:29:04.997 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-21T10:29:04.998 DEBUG:teuthology.orchestra.run.smithi191:> sudo systemctl stop ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a 2024-09-21T10:29:05.332 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 systemd[1]: Stopping Ceph prometheus.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T10:29:05.613 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T10:29:05.614 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.330Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T10:29:05.614 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.331Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T10:29:05.614 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.333Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T10:29:05.614 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.333Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T10:29:05.614 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a[88977]: ts=2024-09-21T10:29:05.333Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T10:29:05.614 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 podman[263062]: 2024-09-21 10:29:05.341099686 +0000 UTC m=+0.117815142 container died eda26012a522f9e7d74d220f464fb8511d96b74cbb416a4edf8d36db89b0d435 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:06.112 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 podman[263062]: 2024-09-21 10:29:05.786405277 +0000 UTC m=+0.563120733 container cleanup eda26012a522f9e7d74d220f464fb8511d96b74cbb416a4edf8d36db89b0d435 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:06.113 INFO:journalctl@ceph.prometheus.a.smithi191.stdout:Sep 21 10:29:05 smithi191 bash[263062]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T10:29:06.250 DEBUG:teuthology.orchestra.run.smithi191:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@prometheus.a.service' 2024-09-21T10:29:06.283 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:29:06.283 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-21T10:29:06.283 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:29:06.433 INFO:teuthology.orchestra.run.smithi076.stdout:Deleting cluster with fsid: e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T10:29:08.118 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:07 smithi076 systemd[1]: Stopping Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:29:08.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:08 smithi076 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a[101515]: ts=2024-09-21T10:29:08.115Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:29:08.373 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:08 smithi076 podman[373802]: 2024-09-21 10:29:08.166996183 +0000 UTC m=+0.165980926 container died 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:08.873 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:08 smithi076 podman[373802]: 2024-09-21 10:29:08.622204835 +0000 UTC m=+0.621189572 container cleanup 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:08.873 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:08 smithi076 bash[373802]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T10:29:09.210 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:08 smithi076 podman[373820]: 2024-09-21 10:29:08.939996399 +0000 UTC m=+0.768473970 container remove 1e9964bd966f198e751ba4062824fd64334fe33daccdc969ef5185674922dcfe (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:09.210 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:09 smithi076 podman[373820]: 2024-09-21 10:29:09.032781835 +0000 UTC m=+0.861259405 volume remove 87951f3be4ce3b79cb42b938ca3b56cec9f91e0d7d33deceb0f2f4b5211a4038 2024-09-21T10:29:09.210 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:09 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T10:29:09.210 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:09 smithi076 systemd[1]: Stopped Ceph alertmanager.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:29:09.210 INFO:journalctl@ceph.alertmanager.a.smithi076.stdout:Sep 21 10:29:09 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 4.112s CPU time. 2024-09-21T10:29:09.623 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:09 smithi076 systemd[1]: Stopping Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:29:09.904 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:09 smithi076 podman[373932]: 2024-09-21 10:29:09.639606975 +0000 UTC m=+0.129562297 container died a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:10.287 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 podman[373932]: 2024-09-21 10:29:10.018386388 +0000 UTC m=+0.508341710 container cleanup a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:10.288 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 bash[373932]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T10:29:10.609 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 podman[373950]: 2024-09-21 10:29:10.286275029 +0000 UTC m=+0.644597450 container remove a75033fa919d1bd29b93f0f00196f69e2dd300b237aa896f45cdf1fa9d4b917b (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:29:10.609 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:29:10.610 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T10:29:10.610 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 systemd[1]: Stopped Ceph node-exporter.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:29:10.610 INFO:journalctl@ceph.node-exporter.a.smithi076.stdout:Sep 21 10:29:10 smithi076 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 8.367s CPU time. 2024-09-21T10:29:15.455 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:29:15.607 INFO:teuthology.orchestra.run.smithi191.stdout:Deleting cluster with fsid: e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T10:29:17.161 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:16 smithi191 systemd[1]: Stopping Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:29:17.449 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=server t=2024-09-21T10:29:17.159486432Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-21T10:29:17.449 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=ticker t=2024-09-21T10:29:17.159547996Z level=info msg=stopped last_tick=2024-09-21T10:29:10Z 2024-09-21T10:29:17.449 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=tracing t=2024-09-21T10:29:17.159575271Z level=info msg="Closing tracing" 2024-09-21T10:29:17.449 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a[93514]: logger=grafana-apiserver t=2024-09-21T10:29:17.170923675Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-21T10:29:17.449 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 podman[263480]: 2024-09-21 10:29:17.180133059 +0000 UTC m=+0.134491802 container died 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:29:17.862 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 podman[263480]: 2024-09-21 10:29:17.648857343 +0000 UTC m=+0.603216157 container cleanup 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:29:17.863 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 bash[263480]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a 2024-09-21T10:29:18.231 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:17 smithi191 podman[263497]: 2024-09-21 10:29:17.908618789 +0000 UTC m=+0.728495170 container remove 61303ed53f7ba792f655518642d5e33b196482361c2386bbe9323c68d8d03c64 (image=quay.io/ceph/grafana:10.4.8, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:29:18.231 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:18 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T10:29:18.231 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:18 smithi191 systemd[1]: Stopped Ceph grafana.a for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:29:18.231 INFO:journalctl@ceph.grafana.a.smithi191.stdout:Sep 21 10:29:18 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@grafana.a.service: Consumed 32.520s CPU time. 2024-09-21T10:29:18.492 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:18 smithi191 systemd[1]: Stopping Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490... 2024-09-21T10:29:18.755 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:18 smithi191 podman[263612]: 2024-09-21 10:29:18.491298229 +0000 UTC m=+0.123083809 container died 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:29:19.113 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:18 smithi191 podman[263612]: 2024-09-21 10:29:18.844271429 +0000 UTC m=+0.476057016 container cleanup 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:29:19.113 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:18 smithi191 bash[263612]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T10:29:19.469 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:19 smithi191 podman[263628]: 2024-09-21 10:29:19.120337468 +0000 UTC m=+0.631146638 container remove 703c0a6bfe5f3ff73d11bd4acde1ff47d7a87157b5a294755c9662bbe617d6c5 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-e97bd93e-77fb-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:29:19.469 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:19 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:29:19.469 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:19 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T10:29:19.469 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:19 smithi191 systemd[1]: Stopped Ceph node-exporter.b for e97bd93e-77fb-11ef-baf6-efdc52797490. 2024-09-21T10:29:19.470 INFO:journalctl@ceph.node-exporter.b.smithi191.stdout:Sep 21 10:29:19 smithi191 systemd[1]: ceph-e97bd93e-77fb-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 7.906s CPU time. 2024-09-21T10:29:23.345 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:29:23.376 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:29:23.405 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T10:29:23.407 DEBUG:teuthology.misc:Transferring archived files from smithi076:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646/remote/smithi076/crash 2024-09-21T10:29:23.408 DEBUG:teuthology.orchestra.run.smithi076:> sudo tar c -f - -C /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:29:23.479 INFO:teuthology.orchestra.run.smithi076.stderr:tar: /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:29:23.479 INFO:teuthology.orchestra.run.smithi076.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:29:23.481 DEBUG:teuthology.misc:Transferring archived files from smithi191:/var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646/remote/smithi191/crash 2024-09-21T10:29:23.482 DEBUG:teuthology.orchestra.run.smithi191:> sudo tar c -f - -C /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:29:23.522 INFO:teuthology.orchestra.run.smithi191.stderr:tar: /var/lib/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:29:23.522 INFO:teuthology.orchestra.run.smithi191.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:29:23.523 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T10:29:23.523 DEBUG:teuthology.orchestra.run.smithi076:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-21T10:29:23.602 INFO:teuthology.orchestra.run.smithi076.stdout:2024-09-21T09:34:28.496699+0000 mon.a (mon.0) 562 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:29:23.607 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-21T10:29:23.607 INFO:tasks.cephadm:Compressing logs... 2024-09-21T10:29:23.607 DEBUG:teuthology.orchestra.run.smithi076:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:29:23.610 DEBUG:teuthology.orchestra.run.smithi191:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:29:23.634 INFO:teuthology.orchestra.run.smithi191.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi191.dhrpzx.log 2024-09-21T10:29:23.635 INFO:teuthology.orchestra.run.smithi191.stderr:: No such file or directory 2024-09-21T10:29:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T10:29:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:29:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi191.dhrpzx.log: 83.6% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi191.dhrpzx.log.gz 2024-09-21T10:29:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T10:29:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:29:23.636 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:29:23.638 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.4.log: /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T10:29:23.638 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T10:29:23.638 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:29:23.639 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mgr.x.log 2024-09-21T10:29:23.639 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.b.log 2024-09-21T10:29:23.639 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:29:23.647 INFO:teuthology.orchestra.run.smithi076.stderr:find: ‘/var/log/rbd-target-api’gzip -5: No such file or directory --verbose -- 2024-09-21T10:29:23.647 INFO:teuthology.orchestra.run.smithi076.stderr: /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T10:29:23.647 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi076.jymebx.log 2024-09-21T10:29:23.648 INFO:teuthology.orchestra.run.smithi076.stderr:gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T10:29:23.648 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.0.log: /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi076.jymebx.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:29:23.648 INFO:teuthology.orchestra.run.smithi076.stderr: 85.8% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mds.foo.smithi076.jymebx.log.gz 2024-09-21T10:29:23.648 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.c.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.a.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mgr.y.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:29:23.651 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T10:29:23.652 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:29:23.662 INFO:teuthology.orchestra.run.smithi076.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.7% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:29:23.859 INFO:teuthology.orchestra.run.smithi191.stderr:/var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.b.log: /var/log/ceph/cephadm.log: 85.8% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:29:23.938 INFO:teuthology.orchestra.run.smithi191.stderr: 87.0% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:29:23.961 INFO:teuthology.orchestra.run.smithi191.stderr: 91.3% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:29:24.011 INFO:teuthology.orchestra.run.smithi191.stderr: 90.5% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:29:24.062 INFO:teuthology.orchestra.run.smithi191.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:29:24.176 INFO:teuthology.orchestra.run.smithi076.stderr: 94.3% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:29:24.252 INFO:teuthology.orchestra.run.smithi076.stderr: 92.0% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:29:24.367 INFO:teuthology.orchestra.run.smithi076.stderr: 94.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:29:24.389 INFO:teuthology.orchestra.run.smithi191.stderr: 89.9% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mgr.x.log.gz 2024-09-21T10:29:24.483 INFO:teuthology.orchestra.run.smithi076.stderr: 91.1% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:29:26.071 INFO:teuthology.orchestra.run.smithi076.stderr: 89.7% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mgr.y.log.gz 2024-09-21T10:29:38.675 INFO:teuthology.orchestra.run.smithi191.stderr: 92.3% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.b.log.gz 2024-09-21T10:29:45.013 INFO:teuthology.orchestra.run.smithi076.stderr: 92.1% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.c.log.gz 2024-09-21T10:29:59.732 INFO:teuthology.orchestra.run.smithi076.stderr: 90.4% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-mon.a.log.gz 2024-09-21T10:30:34.021 INFO:teuthology.orchestra.run.smithi191.stderr: 93.4% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T10:31:30.771 INFO:teuthology.orchestra.run.smithi191.stderr: 93.1% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T10:34:19.046 INFO:teuthology.orchestra.run.smithi191.stderr: 94.0% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T10:35:06.861 INFO:teuthology.orchestra.run.smithi191.stderr: 93.6% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T10:35:06.863 INFO:teuthology.orchestra.run.smithi191.stderr: 2024-09-21T10:35:06.863 INFO:teuthology.orchestra.run.smithi191.stderr:real 5m43.238s 2024-09-21T10:35:06.863 INFO:teuthology.orchestra.run.smithi191.stderr:user 4m57.210s 2024-09-21T10:35:06.863 INFO:teuthology.orchestra.run.smithi191.stderr:sys 0m12.921s 2024-09-21T10:36:56.452 INFO:teuthology.orchestra.run.smithi076.stderr: 92.6% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T10:40:33.863 INFO:teuthology.orchestra.run.smithi076.stderr: 94.1% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T10:49:14.841 INFO:teuthology.orchestra.run.smithi076.stderr: 92.9% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T10:50:28.471 INFO:teuthology.orchestra.run.smithi076.stderr: 92.5% -- replaced with /var/log/ceph/e97bd93e-77fb-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T10:50:28.473 INFO:teuthology.orchestra.run.smithi076.stderr: 2024-09-21T10:50:28.473 INFO:teuthology.orchestra.run.smithi076.stderr:real 21m4.850s 2024-09-21T10:50:28.473 INFO:teuthology.orchestra.run.smithi076.stderr:user 9m46.433s 2024-09-21T10:50:28.474 INFO:teuthology.orchestra.run.smithi076.stderr:sys 0m28.023s 2024-09-21T10:50:28.474 INFO:tasks.cephadm:Archiving logs... 2024-09-21T10:50:28.474 DEBUG:teuthology.misc:Transferring archived files from smithi076:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646/remote/smithi076/log 2024-09-21T10:50:28.475 DEBUG:teuthology.orchestra.run.smithi076:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:51:57.137 DEBUG:teuthology.misc:Transferring archived files from smithi191:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646/remote/smithi191/log 2024-09-21T10:51:57.138 DEBUG:teuthology.orchestra.run.smithi191:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:52:39.207 INFO:tasks.cephadm:Removing cluster... 2024-09-21T10:52:39.207 DEBUG:teuthology.orchestra.run.smithi076:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 --force 2024-09-21T10:52:39.395 INFO:teuthology.orchestra.run.smithi076.stdout:Deleting cluster with fsid: e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T10:52:40.183 DEBUG:teuthology.orchestra.run.smithi191:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e97bd93e-77fb-11ef-baf6-efdc52797490 --force 2024-09-21T10:52:40.335 INFO:teuthology.orchestra.run.smithi191.stdout:Deleting cluster with fsid: e97bd93e-77fb-11ef-baf6-efdc52797490 2024-09-21T10:52:40.959 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T10:52:40.960 DEBUG:teuthology.orchestra.run.smithi076:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:52:40.978 DEBUG:teuthology.orchestra.run.smithi191:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:52:40.994 INFO:tasks.cephadm:Teardown complete 2024-09-21T10:52:40.994 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T10:52:41.010 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T10:52:41.010 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:52:41.021 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:52:41.113 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T10:52:41.121 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T10:52:41.122 DEBUG:teuthology.orchestra.run.smithi076:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:52:41.125 DEBUG:teuthology.orchestra.run.smithi191:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:52:41.139 INFO:teuthology.orchestra.run.smithi191.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:52:41.139 INFO:teuthology.orchestra.run.smithi076.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:52:41.156 INFO:teuthology.orchestra.run.smithi191.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:52:41.156 INFO:teuthology.orchestra.run.smithi191.stdout:=============================================================================== 2024-09-21T10:52:41.156 INFO:teuthology.orchestra.run.smithi191.stdout:^* hv01.front.sepia.ceph.com 3 6 377 51 -174us[ -180us] +/- 49ms 2024-09-21T10:52:41.156 INFO:teuthology.orchestra.run.smithi191.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 52 +226us[ +220us] +/- 58ms 2024-09-21T10:52:41.157 INFO:teuthology.orchestra.run.smithi191.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 -32us[ -37us] +/- 60ms 2024-09-21T10:52:41.157 INFO:teuthology.orchestra.run.smithi191.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:52:41.165 INFO:teuthology.orchestra.run.smithi076.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:52:41.165 INFO:teuthology.orchestra.run.smithi076.stdout:=============================================================================== 2024-09-21T10:52:41.165 INFO:teuthology.orchestra.run.smithi076.stdout:^* hv01.front.sepia.ceph.com 3 6 377 43 -159us[ -157us] +/- 49ms 2024-09-21T10:52:41.166 INFO:teuthology.orchestra.run.smithi076.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 49 +223us[ +225us] +/- 58ms 2024-09-21T10:52:41.166 INFO:teuthology.orchestra.run.smithi076.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 49 -15us[ -13us] +/- 60ms 2024-09-21T10:52:41.166 INFO:teuthology.orchestra.run.smithi076.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:52:41.167 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T10:52:41.176 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T10:52:41.176 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T10:52:41.189 DEBUG:teuthology.orchestra.run.smithi076:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:52:41.314 DEBUG:teuthology.orchestra.run.smithi191:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:52:41.441 DEBUG:teuthology.orchestra.run.smithi076:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:52:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:52:41.472 DEBUG:teuthology.orchestra.run.smithi191:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:52:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:52:41.501 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T10:52:41.513 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T10:52:41.522 INFO:teuthology.task.internal:Duration was 5389.101677 seconds 2024-09-21T10:52:41.523 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T10:52:41.531 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T10:52:41.531 DEBUG:teuthology.orchestra.run.smithi076:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:52:41.534 DEBUG:teuthology.orchestra.run.smithi191:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:52:41.593 INFO:teuthology.orchestra.run.smithi191.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:52:41.597 INFO:teuthology.orchestra.run.smithi076.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:52:41.947 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T10:52:41.947 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi076.front.sepia.ceph.com 2024-09-21T10:52:41.947 DEBUG:teuthology.orchestra.run.smithi076:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:52:41.978 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi191.front.sepia.ceph.com 2024-09-21T10:52:41.978 DEBUG:teuthology.orchestra.run.smithi191:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:52:42.004 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T10:52:42.005 DEBUG:teuthology.orchestra.run.smithi076:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:52:42.020 DEBUG:teuthology.orchestra.run.smithi191:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:52:42.071 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T10:52:42.071 DEBUG:teuthology.orchestra.run.smithi076:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:52:42.089 DEBUG:teuthology.orchestra.run.smithi191:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:52:43.850 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T10:52:43.873 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T10:52:43.873 DEBUG:teuthology.orchestra.run.smithi076:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:52:43.908 DEBUG:teuthology.orchestra.run.smithi191:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:52:43.938 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T10:52:43.950 DEBUG:teuthology.orchestra.run.smithi076:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:52:43.953 DEBUG:teuthology.orchestra.run.smithi191:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:52:43.984 INFO:teuthology.orchestra.run.smithi076.stdout:kernel.core_pattern = core 2024-09-21T10:52:44.019 INFO:teuthology.orchestra.run.smithi191.stdout:kernel.core_pattern = core 2024-09-21T10:52:44.067 DEBUG:teuthology.orchestra.run.smithi076:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:52:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:52:44.083 DEBUG:teuthology.orchestra.run.smithi191:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:52:44.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:52:44.126 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T10:52:44.139 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T10:52:44.140 DEBUG:teuthology.misc:Transferring archived files from smithi076:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646/remote/smithi076 2024-09-21T10:52:44.140 DEBUG:teuthology.orchestra.run.smithi076:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:52:44.211 DEBUG:teuthology.misc:Transferring archived files from smithi191:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913646/remote/smithi191 2024-09-21T10:52:44.211 DEBUG:teuthology.orchestra.run.smithi191:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:52:44.258 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T10:52:44.258 DEBUG:teuthology.orchestra.run.smithi076:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:52:44.260 DEBUG:teuthology.orchestra.run.smithi191:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:52:44.307 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T10:52:44.318 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T10:52:44.318 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T10:52:44.329 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T10:52:44.329 DEBUG:teuthology.orchestra.run.smithi076:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:52:44.331 DEBUG:teuthology.orchestra.run.smithi191:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:52:44.354 INFO:teuthology.orchestra.run.smithi076.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:52 /home/ubuntu/cephtest 2024-09-21T10:52:44.354 INFO:teuthology.orchestra.run.smithi076.stdout: 545827 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:24 /home/ubuntu/cephtest/mnt.0 2024-09-21T10:52:44.354 INFO:teuthology.orchestra.run.smithi076.stdout: 545829 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:25 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T10:52:44.354 INFO:teuthology.orchestra.run.smithi076.stdout: 660886 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:25 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:52:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:52:44.356 INFO:teuthology.orchestra.run.smithi076.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-21T10:52:44.356 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi076 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:52:44.362 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T10:52:44.373 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T10:52:44.383 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi076 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:52:44.387 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 5389.101677179337 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi076 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=0f8b84bb8a9c42d2bc542cfe2114c311 status: fail success: false 2024-09-21T10:52:44.387 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T10:52:44.394 INFO:teuthology.orchestra.run.smithi191.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:52 /home/ubuntu/cephtest 2024-09-21T10:52:44.479 INFO:teuthology.run:FAIL